convex-effect-workflows 0.1.0

package/src/component/logs.ts

@@ -0,0 +1,152 @@
+import { v } from "convex/values";
+import { mutation, query } from "./_generated/server.js";
+import { vLogLevel, vLogSource } from "../shared/validators.js";
+import { nowTs } from "./utils.js";
+import { persistBoundedPayload } from "./payloads.js";
+import { MAX_LOG_DATA_BYTES } from "../shared/constants.js";
+
+export const appendLog = mutation({
+  args: {
+    executionId: v.string(),
+    traceId: v.optional(v.string()),
+    spanId: v.optional(v.string()),
+    stepNumber: v.optional(v.number()),
+    level: vLogLevel,
+    message: v.string(),
+    source: vLogSource,
+    data: v.optional(v.any()),
+    timestamp: v.optional(v.number()),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const dataPayload =
+      args.data === undefined
+        ? undefined
+        : await persistBoundedPayload(
+            ctx,
+            "logData",
+            args.data,
+            MAX_LOG_DATA_BYTES,
+          );
+
+    await ctx.db.insert("logs", {
+      executionId: args.executionId,
+      traceId: args.traceId,
+      spanId: args.spanId,
+      stepNumber: args.stepNumber,
+      level: args.level,
+      message: args.message,
+      source: args.source,
+      data: dataPayload?.preview,
+      timestamp: args.timestamp ?? nowTs(),
+    });
+    return null;
+  },
+});
+
+export const listExecutionLogs = query({
+  args: {
+    executionId: v.string(),
+    limit: v.number(),
+    afterTimestamp: v.optional(v.number()),
+    level: v.optional(vLogLevel),
+    source: v.optional(vLogSource),
+    spanId: v.optional(v.string()),
+  },
+  returns: v.any(),
+  handler: async (ctx, args) => {
+    const limit = Math.max(1, Math.min(args.limit, 500));
+
+    let rows = await ctx.db
+      .query("logs")
+      .withIndex("by_execution_time", (q) => q.eq("executionId", args.executionId))
+      .take(limit + 500);
+
+    if (args.afterTimestamp !== undefined) {
+      rows = rows.filter((row) => row.timestamp > args.afterTimestamp!);
+    }
+    if (args.level) {
+      rows = rows.filter((row) => row.level === args.level);
+    }
+    if (args.source) {
+      rows = rows.filter((row) => row.source === args.source);
+    }
+    if (args.spanId) {
+      rows = rows.filter((row) => row.spanId === args.spanId);
+    }
+
+    rows.sort((a, b) => a.timestamp - b.timestamp);
+    const page = rows.slice(0, limit);
+    const nextAfterTimestamp =
+      rows.length > limit ? page[page.length - 1]?.timestamp : undefined;
+
+    return { page, nextAfterTimestamp };
+  },
+});
+
+export const listSpanLogs = query({
+  args: {
+    spanId: v.string(),
+    limit: v.number(),
+    afterTimestamp: v.optional(v.number()),
+  },
+  returns: v.any(),
+  handler: async (ctx, args) => {
+    const limit = Math.max(1, Math.min(args.limit, 500));
+    let rows = await ctx.db
+      .query("logs")
+      .withIndex("by_span_time", (q) => q.eq("spanId", args.spanId))
+      .take(limit + 200);
+
+    if (args.afterTimestamp !== undefined) {
+      rows = rows.filter((row) => row.timestamp > args.afterTimestamp!);
+    }
+
+    rows.sort((a, b) => a.timestamp - b.timestamp);
+    const page = rows.slice(0, limit);
+    const nextAfterTimestamp =
+      rows.length > limit ? page[page.length - 1]?.timestamp : undefined;
+
+    return { page, nextAfterTimestamp };
+  },
+});
+
+/**
+ * Test helper for creating a simple log entry.
+ */
+export const createLog = mutation({
+  args: {
+    executionId: v.string(),
+    traceId: v.optional(v.string()),
+    level: v.union(
+      v.literal("debug"),
+      v.literal("info"),
+      v.literal("warn"),
+      v.literal("error"),
+    ),
+    message: v.string(),
+    source: v.union(
+      v.literal("workflow"),
+      v.literal("activity"),
+      v.literal("engine"),
+      v.literal("system"),
+    ),
+    timestamp: v.number(),
+    spanId: v.optional(v.string()),
+    stepNumber: v.optional(v.number()),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    await ctx.db.insert("logs", {
+      executionId: args.executionId,
+      traceId: args.traceId,
+      spanId: args.spanId,
+      stepNumber: args.stepNumber,
+      level: args.level,
+      message: args.message,
+      source: args.source,
+      timestamp: args.timestamp,
+    });
+    return null;
+  },
+});

package/src/component/onComplete.ts

@@ -0,0 +1,170 @@
+import { vResult } from "@convex-dev/workpool";
+import { v } from "convex/values";
+import { mutation } from "./_generated/server.js";
+import { scheduleRunnerFanIn } from "./boundaries.js";
+import { nowTs } from "./utils.js";
+
+const vCompletionContext = v.object({
+  executionId: v.string(),
+  stepNumber: v.number(),
+  spanId: v.optional(v.string()),
+  generation: v.number(),
+});
+
+/**
+ * Workpool onComplete handler for activity attempts.
+ *
+ * Invariants:
+ * - idempotent by workId (activityCompletions table)
+ * - stale generation callbacks are ignored
+ * - step and span completion are terminal/idempotent
+ */
+export const handleActivityCompletion = mutation({
+  args: {
+    workId: v.string(),
+    context: vCompletionContext,
+    result: vResult,
+  },
+  returns: v.object({
+    processed: v.boolean(),
+    resumed: v.boolean(),
+    stale: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    const completion = await ctx.db
+      .query("activityCompletions")
+      .withIndex("by_work_id", (q) => q.eq("workId", args.workId))
+      .unique();
+    if (completion) {
+      return { processed: false, resumed: false, stale: false };
+    }
+
+    await ctx.db.insert("activityCompletions", {
+      workId: args.workId,
+      executionId: args.context.executionId,
+      stepNumber: args.context.stepNumber,
+      processedAt: nowTs(),
+    });
+
+    const execution = await ctx.db
+      .query("executions")
+      .withIndex("by_execution_id", (q) =>
+        q.eq("executionId", args.context.executionId),
+      )
+      .unique();
+    if (!execution) {
+      throw new Error(`Execution not found: ${args.context.executionId}`);
+    }
+
+    if (execution.generation !== args.context.generation) {
+      await ctx.db.insert("logs", {
+        executionId: execution.executionId,
+        traceId: execution.traceId,
+        spanId: args.context.spanId,
+        stepNumber: args.context.stepNumber,
+        level: "warn",
+        source: "engine",
+        message: "Ignoring stale activity completion due to generation mismatch",
+        data: {
+          callbackGeneration: args.context.generation,
+          executionGeneration: execution.generation,
+        },
+        timestamp: nowTs(),
+      });
+      return { processed: true, resumed: false, stale: true };
+    }
+
+    const step = await ctx.db
+      .query("journalSteps")
+      .withIndex("by_execution_step", (q) =>
+        q
+          .eq("executionId", args.context.executionId)
+          .eq("stepNumber", args.context.stepNumber),
+      )
+      .unique();
+    if (!step) {
+      throw new Error(
+        `Journal step not found: ${args.context.executionId}/${args.context.stepNumber}`,
+      );
+    }
+
+    if (step.state !== "completed" && step.state !== "failed" && step.state !== "canceled") {
+      await ctx.db.patch(step._id, {
+        state:
+          args.result.kind === "success"
+            ? "completed"
+            : args.result.kind === "failed"
+              ? "failed"
+              : "canceled",
+        runResult:
+          args.result.kind === "success"
+            ? {
+                kind: "success",
+                valuePreview: args.result.returnValue,
+              }
+            : args.result.kind === "failed"
+              ? {
+                  kind: "failed",
+                  error: args.result.error,
+                }
+              : {
+                  kind: "canceled",
+                },
+        completedAt: nowTs(),
+      });
+    }
+
+    if (args.context.spanId) {
+      const span = await ctx.db
+        .query("spans")
+        .withIndex("by_execution_span", (q) =>
+          q
+            .eq("executionId", args.context.executionId)
+            .eq("spanId", args.context.spanId!),
+        )
+        .unique();
+      if (span && span.status !== "ended") {
+        await ctx.db.patch(span._id, {
+          status: "ended",
+          endTime: nowTs(),
+          outputPreview:
+            args.result.kind === "success" ? args.result.returnValue : undefined,
+          error: args.result.kind === "failed" ? args.result.error : undefined,
+        });
+      }
+    }
+
+    await ctx.db.insert("logs", {
+      executionId: args.context.executionId,
+      traceId: execution.traceId,
+      spanId: args.context.spanId,
+      stepNumber: args.context.stepNumber,
+      level: args.result.kind === "failed" ? "error" : "info",
+      source: "activity",
+      message:
+        args.result.kind === "success"
+          ? "Activity completed"
+          : args.result.kind === "failed"
+            ? "Activity failed"
+            : "Activity canceled",
+      data: args.result,
+      timestamp: nowTs(),
+    });
+
+    let resumed = false;
+    if (
+      execution.runnerHandle &&
+      execution.status !== "interrupted" &&
+      execution.status !== "failed" &&
+      execution.status !== "completed"
+    ) {
+      await scheduleRunnerFanIn(ctx.scheduler, {
+        executionId: args.context.executionId,
+        generation: execution.generation,
+      });
+      resumed = true;
+    }
+
+    return { processed: true, resumed, stale: false };
+  },
+});

package/src/component/payloads.ts

@@ -0,0 +1,83 @@
+import { v } from "convex/values";
+import { mutation, query, type MutationCtx, type QueryCtx } from "./_generated/server.js";
+import {
+  MAX_INLINE_PAYLOAD_BYTES,
+  MAX_PREVIEW_STRING_BYTES,
+} from "../shared/constants.js";
+import { valueSize, toPreview, nowTs } from "./utils.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { vPayloadKind } from "../shared/validators.js";
+
+type PayloadRef = {
+  size: number;
+  ref: Id<"payloads"> | undefined;
+  preview: unknown;
+};
+
+type DbReaderCtx = Pick<QueryCtx, "db">;
+
+export async function persistBoundedPayload(
+  ctx: MutationCtx,
+  kind: Doc<"payloads">["kind"],
+  value: unknown,
+  inlineLimit = MAX_INLINE_PAYLOAD_BYTES,
+): Promise<PayloadRef> {
+  const size = valueSize(value);
+  if (size <= inlineLimit) {
+    return { size, ref: undefined, preview: value };
+  }
+  const ref = await ctx.db.insert("payloads", {
+    kind,
+    data: value,
+    size,
+    createdAt: nowTs(),
+  });
+  return {
+    size,
+    ref,
+    preview: toPreview(value, MAX_PREVIEW_STRING_BYTES),
+  };
+}
+
+export async function hydratePayload<T>(
+  ctx: DbReaderCtx,
+  ref: Id<"payloads"> | undefined,
+  preview: T,
+): Promise<T> {
+  if (!ref) {
+    return preview;
+  }
+
+  const payload = await ctx.db.get(ref);
+  if (!payload || !("data" in payload)) {
+    return preview;
+  }
+
+  // Boundary note: payload rows store opaque user values. This helper trusts
+  // caller context for T and falls back to preview when offloaded data is missing.
+  return payload.data as T;
+}
+
+export const getPayload = query({
+  args: { payloadId: v.id("payloads") },
+  returns: v.any(),
+  handler: async (ctx, { payloadId }) => {
+    return await ctx.db.get(payloadId);
+  },
+});
+
+export const createPayload = mutation({
+  args: {
+    kind: vPayloadKind,
+    data: v.any(),
+  },
+  returns: v.id("payloads"),
+  handler: async (ctx, { kind, data }) => {
+    return await ctx.db.insert("payloads", {
+      kind,
+      data,
+      size: valueSize(data),
+      createdAt: nowTs(),
+    });
+  },
+});

package/src/component/queries.ts

@@ -0,0 +1,8 @@
+export {
+  listExecutions,
+  getExecution,
+  listExecutionSteps,
+  listExecutionSpans,
+  listExecutionLogs,
+  executionMetrics,
+} from "./dashboard.js";

package/src/component/runner.ts

@@ -0,0 +1,122 @@
+import { Workpool, vLogLevel, vRetryBehavior } from "@convex-dev/workpool";
+import type { ComponentApi as WorkpoolComponentApi } from "@convex-dev/workpool/_generated/component.js";
+import { componentsGeneric, makeFunctionReference, type FunctionHandle } from "convex/server";
+import { v } from "convex/values";
+import { mutation } from "./_generated/server.js";
+import { deserializeRunnerHandle } from "./boundaries.js";
+
+/**
+ * Fan-in runner trigger for workflow resumption.
+ *
+ * This mutation centralizes generation/terminal guards and schedules
+ * the host-provided runner handle stored on the execution row.
+ */
+const components = componentsGeneric() as unknown as {
+  workpool: WorkpoolComponentApi<"workpool">;
+};
+
+const vWorkpoolOptions = v.optional(
+  v.object({
+    maxParallelism: v.optional(v.number()),
+    logLevel: v.optional(vLogLevel),
+    defaultRetryBehavior: v.optional(vRetryBehavior),
+    retryActionsByDefault: v.optional(v.boolean()),
+  }),
+);
+
+export const run = mutation({
+  args: {
+    executionId: v.string(),
+    generation: v.number(),
+  },
+  returns: v.object({
+    enqueued: v.boolean(),
+    reason: v.optional(v.string()),
+  }),
+  handler: async (ctx, args) => {
+    const execution = await ctx.db
+      .query("executions")
+      .withIndex("by_execution_id", (q) => q.eq("executionId", args.executionId))
+      .unique();
+
+    if (!execution) {
+      return { enqueued: false, reason: "execution_not_found" };
+    }
+
+    if (args.generation < execution.generation) {
+      return { enqueued: false, reason: "stale_generation" };
+    }
+
+    if (
+      execution.status === "completed" ||
+      execution.status === "failed" ||
+      execution.status === "interrupted"
+    ) {
+      return { enqueued: false, reason: "terminal_execution" };
+    }
+
+    if (!execution.runnerHandle) {
+      return { enqueued: false, reason: "runner_handle_missing" };
+    }
+
+    await ctx.scheduler.runAfter(
+      0,
+      deserializeRunnerHandle(execution.runnerHandle),
+      {
+        executionId: execution.executionId,
+        generation: execution.generation,
+      },
+    );
+
+    return { enqueued: true };
+  },
+});
+
+export const enqueueActivity = mutation({
+  args: {
+    executionId: v.string(),
+    generation: v.number(),
+    workflowName: v.string(),
+    activityName: v.string(),
+    attempt: v.number(),
+    stepNumber: v.number(),
+    spanId: v.optional(v.string()),
+    activityWorkerHandle: v.string(),
+    workpoolOptions: vWorkpoolOptions,
+  },
+  returns: v.string(),
+  handler: async (ctx, args) => {
+    const workpool = new Workpool(components.workpool, args.workpoolOptions ?? {});
+    const activityHandle = args.activityWorkerHandle as FunctionHandle<
+      "action",
+      {
+        workflowName: string;
+        activityName: string;
+        executionId: string;
+        attempt: number;
+      },
+      unknown
+    >;
+
+    return await workpool.enqueueAction(
+      ctx,
+      activityHandle,
+      {
+        workflowName: args.workflowName,
+        activityName: args.activityName,
+        executionId: args.executionId,
+        attempt: args.attempt,
+      },
+      {
+        retry: false,
+        onComplete: makeFunctionReference("onComplete:handleActivityCompletion"),
+        context: {
+          executionId: args.executionId,
+          stepNumber: args.stepNumber,
+          spanId: args.spanId,
+          generation: args.generation,
+        },
+      },
+    );
+  },
+});

package/src/component/schema.ts

@@ -0,0 +1,155 @@
+import { defineSchema, defineTable } from "convex/server";
+import { v } from "convex/values";
+import {
+  vDeterminismSignature,
+  vExecutionStatus,
+  vLogLevel,
+  vLogSource,
+  vPayloadKind,
+  vResultEnvelope,
+  vSpanKind,
+  vSpanStatus,
+  vStepKind,
+  vStepState,
+} from "../shared/validators.js";
+
+export default defineSchema({
+  executions: defineTable({
+    workflowName: v.string(),
+    executionId: v.string(),
+    status: vExecutionStatus,
+    generation: v.number(),
+
+    tenantId: v.optional(v.string()),
+    parentExecutionId: v.optional(v.string()),
+
+    payloadRef: v.optional(v.id("payloads")),
+    payloadPreview: v.optional(v.any()),
+    payloadSize: v.number(),
+
+    resultKind: v.optional(v.union(v.literal("success"), v.literal("failure"))),
+    resultRef: v.optional(v.id("payloads")),
+    resultPreview: v.optional(v.any()),
+    error: v.optional(v.string()),
+
+    traceId: v.string(),
+    runnerHandle: v.optional(v.string()),
+    startedAt: v.number(),
+    completedAt: v.optional(v.number()),
+    lastResumedAt: v.optional(v.number()),
+    lastSuspendedAt: v.optional(v.number()),
+  })
+    .index("by_execution_id", ["executionId"])
+    .index("by_workflow_started", ["workflowName", "startedAt"])
+    .index("by_status_started", ["status", "startedAt"])
+    .index("by_started", ["startedAt"])
+    .index("by_tenant_started", ["tenantId", "startedAt"]),
+
+  journalSteps: defineTable({
+    executionId: v.string(),
+    stepNumber: v.number(),
+
+    kind: vStepKind,
+    name: v.string(),
+    signature: vDeterminismSignature,
+    state: vStepState,
+
+    attempt: v.optional(v.number()),
+    workId: v.optional(v.string()),
+    spanId: v.optional(v.string()),
+
+    inputRef: v.optional(v.id("payloads")),
+    inputPreview: v.optional(v.any()),
+    inputSize: v.number(),
+
+    runResult: v.optional(vResultEnvelope),
+
+    startedAt: v.number(),
+    completedAt: v.optional(v.number()),
+    resumeToken: v.optional(v.string()),
+  })
+    .index("by_execution_step", ["executionId", "stepNumber"])
+    .index("by_execution_state", ["executionId", "state"])
+    .index("by_work_id", ["workId"]),
+
+  activityCompletions: defineTable({
+    workId: v.string(),
+    executionId: v.string(),
+    stepNumber: v.number(),
+    processedAt: v.number(),
+  }).index("by_work_id", ["workId"]),
+
+  spans: defineTable({
+    executionId: v.string(),
+    traceId: v.string(),
+    spanId: v.string(),
+    parentSpanId: v.optional(v.string()),
+
+    stepNumber: v.optional(v.number()),
+    name: v.string(),
+    kind: vSpanKind,
+    status: vSpanStatus,
+
+    startTime: v.number(),
+    endTime: v.optional(v.number()),
+    attempt: v.optional(v.number()),
+
+    attributes: v.optional(v.any()),
+
+    inputRef: v.optional(v.id("payloads")),
+    inputPreview: v.optional(v.any()),
+    inputSize: v.optional(v.number()),
+
+    outputRef: v.optional(v.id("payloads")),
+    outputPreview: v.optional(v.any()),
+    outputSize: v.optional(v.number()),
+
+    error: v.optional(v.string()),
+  })
+    .index("by_execution_start", ["executionId", "startTime"])
+    .index("by_trace_start", ["traceId", "startTime"])
+    .index("by_execution_span", ["executionId", "spanId"]),
+
+  logs: defineTable({
+    executionId: v.string(),
+    traceId: v.optional(v.string()),
+    spanId: v.optional(v.string()),
+    stepNumber: v.optional(v.number()),
+
+    level: vLogLevel,
+    message: v.string(),
+    data: v.optional(v.any()),
+    source: vLogSource,
+
+    timestamp: v.number(),
+  })
+    .index("by_execution_time", ["executionId", "timestamp"])
+    .index("by_span_time", ["spanId", "timestamp"])
+    .index("by_level_time", ["level", "timestamp"]),
+
+  deferreds: defineTable({
+    executionId: v.string(),
+    deferredName: v.string(),
+
+    completed: v.boolean(),
+    exitRef: v.optional(v.id("payloads")),
+    exitPreview: v.optional(v.any()),
+    completedAt: v.optional(v.number()),
+  }).index("by_execution_name", ["executionId", "deferredName"]),
+
+  payloads: defineTable({
+    kind: vPayloadKind,
+    data: v.any(),
+    size: v.number(),
+    createdAt: v.number(),
+  }).index("by_created", ["createdAt"]),
+
+  config: defineTable({
+    key: v.string(),
+    maxInlineBytes: v.optional(v.number()),
+    maxLogDataBytes: v.optional(v.number()),
+    maxSpanAttrBytes: v.optional(v.number()),
+    retentionDays: v.optional(v.number()),
+    defaultLogLevel: v.optional(vLogLevel),
+  }).index("by_key", ["key"]),
+});

package/src/component/setup.test.ts

@@ -0,0 +1,15 @@
+/// <reference types="vite/client" />
+import { convexTest } from "convex-test";
+import workpool from "@convex-dev/workpool/test";
+import schema from "./schema.js";
+
+const modules = import.meta.glob("./**/*.ts");
+
+export function initConvexTest() {
+  const t = convexTest(schema, modules);
+  workpool.register(t);
+  return t;
+}
+
+import { test } from "vitest";
+test("setup", () => {});