create-ailk 0.1.0 → 0.2.0

package/templates/apps/api/src/services/flow-engine.ts

@@ -0,0 +1,1296 @@
+/**
+ * @file flow-engine.ts
+ * @description The linear flow engine service (#3956 D3).
+ *
+ * `serveFlow` reads a validated FlowConfig; `submitStep` is the one mutation
+ * path — it persists every landed step server-side IMMEDIATELY (this write is
+ * the partial-state capture the feature exists for, not a batched or
+ * end-of-flow write), and composes the existing `routeLead` service on
+ * completion. Mirrors `lead-routing.ts`'s house shape (ADR 0009 D3
+ * service-layer pattern): type-result returns, mocked-dep tests, PII-safe
+ * logging.
+ *
+ * `submitStep` itself is a thin orchestrator (#3974 AC1) over four isolated
+ * concerns, each a named helper below: `resolveSession` (session
+ * resolution) → `locateStep` (find the submitted step in the snapshot) →
+ * `extractStepValues` (per-step value validation) → `persistStepUpdate`
+ * (the immediate partial-state write) → `completeSubmission` (completion
+ * orchestration: routeLead + deliverable fulfillment). No single function
+ * does more than one of these jobs.
+ *
+ * `cleanupAbandonedSessions` (#3974 AC3) is the FlowSession retention
+ * policy — see its own doc comment below for the full policy + its
+ * coordination with #3962's abandon-email sweep.
+ *
+ * PII boundary: only `{ siteId, slug, stepKey, sessionId, completed }` appear
+ * in logs — never `values` or the assembled `payload` (mirrors
+ * lead-routing.ts's boundary).
+ */
+
+import type { FlowDefinition } from "@working-theory/database";
+import { Prisma } from "@working-theory/database";
+import { logger } from "@working-theory/observability";
+import {
+  aeoScoreResultSchema,
+  deriveFlags,
+  flowAddressValueSchema,
+  flowConfigSchema,
+  hiddenAnswers,
+  isStepVisible,
+  pruneHiddenState,
+  resolveVisibleSteps,
+  submitFlowStepInputSchema,
+  type AeoScoreResult,
+  type FlowConfig,
+  type FlowFlags,
+  type FlowState,
+  type FlowStep,
+} from "@working-theory/validation";
+import { z } from "zod";
+
+import { prisma } from "../lib/prisma.js";
+
+import { loadExampleScore, resolveScoreBudgetMs, scoreUrl } from "./aeo-score.js";
+import { fulfillDeliverable } from "./deliverable-fulfillment.js";
+import { routeLead, type SourceMeta } from "./lead-routing.js";
+import { scoreCompletedSignup } from "./waitlist-scoring.js";
+
+/**
+ * The exact email-format + length check `leadSubmitSchema` (leads.ts:
+ * `email: z.string().email().max(254)`) will re-apply at completion —
+ * reused here VERBATIM (both the format check and the 254 cap) so an
+ * invalid or over-length email is rejected AT SUBMIT TIME (400
+ * invalid_input) rather than silently failing `routeLead` later.
+ */
+const EMAIL_SHAPE = z.string().email().max(254);
+
+/**
+ * text/tel value cap — matches the short-free-text bound `mcp-tools.ts`
+ * already uses for comparable MCP inputs (`name`: max 200, `phone`: max 50);
+ * 256 covers either with headroom (security-review finding, Medium).
+ */
+const TEXT_VALUE_MAX = 256;
+
+/** textarea value cap — matches `submitLeadInputSchema.message`'s max(2000). */
+const TEXTAREA_VALUE_MAX = 2000;
+
+/**
+ * D5 (#4663) — the exact `url` shape check `flowConfigSchema`'s own field
+ * validation implies.
+ *
+ * Security-review finding (#4663, M3): under the installed zod 4.4.3,
+ * `z.string().url()` restricts NEITHER hostname shape NOR protocol — its
+ * internal `$ZodURL` applies those checks only when `def.hostname`/
+ * `def.protocol` are set, and bare `.url()` sets neither. A `javascript:`,
+ * `data:`, or `file:` value would therefore pass this check and reach
+ * persistence in the lead payload (SSRF itself stays blocked downstream,
+ * at `assertUrlSafe` — this is a separate, narrower gap: an unconstrained
+ * value boundary, not a network-reach one). `z.httpUrl()` is zod's own
+ * preset for exactly this — it fixes `protocol` to the http(s)-only regex
+ * and `hostname` to a domain-shaped pattern.
+ */
+const URL_SHAPE = z.httpUrl().max(2048);
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export type FlowEngineErrorKind =
+  "invalid_input" | "flow_not_found" | "step_conflict" | "flows_not_configured";
+
+export type FlowEngineError = { kind: FlowEngineErrorKind; message: string };
+
+export type ServeFlowResult =
+  { ok: true; config: FlowConfig } | { ok: false; error: FlowEngineError };
+
+export type SubmitStepInput = {
+  siteId: string;
+  slug: string;
+  sessionId?: string;
+  stepKey: string;
+  values: Record<string, unknown>;
+  locale?: string;
+};
+
+export type SubmitStepResult =
+  | {
+      ok: true;
+      sessionId: string;
+      landed: string;
+      nextStepKey: string | null;
+      completed: boolean;
+      leadId?: string;
+      /**
+       * Internal-only signal — completion ran but `routeLead` returned
+       * `ok:false` (the session is still the durable record, per D9's
+       * fallback). The route handler reads this to log a PII-safe warning;
+       * it is NOT part of `submitFlowStepOutputSchema`'s public shape and
+       * must be stripped before the response is sent.
+       */
+      leadRoutingFailed?: true;
+      /**
+       * Server-derived segment (#3959 D3/D6) — present from the
+       * segmentation-step response onward (derived this request, or
+       * previously persisted on the session).
+       */
+      segment?: string;
+      /**
+       * D9 (#4663) — present on the `aeo_score` step's own response AND on
+       * the completion response when the flow declares one (the persisted
+       * value, when this submission isn't the one that computed it).
+       */
+      aeoScore?: AeoScoreResult;
+    }
+  | { ok: false; error: FlowEngineError };
+
+export type UpsertFlowDefinitionResult =
+  { ok: true; id: string } | { ok: false; error: FlowEngineError };
+
+// ─── Generated Prisma Client payload types (#3974 AC2) ─────────────────────
+//
+// Narrowed via `select` to exactly the columns this module reads/writes —
+// replacing the former hand-written `as {...}` result casts (code-review
+// finding on #3956's acceptance PR: a schema/select drift is now caught at
+// compile time instead of silently passing through an untyped assertion).
+//
+// `configSnapshot` / `state` / `config` stay `unknown` (never the generated
+// `Prisma.JsonValue`) — this module never trusts a JSON column's TS shape;
+// every read runs through `flowConfigSchema.safeParse` or an explicit
+// `typeof` guard before use, exactly as before this change.
+
+const FLOW_SESSION_SELECT = {
+  id: true,
+  siteId: true,
+  flowId: true,
+  flowSlug: true,
+  configSnapshot: true,
+  state: true,
+  email: true,
+  segment: true,
+  // D2/D6 (#4663) — the derived flag set + the mid-flow aeo_score result.
+  flags: true,
+  aeoScore: true,
+  leadId: true,
+  completedAt: true,
+} as const satisfies Prisma.FlowSessionSelect;
+
+// Session row shapes read from / written to Prisma — narrowed to what this
+// module touches (the JSON columns are widened back to `unknown` — see the
+// section doc comment above).
+type FlowSessionRow = Omit<
+  Prisma.FlowSessionGetPayload<{ select: typeof FLOW_SESSION_SELECT }>,
+  "configSnapshot" | "state" | "flags" | "aeoScore"
+> & { configSnapshot: unknown; state: unknown; flags: unknown; aeoScore: unknown };
+
+const FLOW_DEFINITION_ID_SELECT = {
+  id: true,
+} as const satisfies Prisma.FlowDefinitionSelect;
+
+type FlowDefinitionIdRow = Prisma.FlowDefinitionGetPayload<{
+  select: typeof FLOW_DEFINITION_ID_SELECT;
+}>;
+
+const FLOW_DEFINITION_FOR_SESSION_SELECT = {
+  id: true,
+  config: true,
+  // #5036 D6d — copied onto the session below, so a submission is attributable
+  // to one listing without re-deriving it from a slug that nothing enforces.
+  listingId: true,
+} as const satisfies Prisma.FlowDefinitionSelect;
+
+type FlowDefinitionForSessionRow = Omit<
+  Prisma.FlowDefinitionGetPayload<{
+    select: typeof FLOW_DEFINITION_FOR_SESSION_SELECT;
+  }>,
+  "config"
+> & { config: unknown };
+
+// `serveFlow`'s own lookup passes no `select` (its call args are asserted
+// exactly by flow-engine.test.ts, so no `select` clause may be added) — typed
+// against the full generated row instead, with `config` widened to
+// `unknown` per this section's rule.
+type FlowDefinitionRow = Omit<FlowDefinition, "config"> & { config: unknown };
+
+// ─── DB-required guard (D5) ─────────────────────────────────────────────────────
+
+const FLOWS_NOT_CONFIGURED_ERROR: FlowEngineError = {
+  kind: "flows_not_configured",
+  message: "Flows are not configured.",
+};
+
+/**
+ * A flow session is read-modify-write across requests — the file fallback
+ * `lead-routing.ts` uses for an append-only Lead write would race and
+ * silently lose partial state here. So flows require a configured DB; the
+ * zero-config OSS path keeps its existing static capture via `/v1/leads/*`.
+ */
+function requireDb(): FlowEngineError | null {
+  if (!process.env.DATABASE_URL) return FLOWS_NOT_CONFIGURED_ERROR;
+  return null;
+}
+
+// ─── serveFlow ────────────────────────────────────────────────────────────────
+
+export async function serveFlow(
+  siteId: string,
+  slug: string,
+): Promise<ServeFlowResult> {
+  const dbError = requireDb();
+  if (dbError) return { ok: false, error: dbError };
+
+  const definition: FlowDefinitionRow | null =
+    await prisma.flowDefinition.findUnique({
+      where: { siteId_slug: { siteId, slug } },
+    });
+
+  if (!definition) {
+    return {
+      ok: false,
+      error: { kind: "flow_not_found", message: "Flow not found." },
+    };
+  }
+
+  const parsed = flowConfigSchema.safeParse(definition.config);
+  if (!parsed.success) {
+    // Defensive — upsertFlowDefinition already validates before write; a
+    // parse failure here means the stored config drifted from the schema.
+    return {
+      ok: false,
+      error: {
+        kind: "invalid_input",
+        message: "Stored flow config failed validation.",
+      },
+    };
+  }
+
+  return { ok: true, config: parsed.data };
+}
+
+// ─── upsertFlowDefinition (D6 — service-level only, no authoring route) ────────
+
+export async function upsertFlowDefinition(
+  siteId: string,
+  config: unknown,
+  /**
+   * The `ProjectListing` this definition is projected from (#5036 D6d), when
+   * it is one. Stamped on the row so every session started on this flow can
+   * carry the same reference, instead of the link being the coincidence that
+   * the flow's slug equals the listing's.
+   */
+  listingId?: string,
+): Promise<UpsertFlowDefinitionResult> {
+  const dbError = requireDb();
+  if (dbError) return { ok: false, error: dbError };
+
+  const parsed = flowConfigSchema.safeParse(config);
+  if (!parsed.success) {
+    return {
+      ok: false,
+      error: {
+        kind: "invalid_input",
+        message: parsed.error.issues[0]?.message ?? "invalid flow config",
+      },
+    };
+  }
+
+  const row: FlowDefinitionIdRow = await prisma.flowDefinition.upsert({
+    where: { siteId_slug: { siteId, slug: parsed.data.slug } },
+    create: {
+      siteId,
+      slug: parsed.data.slug,
+      config: parsed.data as unknown as Prisma.InputJsonValue,
+      ...(listingId !== undefined && { listingId }),
+    },
+    update: {
+      config: parsed.data as unknown as Prisma.InputJsonValue,
+      ...(listingId !== undefined && { listingId }),
+    },
+    select: FLOW_DEFINITION_ID_SELECT,
+  });
+
+  return { ok: true, id: row.id };
+}
+
+// ─── seedFlowPresets (#3958 §3.3 — reuse, do not duplicate) ────────────────────
+
+export type SeedFlowPresetsResult =
+  | { ok: true; seeded: string[] }
+  | { ok: false; error: FlowEngineError; seeded: string[] };
+
+/**
+ * Write the given preset flow configs through the existing validated
+ * `upsertFlowDefinition` path, once per preset, for the given site.
+ *
+ * This is the ONLY write path into `flow_definitions` — reusing
+ * `upsertFlowDefinition` (rather than a second Prisma write here) is what
+ * keeps a malformed preset failing at seed time instead of becoming the
+ * engine's defensive "stored flow config failed validation" branch
+ * (`serveFlow` above) at a customer's first request.
+ *
+ * Stops and returns the first `invalid_input` (or `flows_not_configured`)
+ * failure rather than swallowing it, per §3.3 — a caller must not seed a
+ * partial, broken preset set. `seeded` names the slugs successfully written
+ * before any failure, for diagnostics.
+ */
+export async function seedFlowPresets(
+  siteId: string,
+  presets: unknown[],
+): Promise<SeedFlowPresetsResult> {
+  const seeded: string[] = [];
+  for (const preset of presets) {
+    const result = await upsertFlowDefinition(siteId, preset);
+    if (!result.ok) {
+      return { ok: false, error: result.error, seeded };
+    }
+    const slug =
+      typeof preset === "object" && preset !== null && "slug" in preset
+        ? String((preset as { slug: unknown }).slug)
+        : "(unknown)";
+    seeded.push(slug);
+  }
+  return { ok: true, seeded };
+}
+
+// ─── Per-step value validation (D3 — "required fields, select/segmentation
+//     values ∈ declared options") ───────────────────────────────────────────
+
+/**
+ * Validate + extract one step's submitted `values` against its snapshot
+ * definition. Returns only the declared field/question names present in
+ * `values` — an undeclared key is silently dropped, mirroring the leads
+ * route's `buildInput` explicit-destructure discipline (never let an
+ * unexpected key smuggle into persisted state).
+ */
+function extractStepValues(
+  step: FlowStep,
+  values: Record<string, unknown>,
+):
+  { ok: true; data: Record<string, unknown> } | { ok: false; message: string } {
+  if (step.type === "message" || step.type === "aeo_score" || step.type === "custom") {
+    // Interstitial (message) / server-computed (aeo_score, D6) — nothing to
+    // capture from the submission; any submission just lands the step.
+    //
+    // A `custom` step (#5048) lands the same way from the ENGINE's side: its
+    // values are shaped by a renderer this server has no definition for, so
+    // there is nothing here to validate them against. A served flow does not
+    // carry one today — the founder application is client-configured and
+    // never a `FlowSession` — and this branch is what keeps that true by
+    // construction rather than by a fallthrough into the segmentation reader.
+    return { ok: true, data: {} };
+  }
+
+  if (step.type === "field") {
+    const data: Record<string, unknown> = {};
+    for (const field of step.fields) {
+      const raw = values[field.name];
+      const present = raw !== undefined && raw !== null && raw !== "";
+      if (field.required && !present) {
+        return {
+          ok: false,
+          message: `missing required field "${field.name}"`,
+        };
+      }
+      if (!present) continue;
+      // #5279 — an `address` field submits ONE nested parts object under its
+      // own name (six separable values, never a concatenated line), so it is
+      // checked against that shape rather than against any of the string
+      // branches below. `raw` is `unknown` here (boundedStepValues), so this
+      // is the same "type it before it reaches JSONB" discipline the text/tel
+      // cap below documents: an anonymous POST cannot park an arbitrary
+      // object in `flow_sessions.state` by naming an address field.
+      if (field.inputType === "address") {
+        if (!flowAddressValueSchema.safeParse(raw).success) {
+          return {
+            ok: false,
+            message: `field "${field.name}" must be a structured address`,
+          };
+        }
+        data[field.name] = raw;
+        continue;
+      }
+      if (field.inputType === "select") {
+        const allowed = new Set(field.options?.map((o) => o.value) ?? []);
+        if (typeof raw !== "string" || !allowed.has(raw)) {
+          return {
+            ok: false,
+            message: `field "${field.name}" is not one of the declared options`,
+          };
+        }
+      }
+      // A field that becomes session.email (mapsTo:'email') OR is declared
+      // inputType:'email' must actually validate as one — otherwise it
+      // denormalizes onto session.email and later fails routeLead's strict
+      // leadSubmitSchema at completion, where the failure is swallowed (a
+      // silent lost lead, code-review finding on #3956's acceptance verdict).
+      if (
+        (field.inputType === "email" || field.mapsTo === "email") &&
+        (typeof raw !== "string" || !EMAIL_SHAPE.safeParse(raw).success)
+      ) {
+        return {
+          ok: false,
+          message: `field "${field.name}" must be a valid email address`,
+        };
+      }
+      // text/tel/textarea carry no format constraint, but `raw` is
+      // `unknown` (boundedStepValues), so left unchecked an anonymous
+      // POST could smuggle an arbitrarily large string — or a non-string
+      // (nested object/array) — into `flow_sessions.state` JSONB. Bound
+      // by type + length here (security-review finding, Medium: unbounded
+      // + untyped text-field values), mirroring the length caps
+      // ./mcp-tools.ts already uses for comparable free-text MCP inputs
+      // (`name`/`phone` at 200/50, `message` at 2000).
+      if (
+        (field.inputType === "text" || field.inputType === "tel") &&
+        (typeof raw !== "string" || raw.length > TEXT_VALUE_MAX)
+      ) {
+        return {
+          ok: false,
+          message: `field "${field.name}" must be a string of at most ${TEXT_VALUE_MAX} characters`,
+        };
+      }
+      if (
+        field.inputType === "textarea" &&
+        (typeof raw !== "string" || raw.length > TEXTAREA_VALUE_MAX)
+      ) {
+        return {
+          ok: false,
+          message: `field "${field.name}" must be a string of at most ${TEXTAREA_VALUE_MAX} characters`,
+        };
+      }
+      // D5 (#4663) — a `url` field must satisfy the same shape check the
+      // config schema implies (max 2048); the client (StepFlow's
+      // HAS_SCHEME_RE idiom) prefixes a bare-domain value with `https://`
+      // before it ever reaches here, so a scheme-less value is rejected.
+      if (
+        field.inputType === "url" &&
+        (typeof raw !== "string" || !URL_SHAPE.safeParse(raw).success)
+      ) {
+        return {
+          ok: false,
+          message: `field "${field.name}" must be a valid URL`,
+        };
+      }
+      // D5 (#4663) — text/textarea minLength (declared on text/textarea only
+      // — flowConfigSchema's own cross-field check).
+      if (
+        field.minLength !== undefined &&
+        typeof raw === "string" &&
+        raw.length < field.minLength
+      ) {
+        return {
+          ok: false,
+          message: `field "${field.name}" must be at least ${field.minLength} characters`,
+        };
+      }
+      data[field.name] = raw;
+    }
+    return { ok: true, data };
+  }
+
+  // step.type === "segmentation"
+  const data: Record<string, unknown> = {};
+  for (const question of step.questions) {
+    const raw = values[question.name];
+    const allowed = new Set(question.options.map((o) => o.value));
+
+    if (question.multi) {
+      // D4 (#4663) — a non-empty array, ≤ options.length, every element ∈
+      // options, no duplicates, and an exclusive value only as the sole
+      // element (§ D4).
+      if (
+        !Array.isArray(raw) ||
+        raw.length === 0 ||
+        raw.length > question.options.length
+      ) {
+        return {
+          ok: false,
+          message: `question "${question.name}" requires a non-empty array of declared options`,
+        };
+      }
+      const seen = new Set<string>();
+      for (const v of raw) {
+        if (typeof v !== "string" || !allowed.has(v)) {
+          return {
+            ok: false,
+            message: `question "${question.name}" contains an option value that is not declared`,
+          };
+        }
+        if (seen.has(v)) {
+          return {
+            ok: false,
+            message: `question "${question.name}" contains a duplicate option value`,
+          };
+        }
+        seen.add(v);
+      }
+      const exclusive = question.exclusiveValues
+        ? raw.filter((v) => question.exclusiveValues!.includes(v as string))
+        : [];
+      if (exclusive.length > 0 && raw.length > 1) {
+        return {
+          ok: false,
+          message: `question "${question.name}" — an exclusive value must be selected alone`,
+        };
+      }
+      data[question.name] = raw;
+      continue;
+    }
+
+    // single — a deliberate, required single-select.
+    if (typeof raw !== "string" || !allowed.has(raw)) {
+      return {
+        ok: false,
+        message: `question "${question.name}" requires one of the declared options`,
+      };
+    }
+    data[question.name] = raw;
+  }
+  return { ok: true, data };
+}
+
+/** The `mapsTo:'email'` field value landed in this step's data, if any. */
+function extractMappedEmail(
+  step: FlowStep,
+  stepData: Record<string, unknown>,
+): string | undefined {
+  if (step.type !== "field") return undefined;
+  for (const field of step.fields) {
+    if (field.mapsTo === "email") {
+      const value = stepData[field.name];
+      if (typeof value === "string" && value.length > 0) return value;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Derive the segment for a just-landed segmentation step declaring a band
+ * map (#3959 D3). Returns undefined for any non-segmentation step, a
+ * segmentation step with no `segments` block (band maps are opt-in), or —
+ * only possible against a pre-D2 snapshot — a driving-question answer that
+ * doesn't hit any band; D2's totality check guarantees a miss can't happen
+ * against a schema-valid config, and a miss here leaves the session's
+ * segment untouched rather than failing the submission (the snapshot, not
+ * the visitor, is at fault).
+ */
+function deriveSegment(
+  step: FlowStep,
+  stepData: Record<string, unknown>,
+): string | undefined {
+  if (step.type !== "segmentation" || !step.segments) return undefined;
+  const answer = stepData[step.segments.question];
+  if (typeof answer !== "string") return undefined;
+  const band = step.segments.bands.find((b) => b.values.includes(answer));
+  return band?.key;
+}
+
+// ─── Session resolution (#3974 AC1) ────────────────────────────────────────────
+
+type ResolveSessionResult =
+  | { ok: true; session: FlowSessionRow; configSnapshot: FlowConfig }
+  | { ok: false; error: FlowEngineError };
+
+/**
+ * Resolve the session a step submission targets: reuse an in-progress one
+ * (by `sessionId`), or create the first session for a fresh walk. Isolated
+ * from the per-step value validation / persistence / completion-
+ * orchestration concerns `submitStep` composes below (#3974 AC1).
+ */
+async function resolveSession(
+  input: SubmitStepInput,
+): Promise<ResolveSessionResult> {
+  if (input.sessionId) {
+    const existing: FlowSessionRow | null =
+      await prisma.flowSession.findUnique({
+        where: { id: input.sessionId },
+        select: FLOW_SESSION_SELECT,
+      });
+
+    if (
+      !existing ||
+      existing.siteId !== input.siteId ||
+      existing.flowSlug !== input.slug
+    ) {
+      // A sessionId minted under a different flow (flowSlug mismatch) is
+      // rejected here — accepting it would attribute the completing Lead's
+      // `source`/`flowSlug` to the WRONG flow (code-review finding on
+      // #3956's acceptance verdict).
+      return {
+        ok: false,
+        error: { kind: "step_conflict", message: "Unknown session." },
+      };
+    }
+    if (existing.completedAt) {
+      return {
+        ok: false,
+        error: { kind: "step_conflict", message: "Session already completed." },
+      };
+    }
+
+    const parsedSnapshot = flowConfigSchema.safeParse(existing.configSnapshot);
+    if (!parsedSnapshot.success) {
+      return {
+        ok: false,
+        error: {
+          kind: "step_conflict",
+          message: "Session's config snapshot failed validation.",
+        },
+      };
+    }
+    return { ok: true, session: existing, configSnapshot: parsedSnapshot.data };
+  }
+
+  const definition: FlowDefinitionForSessionRow | null =
+    await prisma.flowDefinition.findUnique({
+      where: { siteId_slug: { siteId: input.siteId, slug: input.slug } },
+      select: FLOW_DEFINITION_FOR_SESSION_SELECT,
+    });
+
+  if (!definition) {
+    return {
+      ok: false,
+      error: { kind: "flow_not_found", message: "Flow not found." },
+    };
+  }
+
+  const parsedConfig = flowConfigSchema.safeParse(definition.config);
+  if (!parsedConfig.success) {
+    return {
+      ok: false,
+      error: {
+        kind: "invalid_input",
+        message: "Stored flow config failed validation.",
+      },
+    };
+  }
+  const configSnapshot = parsedConfig.data;
+
+  const session: FlowSessionRow = await prisma.flowSession.create({
+    data: {
+      siteId: input.siteId,
+      flowId: definition.id,
+      flowSlug: input.slug,
+      configSnapshot: configSnapshot as unknown as Prisma.InputJsonValue,
+      state: {},
+      // #3962 D3: persist the locale of the FIRST landed step — an
+      // abandoned session (the only session this feature cares about)
+      // otherwise has no locale anywhere (submitStep only forwarded it
+      // into the completion payload). Set once at create; not updated on
+      // later steps (a mid-walk locale switch is an edge the completion
+      // payload already handles its own way).
+      locale: input.locale ?? configSnapshot.defaultLocale,
+      // #5036 D6d — carried down from the definition, so the session names the
+      // listing it belongs to rather than sharing its slug with it.
+      ...(definition.listingId !== null && { listingId: definition.listingId }),
+    },
+    select: FLOW_SESSION_SELECT,
+  });
+
+  return { ok: true, session, configSnapshot };
+}
+
+// ─── Step lookup ────────────────────────────────────────────────────────────────
+
+type LocateStepResult =
+  | { ok: true; step: FlowStep; index: number }
+  | { ok: false; error: FlowEngineError };
+
+/** Find the submitted `stepKey` in the session's pinned config snapshot. */
+function locateStep(
+  configSnapshot: FlowConfig,
+  stepKey: string,
+): LocateStepResult {
+  const index = configSnapshot.steps.findIndex((s) => s.key === stepKey);
+  if (index === -1) {
+    return {
+      ok: false,
+      error: {
+        kind: "step_conflict",
+        message: `Unknown stepKey "${stepKey}" for this flow.`,
+      },
+    };
+  }
+  return { ok: true, step: configSnapshot.steps[index]!, index };
+}
+
+// ─── Flow state helpers (D1/D2/D3, #4663) ───────────────────────────────────────
+
+/** Normalize a session's `state` JSON column into a typed, defensive value. */
+function normalizeFlowState(value: unknown): FlowState {
+  return value && typeof value === "object" ? (value as FlowState) : {};
+}
+
+/** Merge every landed step's values into one flat name → value map (mirrors flow-visibility.ts's internal helper — engine-local, since that one isn't exported). */
+function flattenFlowState(state: FlowState): Record<string, unknown> {
+  const flat: Record<string, unknown> = {};
+  for (const stepKey of Object.keys(state)) {
+    const stepData = state[stepKey];
+    if (stepData && typeof stepData === "object") Object.assign(flat, stepData);
+  }
+  return flat;
+}
+
+/** A schema-defensive read of a stored `aeoScore` JSON column value. */
+function parseStoredAeoScore(value: unknown): AeoScoreResult | undefined {
+  const parsed = aeoScoreResultSchema.safeParse(value);
+  return parsed.success ? parsed.data : undefined;
+}
+
+// ─── Per-step persistence (#3974 AC1) ───────────────────────────────────────────
+
+type PersistStepUpdateResult = {
+  updatedState: FlowState;
+  email: string | undefined;
+  segment: string | undefined;
+  flags: FlowFlags;
+  /** The resulting `aeoScore` JSON value AFTER this update (unchanged / reset-to-null / newly-computed). */
+  aeoScore: unknown;
+};
+
+/**
+ * Merge one landed step's extracted values into the session's accumulated
+ * state and persist immediately — the per-step partial-state capture this
+ * feature exists for. Isolated from session resolution / value validation /
+ * completion orchestration (#3974 AC1).
+ *
+ * D1/D2 (#4663): re-derives `flags` from the just-updated state and prunes
+ * every now-hidden step's stale state entry (D3) — in the SAME update. D6/D3:
+ * when this landing changes an EARLIER-landed `urlField` value, the
+ * downstream `aeo_score` step's own state entry is dropped and `aeoScore` is
+ * reset to `null`, so the resolver names it as the next step again.
+ * `aeoScoreResult`, when supplied, is this step's OWN aeo_score landing
+ * (D6) — persisted in this SAME update, taking priority over any reset.
+ */
+async function persistStepUpdate(
+  session: FlowSessionRow,
+  configSnapshot: FlowConfig,
+  step: FlowStep,
+  extractedData: Record<string, unknown>,
+  aeoScoreResult: AeoScoreResult | undefined,
+): Promise<PersistStepUpdateResult> {
+  const priorState = normalizeFlowState(session.state);
+  let updatedState: FlowState = {
+    ...priorState,
+    [step.key]: extractedData,
+  };
+
+  const mappedEmail = extractMappedEmail(step, extractedData);
+  const email = mappedEmail ?? session.email ?? undefined;
+
+  // #3959 D3: derive the segment (if this landing step declares a band
+  // map) and hold it in a REQUEST-LOCAL variable — never re-read off
+  // `session`, which is never reassigned after this update, so on the one
+  // submission that lands BOTH segmentation AND completion (segmentation
+  // plausibly last), reading `session.segment` below would see the stale
+  // pre-update null.
+  const derivedSegment = deriveSegment(step, extractedData);
+  const segment = derivedSegment ?? session.segment ?? undefined;
+
+  // D6/D3 (#4663) — a changed `urlField` value resets any downstream
+  // aeo_score result. Detected on the FIELD step landing, before pruning.
+  let aeoScoreReset = false;
+  if (step.type === "field") {
+    const priorStepData = priorState[step.key];
+    for (const field of step.fields) {
+      const scoreStep = configSnapshot.steps.find(
+        (s): s is Extract<FlowStep, { type: "aeo_score" }> =>
+          s.type === "aeo_score" && s.urlField === field.name,
+      );
+      if (!scoreStep) continue;
+      // code-review suggestion (#4663): a step landed for the FIRST time
+      // (no prior state at all) is never a "change" — skip the reset so a
+      // fresh session's first url landing doesn't issue a wasted
+      // Prisma.JsonNull write against a column already null by default.
+      if (priorStepData === undefined) continue;
+      const priorValue = priorStepData[field.name];
+      const newValue = extractedData[field.name];
+      if (priorValue !== newValue) {
+        aeoScoreReset = true;
+        if (scoreStep.key in updatedState) {
+          const rest = { ...updatedState };
+          delete rest[scoreStep.key];
+          updatedState = rest;
+        }
+      }
+    }
+  }
+
+  // D1/D2 (#4663) — re-derive flags from the just-updated state, then prune
+  // any now-hidden step's stale state (persisted in the SAME update).
+  const flags = deriveFlags(configSnapshot, updatedState);
+  updatedState = pruneHiddenState(configSnapshot, updatedState, flags);
+
+  const aeoScore: unknown = aeoScoreResult ?? (aeoScoreReset ? null : undefined);
+
+  await prisma.flowSession.update({
+    where: { id: session.id },
+    data: {
+      state: updatedState as unknown as Prisma.InputJsonValue,
+      flags: flags as unknown as Prisma.InputJsonValue,
+      ...(mappedEmail !== undefined && { email: mappedEmail }),
+      ...(derivedSegment !== undefined && { segment: derivedSegment }),
+      // A JSON column's SQL NULL is written via the `Prisma.JsonNull`
+      // sentinel, never a bare `null` (which Prisma reserves for "leave the
+      // column untouched" on a nullable Json field).
+      ...(aeoScore === null && { aeoScore: Prisma.JsonNull }),
+      ...(aeoScoreResult !== undefined && {
+        aeoScore: aeoScoreResult as unknown as Prisma.InputJsonValue,
+      }),
+    },
+  });
+
+  return {
+    updatedState,
+    email,
+    segment,
+    flags,
+    aeoScore: aeoScore !== undefined ? aeoScore : session.aeoScore,
+  };
+}
+
+// ─── Completion orchestration (#3974 AC1) ───────────────────────────────────────
+
+type CompleteSubmissionParams = {
+  session: FlowSessionRow;
+  configSnapshot: FlowConfig;
+  siteId: string;
+  slug: string;
+  locale: string | undefined;
+  updatedState: FlowState;
+  email: string | undefined;
+  segment: string | undefined;
+  flags: FlowFlags;
+  aeoScore: AeoScoreResult | undefined;
+  sourceMeta: SourceMeta;
+};
+
+type CompleteSubmissionResult = {
+  leadId: string | undefined;
+  leadRoutingFailed: true | undefined;
+};
+
+/**
+ * Runs once every step in the snapshot has landed: assembles the completion
+ * payload, routes it through `routeLead`, stamps `completedAt` (+ `leadId`
+ * on success), and — on success, when the snapshot declares a deliverables
+ * block — fires deliverable fulfillment. Isolated from session resolution /
+ * value validation / per-step persistence (#3974 AC1).
+ */
+async function completeSubmission(
+  params: CompleteSubmissionParams,
+): Promise<CompleteSubmissionResult> {
+  const {
+    session,
+    configSnapshot,
+    siteId,
+    slug,
+    locale,
+    updatedState,
+    email,
+    segment,
+    flags,
+    aeoScore,
+    sourceMeta,
+  } = params;
+
+  if (!email) {
+    // D9: a flow with no email-mapped field completes without a Lead row —
+    // the session is the record. Legal but unused by the #3958 presets.
+    await prisma.flowSession.update({
+      where: { id: session.id },
+      data: { completedAt: new Date() },
+    });
+    return { leadId: undefined, leadRoutingFailed: undefined };
+  }
+
+  // Assemble the payload (D3 order, #4663): landed answers of VISIBLE steps
+  // only, config order → hiddenAnswers of currently-hidden steps →
+  // flowSlug → locale → segment → flags → aeoScore → experimentKey /
+  // waitlistKey (FINAL, per A1 D4 — unchanged). mapsTo-tagged values (incl.
+  // the email field itself) stay in payload too; email is ALSO promoted to
+  // routeLead's top-level required field, per D9 ("email top-level;
+  // everything else in payload").
+  const visibleSteps = resolveVisibleSteps(configSnapshot, updatedState, flags);
+  const payload: Record<string, unknown> = {};
+  for (const s of visibleSteps) {
+    const stepData = updatedState[s.key];
+    if (stepData && typeof stepData === "object") {
+      Object.assign(payload, stepData);
+    }
+  }
+  Object.assign(payload, hiddenAnswers(configSnapshot, updatedState, flags));
+
+  const resolvedLocale = locale ?? configSnapshot.defaultLocale;
+  payload["flowSlug"] = slug;
+  payload["locale"] = resolvedLocale;
+  // #3959 D4: server-owned, written LAST — a smuggled `segment` key in
+  // any submitted values is already dropped by extractStepValues's
+  // declared-names-only extraction (and unrepresentable as a declared
+  // name per D2.5), so this is defense in depth, not the only guard.
+  if (segment !== undefined) {
+    payload["segment"] = segment;
+  }
+  // D2 (#4663) — the derived flag object, written when the snapshot
+  // declares flags (possibly `{}` — every flag evaluated false).
+  if (configSnapshot.flags) {
+    payload["flags"] = flags;
+  }
+  // D6 (#4663) — the persisted aeo_score result, written whenever the
+  // score step is part of the VISIBLE set (it always lands by completion
+  // time in that case — completed requires every visible step in state).
+  if (visibleSteps.some((s) => s.type === "aeo_score") && aeoScore !== undefined) {
+    payload["aeoScore"] = aeoScore;
+  }
+  // #4661 D4: server-owned, written LAST (after `segment`/`flags`/
+  // `aeoScore`) — a smuggled `experimentKey`/`waitlistKey` key in any
+  // submitted values is already dropped by extractStepValues's
+  // declared-names-only extraction (and unrepresentable as a declared name,
+  // per flowConfigSchema's D4 reserved-name check), so this is defense in
+  // depth, not the only guard. The Signup composes the existing Lead
+  // record — no second signup store.
+  if (configSnapshot.experiment) {
+    payload["experimentKey"] = configSnapshot.experiment.key;
+    payload["waitlistKey"] = configSnapshot.experiment.waitlist;
+  }
+
+  const leadResult = await routeLead(
+    {
+      kind: "submit",
+      siteId,
+      source: `flow:${slug}`,
+      email,
+      payload,
+    },
+    sourceMeta,
+  );
+
+  if (!leadResult.ok) {
+    // routeLead's own DB→file fallback already exhausted itself before
+    // returning ok:false. The flow interaction is still complete from
+    // the visitor's perspective — the session itself remains the durable
+    // record (D9's "session is the record" fallback), so we do not fail
+    // this submission over a downstream lead-persistence hiccup.
+    // `leadRoutingFailed` surfaces the fact so the route handler can log
+    // it (PII-safe) — this function has no request-scoped logger of its
+    // own to log through directly.
+    await prisma.flowSession.update({
+      where: { id: session.id },
+      data: { completedAt: new Date() },
+    });
+    return { leadId: undefined, leadRoutingFailed: true };
+  }
+
+  const leadId = leadResult.id;
+  await prisma.flowSession.update({
+    where: { id: session.id },
+    data: { completedAt: new Date(), leadId },
+  });
+
+  // #4662 D6: fires ONLY when the snapshot declares an experiment binding
+  // (opt-in per flow — a flow with no `experiment` block behaves exactly as
+  // merged main, the AC4 regression floor: no read, no score row, unchanged
+  // payload). scoreCompletedSignup is the ONE write path into
+  // waitlist_scores; a scoring failure of ANY kind never fails this
+  // submission (fail-soft, the deliverables-hook posture below).
+  if (configSnapshot.experiment) {
+    try {
+      await scoreCompletedSignup({
+        siteId,
+        leadId,
+        experimentKey: configSnapshot.experiment.key,
+        waitlistKey: configSnapshot.experiment.waitlist,
+        payload,
+        flags: {}, // A3 wires the pre_launch session flag at completion
+      });
+    } catch (err) {
+      logger.error(
+        {
+          siteId,
+          flowSlug: slug,
+          sessionId: session.id,
+          errKind: err instanceof Error ? err.constructor.name : typeof err,
+        },
+        "[flow-engine] waitlist scoring failed — completion still succeeded",
+      );
+    }
+  }
+
+  // #3961 D4: fulfillment fires ONLY on this success path, and only
+  // when the snapshot declares a deliverables block (opt-in per flow
+  // — a flow with no block behaves exactly as merged main, the AC4
+  // regression floor). fulfillDeliverable itself never throws on its
+  // own business-logic misses (no entry, no asset row, no configured
+  // sender — all internal log-and-skip); this try/catch is the
+  // residual safety net for a genuine DB/network error escaping the
+  // service, so a fulfillment failure of ANY kind never fails this
+  // submission (the owner-notification posture).
+  if (configSnapshot.deliverables) {
+    try {
+      await fulfillDeliverable({
+        siteId,
+        flowSlug: slug,
+        configSnapshot,
+        ...(segment !== undefined && { segment }),
+        leadId: leadResult.id,
+        sessionId: session.id,
+        email,
+        locale: resolvedLocale,
+      });
+    } catch (err) {
+      // Swallowed by design (D4) — the submission still succeeds
+      // regardless. fulfillDeliverable logs its own PII-safe
+      // outcomes on every path it controls; reaching HERE means an
+      // exception escaped before it could log, so this is the
+      // residual observability signal — the error's constructor
+      // name only (never `.message`, which could carry an
+      // interpolated PII value from a driver/library exception),
+      // matching the `{siteId, flowSlug, sessionId}` shape the
+      // service's own outcome logs already use (code-review finding
+      // on this issue's own PR: a fully-silent swallow would make a
+      // genuine regression here invisible in production).
+      logger.error(
+        {
+          siteId,
+          flowSlug: slug,
+          sessionId: session.id,
+          errKind:
+            err instanceof Error ? err.constructor.name : typeof err,
+        },
+        "[flow-engine] deliverable fulfillment failed — completion still succeeded",
+      );
+    }
+  }
+
+  return { leadId, leadRoutingFailed: undefined };
+}
+
+// ─── submitStep — the orchestrator (#3974 AC1) ──────────────────────────────────
+
+export async function submitStep(
+  input: SubmitStepInput,
+  sourceMeta: SourceMeta,
+): Promise<SubmitStepResult> {
+  const dbError = requireDb();
+  if (dbError) return { ok: false, error: dbError };
+
+  const parsedInput = submitFlowStepInputSchema.safeParse({
+    sessionId: input.sessionId,
+    stepKey: input.stepKey,
+    values: input.values,
+    locale: input.locale,
+  });
+  if (!parsedInput.success) {
+    return {
+      ok: false,
+      error: {
+        kind: "invalid_input",
+        message: parsedInput.error.issues[0]?.message ?? "invalid input",
+      },
+    };
+  }
+
+  const resolved = await resolveSession(input);
+  if (!resolved.ok) return resolved;
+  const { session, configSnapshot } = resolved;
+
+  const located = locateStep(configSnapshot, input.stepKey);
+  if (!located.ok) return located;
+  const { step, index: stepIndex } = located;
+
+  // D1 (#4663) — a submission for a step not reachable given the CURRENT
+  // (pre-landing) state/flags is a conflict, not a value error.
+  const priorState = normalizeFlowState(session.state);
+  const priorFlags = deriveFlags(configSnapshot, priorState);
+  if (!isStepVisible(configSnapshot, step.key, priorState, priorFlags)) {
+    return {
+      ok: false,
+      error: {
+        kind: "step_conflict",
+        message: "Step is not reachable on this branch.",
+      },
+    };
+  }
+
+  const extracted = extractStepValues(step, input.values);
+  if (!extracted.ok) {
+    return {
+      ok: false,
+      error: { kind: "invalid_input", message: extracted.message },
+    };
+  }
+
+  // D6 (#4663) — an aeo_score landing computes the score BEFORE persisting,
+  // so it lands in the SAME write as the step's own state entry. The url
+  // value was landed on an EARLIER step (the aeo_score step's own extracted
+  // data is always `{}`).
+  let aeoScoreResult: AeoScoreResult | undefined;
+  if (step.type === "aeo_score") {
+    const rawUrl = flattenFlowState(priorState)[step.urlField];
+    const budgetMs = resolveScoreBudgetMs(process.env["AEO_SCORE_BUDGET_MS"], logger);
+    try {
+      aeoScoreResult =
+        typeof rawUrl === "string" && rawUrl.length > 0
+          ? await scoreUrl(rawUrl, { budgetMs })
+          : loadExampleScore();
+      // Security-review finding (#4663, M1): scoreUrl RETURNS (never throws)
+      // on an SSRF-guard rejection, with `message` built from the resolved
+      // address ("Host H resolves to a blocked address: A") — the catch
+      // block below only covers a THROWN exception, so this returned-value
+      // path reached the anonymous caller verbatim and was persisted into
+      // the lead record unsanitized. Scrub it here, in the one caller this
+      // PR owns; the pre-existing POST /v1/aeo/score route (apps/api/src/
+      // routes/aeo/index.ts, #4666, out of #4663's scope) shares the same
+      // gap and is not touched by this fix.
+      if (
+        aeoScoreResult.status === "unavailable" &&
+        aeoScoreResult.reason === "ssrf_blocked"
+      ) {
+        aeoScoreResult = {
+          status: "unavailable",
+          reason: "ssrf_blocked",
+          message: "This URL cannot be scored.",
+        };
+      }
+    } catch (e) {
+      // code-review finding (#4663): never forward a raw caught exception
+      // message to the client — scoreUrl wraps an external-URL crawler, so
+      // an unexpected internal error could leak infrastructure detail (DNS
+      // failure text, timeout internals, an SSRF-guard rejection message).
+      // Log PII-safe server-side (no url), return a generic client message.
+      logger.error(
+        {
+          siteId: input.siteId,
+          slug: input.slug,
+          sessionId: session.id,
+          errKind: e instanceof Error ? e.constructor.name : typeof e,
+        },
+        "[flow-engine] aeo_score crawl threw — landing as unavailable",
+      );
+      aeoScoreResult = {
+        status: "unavailable",
+        reason: "crawl_failed",
+        message: "Could not score this URL.",
+      };
+    }
+  }
+
+  const { updatedState, email, segment, flags, aeoScore } = await persistStepUpdate(
+    session,
+    configSnapshot,
+    step,
+    extracted.data,
+    aeoScoreResult,
+  );
+
+  // D1 (#4663) — resolver-driven successor + completion, replacing the
+  // former linear `steps[stepIndex + 1]` / "every step key in state".
+  const visible = resolveVisibleSteps(configSnapshot, updatedState, flags);
+  const nextVisible = visible.find(
+    (s) => configSnapshot.steps.findIndex((cs) => cs.key === s.key) > stepIndex,
+  );
+  const nextStepKey = nextVisible?.key ?? null;
+  const completed = visible.every((s) => s.key in updatedState);
+
+  let leadId: string | undefined;
+  let leadRoutingFailed: true | undefined;
+
+  if (completed) {
+    const completion = await completeSubmission({
+      session,
+      configSnapshot,
+      siteId: input.siteId,
+      slug: input.slug,
+      locale: input.locale,
+      updatedState,
+      email,
+      segment,
+      flags,
+      aeoScore: aeoScoreResult ?? parseStoredAeoScore(aeoScore),
+      sourceMeta,
+    });
+    leadId = completion.leadId;
+    leadRoutingFailed = completion.leadRoutingFailed;
+  }
+
+  // D9 (#4663) — present on the aeo_score step's own response (just
+  // computed), OR on the completion response (the persisted value, even
+  // when this submission is a LATER step than the one that computed it).
+  const responseAeoScore =
+    aeoScoreResult ?? (completed ? parseStoredAeoScore(aeoScore) : undefined);
+
+  return {
+    ok: true,
+    sessionId: session.id,
+    landed: step.key,
+    nextStepKey,
+    completed,
+    ...(leadId !== undefined && { leadId }),
+    ...(leadRoutingFailed !== undefined && { leadRoutingFailed }),
+    ...(segment !== undefined && { segment }),
+    ...(responseAeoScore !== undefined && { aeoScore: responseAeoScore }),
+  };
+}
+
+// ─── Session retention / cleanup (#3974 AC3) ────────────────────────────────────
+
+/**
+ * FlowSession retention policy. An ABANDONED session (`completedAt: null` —
+ * the visitor never finished the walk) otherwise accumulates indefinitely
+ * with no TTL (security-review finding, Medium, non-blocking, on #3956's
+ * acceptance PR) — `cleanupAbandonedSessions` below is the fix: delete an
+ * abandoned row once its `updatedAt` is older than the retention window.
+ *
+ * A COMPLETED session (`completedAt` set) is NEVER a retention candidate —
+ * it is the durable lead-of-record (D9) and outside this policy's scope;
+ * the delete's `where` clause enforces this structurally.
+ *
+ * Coordinated with #3962's abandon-email sweep (`runAbandonEmailSweep` in
+ * ./abandon-email.ts), which is exactly the natural consumer of these same
+ * abandoned rows (this issue's own "Why"). The default window
+ * (`RETENTION_STALE_DAYS`, 90 days) is chosen to sit far above any realistic
+ * `abandonEmail.delayMinutes` (schema floor: 5 minutes — every shipped
+ * preset and documented example uses a minutes-to-hours delay), so the
+ * sweep always has its full window to run before a row it might still want
+ * becomes retention-eligible — and for a flow with no abandon-email policy
+ * configured at all, the row still gets cleaned up eventually rather than
+ * accumulating forever, instead of waiting on a stamp that will never come.
+ *
+ * Operational wiring (a CLI entry + external scheduler, exactly
+ * `abandon-sweep.ts`'s established shape) lives at
+ * `src/bin/session-retention-sweep.ts`.
+ */
+const RETENTION_STALE_DAYS = 90;
+
+export type CleanupAbandonedSessionsOptions = {
+  /** Override the retention window, in days. Defaults to RETENTION_STALE_DAYS. */
+  staleDays?: number;
+  /** Override "now" — test seam. */
+  now?: Date;
+};
+
+export type CleanupAbandonedSessionsResult =
+  | { ok: true; deleted: number }
+  | { ok: false; error: FlowEngineError };
+
+export async function cleanupAbandonedSessions(
+  opts: CleanupAbandonedSessionsOptions = {},
+): Promise<CleanupAbandonedSessionsResult> {
+  const dbError = requireDb();
+  if (dbError) return { ok: false, error: dbError };
+
+  const staleDays = opts.staleDays ?? RETENTION_STALE_DAYS;
+  const now = opts.now ?? new Date();
+  const cutoff = new Date(now.getTime() - staleDays * 24 * 60 * 60_000);
+
+  const result = await prisma.flowSession.deleteMany({
+    where: {
+      completedAt: null,
+      updatedAt: { lte: cutoff },
+    },
+  });
+
+  logger.info(
+    { deleted: result.count, staleDays },
+    "[flow-engine] abandoned-session cleanup swept",
+  );
+
+  return { ok: true, deleted: result.count };
+}