create-ailk 0.1.0 → 0.2.0

package/templates/apps/api/src/services/listing-csv-sweep.ts

@@ -0,0 +1,455 @@
+/**
+ * @file listing-csv-sweep.ts
+ * @description The weekly founder-CSV sweep (#5001) — the founder's list
+ * arrives without a login.
+ *
+ * `runListingCsvSweep` is the one entry point: for each `live`
+ * `ProjectListing`, select the `WaitlistScore` rows for its experiment that
+ * landed AFTER the last recorded send, join each to its completed
+ * `FlowSession`'s answers, render one CSV, store it as a `DeliverableAsset`,
+ * grant it to the founder's own `Lead`, and email the capability-token
+ * download link with `sendDeliverableEmail` — the same store → grant → link
+ * path `deliverable-fulfillment.ts` (#3961 D5) uses for a flow's promised
+ * asset, so the founder's list is served by `GET /v1/deliverables/:grantId`
+ * with no account, no session, and no second download surface.
+ *
+ * Invoked by the CLI entry (`src/bin/listing-csv-sweep.ts`) on an
+ * operator-supplied cadence — this service is stateless and has no timer of
+ * its own (the `abandon-email.ts` #3962 D1 / `waitlist-followup.ts` #4732 D1
+ * posture, unchanged).
+ *
+ * Two properties the data carries, not just the control flow:
+ *   - NO NEW ROWS ⇒ NO SEND. A listing whose experiment produced no score
+ *     since its last `ListingCsvSend` writes nothing and sends nothing, so a
+ *     second run in the same week is a no-op rather than a duplicate email.
+ *   - The stamp is written only AFTER a successful send. A send failure
+ *     leaves the watermark where it was, so next week's CSV still carries
+ *     those rows rather than losing them to a Resend hiccup.
+ *
+ * Per-listing failures are caught, logged, and skipped — one listing's bad
+ * data or failed send never stops the sweep for the listings after it.
+ *
+ * PII boundary: logs carry only `{ siteId, listingId, slug, outcome }` —
+ * never `founderEmail`, a signup's email/name, or any answer value (the same
+ * discipline `waitlist-followup.ts` and `lead-routing.ts` document). The CSV
+ * itself carries signup PII by design; it reaches the founder through the
+ * opaque grant id alone, never through a log line or a link parameter.
+ */
+
+import { sendDeliverableEmail } from "@working-theory/email";
+import { logger } from "@working-theory/observability";
+import { projectListingQuestionsSchema } from "@working-theory/validation";
+
+import { prisma } from "../lib/prisma.js";
+
+import { listingAnswerLabels } from "./project-listing-flow.js";
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export type ListingCsvSweepSummary = {
+  scanned: number;
+  sent: number;
+  skipped: number;
+  failed: number;
+};
+
+export type ListingCsvSweepError = {
+  kind: "flows_not_configured" | "unexpected_error";
+  message: string;
+};
+
+export type ListingCsvSweepResult =
+  | { ok: true; summary: ListingCsvSweepSummary }
+  | { ok: false; error: ListingCsvSweepError };
+
+export type RunListingCsvSweepOptions = {
+  siteId?: string;
+  now?: Date;
+};
+
+type ListingRow = {
+  id: string;
+  siteId: string;
+  // #5036 — nullable at rest: the Lead is written at submit, and every column
+  // an application fills in is nullable so a half-filled draft is storable.
+  // This sweep only ever selects `active` rows, which reached that state
+  // through `submit` and therefore carry all four — the guards below discharge
+  // the types rather than describing a reachable state.
+  leadId: string | null;
+  slug: string | null;
+  name: string | null;
+  founderEmail: string;
+  experimentKey: string | null;
+  questions: unknown;
+};
+
+type ScoreRow = { leadId: string; tier: string; createdAt: Date };
+type SessionRow = { leadId: string | null; email: string | null; state: unknown };
+
+// ─── DB-required guard (mirrors waitlist-followup.ts's requireDb) ─────────────
+
+function requireDb(): ListingCsvSweepError | null {
+  if (!process.env.DATABASE_URL) {
+    return { kind: "flows_not_configured", message: "Flows are not configured." };
+  }
+  return null;
+}
+
+// ─── PII-safe outcome logging ────────────────────────────────────────────────
+
+type ListingOutcome =
+  | "sent"
+  | "failed"
+  | "no_experiment"
+  | "no_new_signups"
+  | "invalid_questions"
+  | "incomplete_listing"
+  | "sender_unconfigured";
+
+const OUTCOME_MESSAGE: Record<ListingOutcome, string> = {
+  sent: "[listing-csv-sweep] listing selected — CSV sent",
+  failed: "[listing-csv-sweep] listing selected — send failed (unstamped, retried next run)",
+  no_experiment: "[listing-csv-sweep] listing skipped — no experiment generated yet",
+  no_new_signups: "[listing-csv-sweep] listing skipped — no signups since the last send",
+  invalid_questions:
+    "[listing-csv-sweep] listing skipped — stored questions failed schema validation",
+  incomplete_listing:
+    "[listing-csv-sweep] listing skipped — an active listing is missing submit-time fields",
+  sender_unconfigured:
+    "[listing-csv-sweep] listing skipped (unstamped, retried next run) — no from-address configured",
+};
+
+function logOutcome(listing: ListingRow, outcome: ListingOutcome): void {
+  logger.info(
+    { siteId: listing.siteId, listingId: listing.id, slug: listing.slug, outcome },
+    OUTCOME_MESSAGE[outcome],
+  );
+}
+
+// ─── CSV rendering ───────────────────────────────────────────────────────────
+
+/** The column order the founder's weekly CSV always carries (#5001). */
+export const LISTING_CSV_COLUMNS = [
+  "joined_at",
+  "email",
+  "name",
+  "tier",
+  "segment",
+  "q3",
+  "q4",
+  "q5",
+  "q6",
+  "q7",
+  "extra_1",
+  "extra_2",
+  "extra_3",
+] as const;
+
+/** Leading characters a spreadsheet reads as the start of a formula. */
+const FORMULA_LEAD = /^[=+\-@\t\r]/;
+
+/**
+ * One RFC-4180 field. A value a spreadsheet would evaluate as a formula is
+ * prefixed with an apostrophe first — this CSV is emailed to a founder and
+ * opened in Excel/Sheets, and the free-text Q7 column is visitor-supplied.
+ */
+function csvField(value: string): string {
+  const guarded = FORMULA_LEAD.test(value) ? `'${value}` : value;
+  return /[",\r\n]/.test(guarded) ? `"${guarded.replace(/"/g, '""')}"` : guarded;
+}
+
+function csvLine(cells: readonly string[]): string {
+  return cells.map(csvField).join(",");
+}
+
+/** A session's persisted answer for one field, as a plain string. */
+function answerOf(state: Record<string, unknown>, field: string): string {
+  const raw = state[field];
+  return typeof raw === "string" ? raw : "";
+}
+
+// ─── The sweep ───────────────────────────────────────────────────────────────
+
+/**
+ * Build the CSV body for one listing's new signups. Scored/segment answers
+ * are rendered as the founder's OWN option wording (the label) rather than
+ * the stored option value — since #5036 D4 that value IS the founder's own
+ * (`some-slug`), but a slug is still not what a founder reading a
+ * spreadsheet wants to see. An unmapped value falls through verbatim rather
+ * than becoming blank.
+ */
+function renderCsv(
+  scores: readonly ScoreRow[],
+  sessionsByLead: Map<string, SessionRow>,
+  labels: Record<string, Record<string, string>>,
+): string {
+  const lines = [csvLine(LISTING_CSV_COLUMNS)];
+
+  for (const score of scores) {
+    const session = sessionsByLead.get(score.leadId);
+    const state =
+      session && typeof session.state === "object" && session.state !== null
+        ? (session.state as Record<string, unknown>)
+        : {};
+
+    const labelled = (field: string): string => {
+      const value = answerOf(state, field);
+      if (value === "") return "";
+      return labels[field]?.[value] ?? value;
+    };
+
+    lines.push(
+      csvLine([
+        score.createdAt.toISOString(),
+        session?.email ?? answerOf(state, "email"),
+        answerOf(state, "name"),
+        score.tier,
+        labelled("q2"),
+        labelled("q3"),
+        labelled("q4"),
+        labelled("q5"),
+        labelled("q6"),
+        answerOf(state, "q7"),
+        labelled("extra_1"),
+        labelled("extra_2"),
+        labelled("extra_3"),
+      ]),
+    );
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+/**
+ * The unguarded sweep body — mutates `summary` in place as it goes, so a
+ * mid-loop throw still leaves the caller's catch able to log how far it got
+ * (mirrors `waitlist-followup.ts`'s own posture).
+ */
+async function sweepAllListings(
+  opts: RunListingCsvSweepOptions,
+  now: Date,
+  fromAddress: string | undefined,
+  summary: ListingCsvSweepSummary,
+): Promise<void> {
+  const listings = (await prisma.projectListing.findMany({
+    // #5033 — `live` and `approved` collapsed into the single stored `active`
+    // state, and `closed` became COMPUTED rather than stored. The stored
+    // status alone is no longer the whole filter: a listing whose window has
+    // elapsed reads `closed` everywhere else, and this sweep mails the
+    // founder a CSV of their signups' answers, so it must stop at the same
+    // moment the public read does. Without the `windowEndsAt` predicate an
+    // expired listing would keep exporting signup data every run, forever —
+    // and the migration rewrites every historically `closed` row to `active`
+    // with a past window, so the first run after deploy would re-enrol all of
+    // them.
+    //
+    // #5036 D6b narrowed the null-window branch: a FEATURED row has no
+    // meaningful window and never closes, so it stays in scope; a NON-featured
+    // active row with no window is a data error `computeListingStatus` reads
+    // as `closed`, and this filter must agree with it rather than keep mailing
+    // a founder's signup data out of a row nothing bounds.
+    where: {
+      status: "active",
+      OR: [{ featured: true }, { windowEndsAt: { gt: now } }],
+      ...(opts.siteId !== undefined && { siteId: opts.siteId }),
+    },
+    select: {
+      id: true,
+      siteId: true,
+      leadId: true,
+      slug: true,
+      name: true,
+      founderEmail: true,
+      experimentKey: true,
+      questions: true,
+    },
+  })) as ListingRow[];
+
+  const day = now.toISOString().slice(0, 10);
+
+  for (const listing of listings) {
+    summary.scanned += 1;
+    try {
+      const outcome = await sweepOneListing(listing, now, day, fromAddress);
+      if (outcome === "sent") summary.sent += 1;
+      else if (outcome === "failed") summary.failed += 1;
+      else summary.skipped += 1;
+    } catch (err) {
+      // One listing's bad row, malformed session state, or DB hiccup must not
+      // stop the listings after it — the whole point of a sweep over N
+      // independent founders.
+      summary.failed += 1;
+      logger.error(
+        {
+          siteId: listing.siteId,
+          listingId: listing.id,
+          slug: listing.slug,
+          errKind: err instanceof Error ? err.constructor.name : typeof err,
+        },
+        "[listing-csv-sweep] listing failed — continuing with the next listing",
+      );
+    }
+  }
+}
+
+async function sweepOneListing(
+  listing: ListingRow,
+  now: Date,
+  day: string,
+  fromAddress: string | undefined,
+): Promise<ListingOutcome> {
+  // #5036 D6d — the listing stores the experiment's KEY, which is the same
+  // string `WaitlistScore.experimentKey` holds. The `WaitlistExperiment` read
+  // this used to make purely to translate an id into that key is gone.
+  const experimentKey = listing.experimentKey;
+  if (!experimentKey) {
+    logOutcome(listing, "no_experiment");
+    return "no_experiment";
+  }
+
+  if (listing.leadId === null || listing.slug === null || listing.name === null) {
+    logOutcome(listing, "incomplete_listing");
+    return "incomplete_listing";
+  }
+
+  const questions = projectListingQuestionsSchema.safeParse(listing.questions);
+  if (!questions.success) {
+    logOutcome(listing, "invalid_questions");
+    return "invalid_questions";
+  }
+
+  const lastSend = (await prisma.listingCsvSend.findFirst({
+    where: { listingId: listing.id },
+    orderBy: { sentAt: "desc" },
+    select: { sentAt: true },
+  })) as { sentAt: Date } | null;
+
+  const scores = (await prisma.waitlistScore.findMany({
+    where: {
+      siteId: listing.siteId,
+      experimentKey,
+      ...(lastSend && { createdAt: { gt: lastSend.sentAt } }),
+    },
+    select: { leadId: true, tier: true, createdAt: true },
+    orderBy: { createdAt: "asc" },
+  })) as ScoreRow[];
+
+  if (scores.length === 0) {
+    logOutcome(listing, "no_new_signups");
+    return "no_new_signups";
+  }
+
+  if (!fromAddress) {
+    // An env gap must not consume the week's send: nothing is stamped, so a
+    // later configured run still delivers these rows (the `waitlist-followup.ts`
+    // D4 posture).
+    logOutcome(listing, "sender_unconfigured");
+    return "sender_unconfigured";
+  }
+
+  const sessions = (await prisma.flowSession.findMany({
+    where: {
+      siteId: listing.siteId,
+      flowSlug: listing.slug,
+      completedAt: { not: null },
+      leadId: { in: scores.map((s) => s.leadId) },
+    },
+    select: { leadId: true, email: true, state: true },
+  })) as SessionRow[];
+  const sessionsByLead = new Map<string, SessionRow>();
+  for (const session of sessions) {
+    if (session.leadId) sessionsByLead.set(session.leadId, session);
+  }
+
+  const csv = renderCsv(scores, sessionsByLead, listingAnswerLabels(questions.data));
+  const bytes = Buffer.from(csv, "utf8");
+  const key = `listing-${listing.slug}-${day}`;
+  const filename = `${listing.slug}-waitlist-${day}.csv`;
+
+  const asset = (await prisma.deliverableAsset.upsert({
+    where: { siteId_key: { siteId: listing.siteId, key } },
+    create: {
+      siteId: listing.siteId,
+      key,
+      filename,
+      contentType: "text/csv",
+      bytes,
+      size: bytes.byteLength,
+    },
+    update: { filename, contentType: "text/csv", bytes, size: bytes.byteLength },
+    select: { id: true },
+  })) as { id: string };
+
+  // The grant IS the token (#3961 D5). `sessionId` records the listing this
+  // CSV was cut for — a plain-scalar provenance ref (ADR 0009 invariant 1),
+  // the same posture `assetId`/`leadId` beside it already take; this grant is
+  // not born of a flow walk, so there is no FlowSession to name.
+  const grant = (await prisma.deliverableGrant.create({
+    data: {
+      siteId: listing.siteId,
+      assetId: asset.id,
+      leadId: listing.leadId,
+      sessionId: listing.id,
+    },
+    select: { id: true },
+  })) as { id: string };
+
+  // The same link expression `deliverable-fulfillment.ts` builds — the opaque
+  // grant id and nothing else. No PII in the URL.
+  const base = process.env.AILK_PUBLIC_API_URL ?? "";
+  const downloadLink = `${base}/v1/deliverables/${grant.id}`;
+
+  const sendResult = await sendDeliverableEmail({
+    to: listing.founderEmail,
+    fromAddress,
+    subject: `Your ${listing.name} waitlist, week of ${day}`,
+    body: `${scores.length} new ${scores.length === 1 ? "signup" : "signups"} joined the ${listing.name} waitlist this week. Your CSV is ready to download.`,
+    cta: "Download this week's CSV",
+    downloadLink,
+  });
+
+  if (!sendResult.ok) {
+    // UNSTAMPED on purpose — the watermark stays put so these rows are in
+    // next week's CSV rather than lost.
+    logOutcome(listing, "failed");
+    return "failed";
+  }
+
+  await prisma.listingCsvSend.create({
+    data: {
+      listingId: listing.id,
+      sentAt: now,
+      rowCount: scores.length,
+      grantId: grant.id,
+    },
+  });
+
+  logOutcome(listing, "sent");
+  return "sent";
+}
+
+export async function runListingCsvSweep(
+  opts: RunListingCsvSweepOptions = {},
+): Promise<ListingCsvSweepResult> {
+  const dbError = requireDb();
+  if (dbError) return { ok: false, error: dbError };
+
+  const now = opts.now ?? new Date();
+  const fromAddress =
+    process.env.AILK_DELIVERABLE_EMAIL_FROM ?? process.env.AILK_LEAD_NOTIFICATION_FROM;
+
+  const summary: ListingCsvSweepSummary = { scanned: 0, sent: 0, skipped: 0, failed: 0 };
+
+  try {
+    await sweepAllListings(opts, now, fromAddress, summary);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : "Unknown error";
+    logger.error(
+      { partialSummary: summary, err: message },
+      "[listing-csv-sweep] sweep failed mid-run",
+    );
+    return { ok: false, error: { kind: "unexpected_error", message } };
+  }
+
+  return { ok: true, summary };
+}