@tellescope/sdk 2.0.0-beta.1 → 2.0.0-beta.3

package/src/tests/api_tests/form_intake_email_normalization.test.ts

@@ -0,0 +1,489 @@
+require('source-map-support').install();
+
+import { Session } from "../../sdk"
+import {
+  assert_checks,
+  Check,
+  setup_isolated_tenant,
+  IsolatedSuite,
+} from "../setup_isolated"
+import { Form, FormField } from "@tellescope/types-client"
+
+const host = process.env.API_URL || 'http://localhost:8080' as const
+
+const RAND = () => Math.random().toString(36).slice(2, 10)
+
+/** The Related Contacts insert runs AFTER submit_form_response has responded, so unlike the intake
+ *  write it has to be waited for. */
+const poll_for = async <T,>(get: () => Promise<T>, done: (v: T) => boolean, attempts=20): Promise<T> => {
+  let value = await get()
+  for (let i = 0; i < attempts && !done(value); i++) {
+    await new Promise(resolve => setTimeout(resolve, 250))
+    value = await get()
+  }
+  return value
+}
+
+/**
+ * Coverage for CU-86e3kndpg — `normalize_contact_intake_fields` in api/modules/forms.ts.
+ *
+ * The bug: a form question mapped to `intakeField: 'email'` has its answer copied onto the contact
+ * VERBATIM (the `ENDUSER_FIELD_TYPES` branch of `save_enduser_updates_for_form_response`), and the
+ * enduser write goes through `DB.endusers.updateById`, which is a raw Mongo `$set` with no schema
+ * validation. So a contact ends up stored as `John@Example.COM`, which `begin_login_flow`'s exact
+ * `findPage({ email })` never matches — the patient is asked for a password they never set.
+ *
+ * ### Fixture warnings (these make the difference between a real test and a vacuous one)
+ *
+ * 1. **The question type must NOT be `'email'`.** `formResponseAnswerValidator` already lowercases an
+ *    `email`-typed answer at submit, so a fixture built on one passes whether or not the guard
+ *    exists. Every positive case here uses a `'string'` (Text) or `'Dropdown'` question mapped to the
+ *    email intake field — which the webapp Form Editor refuses to save (`intakeFieldError` in
+ *    FormFieldEditor.tsx) but the API accepts, and which is exactly the real customer
+ *    misconfiguration. Section F keeps an `'email'`-typed question as the regression control.
+ *
+ * 2. **The submission must not be a PUBLIC one.** `authFieldIntakeDisabled` deletes `email` and
+ *    `phone` from the intake map outright for an untrusted remote submitter, so a public fixture
+ *    would store nothing either way. Every case drives `prepare_form_response` from the ADMIN
+ *    session, for which `lockAuthFieldIntake` is false.
+ *
+ * Each positive case also asserts that the STORED form response answer still holds the raw typed
+ * text — that is deliberately out of scope, and it proves the normalization observed on the contact
+ * came from the guard rather than from something lowercasing the answer upstream.
+ *
+ * `setup_isolated` rather than `setup_tests` because the cases assert on exact contact field values
+ * and on "which contacts exist", which needs a tenant nobody else writes to.
+ *
+ * Runs standalone or as part of the main suite:
+ *   npm run test:modular -- api_tests/form_intake_email_normalization
+ */
+
+type Fixture = { form: Form, fields: Record<string, FormField> }
+
+/** Every intake question the cases need, on one form. A submission answers only the subset it is
+ *  about — two questions mapped to the same intakeField in one submission would race, last wins. */
+const createFixtureForm = async (sdk: Session): Promise<Fixture> => {
+  const form = await sdk.api.forms.createOne({
+    title: `Intake Contact Normalization ${RAND()}`,
+  })
+
+  const fields: Record<string, FormField> = {}
+  let previousId: string | undefined = undefined
+  const addField = async (key: string, f: { title: string, type: any, intakeField?: string, options?: any }) => {
+    const created: FormField = await sdk.api.form_fields.createOne({
+      formId: form.id,
+      previousFields: previousId
+        ? [{ type: 'after', info: { fieldId: previousId } }]
+        : [{ type: 'root', info: {} }],
+      ...f,
+    } as any)
+    previousId = created.id
+    fields[key] = created
+    return created
+  }
+
+  // the misconfiguration under test: a free-TEXT question mapped to an authentication channel
+  await addField('emailText', { title: 'Email (text)', type: 'string', intakeField: 'email' })
+  await addField('phoneText', { title: 'Phone (text)', type: 'string', intakeField: 'phone' })
+  await addField('landlineText', { title: 'Landline (text)', type: 'string', intakeField: 'landline' })
+  // a custom field, so "every other intake field in the same submission still lands" is observable
+  await addField('color', { title: 'Favorite Color', type: 'string', intakeField: 'favoriteColor' })
+  // single-select: the answer arrives as a ONE-element array even for a built-in field
+  await addField('emailDropdown', {
+    title: 'Email (dropdown)', type: 'Dropdown', intakeField: 'email',
+    options: { radio: true, choices: ['Picked@Example.COM', 'Other@Example.COM'] },
+  })
+  // multi-select: a TWO-element array, which has no single contact value
+  await addField('emailMulti', {
+    title: 'Email (multi)', type: 'multiple_choice', intakeField: 'email',
+    options: { choices: ['First@Example.COM', 'Second@Example.COM'] },
+  })
+  // the regression control — already lowercased at submit, before this ticket
+  await addField('emailTyped', { title: 'Email (email type)', type: 'email', intakeField: 'email' })
+  await addField('lname', { title: 'Last Name', type: 'string', intakeField: 'lname' })
+  // Related Contacts reaches the guard through get_intake_fields_undotted, whose output is spread
+  // OVER each related contact's own typed values — so a SHARED email is the submitter's normalized
+  // one, and an unshared one is left exactly as typed (out of scope, per the ticket).
+  await addField('relatedShared', {
+    title: 'Related (email shared)', type: 'Related Contacts',
+    options: { sharedIntakeFields: ['email'] },
+  })
+  await addField('relatedUnshared', {
+    title: 'Related (email not shared)', type: 'Related Contacts',
+    options: { sharedIntakeFields: ['favoriteColor'] },
+  })
+
+  return { form, fields }
+}
+
+type Answer = { key: string, answer: any }
+
+/** Staff prepare + submit, so `authFieldIntakeDisabled` is false and the intake write actually runs.
+ *  `save_enduser_updates_for_form_response` is awaited inside the submit handler BEFORE it responds,
+ *  so the contact is already updated when this resolves — no polling needed. */
+const submitAnswers = async (
+  sdk: Session,
+  { form, fields }: Fixture,
+  enduserId: string,
+  answers: Answer[],
+) => {
+  const prepared = await sdk.api.form_responses.prepare_form_response({
+    formId: form.id, enduserId,
+  })
+  await sdk.api.form_responses.submit_form_response({
+    accessCode: prepared.accessCode,
+    responses: answers.map(({ key, answer }) => ({
+      fieldId: fields[key].id,
+      fieldTitle: fields[key].title,
+      answer,
+    })),
+  })
+  return prepared.response.id
+}
+
+export const form_intake_email_normalization_checks = async (): Promise<Check[]> => {
+  const checks: Check[] = []
+  const record = (label: string, pass: boolean, detail: string) => checks.push({ label, pass, detail })
+
+  let tenant: Awaited<ReturnType<typeof setup_isolated_tenant>> | undefined
+
+  try {
+    tenant = await setup_isolated_tenant({
+      name: `FormIntakeNormalization ${RAND()}`,
+      users: [{ isAdmin: true }],
+      apiKeys: 0,
+    })
+    const { sdk } = tenant
+
+    const fixture = await createFixtureForm(sdk)
+    const { fields } = fixture
+
+    /** A fresh contact with a known-good baseline on all three contact keys, so "the field was
+     *  omitted" is distinguishable from "the field was written". Numbers are unique per contact
+     *  (endusers are unique on phone, and a random suffix would eventually collide) and shaped so
+     *  `isMobilePhone(.., 'any', { strictMode: true })` accepts them.
+     *
+     *  Careful: the 555-01xx "reserved for fiction" range does NOT pass that check, so
+     *  `(555) 010-1234` is REJECTED by phoneValidator and the guard omits it. Every phone fixture
+     *  here is 555-555-xxxx for that reason. */
+    let phoneSeq = 1000
+    const NEXT_US_PHONE = () => `+1555555${phoneSeq++}`
+    const newContact = async () => {
+      const baselineEmail = `dontsend-base-${RAND()}@tellescope.com`
+      const baselinePhone = NEXT_US_PHONE()
+      const baselineLandline = NEXT_US_PHONE()
+      const enduser = await sdk.api.endusers.createOne({
+        fname: 'Intake', lname: 'Tester',
+        email: baselineEmail,
+        phone: baselinePhone,
+        landline: baselineLandline,
+      })
+      return { enduser, baselineEmail, baselinePhone, baselineLandline }
+    }
+    const reload = (id: string) => sdk.api.endusers.getOne(id)
+
+    /* ================= A. the reported bug: mixed case + surrounding whitespace ================= */
+    {
+      const { enduser } = await newContact()
+      const TYPED = `  John.Doe+Tag-${RAND()}@Example.COM  `
+      const EXPECTED = TYPED.trim().toLowerCase()
+
+      const responseId = await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailText', answer: { type: 'string', value: TYPED } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'A1: a TEXT question mapped to the email intake field stores the address lowercased and trimmed',
+        after.email === EXPECTED,
+        `stored email=${JSON.stringify(after.email)}, expected ${JSON.stringify(EXPECTED)}`
+        + ` (typed ${JSON.stringify(TYPED)}) — without the guard this stores the raw typed string,`
+        + ` which begin_login_flow's exact findPage({ email }) never matches`,
+      )
+
+      // fixture guard: proves the lowercasing observed above is the guard's doing, not something
+      // upstream normalizing a 'string' answer. The stored answer is deliberately out of scope.
+      const stored = await sdk.api.form_responses.getOne(responseId)
+      const storedAnswer = stored.responses?.find(r => r.fieldId === fields.emailText.id)?.answer?.value
+      record(
+        'A2: the stored form response answer still holds the raw typed text (out of scope, unchanged)',
+        storedAnswer === TYPED,
+        `stored answer=${JSON.stringify(storedAnswer)}, typed ${JSON.stringify(TYPED)}`
+        + ` — if this were already trimmed/lowercased, A1 would pass with the guard removed`,
+      )
+    }
+
+    /* ============= B. an invalid value is omitted; the rest of the submission lands ============= */
+    {
+      const { enduser, baselineEmail } = await newContact()
+      const COLOR = `teal-${RAND()}`
+      let submitError: any = undefined
+      try {
+        await submitAnswers(sdk, fixture, enduser.id, [
+          { key: 'emailText', answer: { type: 'string', value: 'not an email at all' } },
+          { key: 'color', answer: { type: 'string', value: COLOR } },
+        ])
+      } catch (err: any) { submitError = err }
+      const after = await reload(enduser.id)
+
+      record(
+        'B1: the submission succeeds — an unparseable answer is never an error for the responder',
+        submitError === undefined,
+        submitError === undefined
+          ? 'submit_form_response resolved normally'
+          : `submit_form_response threw: ${JSON.stringify(submitError?.message ?? submitError)}`,
+      )
+      record(
+        'B2: an invalid email is OMITTED, leaving the existing address untouched',
+        after.email === baselineEmail,
+        `stored email=${JSON.stringify(after.email)}, expected the untouched baseline ${JSON.stringify(baselineEmail)}`
+        + ` — storing the free text here is what would break portal login`,
+      )
+      record(
+        'B3: every other intake field in the SAME submission still lands',
+        (after.fields as any)?.favoriteColor === COLOR,
+        `fields.favoriteColor=${JSON.stringify((after.fields as any)?.favoriteColor)}, expected ${JSON.stringify(COLOR)}`,
+      )
+    }
+
+    /* ======================= C. phone and landline: E.164, or omitted ======================= */
+    {
+      const { enduser } = await newContact()
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'phoneText', answer: { type: 'string', value: '(555) 555-1234' } },
+        { key: 'landlineText', answer: { type: 'string', value: ' 555-555-9876 ' } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'C1: a TEXT question mapped to phone stores the validator E.164 output',
+        after.phone === '+15555551234',
+        `stored phone=${JSON.stringify(after.phone)}, expected "+15555551234" from the typed "(555) 555-1234"`,
+      )
+      record(
+        'C2: landline is normalized the same way (not an auth channel; one stored format)',
+        after.landline === '+15555559876',
+        `stored landline=${JSON.stringify(after.landline)}, expected "+15555559876" from the typed " 555-555-9876 "`,
+      )
+    }
+    {
+      const { enduser, baselinePhone, baselineLandline } = await newContact()
+      const COLOR = `amber-${RAND()}`
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'phoneText', answer: { type: 'string', value: 'call me at work' } },
+        { key: 'landlineText', answer: { type: 'string', value: 'the front desk' } },
+        { key: 'color', answer: { type: 'string', value: COLOR } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'C3: free text that is not a phone number is DROPPED rather than stored',
+        after.phone === baselinePhone && after.landline === baselineLandline,
+        `stored phone=${JSON.stringify(after.phone)} landline=${JSON.stringify(after.landline)},`
+        + ` expected the untouched baselines ${baselinePhone} / ${baselineLandline}`,
+      )
+      record(
+        'C4: the other answers in that same submission still updated',
+        (after.fields as any)?.favoriteColor === COLOR,
+        `fields.favoriteColor=${JSON.stringify((after.fields as any)?.favoriteColor)}, expected ${JSON.stringify(COLOR)}`,
+      )
+    }
+
+    /* ================== D. array-shaped answers: unwrap one, omit the rest ================== */
+    {
+      const { enduser } = await newContact()
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailDropdown', answer: { type: 'Dropdown', value: ['Picked@Example.COM'] } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'D1: a single-select answer is unwrapped to a scalar and lowercased, not stored as a list',
+        after.email === 'picked@example.com',
+        `stored email=${JSON.stringify(after.email)}, expected "picked@example.com"`
+        + ` — the ENDUSER_FIELD_TYPES branch assigns the raw ARRAY, so without the unwrap the contact`
+        + ` holds a bracketed list instead of an address`,
+      )
+    }
+    {
+      const { enduser, baselineEmail } = await newContact()
+      const COLOR = `violet-${RAND()}`
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailMulti', answer: { type: 'multiple_choice', value: ['First@Example.COM', 'Second@Example.COM'] } },
+        { key: 'color', answer: { type: 'string', value: COLOR } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'D2: a MULTI-select answer has no single contact value and is omitted',
+        after.email === baselineEmail,
+        `stored email=${JSON.stringify(after.email)}, expected the untouched baseline ${JSON.stringify(baselineEmail)}`,
+      )
+      record(
+        'D3: the multi-select submission still succeeds and its other answers land',
+        (after.fields as any)?.favoriteColor === COLOR,
+        `fields.favoriteColor=${JSON.stringify((after.fields as any)?.favoriteColor)}, expected ${JSON.stringify(COLOR)}`,
+      )
+    }
+
+    /* ========================== E. over-long value (emailValidator cap) ========================== */
+    {
+      const { enduser, baselineEmail } = await newContact()
+      const COLOR = `indigo-${RAND()}`
+      // 262 characters. emailValidator caps length at 250, but isEmail rejects this first (a local
+      // part over 64 characters is not a valid address) — either way the guard omits it, which is
+      // what the check asserts.
+      const LONG = `${'a'.repeat(250)}@Example.COM`
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailText', answer: { type: 'string', value: LONG } },
+        { key: 'color', answer: { type: 'string', value: COLOR } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'E1: an over-long address is omitted and the submission still succeeds',
+        after.email === baselineEmail && (after.fields as any)?.favoriteColor === COLOR,
+        `stored email=${JSON.stringify(after.email)} (expected the untouched baseline`
+        + ` ${JSON.stringify(baselineEmail)}), fields.favoriteColor=${JSON.stringify((after.fields as any)?.favoriteColor)}`,
+      )
+    }
+
+    /* ============================ F. regressions: unchanged behavior ============================ */
+    {
+      const { enduser } = await newContact()
+      const TYPED = `Regression.${RAND()}@Example.COM`
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailTyped', answer: { type: 'email', value: TYPED } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'F1 (control): an EMAIL-typed question still stores lowercase, exactly as before this ticket',
+        after.email === TYPED.toLowerCase(),
+        `stored email=${JSON.stringify(after.email)}, expected ${JSON.stringify(TYPED.toLowerCase())}`
+        + ` — this path was already lowercased by formResponseAnswerValidator, which is why every`
+        + ` positive case above uses a non-email question type`,
+      )
+    }
+    {
+      const { enduser } = await newContact()
+      // nameValidator only capitalizes the FIRST letter, so MCDONALD survives verbatim
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'lname', answer: { type: 'string', value: 'MCDONALD' } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'F2: name intake is untouched by the guard — capitalization is stored as typed',
+        after.lname === 'MCDONALD',
+        `stored lname=${JSON.stringify(after.lname)}, expected "MCDONALD"`
+        + ` — only email/phone/landline are normalized`,
+      )
+    }
+    {
+      const { enduser, baselineEmail, baselinePhone, baselineLandline } = await newContact()
+      const COLOR = `olive-${RAND()}`
+      // the email question is simply not answered
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'color', answer: { type: 'string', value: COLOR } },
+      ])
+      const after = await reload(enduser.id)
+
+      record(
+        'F3: a submission that answers no contact question leaves email/phone/landline untouched',
+        after.email === baselineEmail && after.phone === baselinePhone && after.landline === baselineLandline,
+        `stored email=${JSON.stringify(after.email)} phone=${JSON.stringify(after.phone)}`
+        + ` landline=${JSON.stringify(after.landline)}; expected the untouched baselines`
+        + ` ${JSON.stringify(baselineEmail)} / ${baselinePhone} / ${baselineLandline}`,
+      )
+    }
+
+    /* ====================== G. Related Contacts (Requirement 9) ====================== */
+    {
+      const { enduser } = await newContact()
+      const TYPED = `Submitter.${RAND()}@Example.COM`
+      const REL_LNAME = `Shared${RAND()}`
+
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailText', answer: { type: 'string', value: TYPED } },
+        {
+          key: 'relatedShared',
+          answer: {
+            type: 'Related Contacts',
+            // the related contact types its OWN email; the shared intake field overwrites it
+            value: [{ fname: 'Related', lname: REL_LNAME, email: `Rel.${RAND()}@Example.COM` }],
+          },
+        },
+      ])
+
+      const related = await poll_for(
+        () => sdk.api.endusers.getSome({ filter: { lname: REL_LNAME } }),
+        e => e.length > 0,
+      )
+      record(
+        'G1: a related contact inheriting a SHARED email gets the submitter normalized address',
+        related.length === 1 && related[0].email === TYPED.toLowerCase(),
+        `found ${related.length} related contact(s), email=${JSON.stringify(related[0]?.email)},`
+        + ` expected ${JSON.stringify(TYPED.toLowerCase())} (submitter typed ${JSON.stringify(TYPED)})`,
+      )
+    }
+    {
+      const { enduser } = await newContact()
+      const TYPED = `Submitter.${RAND()}@Example.COM`
+      const REL_LNAME = `Unshared${RAND()}`
+      const REL_EMAIL = `Rel.${RAND()}@Example.COM`
+
+      await submitAnswers(sdk, fixture, enduser.id, [
+        { key: 'emailText', answer: { type: 'string', value: TYPED } },
+        {
+          key: 'relatedUnshared',
+          answer: {
+            type: 'Related Contacts',
+            value: [{ fname: 'Related', lname: REL_LNAME, email: REL_EMAIL }],
+          },
+        },
+      ])
+
+      const related = await poll_for(
+        () => sdk.api.endusers.getSome({ filter: { lname: REL_LNAME } }),
+        e => e.length > 0,
+      )
+      record(
+        'G2: with email NOT shared, the related contact keeps the address typed for it (out of scope)',
+        related.length === 1 && related[0].email === REL_EMAIL,
+        `found ${related.length} related contact(s), email=${JSON.stringify(related[0]?.email)},`
+        + ` expected the untouched typed ${JSON.stringify(REL_EMAIL)} — normalizing a related contact's`
+        + ` OWN typed email is deliberately out of scope for CU-86e3kndpg and needs its own ticket`,
+      )
+    }
+
+    return checks
+  } finally {
+    await tenant?.teardown()
+  }
+}
+
+/** Asserting wrapper, for standalone runs. */
+export const form_intake_email_normalization_tests = async () => {
+  assert_checks('Form intake contact-field normalization', await form_intake_email_normalization_checks())
+}
+
+/** Enrolls this suite in the auto-discovered parallel pool — see run_discovered_isolated_suites. */
+export const ISOLATED_SUITE: IsolatedSuite = {
+  name: 'Form intake contact-field normalization',
+  run: form_intake_email_normalization_checks,
+}
+
+// Allow running this test file independently
+if (require.main === module) {
+  console.log(`Using API URL: ${host}`)
+  form_intake_email_normalization_tests()
+    .then(() => {
+      console.log("Form intake contact-field normalization test suite completed successfully")
+      process.exit(0)
+    })
+    .catch((error) => {
+      console.error("Form intake contact-field normalization test suite failed:", error)
+      process.exit(1)
+    })
+}