@defra/forms-engine-plugin 4.0.33 → 4.0.35

package/src/server/plugins/engine/components/FileUploadField.ts

@@ -1,10 +1,15 @@
-import { type FileUploadFieldComponent } from '@defra/forms-model'
+import {
+  type FileUploadFieldComponent,
+  type FormMetadata
+} from '@defra/forms-model'
+import Boom from '@hapi/boom'
 import joi, { type ArraySchema } from 'joi'
 
 import {
   FormComponent,
   isUploadState
 } from '~/src/server/plugins/engine/components/FormComponent.js'
+import { InvalidComponentStateError } from '~/src/server/plugins/engine/pageControllers/errors.js'
 import { messageTemplate } from '~/src/server/plugins/engine/pageControllers/validationOptions.js'
 import {
   FileStatus,
@@ -13,6 +18,7 @@ import {
   type FileState,
   type FileUpload,
   type FileUploadMetadata,
+  type FormContext,
   type FormPayload,
   type FormState,
   type FormStateValue,
@@ -26,7 +32,10 @@ import {
   type UploadStatusResponse
 } from '~/src/server/plugins/engine/types.js'
 import { render } from '~/src/server/plugins/nunjucks/index.js'
-import { type FormQuery } from '~/src/server/routes/types.js'
+import {
+  type FormQuery,
+  type FormRequestPayload
+} from '~/src/server/routes/types.js'
 
 export const uploadIdSchema = joi.string().uuid().required()
 
@@ -284,6 +293,56 @@ export class FileUploadField extends FormComponent {
     return FileUploadField.getAllPossibleErrors()
   }
 
+  async onSubmit(
+    request: FormRequestPayload,
+    metadata: FormMetadata,
+    context: FormContext
+  ) {
+    const notificationEmail = metadata.notificationEmail
+
+    if (!notificationEmail) {
+      // this should not happen because notificationEmail is checked further up
+      // the chain in SummaryPageController before submitForm is called.
+      throw new Error('Unexpected missing notificationEmail in metadata')
+    }
+
+    if (!request.app.model?.services.formSubmissionService) {
+      throw new Error('No form submission service available in app model')
+    }
+
+    const { formSubmissionService } = request.app.model.services
+    const values = this.getFormValueFromState(context.state) ?? []
+
+    const files = values.map((value) => ({
+      fileId: value.status.form.file.fileId,
+      initiatedRetrievalKey: value.status.metadata.retrievalKey
+    }))
+
+    if (!files.length) {
+      return
+    }
+
+    try {
+      await formSubmissionService.persistFiles(files, notificationEmail)
+    } catch (error) {
+      if (
+        Boom.isBoom(error) &&
+        (error.output.statusCode === 403 || // Forbidden - retrieval key invalid
+          error.output.statusCode === 410) // Gone - file expired (took to long to submit, etc)
+      ) {
+        // Failed to persist files. We can't recover from this, the only real way we can recover the submissions is
+        // by resetting the problematic components and letting the user re-try.
+        // Scenarios: file missing from S3, invalid retrieval key (timing problem), etc.
+        throw new InvalidComponentStateError(
+          this,
+          'There was a problem with your uploaded files. Re-upload them before submitting the form again.'
+        )
+      }
+
+      throw error
+    }
+  }
+
   /**
    * Static version of getAllPossibleErrors that doesn't require a component instance.
    */

package/src/server/plugins/engine/components/FormComponent.ts

@@ -1,7 +1,15 @@
-import { type FormComponentsDef, type Item } from '@defra/forms-model'
+import {
+  type FormComponentsDef,
+  type FormMetadata,
+  type Item
+} from '@defra/forms-model'
 
 import { ComponentBase } from '~/src/server/plugins/engine/components/ComponentBase.js'
 import { optionalText } from '~/src/server/plugins/engine/components/constants.js'
+import {
+  type FormContext,
+  type FormRequestPayload
+} from '~/src/server/plugins/engine/types/index.js'
 import {
   type ErrorMessageTemplateList,
   type FileState,
@@ -220,6 +228,14 @@ export class FormComponent extends ComponentBase {
       advancedSettingsErrors: []
     }
   }
+
+  onSubmit(
+    _request: FormRequestPayload,
+    _metadata: FormMetadata,
+    _context: FormContext
+  ): Promise<void> {
+    return Promise.resolve()
+  }
 }
 
 /**

package/src/server/plugins/engine/helpers.ts

@@ -321,6 +321,14 @@ export function getError(detail: ValidationErrorItem): FormSubmissionError {
   }
 }
 
+export function createError(componentName: string, message: string) {
+  return {
+    href: `#${componentName}`,
+    name: componentName,
+    text: message
+  }
+}
+
 /**
  * A small helper to safely generate a crumb token.
  * Checks that the crumb plugin is available, that crumb

package/src/server/plugins/engine/index.ts

@@ -31,6 +31,9 @@ const globals = {
 export const VIEW_PATH = 'src/server/plugins/engine/views'
 export const PLUGIN_PATH = 'node_modules/@defra/forms-engine-plugin'
 
+export const STATE_NOT_YET_VALIDATED = '__stateNotYetValidated'
+export const CURRENT_PAGE_PATH_KEY = '__currentPagePath'
+
 export const prepareNunjucksEnvironment = function (
   env: Environment,
   pluginOptions: PluginOptions

package/src/server/plugins/engine/models/FormModel.ts

@@ -48,6 +48,7 @@ import {
   createPage,
   type PageControllerClass
 } from '~/src/server/plugins/engine/pageControllers/helpers/pages.js'
+import { copyNotYetValidatedState } from '~/src/server/plugins/engine/pageControllers/helpers/state.js'
 import { validationOptions as opts } from '~/src/server/plugins/engine/pageControllers/validationOptions.js'
 import * as defaultServices from '~/src/server/plugins/engine/services/index.js'
 import {
@@ -401,6 +402,9 @@ export class FormModel {
     // Add paths for navigation
     this.assignPaths(context)
 
+    // Handle restoration of payload from say a 'save-and-exit' request
+    copyNotYetValidatedState(request, context)
+
     return context
   }
 

package/src/server/plugins/engine/pageControllers/FileUploadPageController.test.ts

@@ -15,6 +15,7 @@ import {
 import { QuestionPageController } from '~/src/server/plugins/engine/pageControllers/QuestionPageController.js'
 import { serverWithSaveAndExit } from '~/src/server/plugins/engine/pageControllers/__stubs__/server.js'
 import * as pageHelpers from '~/src/server/plugins/engine/pageControllers/helpers/index.js'
+import { getFormMetadata } from '~/src/server/plugins/engine/services/formsService.js'
 import * as uploadService from '~/src/server/plugins/engine/services/uploadService.js'
 import {
   FileStatus,
@@ -33,8 +34,11 @@ import {
   type FormResponseToolkit
 } from '~/src/server/routes/types.js'
 import { type CacheService } from '~/src/server/services/index.js'
+import * as fixtures from '~/test/fixtures/index.js'
 import definition from '~/test/form/definitions/file-upload-basic.js'
 
+jest.mock('~/src/server/plugins/engine/services/formsService.js')
+
 type TestableFileUploadPageController = FileUploadPageController & {
   initiateAndStoreNewUpload(
     req: FormRequest,
@@ -65,8 +69,13 @@ describe('FileUploadPageController', () => {
       basePath: 'test'
     })
 
+    jest.mocked(getFormMetadata).mockResolvedValue(fixtures.form.metadata)
+
     controller = new FileUploadPageController(model, pages[0])
     request = {
+      params: {
+        slug: 'test-form'
+      },
       logger: {
         info: jest.fn(),
         error: jest.fn(),

package/src/server/plugins/engine/pageControllers/FileUploadPageController.ts

@@ -371,6 +371,7 @@ export class FileUploadPageController extends QuestionPageController {
       // Flash the error message.
       const { fileUpload } = this
       const cacheService = getCacheService(request.server)
+
       const name = fileUpload.name
       const text = file.errorMessage ?? 'Unknown error'
       const errors: FormSubmissionError[] = [
@@ -423,6 +424,7 @@ export class FileUploadPageController extends QuestionPageController {
   ) {
     const { fileUpload, href, path } = this
     const { options, schema } = fileUpload
+    const { getFormMetadata } = this.model.services.formsService
 
     const files = this.getFilesFromState(state)
 
@@ -433,10 +435,15 @@ export class FileUploadPageController extends QuestionPageController {
     const max = Math.min(schema.max ?? MAX_UPLOADS, MAX_UPLOADS)
 
     if (files.length < max) {
-      const outputEmail =
-        this.model.def.outputEmail ?? 'defraforms@defra.gov.uk'
-
-      const newUpload = await initiateUpload(href, outputEmail, options.accept)
+      const formMetadata = await getFormMetadata(request.params.slug)
+      const notificationEmail =
+        formMetadata.notificationEmail ?? 'defraforms@defra.gov.uk'
+
+      const newUpload = await initiateUpload(
+        href,
+        notificationEmail,
+        options.accept
+      )
 
       if (newUpload === undefined) {
         throw Boom.badRequest('Unexpected empty response from initiateUpload')

package/src/server/plugins/engine/pageControllers/QuestionPageController.ts

@@ -13,6 +13,7 @@ import { type RouteOptions } from '@hapi/hapi'
 import { type ValidationErrorItem } from 'joi'
 
 import {
+  COMPONENT_STATE_ERROR,
   EXTERNAL_STATE_APPENDAGE,
   EXTERNAL_STATE_PAYLOAD
 } from '~/src/server/constants.js'
@@ -28,7 +29,10 @@ import {
 } from '~/src/server/plugins/engine/helpers.js'
 import { type FormModel } from '~/src/server/plugins/engine/models/index.js'
 import { PageController } from '~/src/server/plugins/engine/pageControllers/PageController.js'
-import { prefillStateFromQueryParameters } from '~/src/server/plugins/engine/pageControllers/helpers/state.js'
+import {
+  clearNotYetValidatedState,
+  prefillStateFromQueryParameters
+} from '~/src/server/plugins/engine/pageControllers/helpers/state.js'
 import {
   type AnyFormRequest,
   type FormContext,
@@ -324,7 +328,8 @@ export class QuestionPageController extends PageController {
 
     const cacheService = getCacheService(request.server)
 
-    return cacheService.setState(request, state)
+    // Clear any 'not yet validated' state before saving to cache
+    return cacheService.setState(request, clearNotYetValidatedState(state))
   }
 
   async mergeState(
@@ -413,6 +418,11 @@ export class QuestionPageController extends PageController {
       const viewModel = this.getViewModel(request, context)
       viewModel.errors = collection.getViewErrors(viewModel.errors)
 
+      const flashedError = request.yar.flash(COMPONENT_STATE_ERROR)
+      const flashedErrors = !Array.isArray(flashedError) ? [flashedError] : []
+
+      viewModel.errors = (viewModel.errors ?? []).concat(flashedErrors)
+
       /**
        * Content components can be hidden based on a condition. If the condition evaluates to true, it is safe to be kept, otherwise discard it
        */

package/src/server/plugins/engine/pageControllers/SummaryPageController.test.ts

@@ -83,4 +83,7 @@ describe('SummaryPageController', () => {
       expect(saveAndExitMock).toHaveBeenCalledWith(request, h, context)
     })
   })
+
+  // Note: InvalidComponentStateError handling is comprehensively tested
+  // in the integration test: test/form/component-state-errors.test.js
 })

package/src/server/plugins/engine/pageControllers/SummaryPageController.ts

@@ -7,12 +7,13 @@ import {
 import Boom from '@hapi/boom'
 import { type RouteOptions } from '@hapi/hapi'
 
+import { COMPONENT_STATE_ERROR } from '~/src/server/constants.js'
 import { ComponentCollection } from '~/src/server/plugins/engine/components/ComponentCollection.js'
-import { FileUploadField } from '~/src/server/plugins/engine/components/FileUploadField.js'
 import { getAnswer } from '~/src/server/plugins/engine/components/helpers/components.js'
 import {
   checkEmailAddressForLiveFormSubmission,
   checkFormStatus,
+  createError,
   getCacheService
 } from '~/src/server/plugins/engine/helpers.js'
 import {
@@ -24,11 +25,11 @@ import {
   type DetailItem
 } from '~/src/server/plugins/engine/models/types.js'
 import { QuestionPageController } from '~/src/server/plugins/engine/pageControllers/QuestionPageController.js'
+import { InvalidComponentStateError } from '~/src/server/plugins/engine/pageControllers/errors.js'
 import {
   type FormConfirmationState,
   type FormContext,
-  type FormContextRequest,
-  type FormSubmissionState
+  type FormContextRequest
 } from '~/src/server/plugins/engine/types.js'
 import {
   FormAction,
@@ -136,21 +137,39 @@ export class SummaryPageController extends QuestionPageController {
     const formMetadata = await getFormMetadata(params.slug)
     const { notificationEmail } = formMetadata
     const { isPreview } = checkFormStatus(request.params)
-    const emailAddress = notificationEmail ?? this.model.def.outputEmail
 
-    checkEmailAddressForLiveFormSubmission(emailAddress, isPreview)
+    checkEmailAddressForLiveFormSubmission(notificationEmail, isPreview)
 
     // Send submission email
-    if (emailAddress) {
+    if (notificationEmail) {
       const viewModel = this.getSummaryViewModel(request, context)
-      await submitForm(
-        context,
-        request,
-        viewModel,
-        model,
-        emailAddress,
-        formMetadata
-      )
+
+      try {
+        await submitForm(
+          context,
+          formMetadata,
+          request,
+          viewModel,
+          model,
+          notificationEmail,
+          formMetadata
+        )
+      } catch (error) {
+        if (error instanceof InvalidComponentStateError) {
+          const govukError = createError(
+            error.component.name,
+            error.userMessage
+          )
+
+          request.yar.flash(COMPONENT_STATE_ERROR, govukError, true)
+
+          await cacheService.resetComponentStates(request, error.getStateKeys())
+
+          return this.proceed(request, h, error.component.page?.path)
+        }
+
+        throw error
+      }
     }
 
     await cacheService.setConfirmationState(request, {
@@ -179,13 +198,14 @@ export class SummaryPageController extends QuestionPageController {
 
 export async function submitForm(
   context: FormContext,
+  metadata: FormMetadata,
   request: FormRequestPayload,
   summaryViewModel: SummaryViewModel,
   model: FormModel,
   emailAddress: string,
   formMetadata: FormMetadata
 ) {
-  await extendFileRetention(model, context.state, emailAddress)
+  await finaliseComponents(request, metadata, context)
 
   const formStatus = checkFormStatus(request.params)
   const logTags = ['submit', 'submissionApi']
@@ -222,39 +242,28 @@ export async function submitForm(
   )
 }
 
-async function extendFileRetention(
-  model: FormModel,
-  state: FormSubmissionState,
-  updatedRetrievalKey: string
+/**
+ * Finalises any components that need post-processing before form submission. Candidates usually involve
+ * those that have external state.
+ * Examples include:
+ * - file uploads which are 'persisted' before submission
+ * - payments which are 'captured' before submission
+ */
+async function finaliseComponents(
+  request: FormRequestPayload,
+  metadata: FormMetadata,
+  context: FormContext
 ) {
-  const { formSubmissionService } = model.services
-  const { persistFiles } = formSubmissionService
-  const files: { fileId: string; initiatedRetrievalKey: string }[] = []
-
-  // For each file upload component with files in
-  // state, add the files to the batch getting persisted
-  model.pages.forEach((page) => {
-    const fileUploadComponents = page.collection.fields.filter(
-      (component) => component instanceof FileUploadField
-    )
-
-    fileUploadComponents.forEach((component) => {
-      const values = component.getFormValueFromState(state)
-      if (!values?.length) {
-        return
-      }
-
-      files.push(
-        ...values.map(({ status }) => ({
-          fileId: status.form.file.fileId,
-          initiatedRetrievalKey: status.metadata.retrievalKey
-        }))
-      )
-    })
-  })
+  const relevantFields = context.relevantPages.flatMap(
+    (page) => page.collection.fields
+  )
 
-  if (files.length) {
-    return persistFiles(files, updatedRetrievalKey)
+  for (const component of relevantFields) {
+    /*
+      Each component will throw InvalidComponent if its state is invalid, which is handled
+      by handleFormSubmit
+    */
+    await component.onSubmit(request, metadata, context)
   }
 }
 

package/src/server/plugins/engine/pageControllers/__stubs__/request.ts

@@ -2,20 +2,30 @@ import { server } from '~/src/server/plugins/engine/pageControllers/__stubs__/se
 import { type FormContextRequest } from '~/src/server/plugins/engine/types.js'
 import { type FormRequest } from '~/src/server/routes/types.js'
 
+const mockYar = {
+  flash: jest.fn().mockReturnValue([]),
+  clear: jest.fn(),
+  get: jest.fn(),
+  set: jest.fn(),
+  reset: jest.fn()
+}
+
 export function buildFormRequest(
-  request: Omit<FormRequest, 'server'>
+  request: Omit<FormRequest, 'server' | 'yar'>
 ): FormRequest {
   return {
     ...request,
+    yar: mockYar,
     server
-  } as FormRequest
+  } as unknown as FormRequest
 }
 
 export function buildFormContextRequest(
-  request: Omit<FormContextRequest, 'server'>
+  request: Omit<FormContextRequest, 'server' | 'yar'>
 ): FormContextRequest {
   return {
     ...request,
+    yar: mockYar,
     server
-  } as FormContextRequest
+  } as unknown as FormContextRequest
 }

package/src/server/plugins/engine/pageControllers/errors.test.ts

@@ -0,0 +1,63 @@
+import { ComponentType } from '@defra/forms-model'
+
+import { FileUploadField } from '~/src/server/plugins/engine/components/FileUploadField.js'
+import { TextField } from '~/src/server/plugins/engine/components/TextField.js'
+import { FormModel } from '~/src/server/plugins/engine/models/FormModel.js'
+import { InvalidComponentStateError } from '~/src/server/plugins/engine/pageControllers/errors.js'
+import definition from '~/test/form/definitions/file-upload-basic.js'
+
+describe('InvalidComponentStateError', () => {
+  let model: FormModel
+
+  beforeEach(() => {
+    model = new FormModel(definition, {
+      basePath: 'test'
+    })
+  })
+
+  describe('getStateKeys', () => {
+    it('should return component name and upload for FileUploadField', () => {
+      const page = model.pages.find((p) => p.path === '/file-upload-component')
+      const component = new FileUploadField(
+        {
+          name: 'fileUpload',
+          title: 'Upload something',
+          type: ComponentType.FileUploadField,
+          options: {},
+          schema: {}
+        },
+        { model, page }
+      )
+
+      const error = new InvalidComponentStateError(
+        component,
+        'Test error message'
+      )
+      const stateKeys = error.getStateKeys()
+
+      expect(stateKeys).toEqual(['fileUpload', 'upload'])
+    })
+
+    it('should return only component name for non-FileUploadField components', () => {
+      const page = model.pages.find((p) => p.path === '/file-upload-component')
+      const component = new TextField(
+        {
+          name: 'textField',
+          title: 'Text field',
+          type: ComponentType.TextField,
+          options: {},
+          schema: {}
+        },
+        { model, page }
+      )
+
+      const error = new InvalidComponentStateError(
+        component,
+        'Test error message'
+      )
+      const stateKeys = error.getStateKeys()
+
+      expect(stateKeys).toEqual(['textField'])
+    })
+  })
+})

package/src/server/plugins/engine/pageControllers/errors.ts

@@ -0,0 +1,30 @@
+import { FileUploadField } from '~/src/server/plugins/engine/components/FileUploadField.js'
+import { type FormComponent } from '~/src/server/plugins/engine/components/FormComponent.js'
+
+/**
+ * Thrown when a component has an invalid state. This is typically only required where state needs
+ * to be checked against an external source upon submission of a form. For example: file upload
+ * has internal state (file upload IDs) but also external state (files in S3). The internal state
+ * is always validated by the engine, but the external state needs validating too.
+ *
+ * This should be used within a formComponent.onSubmit(...).
+ */
+export class InvalidComponentStateError extends Error {
+  public readonly component: FormComponent
+  public readonly userMessage: string
+
+  constructor(component: FormComponent, userMessage: string) {
+    const message = `Invalid component state for: ${component.name}`
+    super(message)
+    this.name = 'InvalidComponentStateError'
+    this.component = component
+    this.userMessage = userMessage
+  }
+
+  getStateKeys() {
+    const extraStateKeys =
+      this.component instanceof FileUploadField ? ['upload'] : []
+
+    return [this.component.name].concat(extraStateKeys)
+  }
+}

package/src/server/plugins/engine/pageControllers/helpers/state.test.ts

@@ -3,10 +3,15 @@ import { ComponentType, type Page } from '@defra/forms-model'
 import { type FormModel } from '~/src/server/plugins/engine/models/FormModel.js'
 import { type PageControllerClass } from '~/src/server/plugins/engine/pageControllers/helpers/pages.js'
 import {
+  copyNotYetValidatedState,
   prefillStateFromQueryParameters,
   stripParam
 } from '~/src/server/plugins/engine/pageControllers/helpers/state.js'
-import { type AnyFormRequest } from '~/src/server/plugins/engine/types.js'
+import {
+  type AnyFormRequest,
+  type FormContext,
+  type FormContextRequest
+} from '~/src/server/plugins/engine/types.js'
 import { type FormsService, type Services } from '~/src/server/types.js'
 
 function buildMockPage(
@@ -218,4 +223,73 @@ describe('State helpers', () => {
       expect(stripParam(params, 'anyParam')).toBeUndefined()
     })
   })
+
+  describe('copyNotYetValidatedState', () => {
+    it('should ignore if no invalid state', () => {
+      const mockRequest = {} as FormContextRequest
+      const mockContext = {
+        state: { abc: '123' },
+        payload: {}
+      } as unknown as FormContext
+      copyNotYetValidatedState(mockRequest, mockContext)
+      expect(mockContext.state).toEqual({ abc: '123' })
+      expect(mockContext.payload).toEqual({})
+    })
+
+    it('should ignore if wrong path', () => {
+      const mockRequest = {
+        url: {
+          pathname: '/form-page1'
+        }
+      } as unknown as FormContextRequest
+      const mockContext = {
+        state: {
+          abc: '123',
+          __stateNotYetValidated: {
+            def: '456',
+            __currentPagePath: '/root'
+          }
+        },
+        payload: {}
+      } as unknown as FormContext
+      copyNotYetValidatedState(mockRequest, mockContext)
+      expect(mockContext.state).toEqual({
+        abc: '123',
+        __stateNotYetValidated: {
+          def: '456',
+          __currentPagePath: '/root'
+        }
+      })
+      expect(mockContext.payload).toEqual({})
+    })
+
+    it('should apply if correct path', () => {
+      const mockRequest = {
+        url: {
+          pathname: '/form-page1'
+        }
+      } as unknown as FormContextRequest
+      const mockContext = {
+        state: {
+          abc: '123',
+          __stateNotYetValidated: {
+            def: '456',
+            __currentPagePath: '/form-page1'
+          }
+        },
+        payload: {}
+      } as unknown as FormContext
+      copyNotYetValidatedState(mockRequest, mockContext)
+      expect(mockContext.state).toEqual({
+        abc: '123',
+        __stateNotYetValidated: {
+          def: '456',
+          __currentPagePath: '/form-page1'
+        }
+      })
+      expect(mockContext.payload).toEqual({
+        def: '456'
+      })
+    })
+  })
 })