@devtion/backend 0.0.0-4088679 → 0.0.0-477457c

package/src/functions/circuit.ts

@@ -40,6 +40,8 @@ import {
 } from "@devtion/actions"
 import { zKey } from "snarkjs"
 import { CommandInvocationStatus, SSMClient } from "@aws-sdk/client-ssm"
+import { EC2Client } from "@aws-sdk/client-ec2"
+import { HttpsError } from "firebase-functions/v2/https"
 import { FinalizeCircuitData, VerifyContributionData } from "../types/index"
 import { LogLevel } from "../types/enums"
 import { COMMON_ERRORS, logAndThrowError, makeError, printLog, SPECIFIC_ERRORS } from "../lib/errors"
@@ -57,7 +59,6 @@ import {
     sleep,
     uploadFileToBucket
 } from "../lib/utils"
-import { EC2Client } from "@aws-sdk/client-ec2"
 
 dotenv.config()
 
@@ -114,7 +115,7 @@ const coordinate = async (
     if (isSingleParticipantCoordination) {
         // Scenario (A).
         if (emptyWaitingQueue) {
-            printLog(`Coordinate - executing scenario A - emptyWaitingQueue`, LogLevel.DEBUG)
+            printLog(`Coordinate - executing scenario A - emptyWaitingQueue`, LogLevel.INFO)
 
             // Update.
             newCurrentContributorId = participant.id
@@ -126,7 +127,7 @@ const coordinate = async (
         else if (participantResumingAfterTimeoutExpiration) {
             printLog(
                 `Coordinate - executing scenario A - single - participantResumingAfterTimeoutExpiration`,
-                LogLevel.DEBUG
+                LogLevel.INFO
             )
 
             newParticipantStatus = ParticipantStatus.CONTRIBUTING
@@ -135,7 +136,7 @@ const coordinate = async (
         }
         // Scenario (B).
         else if (participantIsNotCurrentContributor) {
-            printLog(`Coordinate - executing scenario B - single - participantIsNotCurrentContributor`, LogLevel.DEBUG)
+            printLog(`Coordinate - executing scenario B - single - participantIsNotCurrentContributor`, LogLevel.INFO)
 
             newCurrentContributorId = currentContributor
             newParticipantStatus = ParticipantStatus.WAITING
@@ -159,7 +160,7 @@ const coordinate = async (
     } else if (participantIsCurrentContributor && participantCompletedOneOrAllContributions && !!ceremonyId) {
         printLog(
             `Coordinate - executing scenario C - multi - participantIsCurrentContributor && participantCompletedOneOrAllContributions`,
-            LogLevel.DEBUG
+            LogLevel.INFO
         )
 
         newParticipantStatus = ParticipantStatus.CONTRIBUTING
@@ -189,7 +190,7 @@ const coordinate = async (
 
             printLog(
                 `Participant ${newCurrentContributorId} is the new current contributor for circuit ${circuit.id}`,
-                LogLevel.DEBUG
+                LogLevel.INFO
             )
         }
     }
@@ -214,57 +215,80 @@ const coordinate = async (
  * Wait until the command has completed its execution inside the VM.
  * @dev this method implements a custom interval to check 5 times after 1 minute if the command execution
  * has been completed or not by calling the `retrieveCommandStatus` method.
- * @param {any} resolve the promise.
- * @param {any} reject the promise.
  * @param {SSMClient} ssm the SSM client.
  * @param {string} vmInstanceId the unique identifier of the VM instance.
  * @param {string} commandId the unique identifier of the VM command.
  * @returns <Promise<void>> true when the command execution succeed; otherwise false.
  */
-const waitForVMCommandExecution = (
-    resolve: any,
-    reject: any,
-    ssm: SSMClient,
-    vmInstanceId: string,
-    commandId: string
-) => {
-    const interval = setInterval(async () => {
-        try {
-            // Get command status.
-            const cmdStatus = await retrieveCommandStatus(ssm, vmInstanceId, commandId)
-            printLog(`Checking command ${commandId} status => ${cmdStatus}`, LogLevel.DEBUG)
+const waitForVMCommandExecution = (ssm: SSMClient, vmInstanceId: string, commandId: string): Promise<void> =>
+    new Promise((resolve, reject) => {
+        const poll = async () => {
+            try {
+                // Get command status.
+                const cmdStatus = await retrieveCommandStatus(ssm, vmInstanceId, commandId)
+                printLog(`Checking command ${commandId} status => ${cmdStatus}`, LogLevel.DEBUG)
+
+                let error: HttpsError | undefined
+                switch (cmdStatus) {
+                    case CommandInvocationStatus.CANCELLING:
+                    case CommandInvocationStatus.CANCELLED: {
+                        error = SPECIFIC_ERRORS.SE_VM_CANCELLED_COMMAND_EXECUTION
+                        break
+                    }
+                    case CommandInvocationStatus.DELAYED: {
+                        error = SPECIFIC_ERRORS.SE_VM_DELAYED_COMMAND_EXECUTION
+                        break
+                    }
+                    case CommandInvocationStatus.FAILED: {
+                        error = SPECIFIC_ERRORS.SE_VM_FAILED_COMMAND_EXECUTION
+                        break
+                    }
+                    case CommandInvocationStatus.TIMED_OUT: {
+                        error = SPECIFIC_ERRORS.SE_VM_TIMEDOUT_COMMAND_EXECUTION
+                        break
+                    }
+                    case CommandInvocationStatus.IN_PROGRESS:
+                    case CommandInvocationStatus.PENDING: {
+                        // wait a minute and poll again
+                        setTimeout(poll, 60000)
+                        return
+                    }
+                    case CommandInvocationStatus.SUCCESS: {
+                        printLog(`Command ${commandId} successfully completed`, LogLevel.DEBUG)
+
+                        // Resolve the promise.
+                        resolve()
+                        return
+                    }
+                    default: {
+                        logAndThrowError(SPECIFIC_ERRORS.SE_VM_UNKNOWN_COMMAND_STATUS)
+                    }
+                }
+
+                if (error) {
+                    logAndThrowError(error)
+                }
+            } catch (error: any) {
+                printLog(`Invalid command ${commandId} execution`, LogLevel.DEBUG)
 
-            if (cmdStatus === CommandInvocationStatus.SUCCESS) {
-                printLog(`Command ${commandId} successfully completed`, LogLevel.DEBUG)
+                const ec2 = await createEC2Client()
 
-                // Resolve the promise.
-                resolve()
-            } else if (cmdStatus === CommandInvocationStatus.FAILED) {
-                logAndThrowError(SPECIFIC_ERRORS.SE_VM_FAILED_COMMAND_EXECUTION)
-                reject()
-            } else if (cmdStatus === CommandInvocationStatus.TIMED_OUT) {
-                logAndThrowError(SPECIFIC_ERRORS.SE_VM_TIMEDOUT_COMMAND_EXECUTION)
-                reject()
-            } else if (cmdStatus === CommandInvocationStatus.CANCELLED) {
-                logAndThrowError(SPECIFIC_ERRORS.SE_VM_CANCELLED_COMMAND_EXECUTION)
-                reject()
-            } else if (cmdStatus === CommandInvocationStatus.DELAYED) {
-                logAndThrowError(SPECIFIC_ERRORS.SE_VM_DELAYED_COMMAND_EXECUTION)
-                reject()
-            }
-        } catch (error: any) {
-            printLog(`Invalid command ${commandId} execution`, LogLevel.DEBUG)
+                // if it errors out, let's just log it as a warning so the coordinator is aware
+                try {
+                    await stopEC2Instance(ec2, vmInstanceId)
+                } catch (stopError: any) {
+                    printLog(`Error while stopping VM instance ${vmInstanceId} - Error ${stopError}`, LogLevel.WARN)
+                }
 
-            if (!error.toString().includes(commandId)) logAndThrowError(COMMON_ERRORS.CM_INVALID_COMMAND_EXECUTION)
+                if (!error.toString().includes(commandId)) logAndThrowError(COMMON_ERRORS.CM_INVALID_COMMAND_EXECUTION)
 
-            // Reject the promise.
-            reject()
-        } finally {
-            // Clear the interval.
-            clearInterval(interval)
+                // Reject the promise.
+                reject()
+            }
         }
-    }, 60000) // 1 minute.
-}
+
+        setTimeout(poll, 60000)
+    })
 
 /**
  * This method is used to coordinate the waiting queues of ceremony circuits.
@@ -286,9 +310,9 @@ const waitForVMCommandExecution = (
  * - Just completed a contribution or all contributions for each circuit. If yes, coordinate (multi-participant scenario).
  */
 export const coordinateCeremonyParticipant = functionsV1
-    .region('europe-west1')
+    .region("europe-west1")
     .runWith({
-        memory: "512MB"
+        memory: "1GB"
     })
     .firestore.document(
         `${commonTerms.collections.ceremonies.name}/{ceremonyId}/${commonTerms.collections.participants.name}/{participantId}`
@@ -322,10 +346,10 @@ export const coordinateCeremonyParticipant = functionsV1
             contributionStep: changedContributionStep
         } = changedParticipant.data()!
 
-        printLog(`Coordinate participant ${exParticipant.id} for ceremony ${ceremonyId}`, LogLevel.DEBUG)
+        printLog(`Coordinate participant ${exParticipant.id} for ceremony ${ceremonyId}`, LogLevel.INFO)
         printLog(
             `Participant status: ${prevStatus} => ${changedStatus} - Participant contribution step: ${prevContributionStep} => ${changedContributionStep}`,
-            LogLevel.DEBUG
+            LogLevel.INFO
         )
 
         // Define pre-conditions.
@@ -346,8 +370,8 @@ export const coordinateCeremonyParticipant = functionsV1
 
         const participantCompletedContribution =
             prevContributionProgress === changedContributionProgress &&
-            prevStatus === ParticipantStatus.CONTRIBUTING &&
-            prevContributionStep === ParticipantContributionStep.VERIFYING &&
+            (prevStatus === ParticipantStatus.CONTRIBUTING ||
+                prevContributionStep === ParticipantContributionStep.VERIFYING) &&
             changedStatus === ParticipantStatus.CONTRIBUTED &&
             changedContributionStep === ParticipantContributionStep.COMPLETED
 
@@ -360,7 +384,7 @@ export const coordinateCeremonyParticipant = functionsV1
             // Step (2.A).
             printLog(
                 `Participant is ready for first contribution (${participantReadyForFirstContribution}) or for the next contribution (${participantReadyForNextContribution}) or is resuming after a timeout expiration (${participantResumingContributionAfterTimeout})`,
-                LogLevel.DEBUG
+                LogLevel.INFO
             )
 
             // Get the circuit.
@@ -374,7 +398,7 @@ export const coordinateCeremonyParticipant = functionsV1
             // Step (2.B).
             printLog(
                 `Participant completed a contribution (${participantCompletedContribution}) or every contribution for each circuit (${participantCompletedEveryCircuitContribution})`,
-                LogLevel.DEBUG
+                LogLevel.INFO
             )
 
             // Get the circuit.
@@ -387,7 +411,6 @@ export const coordinateCeremonyParticipant = functionsV1
         }
     })
 
-
 /**
  * Recursive function to check whether an EC2 is in a running state
  * @notice required step to run commands
@@ -396,23 +419,18 @@ export const coordinateCeremonyParticipant = functionsV1
  * @param attempts <number> - how many times to retry before failing
  * @returns <Promise<boolean>> - whether the VM was started
  */
-const checkIfVMRunning = async (
-    ec2: EC2Client, 
-    vmInstanceId: string, 
-    attempts = 5
-    ): Promise<boolean> => {
+const checkIfVMRunning = async (ec2: EC2Client, vmInstanceId: string, attempts = 5): Promise<boolean> => {
     // if we tried 5 times, then throw an error
     if (attempts <= 0) logAndThrowError(SPECIFIC_ERRORS.SE_VM_NOT_RUNNING)
 
-    await sleep(60000); // Wait for 1 min
-    const isVMRunning = await checkIfRunning(ec2, vmInstanceId) 
+    await sleep(60000) // Wait for 1 min
+    const isVMRunning = await checkIfRunning(ec2, vmInstanceId)
 
     if (!isVMRunning) {
         printLog(`VM not running, ${attempts - 1} attempts remaining. Retrying in 1 minute...`, LogLevel.DEBUG)
-        return await checkIfVMRunning(ec2, vmInstanceId, attempts - 1)
-    } else {
-        return true
+        return checkIfVMRunning(ec2, vmInstanceId, attempts - 1)
     }
+    return true
 }
 
 /**
@@ -442,367 +460,391 @@ const checkIfVMRunning = async (
  * 2) Send all updates atomically to the Firestore database.
  */
 export const verifycontribution = functionsV2.https.onCall(
-    { memory: "16GiB", timeoutSeconds: 3600, region: 'europe-west1' },
+    { memory: "32GiB", timeoutSeconds: 3600, region: "europe-west1", cpu: 8 },
     async (request: functionsV2.https.CallableRequest<VerifyContributionData>): Promise<any> => {
-        if (!request.auth || (!request.auth.token.participant && !request.auth.token.coordinator))
-            logAndThrowError(SPECIFIC_ERRORS.SE_AUTH_NO_CURRENT_AUTH_USER)
-
-        if (
-            !request.data.ceremonyId ||
-            !request.data.circuitId ||
-            !request.data.contributorOrCoordinatorIdentifier ||
-            !request.data.bucketName
-        )
-            logAndThrowError(COMMON_ERRORS.CM_MISSING_OR_WRONG_INPUT_DATA)
-
-        if (
-            !process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_NAME ||
-            !process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_VERSION ||
-            !process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_COMMIT_HASH
-        )
-            logAndThrowError(COMMON_ERRORS.CM_WRONG_CONFIGURATION)
-
-        // Step (0).
-
-        // Prepare and start timer.
-        const verifyContributionTimer = new Timer({ label: commonTerms.cloudFunctionsNames.verifyContribution })
-        verifyContributionTimer.start()
-
-        // Get DB.
-        const firestore = admin.firestore()
-        // Prepare batch of txs.
-        const batch = firestore.batch()
-
-        // Extract data.
-        const { ceremonyId, circuitId, contributorOrCoordinatorIdentifier, bucketName } = request.data
-        const userId = request.auth?.uid
-
-        // Look for the ceremony, circuit and participant document.
-        const ceremonyDoc = await getDocumentById(commonTerms.collections.ceremonies.name, ceremonyId)
-        const circuitDoc = await getDocumentById(getCircuitsCollectionPath(ceremonyId), circuitId)
-        const participantDoc = await getDocumentById(getParticipantsCollectionPath(ceremonyId), userId!)
+        try {
+            if (!request.auth || (!request.auth.token.participant && !request.auth.token.coordinator))
+                logAndThrowError(SPECIFIC_ERRORS.SE_AUTH_NO_CURRENT_AUTH_USER)
+
+            if (
+                !request.data.ceremonyId ||
+                !request.data.circuitId ||
+                !request.data.contributorOrCoordinatorIdentifier ||
+                !request.data.bucketName
+            )
+                logAndThrowError(COMMON_ERRORS.CM_MISSING_OR_WRONG_INPUT_DATA)
 
-        if (!ceremonyDoc.data() || !circuitDoc.data() || !participantDoc.data())
-            logAndThrowError(COMMON_ERRORS.CM_INEXISTENT_DOCUMENT_DATA)
+            if (
+                !process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_NAME ||
+                !process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_VERSION ||
+                !process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_COMMIT_HASH
+            )
+                logAndThrowError(COMMON_ERRORS.CM_WRONG_CONFIGURATION)
+
+            const BUCKET_NAME_REGEX = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/
+            if (!BUCKET_NAME_REGEX.test(request.data.bucketName)) logAndThrowError(SPECIFIC_ERRORS.WRONG_BUCKET_NAME)
+
+            // Step (0).
+
+            // Prepare and start timer.
+            const verifyContributionTimer = new Timer({ label: commonTerms.cloudFunctionsNames.verifyContribution })
+            verifyContributionTimer.start()
+
+            // Get DB.
+            const firestore = admin.firestore()
+            // Prepare batch of txs.
+            const batch = firestore.batch()
+
+            // Extract data.
+            const { ceremonyId, circuitId, contributorOrCoordinatorIdentifier, bucketName } = request.data
+            const userId = request.auth?.uid
+
+            // Look for the ceremony, circuit and participant document.
+            const ceremonyDoc = await getDocumentById(commonTerms.collections.ceremonies.name, ceremonyId)
+            const circuitDoc = await getDocumentById(getCircuitsCollectionPath(ceremonyId), circuitId)
+            const participantDoc = await getDocumentById(getParticipantsCollectionPath(ceremonyId), userId!)
+
+            if (!ceremonyDoc.data() || !circuitDoc.data() || !participantDoc.data())
+                logAndThrowError(COMMON_ERRORS.CM_INEXISTENT_DOCUMENT_DATA)
+
+            // Extract documents data.
+            const { state } = ceremonyDoc.data()!
+            const { status, contributions, verificationStartedAt, contributionStartedAt } = participantDoc.data()!
+            const { waitingQueue, prefix, avgTimings, verification, files } = circuitDoc.data()!
+            const { completedContributions, failedContributions } = waitingQueue
+            const {
+                contributionComputation: avgContributionComputationTime,
+                fullContribution: avgFullContributionTime,
+                verifyCloudFunction: avgVerifyCloudFunctionTime
+            } = avgTimings
+            const { cfOrVm, vm } = verification
+            // we might not have it if the circuit is not using VM.
+            let vmInstanceId: string = ""
+            if (vm) vmInstanceId = vm.vmInstanceId
 
-        // Extract documents data.
-        const { state } = ceremonyDoc.data()!
-        const { status, contributions, verificationStartedAt, contributionStartedAt } = participantDoc.data()!
-        const { waitingQueue, prefix, avgTimings, verification, files } = circuitDoc.data()!
-        const { completedContributions, failedContributions } = waitingQueue
-        const {
-            contributionComputation: avgContributionComputationTime,
-            fullContribution: avgFullContributionTime,
-            verifyCloudFunction: avgVerifyCloudFunctionTime
-        } = avgTimings
-        const { cfOrVm, vm } = verification
-        // we might not have it if the circuit is not using VM.
-        let vmInstanceId: string = ""
-        if (vm) vmInstanceId = vm.vmInstanceId
+            // Define pre-conditions.
+            const isFinalizing = state === CeremonyState.CLOSED && request.auth && request.auth.token.coordinator // true only when the coordinator verifies the final contributions.
+            const isContributing = status === ParticipantStatus.CONTRIBUTING
+            const isUsingVM = cfOrVm === CircuitContributionVerificationMechanism.VM && !!vmInstanceId
+
+            // Prepare state.
+            let isContributionValid = false
+            let verifyCloudFunctionExecutionTime = 0 // time spent while executing the verify contribution cloud function.
+            let verifyCloudFunctionTime = 0 // time spent while executing the core business logic of this cloud function.
+            let fullContributionTime = 0 // time spent while doing non-verification contributions tasks (download, compute, upload).
+            let contributionComputationTime = 0 // time spent while computing the contribution.
+            let lastZkeyBlake2bHash: string = "" // the Blake2B hash of the last zKey.
+            let verificationTranscriptTemporaryLocalPath: string = "" // the local temporary path for the verification transcript.
+            let transcriptBlake2bHash: string = "" // the Blake2B hash of the verification transcript.
+            let commandId: string = "" // the unique identifier of the VM command.
+
+            // Derive necessary data.
+            const lastZkeyIndex = formatZkeyIndex(completedContributions + 1)
+            const verificationTranscriptCompleteFilename = `${prefix}_${
+                isFinalizing
+                    ? `${contributorOrCoordinatorIdentifier}_${finalContributionIndex}_verification_transcript.log`
+                    : `${lastZkeyIndex}_${contributorOrCoordinatorIdentifier}_verification_transcript.log`
+            }`
+
+            const lastZkeyFilename = `${prefix}_${isFinalizing ? finalContributionIndex : lastZkeyIndex}.zkey`
+
+            // Prepare state for VM verification (if needed).
+            const ec2 = await createEC2Client()
+            const ssm = await createSSMClient()
+
+            // Step (1.A.1).
+            // Get storage paths.
+            const verificationTranscriptStoragePathAndFilename = getTranscriptStorageFilePath(
+                prefix,
+                verificationTranscriptCompleteFilename
+            )
+            // the zKey storage path is required to be sent to the VM api
+            const lastZkeyStoragePath = getZkeyStorageFilePath(
+                prefix,
+                `${prefix}_${isFinalizing ? finalContributionIndex : lastZkeyIndex}.zkey`
+            )
 
-        // Define pre-conditions.
-        const isFinalizing = state === CeremonyState.CLOSED && request.auth && request.auth.token.coordinator // true only when the coordinator verifies the final contributions.
-        const isContributing = status === ParticipantStatus.CONTRIBUTING
-        const isUsingVM = cfOrVm === CircuitContributionVerificationMechanism.VM && !!vmInstanceId
-
-        // Prepare state.
-        let isContributionValid = false
-        let verifyCloudFunctionExecutionTime = 0 // time spent while executing the verify contribution cloud function.
-        let verifyCloudFunctionTime = 0 // time spent while executing the core business logic of this cloud function.
-        let fullContributionTime = 0 // time spent while doing non-verification contributions tasks (download, compute, upload).
-        let contributionComputationTime = 0 // time spent while computing the contribution.
-        let lastZkeyBlake2bHash: string = "" // the Blake2B hash of the last zKey.
-        let verificationTranscriptTemporaryLocalPath: string = "" // the local temporary path for the verification transcript.
-        let transcriptBlake2bHash: string = "" // the Blake2B hash of the verification transcript.
-        let commandId: string = "" // the unique identifier of the VM command.
-
-        // Derive necessary data.
-        const lastZkeyIndex = formatZkeyIndex(completedContributions + 1)
-        const verificationTranscriptCompleteFilename = `${prefix}_${
-            isFinalizing
-                ? `${contributorOrCoordinatorIdentifier}_${finalContributionIndex}_verification_transcript.log`
-                : `${lastZkeyIndex}_${contributorOrCoordinatorIdentifier}_verification_transcript.log`
-        }`
-
-        const lastZkeyFilename = `${prefix}_${isFinalizing ? finalContributionIndex : lastZkeyIndex}.zkey`
-
-        // Prepare state for VM verification (if needed).
-        const ec2 = await createEC2Client()
-        const ssm = await createSSMClient()
-
-        // Step (1.A.1).
-        // Get storage paths.
-        const verificationTranscriptStoragePathAndFilename = getTranscriptStorageFilePath(
-            prefix,
-            verificationTranscriptCompleteFilename
-        )
-        // the zKey storage path is required to be sent to the VM api
-        const lastZkeyStoragePath = getZkeyStorageFilePath(
-            prefix,
-            `${prefix}_${isFinalizing ? finalContributionIndex : lastZkeyIndex}.zkey`
-        )
+            const verificationTaskTimer = new Timer({ label: `${ceremonyId}-${circuitId}-${participantDoc.id}` })
 
-        const verificationTaskTimer = new Timer({ label: `${ceremonyId}-${circuitId}-${participantDoc.id}` })
+            const dumpLog = async (path: string): Promise<void> => {
+                printLog(`transcript >>>>>>`, LogLevel.DEBUG)
+                try {
+                    const data = await fs.promises.readFile(path, "utf8")
+                    printLog(data, LogLevel.DEBUG)
+                } catch (readError: any) {
+                    printLog(readError, LogLevel.ERROR)
+                }
+            }
 
-        const completeVerification = async () => {
-            // Stop verification task timer.
-            printLog("Completing verification", LogLevel.DEBUG)
-            verificationTaskTimer.stop()
-            verifyCloudFunctionExecutionTime = verificationTaskTimer.ms()
+            const completeVerification = async () => {
+                // Stop verification task timer.
+                printLog("Completing verification", LogLevel.DEBUG)
+                verificationTaskTimer.stop()
+                verifyCloudFunctionExecutionTime = verificationTaskTimer.ms()
 
-            if (isUsingVM) {
-                // Create temporary path.
-                verificationTranscriptTemporaryLocalPath = createTemporaryLocalPath(
-                    `${circuitId}_${participantDoc.id}.log`
-                )
+                if (isUsingVM) {
+                    // Create temporary path.
+                    verificationTranscriptTemporaryLocalPath = createTemporaryLocalPath(
+                        `${circuitId}_${participantDoc.id}.log`
+                    )
 
-                await sleep(1000) // wait 1s for file creation.
+                    await sleep(1000) // wait 1s for file creation.
 
-                // Download from bucket.
-                // nb. the transcript MUST be uploaded from the VM by verification commands.
-                await downloadArtifactFromS3Bucket(
-                    bucketName,
-                    verificationTranscriptStoragePathAndFilename,
-                    verificationTranscriptTemporaryLocalPath
-                )
+                    // Download from bucket.
+                    // nb. the transcript MUST be uploaded from the VM by verification commands.
+                    await downloadArtifactFromS3Bucket(
+                        bucketName,
+                        verificationTranscriptStoragePathAndFilename,
+                        verificationTranscriptTemporaryLocalPath
+                    )
 
-                // Read the verification trascript and validate data by checking for core info ("ZKey Ok!").
-                const content = fs.readFileSync(verificationTranscriptTemporaryLocalPath, "utf-8")
+                    // Read the verification trascript and validate data by checking for core info ("ZKey Ok!").
+                    const content = fs.readFileSync(verificationTranscriptTemporaryLocalPath, "utf-8")
 
-                if (content.includes("ZKey Ok!")) isContributionValid = true
+                    if (content.includes("ZKey Ok!")) isContributionValid = true
 
-                // If the contribution is valid, then format and store the trascript.
-                if (isContributionValid) {
-                    // eslint-disable-next-line no-control-regex
-                    const updated = content.replace(/\x1b[[0-9;]*m/g, "")
+                    // If the contribution is valid, then format and store the trascript.
+                    if (isContributionValid) {
+                        // eslint-disable-next-line no-control-regex
+                        const updated = content.replace(/\x1b[[0-9;]*m/g, "")
 
-                    fs.writeFileSync(verificationTranscriptTemporaryLocalPath, updated)
+                        fs.writeFileSync(verificationTranscriptTemporaryLocalPath, updated)
+                    }
                 }
-            }
-
-            printLog(`The contribution has been verified - Result ${isContributionValid}`, LogLevel.DEBUG)
-
-            // Create a new contribution document.
-            const contributionDoc = await firestore
-                .collection(getContributionsCollectionPath(ceremonyId, circuitId))
-                .doc()
-                .get()
 
-            // Step (1.A.4).
-            if (isContributionValid) {
-                // Sleep ~3 seconds to wait for verification transcription.
-                await sleep(3000)
+                printLog(`The contribution has been verified - Result ${isContributionValid}`, LogLevel.DEBUG)
 
-                // Step (1.A.4.A.1).
-                if (isUsingVM) {
-                    // Retrieve the contribution hash from the command output.
-                    lastZkeyBlake2bHash = await retrieveCommandOutput(ssm, vmInstanceId, commandId)
-
-                    const hashRegex = /[a-fA-F0-9]{64}/
-                    const match = lastZkeyBlake2bHash.match(hashRegex)!
+                // Create a new contribution document.
+                const contributionDoc = await firestore
+                    .collection(getContributionsCollectionPath(ceremonyId, circuitId))
+                    .doc()
+                    .get()
 
-                    lastZkeyBlake2bHash = match.at(0)!
-
-                    // re upload the formatted verification transcript
-                    await uploadFileToBucket(
-                        bucketName,
-                        verificationTranscriptStoragePathAndFilename,
-                        verificationTranscriptTemporaryLocalPath,
-                        true
+                // Step (1.A.4).
+                if (isContributionValid) {
+                    // Sleep ~3 seconds to wait for verification transcription.
+                    await sleep(3000)
+
+                    // Step (1.A.4.A.1).
+                    if (isUsingVM) {
+                        // Retrieve the contribution hash from the command output.
+                        lastZkeyBlake2bHash = await retrieveCommandOutput(ssm, vmInstanceId, commandId)
+
+                        const hashRegex = /[a-fA-F0-9]{64}/
+                        const match = lastZkeyBlake2bHash.match(hashRegex)!
+
+                        lastZkeyBlake2bHash = match.at(0)!
+
+                        // re upload the formatted verification transcript
+                        await uploadFileToBucket(
+                            bucketName,
+                            verificationTranscriptStoragePathAndFilename,
+                            verificationTranscriptTemporaryLocalPath,
+                            true
+                        )
+                    } else {
+                        // Upload verification transcript.
+                        /// nb. do not use multi-part upload here due to small file size.
+                        await uploadFileToBucket(
+                            bucketName,
+                            verificationTranscriptStoragePathAndFilename,
+                            verificationTranscriptTemporaryLocalPath,
+                            true
+                        )
+                    }
+
+                    // Compute verification transcript hash.
+                    transcriptBlake2bHash = await blake512FromPath(verificationTranscriptTemporaryLocalPath)
+
+                    // Free resources by unlinking transcript temporary file.
+                    fs.unlinkSync(verificationTranscriptTemporaryLocalPath)
+
+                    // Filter participant contributions to find the data related to the one verified.
+                    const participantContributions = contributions.filter(
+                        (contribution: Contribution) =>
+                            !!contribution.hash && !!contribution.computationTime && !contribution.doc
                     )
 
+                    /// @dev (there must be only one contribution with an empty 'doc' field).
+                    if (participantContributions.length !== 1)
+                        logAndThrowError(SPECIFIC_ERRORS.SE_VERIFICATION_NO_PARTICIPANT_CONTRIBUTION_DATA)
+
+                    // Get contribution computation time.
+                    contributionComputationTime = contributions.at(0).computationTime
+
+                    // Step (1.A.4.A.2).
+                    batch.create(contributionDoc.ref, {
+                        participantId: participantDoc.id,
+                        contributionComputationTime,
+                        verificationComputationTime: verifyCloudFunctionExecutionTime,
+                        zkeyIndex: isFinalizing ? finalContributionIndex : lastZkeyIndex,
+                        files: {
+                            transcriptFilename: verificationTranscriptCompleteFilename,
+                            lastZkeyFilename,
+                            transcriptStoragePath: verificationTranscriptStoragePathAndFilename,
+                            lastZkeyStoragePath,
+                            transcriptBlake2bHash,
+                            lastZkeyBlake2bHash
+                        },
+                        verificationSoftware: {
+                            name: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_NAME),
+                            version: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_VERSION),
+                            commitHash: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_COMMIT_HASH)
+                        },
+                        valid: isContributionValid,
+                        lastUpdated: getCurrentServerTimestampInMillis()
+                    })
+
+                    verifyContributionTimer.stop()
+                    verifyCloudFunctionTime = verifyContributionTimer.ms()
                 } else {
-                    // Upload verification transcript.
-                    /// nb. do not use multi-part upload here due to small file size.
-                    await uploadFileToBucket(
-                        bucketName,
-                        verificationTranscriptStoragePathAndFilename,
-                        verificationTranscriptTemporaryLocalPath,
-                        true
-                    )
+                    // Step (1.A.4.B).
+
+                    // Free-up storage by deleting invalid contribution.
+                    await deleteObject(bucketName, lastZkeyStoragePath)
+
+                    // Step (1.A.4.B.1).
+                    batch.create(contributionDoc.ref, {
+                        participantId: participantDoc.id,
+                        verificationComputationTime: verifyCloudFunctionExecutionTime,
+                        zkeyIndex: isFinalizing ? finalContributionIndex : lastZkeyIndex,
+                        verificationSoftware: {
+                            name: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_NAME),
+                            version: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_VERSION),
+                            commitHash: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_COMMIT_HASH)
+                        },
+                        valid: isContributionValid,
+                        lastUpdated: getCurrentServerTimestampInMillis()
+                    })
                 }
 
-                // Compute verification transcript hash.
-                transcriptBlake2bHash = await blake512FromPath(verificationTranscriptTemporaryLocalPath)
+                // Stop VM instance
+                if (isUsingVM) {
+                    // using try and catch as the VM stopping function can throw
+                    // however we want to continue without stopping as the
+                    // verification was valid, and inform the coordinator
+                    try {
+                        await stopEC2Instance(ec2, vmInstanceId)
+                    } catch (error: any) {
+                        printLog(`Error while stopping VM instance ${vmInstanceId} - Error ${error}`, LogLevel.WARN)
+                    }
+                }
+                // Step (1.A.4.C)
+                if (!isFinalizing) {
+                    // Step (1.A.4.C.1)
+                    // Compute new average contribution/verification time.
+                    fullContributionTime = Number(verificationStartedAt) - Number(contributionStartedAt)
+
+                    const newAvgContributionComputationTime =
+                        avgContributionComputationTime > 0
+                            ? (avgContributionComputationTime + contributionComputationTime) / 2
+                            : contributionComputationTime
+                    const newAvgFullContributionTime =
+                        avgFullContributionTime > 0
+                            ? (avgFullContributionTime + fullContributionTime) / 2
+                            : fullContributionTime
+                    const newAvgVerifyCloudFunctionTime =
+                        avgVerifyCloudFunctionTime > 0
+                            ? (avgVerifyCloudFunctionTime + verifyCloudFunctionTime) / 2
+                            : verifyCloudFunctionTime
+
+                    // Prepare tx to update circuit average contribution/verification time.
+                    const updatedCircuitDoc = await getDocumentById(getCircuitsCollectionPath(ceremonyId), circuitId)
+                    const { waitingQueue: updatedWaitingQueue } = updatedCircuitDoc.data()!
+                    /// @dev this must happen only for valid contributions.
+                    batch.update(circuitDoc.ref, {
+                        avgTimings: {
+                            contributionComputation: isContributionValid
+                                ? newAvgContributionComputationTime
+                                : avgContributionComputationTime,
+                            fullContribution: isContributionValid
+                                ? newAvgFullContributionTime
+                                : avgFullContributionTime,
+                            verifyCloudFunction: isContributionValid
+                                ? newAvgVerifyCloudFunctionTime
+                                : avgVerifyCloudFunctionTime
+                        },
+                        waitingQueue: {
+                            ...updatedWaitingQueue,
+                            completedContributions: isContributionValid
+                                ? completedContributions + 1
+                                : completedContributions,
+                            failedContributions: isContributionValid ? failedContributions : failedContributions + 1
+                        },
+                        lastUpdated: getCurrentServerTimestampInMillis()
+                    })
+                }
 
-                // Free resources by unlinking transcript temporary file.
-                fs.unlinkSync(verificationTranscriptTemporaryLocalPath)
+                // Step (2).
+                await batch.commit()
 
-                // Filter participant contributions to find the data related to the one verified.
-                const participantContributions = contributions.filter(
-                    (contribution: Contribution) =>
-                        !!contribution.hash && !!contribution.computationTime && !contribution.doc
+                printLog(
+                    `The contribution #${
+                        isFinalizing ? finalContributionIndex : lastZkeyIndex
+                    } of circuit ${circuitId} (ceremony ${ceremonyId}) has been verified as ${
+                        isContributionValid ? "valid" : "invalid"
+                    } for the participant ${participantDoc.id}`,
+                    LogLevel.INFO
                 )
-
-                /// @dev (there must be only one contribution with an empty 'doc' field).
-                if (participantContributions.length !== 1)
-                    logAndThrowError(SPECIFIC_ERRORS.SE_VERIFICATION_NO_PARTICIPANT_CONTRIBUTION_DATA)
-
-                // Get contribution computation time.
-                contributionComputationTime = contributions.at(0).computationTime
-
-                // Step (1.A.4.A.2).
-                batch.create(contributionDoc.ref, {
-                    participantId: participantDoc.id,
-                    contributionComputationTime,
-                    verificationComputationTime: verifyCloudFunctionExecutionTime,
-                    zkeyIndex: isFinalizing ? finalContributionIndex : lastZkeyIndex,
-                    files: {
-                        transcriptFilename: verificationTranscriptCompleteFilename,
-                        lastZkeyFilename,
-                        transcriptStoragePath: verificationTranscriptStoragePathAndFilename,
-                        lastZkeyStoragePath,
-                        transcriptBlake2bHash,
-                        lastZkeyBlake2bHash
-                    },
-                    verificationSoftware: {
-                        name: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_NAME),
-                        version: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_VERSION),
-                        commitHash: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_COMMIT_HASH)
-                    },
-                    valid: isContributionValid,
-                    lastUpdated: getCurrentServerTimestampInMillis()
-                })
-
-                verifyContributionTimer.stop()
-                verifyCloudFunctionTime = verifyContributionTimer.ms()
-            } else {
-                // Step (1.A.4.B).
-
-                // Free-up storage by deleting invalid contribution.
-                await deleteObject(bucketName, lastZkeyStoragePath)
-
-                // Step (1.A.4.B.1).
-                batch.create(contributionDoc.ref, {
-                    participantId: participantDoc.id,
-                    verificationComputationTime: verifyCloudFunctionExecutionTime,
-                    zkeyIndex: isFinalizing ? finalContributionIndex : lastZkeyIndex,
-                    verificationSoftware: {
-                        name: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_NAME),
-                        version: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_VERSION),
-                        commitHash: String(process.env.CUSTOM_CONTRIBUTION_VERIFICATION_SOFTWARE_COMMIT_HASH)
-                    },
-                    valid: isContributionValid,
-                    lastUpdated: getCurrentServerTimestampInMillis()
-                })
-            }
-
-            // Stop VM instance
-            if (isUsingVM) await stopEC2Instance(ec2, vmInstanceId)
-
-            // Step (1.A.4.C)
-            if (!isFinalizing) {
-                // Step (1.A.4.C.1)
-                // Compute new average contribution/verification time.
-                fullContributionTime = Number(verificationStartedAt) - Number(contributionStartedAt)
-
-                const newAvgContributionComputationTime =
-                    avgContributionComputationTime > 0
-                        ? (avgContributionComputationTime + contributionComputationTime) / 2
-                        : contributionComputationTime
-                const newAvgFullContributionTime =
-                    avgFullContributionTime > 0
-                        ? (avgFullContributionTime + fullContributionTime) / 2
-                        : fullContributionTime
-                const newAvgVerifyCloudFunctionTime =
-                    avgVerifyCloudFunctionTime > 0
-                        ? (avgVerifyCloudFunctionTime + verifyCloudFunctionTime) / 2
-                        : verifyCloudFunctionTime
-
-                // Prepare tx to update circuit average contribution/verification time.    
-                const updatedCircuitDoc = await getDocumentById(getCircuitsCollectionPath(ceremonyId), circuitId)
-                const { waitingQueue: updatedWaitingQueue } = updatedCircuitDoc.data()!
-                /// @dev this must happen only for valid contributions.
-                batch.update(circuitDoc.ref, {
-                    avgTimings: {
-                        contributionComputation: isContributionValid
-                            ? newAvgContributionComputationTime
-                            : avgContributionComputationTime,
-                        fullContribution: isContributionValid ? newAvgFullContributionTime : avgFullContributionTime,
-                        verifyCloudFunction: isContributionValid
-                            ? newAvgVerifyCloudFunctionTime
-                            : avgVerifyCloudFunctionTime
-                    },
-                    waitingQueue: {
-                        ...updatedWaitingQueue,
-                        completedContributions: isContributionValid
-                            ? completedContributions + 1
-                            : completedContributions,
-                        failedContributions: isContributionValid ? failedContributions : failedContributions + 1
-                    },
-                    lastUpdated: getCurrentServerTimestampInMillis()
-                })
             }
 
-            // Step (2).
-            await batch.commit()
+            // Step (1).
+            if (isContributing || isFinalizing) {
+                // Prepare timer.
+                verificationTaskTimer.start()
 
-            printLog(
-                `The contribution #${
-                    isFinalizing ? finalContributionIndex : lastZkeyIndex
-                } of circuit ${circuitId} (ceremony ${ceremonyId}) has been verified as ${
-                    isContributionValid ? "valid" : "invalid"
-                } for the participant ${participantDoc.id}`,
-                LogLevel.DEBUG
-            )
-        }
-
-        // Step (1).
-        if (isContributing || isFinalizing) {
-            // Prepare timer.
-            verificationTaskTimer.start()
+                // Step (1.A.3.0).
+                if (isUsingVM) {
+                    printLog(`Starting the VM mechanism`, LogLevel.DEBUG)
 
-            // Step (1.A.3.0).
-            if (isUsingVM) {
-                printLog(`Starting the VM mechanism`, LogLevel.DEBUG)
+                    // Prepare for VM execution.
+                    let isVMRunning = false // true when the VM is up, otherwise false.
 
-                // Prepare for VM execution.
-                let isVMRunning = false // true when the VM is up, otherwise false.
+                    // Step (1.A.3.1).
+                    await startEC2Instance(ec2, vmInstanceId)
 
-                // Step (1.A.3.1).
-                await startEC2Instance(ec2, vmInstanceId)
+                    await sleep(60000) // nb. wait for VM startup (1 mins + retry).
 
-                await sleep(60000) // nb. wait for VM startup (1 mins + retry).
+                    // Check if the startup is running.
+                    isVMRunning = await checkIfVMRunning(ec2, vmInstanceId)
 
-                // Check if the startup is running.
-                isVMRunning = await checkIfVMRunning(ec2, vmInstanceId)
+                    printLog(`VM running: ${isVMRunning}`, LogLevel.DEBUG)
 
-                printLog(`VM running: ${isVMRunning}`, LogLevel.DEBUG)
+                    // Step (1.A.3.2).
+                    // Prepare.
+                    const verificationCommand = vmContributionVerificationCommand(
+                        bucketName,
+                        lastZkeyStoragePath,
+                        verificationTranscriptStoragePathAndFilename
+                    )
 
-                // Step (1.A.3.2).
-                // Prepare.
-                const verificationCommand = vmContributionVerificationCommand(
-                    bucketName,
-                    lastZkeyStoragePath,
-                    verificationTranscriptStoragePathAndFilename
-                )
+                    // Run.
+                    commandId = await runCommandUsingSSM(ssm, vmInstanceId, verificationCommand)
 
-                // Run.
-                commandId = await runCommandUsingSSM(ssm, vmInstanceId, verificationCommand)
+                    printLog(`Starting the execution of command ${commandId}`, LogLevel.DEBUG)
 
-                printLog(`Starting the execution of command ${commandId}`, LogLevel.DEBUG)
+                    // Step (1.A.3.3).
+                    return await waitForVMCommandExecution(ssm, vmInstanceId, commandId)
+                        .then(async () => {
+                            // Command execution successfully completed.
+                            printLog(`Command ${commandId} execution has been successfully completed`, LogLevel.DEBUG)
+                            await completeVerification()
+                        })
+                        .catch((error: any) => {
+                            // Command execution aborted.
+                            printLog(`Command ${commandId} execution has been aborted - Error ${error}`, LogLevel.WARN)
 
-                // Step (1.A.3.3).
-                return new Promise<void>((resolve, reject) =>
-                    waitForVMCommandExecution(resolve, reject, ssm, vmInstanceId, commandId)
-                )
-                    .then(async () => {
-                        // Command execution successfully completed.
-                        printLog(`Command ${commandId} execution has been successfully completed`, LogLevel.DEBUG)
-                        await completeVerification()
-                    })
-                    .catch((error: any) => {
-                        // Command execution aborted.
-                        printLog(`Command ${commandId} execution has been aborted - Error ${error}`, LogLevel.DEBUG)
+                            logAndThrowError(COMMON_ERRORS.CM_INVALID_COMMAND_EXECUTION)
+                        })
+                }
 
-                        logAndThrowError(COMMON_ERRORS.CM_INVALID_COMMAND_EXECUTION)
-                    })
-            } else {
                 // CF approach.
                 printLog(`CF mechanism`, LogLevel.DEBUG)
 
                 const potStoragePath = getPotStorageFilePath(files.potFilename)
                 const firstZkeyStoragePath = getZkeyStorageFilePath(prefix, `${prefix}_${genesisZkeyIndex}.zkey`)
+                printLog(`pot file: ${potStoragePath}`, LogLevel.DEBUG)
+                printLog(`zkey file: ${firstZkeyStoragePath}`, LogLevel.DEBUG)
                 // Prepare temporary file paths.
                 // (nb. these are needed to download the necessary artifacts for verification from AWS S3).
                 verificationTranscriptTemporaryLocalPath = createTemporaryLocalPath(
@@ -811,6 +853,9 @@ export const verifycontribution = functionsV2.https.onCall(
                 const potTempFilePath = createTemporaryLocalPath(`${circuitId}_${participantDoc.id}.pot`)
                 const firstZkeyTempFilePath = createTemporaryLocalPath(`${circuitId}_${participantDoc.id}_genesis.zkey`)
                 const lastZkeyTempFilePath = createTemporaryLocalPath(`${circuitId}_${participantDoc.id}_last.zkey`)
+                printLog(`pot file: ${potTempFilePath}`, LogLevel.DEBUG)
+                printLog(`firstZkey file: ${firstZkeyTempFilePath}`, LogLevel.DEBUG)
+                printLog(`last zkey file: ${lastZkeyTempFilePath}`, LogLevel.DEBUG)
 
                 // Create and populate transcript.
                 const transcriptLogger = createCustomLoggerForFile(verificationTranscriptTemporaryLocalPath)
@@ -834,7 +879,9 @@ export const verifycontribution = functionsV2.https.onCall(
                     lastZkeyTempFilePath,
                     transcriptLogger
                 )
-                
+
+                await dumpLog(verificationTranscriptTemporaryLocalPath)
+
                 // Compute contribution hash.
                 lastZkeyBlake2bHash = await blake512FromPath(lastZkeyTempFilePath)
 
@@ -846,10 +893,15 @@ export const verifycontribution = functionsV2.https.onCall(
                     fs.unlinkSync(lastZkeyTempFilePath)
                 } catch (error: any) {
                     printLog(`Error while unlinking temporary files - Error ${error}`, LogLevel.WARN)
-                }  
+                }
 
                 await completeVerification()
             }
+
+            return null
+        } catch (error: any) {
+            logAndThrowError(makeError("unknown", error))
+            return null
         }
     }
 )
@@ -860,9 +912,9 @@ export const verifycontribution = functionsV2.https.onCall(
  * this does not happen if the participant is actually the coordinator who is finalizing the ceremony.
  */
 export const refreshParticipantAfterContributionVerification = functionsV1
-    .region('europe-west1')
+    .region("europe-west1")
     .runWith({
-        memory: "512MB"
+        memory: "1GB"
     })
     .firestore.document(
         `/${commonTerms.collections.ceremonies.name}/{ceremony}/${commonTerms.collections.circuits.name}/{circuit}/${commonTerms.collections.contributions.name}/{contributions}`
@@ -932,8 +984,8 @@ export const refreshParticipantAfterContributionVerification = functionsV1
         await batch.commit()
 
         printLog(
-            `Participant ${participantId} refreshed after contribution ${createdContribution.id} - The participant was finalizing the ceremony ${isFinalizing}`,
-            LogLevel.DEBUG
+            `Participant ${participantId} refreshed after contribution ${createdContribution.id} - The participant was finalizing the ceremony? ${isFinalizing}`,
+            LogLevel.INFO
         )
     })
 
@@ -943,9 +995,9 @@ export const refreshParticipantAfterContributionVerification = functionsV1
  * and verification key extracted from the circuit final contribution (as part of the ceremony finalization process).
  */
 export const finalizeCircuit = functionsV1
-    .region('europe-west1')
+    .region("europe-west1")
     .runWith({
-        memory: "512MB"
+        memory: "1GB"
     })
     .https.onCall(async (data: FinalizeCircuitData, context: functionsV1.https.CallableContext) => {
         if (!context.auth || !context.auth.token.coordinator) logAndThrowError(COMMON_ERRORS.CM_NOT_COORDINATOR_ROLE)