@devicecloud.dev/dcd 5.2.0-beta.2 → 5.2.0-beta.4

package/dist/methods.js

@@ -11,6 +11,7 @@ import { inferEnvFromApiUrl } from './config/environments.js';
 import { ApiError, ApiGateway } from './gateways/api-gateway.js';
 import { SupabaseGateway } from './gateways/supabase-gateway.js';
 import { MetadataExtractorService } from './services/metadata-extractor.service.js';
+import { encryptFileToPath, generateDek, isEncryptionEnabled, resolveKekPublicKey, wrapDek, } from './utils/envelope.js';
 import { colors, formatId } from './utils/styling.js';
 const mimeTypeLookupByExtension = {
     apk: 'application/vnd.android.package-archive',
@@ -68,7 +69,7 @@ export const compressFilesFromRelativePath = async (basePath, files, commonRoot)
     return zipToBuffer(zipfile);
 };
 export const verifyAppZip = async (zipPath) => {
-    // eslint-disable-next-line import/namespace, new-cap
+    // eslint-disable-next-line new-cap
     const zip = await new StreamZip.async({
         file: zipPath,
         storeEntries: true,
@@ -94,6 +95,7 @@ export const verifyAppZip = async (zipPath) => {
 };
 export const uploadBinary = async (config) => {
     const { filePath, apiUrl, auth, ignoreShaCheck = false, log = true, debug = false } = config;
+    const encrypt = isEncryptionEnabled(config.encrypt);
     if (log) {
         ux.action.start(colors.bold('Checking and uploading binary'), colors.dim('Initializing'), {
             stdout: true,
@@ -107,14 +109,20 @@ export const uploadBinary = async (config) => {
     }
     const startTime = Date.now();
     let source;
+    let encCleanupDir;
+    let enc;
     try {
         // Prepare file for upload
         source = await prepareFileForUpload(filePath, debug, startTime);
-        // Calculate SHA hash
-        const sha = await calculateFileHash(source, debug, log);
-        // Check for existing upload with same SHA
-        if (!ignoreShaCheck && sha) {
-            const { exists, binaryId } = await checkExistingUpload(apiUrl, auth, sha, debug);
+        // Hash the PLAINTEXT first, before any encryption (dcd#1168). Encryption
+        // uses a fresh random DEK per upload, so the ciphertext hash differs every
+        // time and cannot dedup — the plaintext hash is the only stable key. Doing it
+        // in this order also means a dedup hit skips the encryption work entirely,
+        // not just the upload.
+        const shaPlain = await calculateFileHash(source, debug, log);
+        // Check for an existing upload before spending anything on encryption.
+        if (!ignoreShaCheck && shaPlain) {
+            const { exists, binaryId } = await checkExistingUpload(apiUrl, auth, encrypt ? { encrypted: true, shaPlain } : { sha: shaPlain }, debug);
             if (exists && binaryId) {
                 if (log) {
                     ux.info(colors.dim('SHA hash matches existing binary with ID: ') + formatId(binaryId) + colors.dim(', skipping upload. Force upload with --ignore-sha-check'));
@@ -123,8 +131,33 @@ export const uploadBinary = async (config) => {
                 return binaryId;
             }
         }
+        // Encrypt after the dedup check, so the SHA sent at finalise, and both
+        // uploaders, operate on ciphertext (binaries.sha = ciphertext hash, #1138).
+        if (encrypt) {
+            const encrypted = await encryptUploadSource(source, apiUrl, debug);
+            enc = encrypted.enc;
+            encCleanupDir = encrypted.cleanupDir;
+            if (log) {
+                ux.info(colors.dim(`Encrypting binary before upload (KEK v${enc.kek})`));
+            }
+        }
+        // Re-hash once encrypted: what lands in storage is the ciphertext, and every
+        // downstream verifySha hashes the bytes it actually holds (no DEK required).
+        const sha = encrypt ? await calculateFileHash(source, debug, false) : shaPlain;
         // Perform the upload
-        const uploadId = await performUpload({ auth, apiUrl, debug, filePath, sha, source, startTime });
+        const uploadId = await performUpload({
+            auth,
+            apiUrl,
+            debug,
+            enc,
+            filePath,
+            sha,
+            // Only encrypted uploads record a plaintext hash; it is what makes the
+            // next encrypted run of this binary dedupable.
+            shaPlain: encrypt ? shaPlain : undefined,
+            source,
+            startTime,
+        });
         if (log) {
             ux.action.stop(colors.success('\n✓ Binary uploaded with ID: ') + formatId(uploadId));
         }
@@ -150,8 +183,41 @@ export const uploadBinary = async (config) => {
         if (source?.cleanupDir) {
             await rm(source.cleanupDir, { recursive: true, force: true }).catch(() => { });
         }
+        if (encCleanupDir) {
+            await rm(encCleanupDir, { recursive: true, force: true }).catch(() => { });
+        }
     }
 };
+/**
+ * Encrypt the prepared upload source in place (dcd#1138): generate a per-upload
+ * DEK, stream-encrypt `source.diskPath` into a temp ciphertext file, wrap the
+ * DEK with the environment's pinned KEK public key, and repoint `source` at the
+ * ciphertext so the SHA, dedup check, and both uploaders operate on ciphertext.
+ * Returns the envelope metadata (for `binaries.metadata.enc`) plus the temp dir
+ * to clean up. Throws if no KEK is available for the environment.
+ */
+async function encryptUploadSource(source, apiUrl, debug) {
+    const kek = resolveKekPublicKey(apiUrl);
+    if (!kek) {
+        throw new Error('Binary encryption was requested but no KEK public key is configured for this environment. ' +
+            'Set DCD_BINARY_KEK_PUBLIC=<version>:<base64> or pin one in src/config/environments.ts.');
+    }
+    const dek = generateDek();
+    const enc = wrapDek(dek, kek);
+    const cleanupDir = await mkdtemp(path.join(os.tmpdir(), 'dcd-enc-'));
+    const cipherPath = path.join(cleanupDir, 'binary.enc');
+    if (debug) {
+        console.log(`[DEBUG] Encrypting binary with KEK v${kek.version} -> ${cipherPath}`);
+    }
+    await encryptFileToPath(source.diskPath, cipherPath, dek, kek.version);
+    const { size } = await stat(cipherPath);
+    source.diskPath = cipherPath;
+    source.size = size;
+    if (debug) {
+        console.log(`[DEBUG] Ciphertext size: ${(size / 1024 / 1024).toFixed(2)} MB`);
+    }
+    return { enc, cleanupDir };
+}
 /**
  * Prepares a file for upload: .app directories are zipped to a temp file on
  * disk; everything else is described in place. Nothing is read into memory.
@@ -256,21 +322,34 @@ async function calculateFileHash(source, debug, log) {
     }
 }
 /**
- * Checks if an upload with the same SHA already exists
+ * Checks whether a matching binary has already been uploaded.
+ *
+ * `lookup` is `{ sha }` for a plaintext upload, or `{ shaPlain, encrypted: true }`
+ * for an encrypted one (dcd#1168) — see {@link ApiGateway.checkForExistingUpload}.
+ *
+ * When asking as an encrypting client, a hit is only honoured if the server
+ * confirms the matched binary is itself encrypted. Any binary is a *plausible*
+ * match on plaintext hash, including a previously-uploaded plaintext copy of the
+ * same app, and reusing that would hand back an unencrypted binary while the user
+ * had asked for encryption. The server applies the same predicate; this is the
+ * client refusing to depend on that, so an older or misbehaving deployment
+ * degrades into a redundant upload rather than a silent loss of encryption.
+ *
  * @param apiUrl API base URL
  * @param auth AuthContext carrying request headers
- * @param sha SHA-256 hash to check
+ * @param lookup Dedup key — plaintext hash for encrypted uploads, else the sha
  * @param debug Whether debug logging is enabled
  * @returns Promise resolving to object with exists flag and optional binaryId
  */
-async function checkExistingUpload(apiUrl, auth, sha, debug) {
+async function checkExistingUpload(apiUrl, auth, lookup, debug) {
     try {
         if (debug) {
             console.log('[DEBUG] Checking for existing upload with matching SHA...');
+            console.log(`[DEBUG] Lookup: ${JSON.stringify(lookup)}`);
             console.log(`[DEBUG] Target endpoint: ${apiUrl}/uploads/checkForExistingUpload`);
         }
         const shaCheckStartTime = Date.now();
-        const { appBinaryId, exists } = await ApiGateway.checkForExistingUpload(apiUrl, auth, sha);
+        const { appBinaryId, encrypted, exists } = await ApiGateway.checkForExistingUpload(apiUrl, auth, lookup);
         if (debug) {
             console.log(`[DEBUG] SHA check completed in ${Date.now() - shaCheckStartTime}ms`);
             console.log(`[DEBUG] Existing binary found: ${exists}`);
@@ -278,6 +357,12 @@ async function checkExistingUpload(apiUrl, auth, sha, debug) {
                 console.log(`[DEBUG] Existing binary ID: ${appBinaryId}`);
             }
         }
+        if (exists && lookup.encrypted && encrypted !== true) {
+            if (debug) {
+                console.log('[DEBUG] Ignoring dedup hit: encryption was requested but the matched binary is not encrypted');
+            }
+            return { exists: false };
+        }
         return { binaryId: appBinaryId, exists };
     }
     catch (error) {
@@ -504,11 +589,15 @@ function validateUploadResults(supabaseSuccess, backblazeSuccess, lastError, b2,
  * @returns Promise resolving to upload ID
  */
 async function performUpload(config) {
-    const { filePath, apiUrl, auth, source, sha, debug, startTime } = config;
+    const { filePath, apiUrl, auth, enc, source, sha, shaPlain, debug, startTime } = config;
     // Request upload URL and paths
     const { id, tempPath, finalPath, b2 } = await requestUploadPaths(apiUrl, auth, filePath, source.size, debug);
-    // Extract app metadata
+    // Extract app metadata from the original (plaintext) file. Attach the
+    // envelope so it lands on binaries.metadata.enc (#1138).
     const metadata = await extractBinaryMetadata(filePath, debug);
+    if (enc) {
+        metadata.enc = enc;
+    }
     const env = inferEnvFromApiUrl(apiUrl);
     // Upload to Backblaze first (primary)
     const backblazeResult = await handleBackblazeUpload({
@@ -554,6 +643,7 @@ async function performUpload(config) {
         path: tempPath,
         // sha is undefined when hash calculation failed — omit it explicitly
         ...(sha ? { sha } : {}),
+        ...(shaPlain ? { shaPlain } : {}),
         supabaseSuccess: supabaseResult.success,
     });
     if (debug) {

package/dist/services/test-submission.service.d.ts

@@ -4,6 +4,7 @@ export interface TestSubmissionConfig {
     androidApiLevel?: string;
     androidDevice?: string;
     androidNoSnapshot?: boolean;
+    apiUrl?: string;
     appBinaryId: string;
     cliVersion: string;
     commonRoot: string;
@@ -12,6 +13,12 @@ export interface TestSubmissionConfig {
     deviceLocale?: string;
     deviceMatrix?: DeviceMatrixConfig[];
     disableAnimations?: boolean;
+    /**
+     * Encrypt the flow zip and env vars before upload (#1151/#1152), each with its
+     * own per-upload DEK wrapped under the environment KEK. Requires `apiUrl` to
+     * resolve the pinned KEK public key.
+     */
+    encrypt?: boolean;
     env?: string[];
     executionPlan: IExecutionPlan;
     flowFile: string;
@@ -22,8 +29,6 @@ export interface TestSubmissionConfig {
     maestroChromeOnboarding?: boolean;
     maestroVersion: string;
     metadata?: string[];
-    mitmHost?: string;
-    mitmPath?: string;
     name?: string;
     orientation?: string;
     raw?: unknown;

package/dist/services/test-submission.service.js

@@ -1,6 +1,7 @@
 import { createHash } from 'node:crypto';
 import * as path from 'node:path';
 import { compressFilesFromRelativePath } from '../methods.js';
+import { encryptEnv, encryptFlowBuffer, resolveKekPublicKey, } from '../utils/envelope.js';
 import { toPortableRelativePath } from '../utils/paths.js';
 const mimeTypeLookupByExtension = {
     zip: 'application/zip',
@@ -19,7 +20,7 @@ export class TestSubmissionService {
      * @returns The flow zip buffer, its SHA-256, and the string-encoded fields
      */
     async buildTestPayload(config) {
-        const { appBinaryId, flowFile, executionPlan, commonRoot, cliVersion, env = [], metadata = [], googlePlay = false, androidApiLevel, androidDevice, androidNoSnapshot, iOSVersion, iOSDevice, name, runnerType, maestroVersion, deviceLocale, orientation, mitmHost, mitmPath, retry, continueOnFailure = true, report, showCrosshairs, maestroChromeOnboarding, raw, disableAnimations, deviceMatrix, debug = false, logger, } = config;
+        const { apiUrl, appBinaryId, encrypt = false, flowFile, executionPlan, commonRoot, cliVersion, env = [], metadata = [], googlePlay = false, androidApiLevel, androidDevice, androidNoSnapshot, iOSVersion, iOSDevice, name, runnerType, maestroVersion, deviceLocale, orientation, retry, continueOnFailure = true, report, showCrosshairs, maestroChromeOnboarding, raw, disableAnimations, deviceMatrix, debug = false, logger, } = config;
         const { allExcludeTags, allIncludeTags, flowMetadata, flowOverrides, flowsToRun: testFileNames, referencedFiles, sequence, workspaceConfig, } = executionPlan;
         const { flows: sequentialFlows = [] } = sequence ?? {};
         const envObject = this.parseKeyValuePairs(env);
@@ -42,7 +43,7 @@ export class TestSubmissionService {
             }
         }
         this.logDebug(debug, logger, `[DEBUG] Compressing files from path: ${flowFile}`);
-        const buffer = await compressFilesFromRelativePath(flowFile?.endsWith('.yaml') || flowFile?.endsWith('.yml')
+        const plaintextZip = await compressFilesFromRelativePath(flowFile?.endsWith('.yaml') || flowFile?.endsWith('.yml')
             ? path.dirname(flowFile)
             : flowFile, [
             ...new Set([
@@ -51,8 +52,37 @@ export class TestSubmissionService {
                 ...sequentialFlows,
             ]),
         ], commonRoot);
-        this.logDebug(debug, logger, `[DEBUG] Compressed file size: ${buffer.length} bytes`);
-        // Calculate SHA-256 hash of the flow ZIP
+        this.logDebug(debug, logger, `[DEBUG] Compressed file size: ${plaintextZip.length} bytes`);
+        // Client-side envelope encryption (#1151 flow zip, #1152 env vars). Each
+        // gets its own per-upload DEK wrapped under the environment KEK; the flow
+        // zip becomes a DCDE container and `sha` is the CIPHERTEXT hash (uploads.sha
+        // stays a ciphertext hash, mirroring binaries.sha). `enc` rides `fields`
+        // as a JSON string like every other field, so both the JSON submitFlowTest
+        // body and the legacy multipart form carry it identically.
+        let buffer = plaintextZip;
+        let envObjectToSend = envObject;
+        let flowEnc;
+        if (encrypt) {
+            if (!apiUrl) {
+                throw new Error('Encryption requires apiUrl to resolve the KEK public key');
+            }
+            const kek = resolveKekPublicKey(apiUrl);
+            if (!kek) {
+                throw new Error('Encryption was requested but no KEK public key is configured for this environment. ' +
+                    'Set DCD_BINARY_KEK_PUBLIC=<version>:<base64> or pin one in src/config/environments.ts.');
+            }
+            const flow = encryptFlowBuffer(plaintextZip, kek);
+            buffer = flow.ciphertext;
+            flowEnc = flow.enc;
+            this.logDebug(debug, logger, `[DEBUG] Encrypting flow zip before upload (KEK v${kek.version}); ciphertext ${buffer.length} bytes`);
+            // Only encrypt when there are env vars; an empty map has no secret to
+            // protect and stays a plaintext `{}` (no `enc` marker, passes through).
+            if (Object.keys(envObject).length > 0) {
+                envObjectToSend = { enc: encryptEnv(envObject, kek) };
+                this.logDebug(debug, logger, `[DEBUG] Encrypting ${Object.keys(envObject).length} env var(s)`);
+            }
+        }
+        // SHA-256 of what actually gets uploaded (ciphertext when encrypted).
         const sha = createHash('sha256').update(buffer).digest('hex');
         this.logDebug(debug, logger, `[DEBUG] Flow ZIP SHA-256: ${sha}`);
         // String-encoded fields, in the same order and with the same encoding as
@@ -64,7 +94,12 @@ export class TestSubmissionService {
         fields.flowMetadata = JSON.stringify(this.normalizePathMap(flowMetadata, commonRoot));
         fields.testFileOverrides = JSON.stringify(this.normalizePathMap(flowOverrides, commonRoot));
         fields.sequentialFlows = JSON.stringify(this.normalizePaths(sequentialFlows, commonRoot));
-        fields.env = JSON.stringify(envObject);
+        fields.env = JSON.stringify(envObjectToSend);
+        // Flow-zip envelope for uploads.metadata.enc (#1151). JSON string so both
+        // submission paths carry it like every other field; the API parses it.
+        if (flowEnc) {
+            fields.enc = JSON.stringify(flowEnc);
+        }
         // Note: googlePlay is now included in configPayload below instead of as a separate field
         // to work around a FormData parsing issue in the API
         // Explicit device matrix (one upload, N cells). Only sent when present, so
@@ -94,8 +129,6 @@ export class TestSubmissionService {
             deviceLocale,
             googlePlay,
             maestroVersion,
-            mitmHost,
-            mitmPath,
             orientation,
             raw: JSON.stringify(raw),
             report,

package/dist/types/domain/device.types.d.ts

@@ -31,6 +31,7 @@ export declare enum EAndroidApiLevels {
     'thirtyFive' = "35",
     'thirtyFour' = "34",
     'thirtyOne' = "31",
+    'thirtySeven' = "37",
     'thirtySix' = "36",
     'thirtyThree' = "33",
     'thirtyTwo' = "32",

package/dist/types/domain/device.types.js

@@ -35,6 +35,7 @@ export var EAndroidApiLevels;
     EAndroidApiLevels["thirtyFive"] = "35";
     EAndroidApiLevels["thirtyFour"] = "34";
     EAndroidApiLevels["thirtyOne"] = "31";
+    EAndroidApiLevels["thirtySeven"] = "37";
     EAndroidApiLevels["thirtySix"] = "36";
     EAndroidApiLevels["thirtyThree"] = "33";
     EAndroidApiLevels["thirtyTwo"] = "32";