@appsemble/node-utils 0.36.10-test.3 → 0.36.10-test.4

package/README.md

@@ -1,9 +1,9 @@
-# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.36.10-test.3/config/assets/logo.svg) Appsemble Node Utilities
+# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.36.10-test.4/config/assets/logo.svg) Appsemble Node Utilities
 
 > NodeJS utilities used by Appsemble internally.
 
 [![npm](https://img.shields.io/npm/v/@appsemble/node-utils)](https://www.npmjs.com/package/@appsemble/node-utils)
-[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.36.10-test.3/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.36.10-test.3)
+[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.36.10-test.4/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.36.10-test.4)
 [![Prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://prettier.io)
 
 ## Table of Contents
@@ -26,5 +26,5 @@ compatibility is not guaranteed.
 
 ## License
 
-[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.36.10-test.3/LICENSE.md) ©
+[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.36.10-test.4/LICENSE.md) ©
 [Appsemble](https://appsemble.com)

package/app.js

@@ -9,11 +9,37 @@ import { copy, ensureDir } from 'fs-extra';
 import { IntlMessageFormat } from 'intl-messageformat';
 import lodash from 'lodash';
 import { format, resolveConfig } from 'prettier';
-import { parseDocument } from 'yaml';
+import { Alias, isCollection, parseDocument } from 'yaml';
 import { opendirSafe, readData, writeData } from './fs.js';
 import { logger } from './logger.js';
 const getNumberFormat = memoize((locale, opts) => new Intl.NumberFormat(locale, opts));
 const getPluralRules = memoize((locale, opts) => new Intl.PluralRules(locale, opts));
+const aliasReferencePattern = /^\*(?!$)[^\s,[\]{}]+$/u;
+function expandPatchParentCollections(doc, path) {
+    for (let index = 0; index < path.length; index += 1) {
+        const node = doc.getIn(path.slice(0, index), true);
+        if (isCollection(node)) {
+            node.flow = false;
+        }
+    }
+}
+function createPatchValue(doc, value) {
+    if (typeof value === 'string' && aliasReferencePattern.test(value)) {
+        return new Alias(value.slice(1));
+    }
+    if (Array.isArray(value)) {
+        return doc.createNode(value.map((item) => createPatchValue(doc, item)), {
+            aliasDuplicateObjects: false,
+        });
+    }
+    if (value && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype) {
+        return doc.createNode(Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [
+            entryKey,
+            createPatchValue(doc, entryValue),
+        ])), { aliasDuplicateObjects: false });
+    }
+    return value;
+}
 /**
  * Get a context for remappers based on an app definition.
  *
@@ -109,7 +135,8 @@ export async function patchDefinition(appPath, patches) {
                 doc.deleteIn(key);
             }
             else {
-                doc.setIn(key, value);
+                expandPatchParentCollections(doc, key);
+                doc.setIn(key, createPatchValue(doc, value));
             }
         }
         const prettierOptions = (await resolveConfig(path, { editorconfig: true }));

package/container/helpers.d.ts

@@ -4,15 +4,15 @@ export declare const maxCPU: number;
 export declare const maxMemoryGi: number;
 export declare const appIdLabel = "appId";
 export declare const resourceDefaults: ContainerResourceProps;
+export declare function getContainerNamespace(): string;
+export declare function formatServiceName(containerName: string, appName: string, appId: string): string;
+export declare function formatSecretName(appName: string, appId: string): string;
+export declare function handleKubernetesError(error: unknown): void;
 export declare function getKubeConfig(): {
     appsApi: AppsV1Api;
     coreApi: CoreV1Api;
     kubeconfig: KubeConfig;
 };
-export declare function getContainerNamespace(): string;
-export declare function formatServiceName(containerName: string, appName: string, appId: string): string;
-export declare function formatSecretName(appName: string, appId: string): string;
-export declare function handleKubernetesError(error: unknown): void;
 export declare function deleteResource(type: 'deployment' | 'secret' | 'service', namespace: string, name: string): Promise<void>;
 /**
  * Accepts a url used to call a companion container.

package/container/helpers.js

@@ -5,13 +5,6 @@ export const maxCPU = process.env.MAX_CONTAINER_CPU ?? 3;
 export const maxMemoryGi = process.env.MAX_CONTAINER_MEMORY ?? 3;
 export const appIdLabel = 'appId';
 export const resourceDefaults = { memory: '128Mi', cpu: '0.1' };
-export function getKubeConfig() {
-    const kubeconfig = new KubeConfig();
-    kubeconfig.loadFromDefault();
-    const appsApi = kubeconfig.makeApiClient(AppsV1Api);
-    const coreApi = kubeconfig.makeApiClient(CoreV1Api);
-    return { appsApi, coreApi, kubeconfig };
-}
 export function getContainerNamespace() {
     return `companion-containers-${process.env.SERVICE_NAME ?? 'appsemble'}`;
 }
@@ -32,6 +25,54 @@ export function handleKubernetesError(error) {
     }
     logger.error(error);
 }
+function sleep(timeout) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, timeout);
+    });
+}
+function isTransientKubernetesError(error) {
+    if (!(error instanceof ApiException)) {
+        return true;
+    }
+    return (typeof error.code === 'number' &&
+        (error.code === 408 || error.code === 429 || error.code >= 500));
+}
+async function withKubernetesRetry(description, operation) {
+    const attempts = Number(process.env.KUBERNETES_REQUEST_RETRIES ?? 3);
+    const delay = Number(process.env.KUBERNETES_RETRY_DELAY_MS ?? 1000);
+    for (let attempt = 1;; attempt += 1) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            if (attempt >= attempts || !isTransientKubernetesError(error)) {
+                throw error;
+            }
+            logger.warn(`Kubernetes request failed while trying to ${description}. Retrying ${attempt}/${attempts}`);
+            if (delay > 0) {
+                await sleep(delay * attempt);
+            }
+        }
+    }
+}
+function withKubernetesRetries(description, client) {
+    return new Proxy(client, {
+        get(target, property) {
+            const value = Reflect.get(target, property, target);
+            if (typeof value !== 'function') {
+                return value;
+            }
+            return (...args) => withKubernetesRetry(`${description}.${String(property)}`, () => value.apply(target, args));
+        },
+    });
+}
+export function getKubeConfig() {
+    const kubeconfig = new KubeConfig();
+    kubeconfig.loadFromDefault();
+    const appsApi = withKubernetesRetries('AppsV1Api', kubeconfig.makeApiClient(AppsV1Api));
+    const coreApi = withKubernetesRetries('CoreV1Api', kubeconfig.makeApiClient(CoreV1Api));
+    return { appsApi, coreApi, kubeconfig };
+}
 export async function deleteResource(type, namespace, name) {
     if (process.env.TEST) {
         return;

package/container/operations.js

@@ -212,6 +212,7 @@ export async function updateCompanionContainers(definitions, appName, appId, reg
                     logger.silly(`Deployment ${serviceName} has no matching pod with metadata.name, skipping`);
                     continue;
                 }
+                const podName = pod.metadata.name;
                 // Set base labels, such as selector, pod name, pod hash
                 const props = {
                     ...def.metadata,
@@ -221,7 +222,7 @@ export async function updateCompanionContainers(definitions, appName, appId, reg
                         'pod-template-hash': pod?.metadata?.labels?.['pod-template-hash'],
                         appId,
                     },
-                    name: pod.metadata.name,
+                    name: podName,
                 };
                 const podPatch = [
                     {
@@ -231,7 +232,7 @@ export async function updateCompanionContainers(definitions, appName, appId, reg
                     },
                 ];
                 const updatedPod = await coreApi.patchNamespacedPod({
-                    name: pod.metadata.name,
+                    name: podName,
                     namespace,
                     body: podPatch,
                 });

package/icon.js

@@ -61,10 +61,10 @@ export async function serveIcon(ctx, { background, cache, fallback, height, icon
         if (background) {
             img.flatten({ background });
         }
-        // Cache app icons for 1 week.
-        if (cache) {
-            ctx.set('cache-control', `public, max-age=${60 * 60 * 24 * 7},immutable`);
-        }
+    }
+    // Cache app icons for 1 week.
+    if (cache) {
+        ctx.set('cache-control', `public, max-age=${60 * 60 * 24 * 7},immutable`);
     }
     ctx.body = await img.toFormat('png').toBuffer();
     ctx.type = 'image/png';

package/index.d.ts

@@ -47,3 +47,4 @@ export * from './PhoneNumberValidationError.js';
 export * from './createUser.js';
 export * from './deleteUser.js';
 export * from './updateSubscription.js';
+export * from './server/utils/ssrf.js';

package/index.js

@@ -47,4 +47,5 @@ export * from './PhoneNumberValidationError.js';
 export * from './createUser.js';
 export * from './deleteUser.js';
 export * from './updateSubscription.js';
+export * from './server/utils/ssrf.js';
 //# sourceMappingURL=index.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appsemble/node-utils",
-  "version": "0.36.10-test.3",
+  "version": "0.36.10-test.4",
   "description": "NodeJS utilities used by Appsemble internally.",
   "keywords": [
     "app",
@@ -40,9 +40,9 @@
     "test": "vitest"
   },
   "dependencies": {
-    "@appsemble/lang-sdk": "0.36.10-test.3",
-    "@appsemble/types": "0.36.10-test.3",
-    "@appsemble/utils": "0.36.10-test.3",
+    "@appsemble/lang-sdk": "0.36.10-test.4",
+    "@appsemble/types": "0.36.10-test.4",
+    "@appsemble/utils": "0.36.10-test.4",
     "@formatjs/fast-memoize": "^2.0.0",
     "@fortawesome/fontawesome-free": "^6.0.0",
     "@inquirer/prompts": "^8.0.0",
@@ -67,6 +67,7 @@
     "form-data": "^4.0.4",
     "fs-extra": "^11.0.0",
     "intl-messageformat": "^11.0.0",
+    "ipaddr.js": "^2.4.0",
     "jsonschema": "~1.4.1",
     "keytar": "^7.0.0",
     "koa": "^3.0.0",
@@ -82,7 +83,7 @@
     "lodash": "^4.0.0",
     "lodash-es": "^4.0.0",
     "logform": "^2.0.0",
-    "memfs": "4.57.2",
+    "memfs": "4.57.8",
     "mime-types": "^2.0.0",
     "minio": "^8.0.3",
     "mustache": "^4.0.0",

package/s3.d.ts

@@ -13,6 +13,6 @@ export declare function uploadS3FileFromPath(bucket: string, key: string, path:
 export declare function getS3File(bucket: string, key: string): Promise<Readable>;
 export declare function getS3FileBuffer(bucket: string, key: string): Promise<Buffer>;
 export declare function getS3FileStats(bucket: string, key: string): Promise<BucketItemStat>;
-export declare function deleteS3File(bucket: string, key: string): Promise<void>;
 export declare function deleteS3Files(bucket: string, keys: string[]): Promise<void>;
+export declare function deleteS3File(bucket: string, key: string): Promise<void>;
 export declare function clearAllS3Buckets(): Promise<void>;

package/s3.js

@@ -92,24 +92,22 @@ export async function getS3FileStats(bucket, key) {
         throw error;
     }
 }
-export async function deleteS3File(bucket, key) {
-    try {
-        await s3Client.removeObject(bucket, key);
-    }
-    catch (error) {
-        logger.error(error);
-        throw error;
-    }
-}
 export async function deleteS3Files(bucket, keys) {
     try {
         await s3Client.removeObjects(bucket, keys);
     }
     catch (error) {
+        if (error instanceof S3Error && error.code === 'NoSuchBucket') {
+            logger.warn(`S3 bucket "${bucket}" does not exist; skipping deletion`);
+            return;
+        }
         logger.error(error);
         throw error;
     }
 }
+export async function deleteS3File(bucket, key) {
+    await deleteS3Files(bucket, [key]);
+}
 export async function clearAllS3Buckets() {
     try {
         const buckets = await s3Client.listBuckets();

package/server/utils/actions.js

@@ -1,8 +1,7 @@
 import { defaultLocale, remap, } from '@appsemble/lang-sdk';
-import { assertKoaCondition, createFormData, EmailQuotaExceededError, getContainerNamespace, getRemapperContext, logger, parseServiceUrl, scaleDeployment, setLastRequestAnnotation, throwKoaError, version, waitForPodReadiness, } from '@appsemble/node-utils';
+import { assertKoaCondition, createFormData, EmailQuotaExceededError, getContainerNamespace, getRemapperContext, getSSRFProtectedAgents, logger, parseServiceUrl, scaleDeployment, setLastRequestAnnotation, throwKoaError, version, waitForPodReadiness, } from '@appsemble/node-utils';
 import axios from 'axios';
 import { get, mapValues, pick } from 'lodash-es';
-import { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } from 'request-filtering-agent';
 /**
  * These response headers are forwarded when proxying requests.
  */
@@ -147,23 +146,13 @@ async function handleRequestProxy(ctx, app, action, useBody, options) {
         // Apply SSRF protection for non-companion-container URLs
         // This blocks requests to private IPs, localhost, link-local addresses,
         // and hostnames that resolve to private IPs (prevents DNS rebinding attacks)
-        //
-        // VITEST_CONF_ALLOW_PRIVATE_IP_PROXY: Test-only env var set in vitest.setup.ts
-        // to allow proxy tests to use localhost servers. SSRF tests explicitly unset this.
-        // This env var should NEVER be set in production.
-        const allowPrivateIPAddress = process.env.VITEST_CONF_ALLOW_PRIVATE_IP_PROXY === '1';
-        // Preserve any existing agent options (e.g., client certs from applyAppServiceSecrets)
-        // while still applying SSRF protection
-        const existingHttpsOptions = axiosConfig.httpsAgent?.options ?? {};
-        const existingHttpOptions = axiosConfig.httpAgent?.options ?? {};
-        axiosConfig.httpAgent = new RequestFilteringHttpAgent({
-            ...existingHttpOptions,
-            allowPrivateIPAddress,
-        });
-        axiosConfig.httpsAgent = new RequestFilteringHttpsAgent({
-            ...existingHttpsOptions,
-            allowPrivateIPAddress,
+        const { httpAgent, httpsAgent } = await getSSRFProtectedAgents({
+            hostname: proxyUrl.hostname,
+            httpAgent: axiosConfig.httpAgent,
+            httpsAgent: axiosConfig.httpsAgent,
         });
+        axiosConfig.httpAgent = httpAgent;
+        axiosConfig.httpsAgent = httpsAgent;
     }
     logger.verbose(`Forwarding request to ${axios.getUri(axiosConfig)}`);
     logger.verbose('Axios Config:');

package/server/utils/ssrf.d.ts

@@ -0,0 +1,11 @@
+import { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } from 'request-filtering-agent';
+export interface SSRFOptions {
+    allowPrivateIPAddress?: boolean;
+    hostname?: string;
+    httpAgent?: any;
+    httpsAgent?: any;
+}
+export declare function getSSRFProtectedAgents(options?: SSRFOptions): Promise<{
+    httpAgent: RequestFilteringHttpAgent;
+    httpsAgent: RequestFilteringHttpsAgent;
+}>;

package/server/utils/ssrf.js

@@ -0,0 +1,88 @@
+import dns from 'node:dns';
+import { isIP } from 'node:net';
+import ipaddr from 'ipaddr.js';
+import { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } from 'request-filtering-agent';
+/**
+ * Detect NAT64 (RFC 6052 well-known prefix) and IPv4-mapped IPv6 addresses whose embedded IPv4 is a
+ * public unicast address.
+ *
+ * `request-filtering-agent` blocks these IPv6 forms, so when they wrap a legitimate public address
+ * we return it to be added to the allow-list. Addresses embedding a private/loopback/link-local
+ * IPv4 (e.g. `64:ff9b::127.0.0.1`) return `undefined` and stay blocked.
+ *
+ * @param address An IP address string (not a hostname).
+ * @returns The address when it should be allow-listed, otherwise `undefined`.
+ */
+function getAllowableEmbeddedAddress(address) {
+    let parsed;
+    try {
+        parsed = ipaddr.parse(address);
+    }
+    catch {
+        return undefined;
+    }
+    if (parsed.kind() !== 'ipv6') {
+        return undefined;
+    }
+    const ipv6 = parsed;
+    const range = ipv6.range();
+    let ipv4;
+    if (range === 'rfc6052') {
+        const parts = ipv6.toByteArray();
+        // For the well-known prefix /96, the IPv4 is in parts[12...15]
+        if (parts[0] === 0x00 && parts[1] === 0x64 && parts[2] === 0xff && parts[3] === 0x9b) {
+            ipv4 = new ipaddr.IPv4(parts.slice(12));
+        }
+    }
+    else if (range === 'ipv4Mapped') {
+        ipv4 = ipv6.toIPv4Address();
+    }
+    return ipv4 && ipv4.range() === 'unicast' ? address : undefined;
+}
+export async function getSSRFProtectedAgents(options = {}) {
+    const { allowPrivateIPAddress = process.env.VITEST_CONF_ALLOW_PRIVATE_IP_PROXY === '1', hostname, } = options;
+    const allowIPAddressList = process.env.SSRF_ALLOW_IP_ADDRESS_LIST?.split(',') ?? [];
+    // `URL.hostname` wraps IPv6 literals in brackets (e.g. `[64:ff9b::1]`); strip them so the value
+    // can be parsed as an IP and resolved by DNS.
+    const normalizedHostname = hostname?.replace(/^\[|]$/g, '');
+    if (normalizedHostname) {
+        if (isIP(normalizedHostname)) {
+            // IP literal: inspect it directly. No DNS lookup is needed (which also avoids resolving on top
+            // of the lookup request-filtering-agent already performs at connect time).
+            const allowable = getAllowableEmbeddedAddress(normalizedHostname);
+            if (allowable) {
+                allowIPAddressList.push(allowable);
+            }
+        }
+        else {
+            // Real hostname: resolve to detect NAT64/DNS64-synthesised addresses.
+            try {
+                const addresses = await dns.promises.lookup(normalizedHostname, { all: true });
+                for (const { address } of addresses) {
+                    const allowable = getAllowableEmbeddedAddress(address);
+                    if (allowable) {
+                        allowIPAddressList.push(allowable);
+                    }
+                }
+            }
+            catch {
+                // Ignore DNS lookup errors, the agent will handle them
+            }
+        }
+    }
+    const existingHttpOptions = options.httpAgent?.options ?? {};
+    const existingHttpsOptions = options.httpsAgent?.options ?? {};
+    return {
+        httpAgent: new RequestFilteringHttpAgent({
+            ...existingHttpOptions,
+            allowPrivateIPAddress,
+            allowIPAddressList,
+        }),
+        httpsAgent: new RequestFilteringHttpsAgent({
+            ...existingHttpsOptions,
+            allowPrivateIPAddress,
+            allowIPAddressList,
+        }),
+    };
+}
+//# sourceMappingURL=ssrf.js.map