@ts-cloud/core 0.16.16 → 0.16.18

package/dist/modules/cdn-failover.d.ts

@@ -23,8 +23,15 @@
  * }
  * ```
  *
+ * {@link validateOriginGroups} checks a whole distribution config the same way
+ * CloudFront does, so a distribution assembled by hand fails at template
+ * generation instead of half way through a CloudFormation deploy.
+ *
+ * Verified against a live distribution (S3 origins in two regions, failover on
+ * a missing object and on a primary that denies every read) for
+ * stacksjs/stacks#1159.
+ *
  * @see https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/high_availability_origin_failover.html
- * @experimental Built against the AWS documentation; not yet exercised against a live distribution.
  */
 /** Every status code CloudFront accepts as a failover criterion. */
 export declare const CLOUDFRONT_FAILOVER_STATUS_CODES: readonly number[];
@@ -38,6 +45,27 @@ export declare const DEFAULT_FAILOVER_STATUS_CODES: readonly number[];
 export declare const ORIGIN_GROUP_ALLOWED_METHODS: readonly string[];
 /** Suffix appended to the primary origin id to name its origin group. */
 export declare const ORIGIN_GROUP_ID_SUFFIX = "-failover-group";
+/**
+ * How CloudFront picks the origin it asks first.
+ *
+ * - `default`: always the primary (the first member), falling back to the
+ *   secondary on a failover status code.
+ * - `media-quality-based`: CloudFront scores both origins and asks the better
+ *   one first. Only valid when both members are AWS Elemental MediaPackage v2
+ *   endpoints.
+ */
+export type OriginGroupSelectionCriteria = 'default' | 'media-quality-based';
+export declare const ORIGIN_GROUP_SELECTION_CRITERIA: readonly OriginGroupSelectionCriteria[];
+/** `ConnectionAttempts` CloudFront accepts on an origin (its default is 3). */
+export declare const ORIGIN_CONNECTION_ATTEMPTS: {
+    readonly min: 1;
+    readonly max: 3;
+};
+/** `ConnectionTimeout`, in seconds, CloudFront accepts on an origin (its default is 10). */
+export declare const ORIGIN_CONNECTION_TIMEOUT: {
+    readonly min: 1;
+    readonly max: 10;
+};
 export interface OriginGroupOptions {
     /** Id of the origin CloudFront asks first. */
     primaryOriginId: string;
@@ -47,6 +75,11 @@ export interface OriginGroupOptions {
     statusCodes?: readonly number[];
     /** Origin group id, which cache behaviors use as their `TargetOriginId`. @default `<primaryOriginId>-failover-group` */
     groupId?: string;
+    /**
+     * How CloudFront picks the origin it asks first. Omitted from the template
+     * unless set, which CloudFront treats as `default`.
+     */
+    selectionCriteria?: OriginGroupSelectionCriteria;
 }
 export interface CloudFrontOriginGroup {
     Id: string;
@@ -62,6 +95,14 @@ export interface CloudFrontOriginGroup {
             OriginId: string;
         }>;
     };
+    SelectionCriteria?: OriginGroupSelectionCriteria;
+}
+/** Connection tuning for one origin. Unset keys keep CloudFront's defaults (3 attempts, 10 seconds). */
+export interface OriginConnectionOptions {
+    /** Times CloudFront tries to connect to the origin, 1-3. */
+    connectionAttempts?: number;
+    /** Seconds CloudFront waits to establish a connection, 1-10. */
+    connectionTimeout?: number;
 }
 export interface CloudFrontOriginGroups {
     Quantity: number;
@@ -76,6 +117,22 @@ export interface CloudFrontOriginGroups {
 export declare function resolveFailoverStatusCodes(statusCodes?: readonly number[]): number[];
 /** Build one origin group (primary first, secondary second). */
 export declare function buildOriginGroup(options: OriginGroupOptions): CloudFrontOriginGroup;
+/**
+ * Validate an origin's connection tuning and return it in template form
+ * (`ConnectionAttempts` / `ConnectionTimeout`), holding only the keys that were
+ * set so an untuned origin's template is unchanged.
+ *
+ * Lowering these on the primary is how failover gets faster: by default
+ * CloudFront spends up to 30 seconds (3 attempts of 10 seconds) on an
+ * unreachable primary before it asks the secondary.
+ *
+ * @param where Names the origin in the error, e.g. `infrastructure.cdn.main`.
+ * @throws when a value is not a whole number in CloudFront's range.
+ */
+export declare function resolveOriginConnection(options: OriginConnectionOptions | undefined, where?: string): {
+    ConnectionAttempts?: number;
+    ConnectionTimeout?: number;
+};
 /** Build a distribution's `OriginGroups` block holding a single failover group. */
 export declare function buildOriginGroups(options: OriginGroupOptions): CloudFrontOriginGroups;
 /**
@@ -95,3 +152,26 @@ export declare function assertOriginGroupMethods(allowedMethods: readonly string
  * S3 *website* endpoints only speak HTTP and are custom origins.
  */
 export declare function isS3RestEndpoint(domainName: string): boolean;
+/**
+ * Check every origin group in a distribution config against the rules
+ * CloudFront enforces, and throw a message that names the broken piece.
+ *
+ * Accepts the CloudFormation `DistributionConfig` (arrays) and the CloudFront
+ * API's (`{ Quantity, Items }`), and skips comparisons on values that are
+ * CloudFormation intrinsics. Checked:
+ *
+ * - each group has exactly two members, both existing origins, and distinct;
+ * - group ids are unique and do not shadow an origin id;
+ * - failover status codes are ones CloudFront accepts, and `Quantity` matches;
+ * - `SelectionCriteria` is `default` or `media-quality-based`, the latter only
+ *   between MediaPackage v2 origins;
+ * - every cache behavior (default and path) that targets a group allows only
+ *   GET, HEAD and OPTIONS;
+ * - every cache behavior targets an origin or group that exists;
+ * - every origin's `ConnectionAttempts` (1-3) and `ConnectionTimeout` (1-10).
+ *
+ * A distribution with no origin groups passes the group checks trivially.
+ *
+ * @param where Names the distribution in errors, e.g. `infrastructure.cdn.main`.
+ */
+export declare function validateOriginGroups(distributionConfig: Record<string, any>, where?: string): void;

package/dist/modules/cdn.d.ts

@@ -1,5 +1,6 @@
 import type { CloudFrontDistribution, CloudFrontOriginAccessControl, IAMRole, LambdaFunction } from '@ts-cloud/aws-types';
 import type { EnvironmentType } from '../types';
+import type { OriginConnectionOptions, OriginGroupSelectionCriteria } from './cdn-failover';
 export interface DistributionOptions {
     slug: string;
     environment: EnvironmentType;
@@ -14,15 +15,14 @@ export interface DistributionOptions {
     /**
      * Secondary origin CloudFront fails over to when the primary errors.
      * Adds an origin group and points the default cache behavior at it.
-     * @experimental See {@link CDN.addOriginFailover}.
+     * See {@link CDN.addOriginFailover}.
      */
     failoverOrigin?: FailoverOriginConfig;
 }
 /**
  * Secondary origin for CloudFront origin failover.
- * @experimental Built against the AWS documentation; not yet exercised against a live distribution.
  */
-export interface FailoverOriginConfig {
+export interface FailoverOriginConfig extends OriginConnectionOptions {
     /** Domain of the secondary origin, e.g. a replica bucket in another region. */
     domainName: string;
     /** Origin kind. @default 's3' for an S3 REST endpoint, otherwise 'custom' */
@@ -35,8 +35,17 @@ export interface FailoverOriginConfig {
      * 416, 429, 500, 502, 503, 504. @default [500, 502, 503, 504]
      */
     statusCodes?: number[];
+    /** How CloudFront picks the origin it asks first. @default CloudFront's 'default' (primary first) */
+    selectionCriteria?: OriginGroupSelectionCriteria;
+    /**
+     * Connection tuning for the *primary* origin. Lower values make failover
+     * faster: CloudFront otherwise spends up to 30 seconds (3 attempts of 10
+     * seconds) on an unreachable primary. `connectionAttempts` and
+     * `connectionTimeout` on this object tune the secondary.
+     */
+    primary?: OriginConnectionOptions;
 }
-export interface OriginConfig {
+export interface OriginConfig extends OriginConnectionOptions {
     type?: 's3' | 'alb' | 'custom';
     id?: string;
     originId?: string;
@@ -79,9 +88,8 @@ export declare class CDN {
      *
      * CloudFront only fails over GET, HEAD and OPTIONS, and rejects an origin
      * group behind a cache behavior that allows writes, so this throws if the
-     * default cache behavior allows any other method.
-     *
-     * @experimental Built against the AWS documentation; not yet exercised against a live distribution.
+     * default cache behavior allows any other method. The finished config is
+     * checked with {@link validateOriginGroups}.
      */
     static addOriginFailover(distribution: CloudFrontDistribution, failover: FailoverOriginConfig): CloudFrontDistribution;
     /**

package/dist/modules/index.d.ts

@@ -6,6 +6,7 @@ export * from './storage';
 export * from './registry';
 export * from './cdn';
 export * from './cdn-failover';
+export * from './storage-failover';
 export * from './dns';
 export * from './security';
 export * from './compute';

package/dist/modules/storage-failover.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Cross-region failover for a website bucket (stacksjs/stacks#1159).
+ *
+ * The CloudFront half is an origin group (see `cdn-failover.ts`): the primary
+ * bucket first, a replica bucket in another region second. This module holds
+ * the pure pieces the bucket half needs:
+ *
+ * - {@link resolveStorageFailover} turns `infrastructure.storage.<name>.failover`
+ *   into a concrete plan (replica name and region, status codes, connection
+ *   tuning) and rejects configs CloudFront or S3 would refuse later.
+ * - {@link buildReplicationRole} and {@link buildReplicationConfiguration}
+ *   emit the IAM role and `ReplicationConfiguration` that copy every write to
+ *   the replica.
+ * - {@link STORAGE_FAILOVER_METADATA_KEY} names the template `Metadata` entry
+ *   that tells a deployer which replicas to create before the stack and grant
+ *   CloudFront access to after it. The replica cannot live in the stack: a
+ *   CloudFormation stack only creates buckets in its own region.
+ */
+import type { StorageFailoverConfig } from '../types';
+/**
+ * Failover codes for a bucket origin when none are given. An S3 origin behind
+ * origin access control answers a missing object with 403 (the distribution
+ * may not list the bucket), so 403 and 404 cover an object the primary lost;
+ * the 5xx codes cover the bucket or its region failing.
+ */
+export declare const DEFAULT_STORAGE_FAILOVER_STATUS_CODES: readonly number[];
+/** Template `Metadata` key listing the failover replicas a stack depends on. */
+export declare const STORAGE_FAILOVER_METADATA_KEY = "TsCloud::StorageFailover";
+/** Sid of the replica bucket-policy statement that lets the distribution read it. */
+export declare const FAILOVER_REPLICA_POLICY_SID = "AllowCloudFrontFailoverRead";
+/** One replica, as recorded in the template metadata and consumed by the deployer. */
+export interface StorageFailoverReplica {
+    /** `infrastructure.storage` key. */
+    name: string;
+    primaryBucket: string;
+    primaryRegion: string;
+    replicaBucket: string;
+    replicaRegion: string;
+    /** Whether S3 replication copies writes to the replica (and so whether it must be versioned). */
+    replicate: boolean;
+    /** Stack output holding the distribution ARN the replica policy is scoped to. */
+    distributionArnOutput: string;
+}
+export interface ResolvedStorageFailover {
+    replicaBucket: string;
+    replicaRegion: string;
+    replicate: boolean;
+    statusCodes: number[];
+    primaryConnection: {
+        ConnectionAttempts?: number;
+        ConnectionTimeout?: number;
+    };
+}
+/**
+ * Resolve and validate a bucket's failover config.
+ *
+ * @param where Names the bucket in errors, e.g. `infrastructure.storage.public`.
+ * @throws when the region is missing, malformed or the primary's own region,
+ * when the replica name is not a valid bucket name (or collides with the
+ * primary), or when a status code or connection setting is out of range.
+ */
+export declare function resolveStorageFailover(options: {
+    failover: StorageFailoverConfig;
+    primaryBucket: string;
+    primaryRegion: string;
+    where?: string;
+}): ResolvedStorageFailover;
+/** S3 REST endpoint of a bucket, the domain a CloudFront S3 origin uses. */
+export declare function s3RegionalDomain(bucket: string, region: string): string;
+/**
+ * IAM role S3 assumes to replicate `primaryBucket` into `replicaBucket`. The
+ * permissions are the minimum the S3 docs list for live replication,
+ * including delete markers.
+ */
+export declare function buildReplicationRole(options: {
+    primaryBucket: string;
+    replicaBucket: string;
+    roleName?: string;
+}): any;
+/**
+ * `ReplicationConfiguration` for the primary bucket: one rule copying every
+ * object, and its deletes, to the replica.
+ *
+ * @param roleArn The replication role's ARN, usually `{ 'Fn::GetAtt': [roleLogicalId, 'Arn'] }`.
+ */
+export declare function buildReplicationConfiguration(options: {
+    replicaBucket: string;
+    roleArn: unknown;
+}): any;
+/**
+ * The replica bucket-policy statement that lets one distribution, through
+ * origin access control, read the replica.
+ */
+export declare function buildFailoverReplicaPolicyStatement(options: {
+    replicaBucket: string;
+    distributionArn: string;
+}): any;
+/** Read the failover replicas a generated template depends on (empty when none). */
+export declare function storageFailoverReplicasFromTemplate(template: unknown): StorageFailoverReplica[];

package/dist/template-builder.d.ts

@@ -18,6 +18,10 @@ export declare class TemplateBuilder {
      * Add an output to the template
      */
     addOutput(name: string, output: NonNullable<CloudFormationTemplate['Outputs']>[string]): this;
+    /**
+     * Set a template-level `Metadata` entry
+     */
+    addMetadata(key: string, value: unknown): this;
     /**
      * Check if a resource already exists in the template
      */

package/dist/types.d.ts

@@ -2194,6 +2194,18 @@ export interface StorageItemConfig {
      * Defaults to directory, which matches most blog/static-export outputs.
      */
     pathRewriteStyle?: 'directory' | 'flat';
+    /**
+     * CloudFront origin failover to a replica of this bucket in another region.
+     * Only applies to a website bucket served through CloudFront. Off unless set.
+     *
+     * @example
+     * ```ts
+     * storage: {
+     *   public: { website: true, failover: { region: 'us-west-2' } },
+     * }
+     * ```
+     */
+    failover?: StorageFailoverConfig;
     /**
      * Whether this bucket serves a single-page application (SPA).
      * When true: 403/404 errors return index.html with status 200 (for client-side routing).
@@ -3145,6 +3157,107 @@ export interface ComputeConfig {
      * simple as adding an entry here and redeploying.
      */
     sshKeys?: SshKeyConfig[];
+    /**
+     * Bounded disk retention on box hosts (Hetzner and ssh providers, and AWS
+     * EC2 boxes deployed over SSM).
+     *
+     * ts-cloud prunes what it leaves behind on a box: abandoned upload staging,
+     * the release artifact cache, the Bun download cache, journald, unused
+     * container images and the apt cache. Current and rollback releases are never
+     * touched; `keepReleases` governs those. The cleanup runs after every deploy,
+     * after a failed one, and on a systemd timer, so a box that stops deploying
+     * still cleans. It reads root-filesystem usage first: below
+     * `pressure.lowWaterPercent` it skips the expensive rules, and at or above
+     * `pressure.highWaterPercent` it prunes on shorter windows and warns.
+     *
+     * Omit for the defaults. `false` turns the whole mechanism off, including a
+     * timer an earlier deploy installed.
+     */
+    hostCleanup?: boolean | ComputeHostCleanupConfig;
+}
+/**
+ * Retention windows for {@link ComputeConfig.hostCleanup}. Each field is an age
+ * past which the matching files are deleted; omit one to keep its default.
+ */
+export interface HostCleanupRetention {
+    /**
+     * Abandoned upload staging (`/var/ts-cloud/staging`, `/tmp/*-release.tar.gz`).
+     * Doubles as the in-flight bound for a concurrent deploy, so it cannot go
+     * below 15. @default 60
+     */
+    stagingMaxAgeMinutes?: number;
+    /** Completed `*.tar.gz` in the release artifact cache, `-mtime` days. @default 2 */
+    artifactMaxAgeDays?: number;
+    /**
+     * `.tmp` uploads to the artifact cache stranded by a dropped transfer. An
+     * upload in flight is younger than this, so it cannot go below 15. @default 60
+     */
+    artifactUploadMaxAgeMinutes?: number;
+    /** Bun's install (download) cache, `-mtime` days. @default 7 */
+    bunCacheMaxAgeDays?: number;
+    /** journald retention by age. @default 14 */
+    journalMaxAgeDays?: number;
+    /** journald retention by size. @default 512 */
+    journalMaxSizeMb?: number;
+    /** Unused docker/podman images, by age. @default 168 */
+    containerImageMaxAgeHours?: number;
+}
+/** How host cleanup reacts to a filling disk. See {@link ComputeHostCleanupConfig.pressure}. */
+export interface HostCleanupPressureConfig {
+    /**
+     * Below this root-filesystem usage the cleanup runs only the cheap rules
+     * (staging, artifacts, journald, registered paths) and skips the Bun cache
+     * walk, image prunes and `apt-get clean`. `0` always runs everything.
+     * @default 50
+     */
+    lowWaterPercent?: number;
+    /**
+     * At or above this usage the cleanup prunes with {@link escalated} windows,
+     * prints a warning, and notifies the on-box notifier if it is still above
+     * after cleaning. @default 85
+     */
+    highWaterPercent?: number;
+    /**
+     * Windows used at or above the high-water mark. Never looser than the normal
+     * ones: each field is the smaller of the two.
+     * @default { artifactMaxAgeDays: 0, bunCacheMaxAgeDays: 1, journalMaxAgeDays: 3, journalMaxSizeMb: 256, containerImageMaxAgeHours: 24 }
+     */
+    escalated?: HostCleanupRetention;
+}
+/**
+ * A directory another component writes timestamped files into, pruned by the
+ * same cleanup on the same schedule. Only direct children of `path` whose name
+ * matches `pattern` are candidates, and only on the same filesystem.
+ */
+export interface HostCleanupPathRule {
+    /** Absolute directory, e.g. `/root/rpx-backups`. Not `/`, and not under `/var/www`. */
+    path: string;
+    /** `find -name` glob for the entries to prune, e.g. `cert-backup-*`. */
+    pattern: string;
+    /** Delete entries older than this many days (`-mtime`). */
+    maxAgeDays: number;
+    /** Prune matching files, or matching directories with their contents. @default 'file' */
+    type?: 'file' | 'directory';
+}
+/** Host cleanup settings. See {@link ComputeConfig.hostCleanup}. */
+export interface ComputeHostCleanupConfig {
+    /**
+     * Install the `ts-cloud-host-cleanup.timer` systemd timer, so cleanup runs on
+     * a schedule rather than only when something deploys. `false` removes a timer
+     * an earlier deploy installed and keeps the deploy-time run. @default true
+     */
+    timer?: boolean;
+    /** systemd `OnCalendar=` expression for the timer. @default 'daily' */
+    schedule?: string;
+    /** Retention windows; omit a field to keep its default. */
+    retention?: HostCleanupRetention;
+    /**
+     * Disk-pressure thresholds. `false` runs every rule with the normal windows
+     * regardless of usage, as ts-cloud did before the thresholds existed.
+     */
+    pressure?: boolean | HostCleanupPressureConfig;
+    /** Extra directories to prune, e.g. a proxy's own backups. */
+    paths?: HostCleanupPathRule[];
 }
 /** An operator SSH key authorized on the box. See {@link ComputeConfig.sshKeys}. */
 export interface SshKeyConfig {
@@ -4110,6 +4223,53 @@ export interface DatabaseItemConfig {
         }>;
     }>;
 }
+/** Object form of {@link CdnItemConfig.failoverOrigin}. */
+export interface CdnFailoverOriginConfig {
+    /** Domain of the secondary origin. */
+    domain: string;
+    /** Path CloudFront prepends to requests sent to the secondary. */
+    originPath?: string;
+    /** Times CloudFront tries to connect to the secondary, 1-3. */
+    connectionAttempts?: number;
+    /** Seconds CloudFront waits to connect to the secondary, 1-10. */
+    connectionTimeout?: number;
+}
+/**
+ * CloudFront origin failover for a website bucket: a replica bucket in a
+ * second region that CloudFront serves from when the primary bucket fails.
+ *
+ * ts-cloud creates the replica (versioned, private, encrypted) in
+ * {@link region} before the stack deploys, replicates every write to it with
+ * S3 replication, copies objects that predate replication across, and lets
+ * the distribution read it through the same origin access control as the
+ * primary. The replica lives outside the CloudFormation stack, because a stack
+ * can only create buckets in its own region, so removing the stack leaves it
+ * behind.
+ */
+export interface StorageFailoverConfig {
+    /** Region of the replica bucket. Must differ from the stack's region. */
+    region: string;
+    /** Replica bucket name. @default `<primary bucket>-<region>` */
+    bucket?: string;
+    /**
+     * Primary status codes that send a request to the replica. CloudFront
+     * accepts 400, 403, 404, 416, 429, 500, 502, 503 and 504.
+     * @default [403, 404, 500, 502, 503, 504] - an S3 origin behind origin access
+     * control answers a missing object with 403, so 403 and 404 cover an object
+     * the primary lost, and the 5xx codes cover the bucket or region failing.
+     */
+    statusCodes?: number[];
+    /** Times CloudFront tries to connect to the primary bucket, 1-3. @default CloudFront's 3 */
+    connectionAttempts?: number;
+    /** Seconds CloudFront waits to connect to the primary bucket, 1-10. @default CloudFront's 10 */
+    connectionTimeout?: number;
+    /**
+     * Replicate the primary bucket to the replica with S3 replication (and turn
+     * on the versioning that requires). Set false to keep the replica in sync
+     * yourself. @default true
+     */
+    replicate?: boolean;
+}
 export interface CdnItemConfig {
     origin?: string;
     customDomain?: string | {
@@ -4129,6 +4289,15 @@ export interface CdnItemConfig {
     originShield?: boolean;
     /** AWS region used by Origin Shield. Defaults to the deployment region. */
     originShieldRegion?: string;
+    /**
+     * Times CloudFront tries to connect to {@link origin}, 1-3. CloudFront's
+     * default is 3. Lower it, with {@link connectionTimeout}, to fail over
+     * faster: an unreachable primary otherwise costs up to 30 seconds before
+     * CloudFront asks {@link failoverOrigin}.
+     */
+    connectionAttempts?: number;
+    /** Seconds CloudFront waits to connect to {@link origin}, 1-10. CloudFront's default is 10. */
+    connectionTimeout?: number;
     /**
      * Secondary origin CloudFront fails over to when {@link origin} errors,
      * e.g. a replica bucket in another region. Adds a CloudFront origin group
@@ -4136,16 +4305,17 @@ export interface CdnItemConfig {
      * CloudFront only fails over GET, HEAD and OPTIONS requests. An S3 REST
      * endpoint becomes an S3 origin; any other host an HTTPS-only custom origin.
      *
-     * @experimental Generated from the AWS documentation; not yet verified
-     * against a live distribution.
+     * A string is the secondary's domain. The object form also tunes the
+     * secondary's connection settings and sets its origin path.
      */
-    failoverOrigin?: string;
+    failoverOrigin?: string | CdnFailoverOriginConfig;
     /**
      * Status codes from {@link origin} that trigger failover to
      * {@link failoverOrigin}. CloudFront accepts 400, 403, 404, 416, 429, 500,
-     * 502, 503 and 504; anything else throws at template generation.
+     * 502, 503 and 504; anything else throws at template generation. Add 403
+     * and 404 to fail over on an object the primary does not have (an S3 origin
+     * behind origin access control answers a missing object with 403).
      * @default [500, 502, 503, 504]
-     * @experimental
      */
     failoverStatusCodes?: number[];
     /**

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ts-cloud/core",
   "type": "module",
-  "version": "0.16.16",
+  "version": "0.16.18",
   "description": "Core CloudFormation generation library for ts-cloud",
   "author": "Chris Breuer <chris@stacksjs.com>",
   "license": "MIT",
@@ -31,7 +31,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@ts-cloud/aws-types": "0.16.16"
+    "@ts-cloud/aws-types": "0.16.18"
   },
   "devDependencies": {
     "typescript": "^7.0.2"