@stacksjs/ts-cloud 0.2.22 → 0.2.23

package/dist/index.d.ts

@@ -6,6 +6,8 @@ export { AWSClient, CloudFormationClient, CloudFormationClient as AWSCloudFormat
 export type { AWSRequestOptions, AWSClientConfig, AWSError, AWSCredentials as AWSClientCredentials, StackParameter, StackTag, CreateStackOptions, UpdateStackOptions, DescribeStacksOptions, StackEvent, Stack, InvalidationOptions, Distribution, S3SyncOptions, S3CopyOptions, S3ListOptions, S3Object, CertificateDetail, Certificate as ELBv2Certificate, RekognitionS3Object, RekognitionBoundingBox, TextractS3Object, TextractBoundingBox, CountryCode, ContactType, ContactDetail, KendraCreateDataSourceCommandInput, KendraCreateDataSourceCommandOutput, KendraListDataSourcesCommandInput, KendraListDataSourcesCommandOutput, InvokeModelCommandInput, InvokeModelCommandOutput, InvokeModelWithResponseStreamCommandInput, InvokeModelWithResponseStreamCommandOutput, CreateModelCustomizationJobCommandInput, CreateModelCustomizationJobCommandOutput, GetModelCustomizationJobCommandInput, GetModelCustomizationJobCommandOutput, ListFoundationModelsCommandInput, ListFoundationModelsCommandOutput, AttributeValue as DynamoDBAttributeValue, KeySchemaElement, AttributeDefinition as DynamoDBAttributeDefinition, } from './aws';
 export { createObjectStorageClient, providerEndpoint, resolveObjectStorage, } from './object-storage';
 export type { ObjectStorageConfig, ObjectStorageCredentials, ObjectStorageProvider, ResolvedObjectStorage, } from './object-storage';
+export { keyMatchesFilters, migrateObjectStorage, remapKey, } from './object-storage/migrate';
+export type { MigrateEndpoint, MigrateError, MigrateOptions, MigratePlanItem, MigrateProgress, MigrateResult, MigrateVerification, } from './object-storage/migrate';
 export * from './ssl';
 export { deployStaticSite, deployStaticSiteFull, uploadStaticFiles, invalidateCache, deleteStaticSite, generateStaticSiteTemplate, deployStaticSiteWithExternalDns, deployStaticSiteWithExternalDnsFull, generateExternalDnsStaticSiteTemplate, deploySite, resolveSiteDeployTarget, resolveSiteKind, validateDeploymentConfig, } from './deploy';
 export type { StaticSiteConfig, DeployResult, UploadOptions, ExternalDnsStaticSiteConfig, ExternalDnsDeployResult, DeploySiteConfig, DeploySiteResult, StaticSiteDnsProvider, SiteDeployKind, DeploymentValidationResult, } from './deploy';

package/dist/index.js

@@ -2123,6 +2123,67 @@ class S3Client2 {
     });
     return result;
   }
+  async getObjectBytes(bucket, key) {
+    const { accessKeyId, secretAccessKey, sessionToken } = this.getCredentials();
+    const host = this.s3VirtualHost(bucket);
+    const encodedKey = key.split("/").map((seg) => encodeURIComponent(seg)).join("/");
+    const canonicalUri = this.forcePathStyle ? `/${bucket}/${encodedKey}` : `/${encodedKey}`;
+    const url = `https://${host}${canonicalUri}`;
+    const now = new Date;
+    const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
+    const dateStamp = now.toISOString().slice(0, 10).replace(/-/g, "");
+    const payloadHash = crypto3.createHash("sha256").update("").digest("hex");
+    const requestHeaders = {
+      host,
+      "x-amz-date": amzDate,
+      "x-amz-content-sha256": payloadHash
+    };
+    if (sessionToken) {
+      requestHeaders["x-amz-security-token"] = sessionToken;
+    }
+    const canonicalHeaders = Object.keys(requestHeaders).sort().map((k) => `${k.toLowerCase()}:${requestHeaders[k].trim()}
+`).join("");
+    const signedHeaders = Object.keys(requestHeaders).sort().map((k) => k.toLowerCase()).join(";");
+    const canonicalRequest = [
+      "GET",
+      canonicalUri,
+      "",
+      canonicalHeaders,
+      signedHeaders,
+      payloadHash
+    ].join(`
+`);
+    const algorithm = "AWS4-HMAC-SHA256";
+    const credentialScope = `${dateStamp}/${this.region}/s3/aws4_request`;
+    const stringToSign = [
+      algorithm,
+      amzDate,
+      credentialScope,
+      crypto3.createHash("sha256").update(canonicalRequest).digest("hex")
+    ].join(`
+`);
+    const kDate = crypto3.createHmac("sha256", `AWS4${secretAccessKey}`).update(dateStamp).digest();
+    const kRegion = crypto3.createHmac("sha256", kDate).update(this.region).digest();
+    const kService = crypto3.createHmac("sha256", kRegion).update("s3").digest();
+    const kSigning = crypto3.createHmac("sha256", kService).update("aws4_request").digest();
+    const signature = crypto3.createHmac("sha256", kSigning).update(stringToSign).digest("hex");
+    const authorizationHeader = `${algorithm} Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
+    const response = await fetch(url, {
+      method: "GET",
+      headers: { ...requestHeaders, Authorization: authorizationHeader }
+    });
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`S3 GET failed: ${response.status} ${errorText}`);
+    }
+    const buffer = await response.arrayBuffer();
+    const contentLengthHeader = response.headers.get("content-length");
+    return {
+      body: new Uint8Array(buffer),
+      contentType: response.headers.get("content-type") ?? undefined,
+      contentLength: contentLengthHeader ? Number.parseInt(contentLengthHeader, 10) : undefined
+    };
+  }
   async copyObject(options) {
     const headers = {
       "x-amz-copy-source": `/${options.sourceBucket}/${options.sourceKey}`
@@ -65570,6 +65631,165 @@ init_client();
 
 // src/object-storage/index.ts
 init_s3();
+
+// src/object-storage/migrate.ts
+function stripPrefix(key, prefix) {
+  if (!prefix)
+    return key;
+  return key.startsWith(prefix) ? key.slice(prefix.length) : key;
+}
+function remapKey(sourceKey, fromPrefix, toPrefix) {
+  const stripped = stripPrefix(sourceKey, fromPrefix);
+  return `${toPrefix ?? ""}${stripped}`;
+}
+function keyMatchesFilters(key, include, exclude) {
+  if (exclude && exclude.some((p) => key.startsWith(p)))
+    return false;
+  if (include && include.length > 0)
+    return include.some((p) => key.startsWith(p));
+  return true;
+}
+async function mapWithConcurrency(items, limit, worker) {
+  let cursor = 0;
+  const runners = [];
+  const size = Math.max(1, Math.min(limit, items.length || 1));
+  for (let i = 0;i < size; i++) {
+    runners.push((async () => {
+      while (true) {
+        const index = cursor++;
+        if (index >= items.length)
+          return;
+        await worker(items[index], index);
+      }
+    })());
+  }
+  await Promise.all(runners);
+}
+function clientFor(endpoint) {
+  if (endpoint.client)
+    return endpoint.client;
+  return createObjectStorageClient({
+    provider: endpoint.provider,
+    region: endpoint.region,
+    endpoint: endpoint.endpoint,
+    forcePathStyle: endpoint.forcePathStyle,
+    credentials: endpoint.credentials
+  });
+}
+async function migrateObjectStorage(options) {
+  const concurrency = options.concurrency ?? 8;
+  const fromClient = clientFor(options.from);
+  const toClient = clientFor(options.to);
+  const sourceObjects = await fromClient.listAllObjects({
+    bucket: options.from.bucket,
+    prefix: options.from.prefix
+  });
+  const result = {
+    copied: 0,
+    skipped: 0,
+    excluded: 0,
+    bytesCopied: 0,
+    errors: [],
+    excludedKeys: [],
+    deleted: []
+  };
+  const toCopy = [];
+  for (const obj of sourceObjects) {
+    if (!keyMatchesFilters(obj.Key, options.include, options.exclude)) {
+      result.excluded++;
+      result.excludedKeys.push(obj.Key);
+      continue;
+    }
+    toCopy.push({ source: obj, destKey: remapKey(obj.Key, options.from.prefix, options.to.prefix) });
+  }
+  if (options.dryRun) {
+    result.plan = toCopy.map(({ source, destKey }) => ({ key: source.Key, destKey, size: source.Size }));
+    const total2 = sourceObjects.length;
+    let index = 0;
+    for (const obj of sourceObjects) {
+      index++;
+      const excluded = !keyMatchesFilters(obj.Key, options.include, options.exclude);
+      options.onProgress?.({
+        key: obj.Key,
+        destKey: excluded ? "" : remapKey(obj.Key, options.from.prefix, options.to.prefix),
+        size: obj.Size,
+        action: excluded ? "excluded" : "planned",
+        index,
+        total: total2
+      });
+    }
+    return result;
+  }
+  const total = toCopy.length;
+  let processed = 0;
+  await mapWithConcurrency(toCopy, concurrency, async ({ source, destKey }) => {
+    const myIndex = ++processed;
+    try {
+      if (!options.force) {
+        const head = await toClient.headObject(options.to.bucket, destKey);
+        if (head && head.ContentLength === source.Size) {
+          result.skipped++;
+          options.onProgress?.({ key: source.Key, destKey, size: source.Size, action: "skipped", index: myIndex, total });
+          return;
+        }
+      }
+      const { body, contentType } = await fromClient.getObjectBytes(options.from.bucket, source.Key);
+      await toClient.putObject({
+        bucket: options.to.bucket,
+        key: destKey,
+        body,
+        contentType
+      });
+      result.copied++;
+      result.bytesCopied += body.byteLength;
+      options.onProgress?.({ key: source.Key, destKey, size: source.Size, action: "copied", index: myIndex, total });
+    } catch (err) {
+      result.errors.push({ key: source.Key, message: err?.message ?? String(err) });
+      options.onProgress?.({ key: source.Key, destKey, size: source.Size, action: "error", index: myIndex, total });
+    }
+  });
+  const copiedDestKeys = new Set(toCopy.map((c) => c.destKey));
+  if (options.deleteExtraneous) {
+    const destObjects = await toClient.listAllObjects({ bucket: options.to.bucket, prefix: options.to.prefix });
+    const extraneous = destObjects.filter((o) => !copiedDestKeys.has(o.Key)).map((o) => o.Key);
+    for (const key of extraneous) {
+      try {
+        await toClient.deleteObject(options.to.bucket, key);
+        result.deleted.push(key);
+      } catch (err) {
+        result.errors.push({ key, message: `delete failed: ${err?.message ?? String(err)}` });
+      }
+    }
+  }
+  if (options.verify) {
+    const destObjects = await toClient.listAllObjects({ bucket: options.to.bucket, prefix: options.to.prefix });
+    const destBySizeKey = new Map(destObjects.map((o) => [o.Key, o.Size]));
+    const missing = [];
+    const sizeMismatches = [];
+    let matched = 0;
+    for (const { source, destKey } of toCopy) {
+      if (!destBySizeKey.has(destKey)) {
+        missing.push(destKey);
+        continue;
+      }
+      const actual = destBySizeKey.get(destKey);
+      if (actual !== source.Size) {
+        sizeMismatches.push({ key: destKey, expected: source.Size, actual });
+        continue;
+      }
+      matched++;
+    }
+    result.verification = {
+      ok: missing.length === 0 && sizeMismatches.length === 0,
+      matched,
+      missing,
+      sizeMismatches
+    };
+  }
+  return result;
+}
+
+// src/object-storage/index.ts
 var DEFAULT_REGION = {
   aws: "us-east-1",
   backblaze: "us-west-004",
@@ -82396,6 +82616,7 @@ export {
   resolveCaddyfile,
   requiresReplacement,
   replicaManager,
+  remapKey,
   regionPairManager,
   quickHash,
   queueManagementManager,
@@ -82417,6 +82638,7 @@ export {
   multiRegionManager,
   multiAccountManager,
   migrationManager,
+  migrateObjectStorage,
   metricsManager,
   mergeInfrastructure,
   makeAWSRequestOnce,
@@ -82430,6 +82652,7 @@ export {
   lambdaDestinationsManager,
   lambdaDLQManager,
   lambdaConcurrencyManager,
+  keyMatchesFilters,
   isWebCryptoAvailable,
   isValidRegion,
   isOnDemandDomain,

package/dist/object-storage/index.d.ts

@@ -23,6 +23,7 @@
  * ```
  */
 import { S3Client } from '../aws/s3';
+export * from './migrate';
 export type ObjectStorageProvider = 'aws' | 'backblaze' | 'hetzner';
 export interface ObjectStorageCredentials {
     accessKeyId: string;

package/dist/object-storage/migrate.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Cross-provider object-storage migration.
+ *
+ * Copies objects from one S3-compatible bucket to another — AWS S3, Backblaze
+ * B2, Hetzner Object Storage, in any direction. Both sides are driven by the
+ * same {@link createObjectStorageClient}, so the only thing that changes per
+ * side is the provider/region/endpoint/credentials.
+ *
+ * Bytes are copied (not strings) via {@link S3Client.getObjectBytes}, so binary
+ * payloads (images, archives, mail attachments) survive intact. Content-Type is
+ * preserved when the source reports it.
+ *
+ * The copy is idempotent: an object already present at the destination with the
+ * same size is skipped unless `force` is set. Keys may be remapped by stripping
+ * `fromPrefix` and prepending `toPrefix`, and filtered with include/exclude
+ * prefix lists so an operator can clearly see what was migrated vs. deliberately
+ * left behind.
+ */
+import type { S3Client } from '../aws/s3';
+import type { ObjectStorageProvider } from './index';
+/** One side (source or destination) of a migration. */
+export interface MigrateEndpoint {
+    provider: ObjectStorageProvider;
+    bucket: string;
+    region?: string;
+    /** Endpoint host override (no scheme). Defaults to the provider's standard endpoint. */
+    endpoint?: string;
+    forcePathStyle?: boolean;
+    /** Key prefix. On the source it scopes/strips; on the dest it is prepended. */
+    prefix?: string;
+    /** Explicit credentials. When omitted, resolved from the provider's env vars. */
+    credentials?: {
+        accessKeyId: string;
+        secretAccessKey: string;
+        sessionToken?: string;
+    };
+    /** Pre-built client (used by tests to inject an in-memory mock). */
+    client?: S3Client;
+}
+export interface MigrateOptions {
+    from: MigrateEndpoint;
+    to: MigrateEndpoint;
+    /** Only copy keys whose (source) key starts with one of these prefixes. */
+    include?: string[];
+    /** Skip keys whose (source) key starts with one of these prefixes. */
+    exclude?: string[];
+    /** Plan only — do not write to the destination. */
+    dryRun?: boolean;
+    /** Re-copy even if the destination already has an object of the same size. */
+    force?: boolean;
+    /** Delete destination keys (within the dest prefix) that are not in the copied set. Default OFF. */
+    deleteExtraneous?: boolean;
+    /** Max in-flight copies. Default 8. */
+    concurrency?: number;
+    /** After copying, re-list the destination and assert counts + sizes match the copied set. */
+    verify?: boolean;
+    /** Optional progress callback, invoked per object after it is copied/skipped. */
+    onProgress?: (event: MigrateProgress) => void;
+}
+export interface MigrateProgress {
+    /** Source key. */
+    key: string;
+    /** Destination key the object maps to. */
+    destKey: string;
+    size: number;
+    action: 'copied' | 'skipped' | 'excluded' | 'error' | 'planned';
+    /** 1-based index of this object within the full source listing. */
+    index: number;
+    total: number;
+}
+export interface MigrateError {
+    key: string;
+    message: string;
+}
+export interface MigratePlanItem {
+    key: string;
+    destKey: string;
+    size: number;
+}
+export interface MigrateResult {
+    copied: number;
+    skipped: number;
+    excluded: number;
+    bytesCopied: number;
+    errors: MigrateError[];
+    /** Keys that were excluded by include/exclude filters (source keys). */
+    excludedKeys: string[];
+    /** Keys deleted from the destination via `deleteExtraneous`. */
+    deleted: string[];
+    /** When `dryRun` is set, the objects that would be copied. */
+    plan?: MigratePlanItem[];
+    /** Verification outcome when `verify` is set. */
+    verification?: MigrateVerification;
+}
+export interface MigrateVerification {
+    ok: boolean;
+    /** Number of (key,size) pairs that matched at the destination. */
+    matched: number;
+    /** Copied keys missing at the destination. */
+    missing: string[];
+    /** Copied keys present at the destination but with a different size. */
+    sizeMismatches: Array<{
+        key: string;
+        expected: number;
+        actual: number;
+    }>;
+}
+/**
+ * Compute the destination key for a source key: strip the source prefix, then
+ * prepend the destination prefix. Exported for unit testing.
+ */
+export declare function remapKey(sourceKey: string, fromPrefix?: string, toPrefix?: string): string;
+/**
+ * Decide whether a source key passes the include/exclude prefix filters.
+ * `include` (when non-empty) is a whitelist; `exclude` always wins. Exported for
+ * unit testing.
+ */
+export declare function keyMatchesFilters(key: string, include?: string[], exclude?: string[]): boolean;
+/**
+ * Migrate objects from one S3-compatible bucket to another.
+ *
+ * @returns a structured {@link MigrateResult} so callers (CLI, buddy, scripts)
+ * can report exactly what was copied, skipped, excluded, deleted and verified.
+ */
+export declare function migrateObjectStorage(options: MigrateOptions): Promise<MigrateResult>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@stacksjs/ts-cloud",
   "type": "module",
-  "version": "0.2.22",
+  "version": "0.2.23",
   "description": "A lightweight, performant infrastructure-as-code library and CLI for deploying both server-based (EC2) and serverless applications.",
   "author": "Chris Breuer <chris@stacksjs.com>",
   "license": "MIT",
@@ -89,8 +89,8 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ts-cloud/aws-types": "0.2.22",
-    "@ts-cloud/core": "0.2.22",
+    "@ts-cloud/aws-types": "0.2.23",
+    "@ts-cloud/core": "0.2.23",
     "@stacksjs/ts-xml": "^0.1.0"
   },
   "devDependencies": {