@git.zone/tsdocker 1.14.0 → 1.15.1

package/ts/classes.registrycopy.ts

@@ -0,0 +1,511 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { logger } from './tsdocker.logging.js';
+
+interface IRegistryCredentials {
+  username: string;
+  password: string;
+}
+
+interface ITokenCache {
+  [scope: string]: { token: string; expiry: number };
+}
+
+/**
+ * OCI Distribution API client for copying images between registries.
+ * Supports manifest lists (multi-arch) and single-platform manifests.
+ * Uses native fetch (Node 18+).
+ */
+export class RegistryCopy {
+  private tokenCache: ITokenCache = {};
+
+  /**
+   * Reads Docker credentials from ~/.docker/config.json for a given registry.
+   * Supports base64-encoded "auth" field in the config.
+   */
+  public static getDockerConfigCredentials(registryUrl: string): IRegistryCredentials | null {
+    try {
+      const configPath = path.join(os.homedir(), '.docker', 'config.json');
+      if (!fs.existsSync(configPath)) return null;
+
+      const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+      const auths = config.auths || {};
+
+      // Try exact match first, then common variations
+      const keys = [
+        registryUrl,
+        `https://${registryUrl}`,
+        `http://${registryUrl}`,
+      ];
+
+      // Docker Hub special cases
+      if (registryUrl === 'docker.io' || registryUrl === 'registry-1.docker.io') {
+        keys.push(
+          'https://index.docker.io/v1/',
+          'https://index.docker.io/v2/',
+          'index.docker.io',
+          'docker.io',
+          'registry-1.docker.io',
+        );
+      }
+
+      for (const key of keys) {
+        if (auths[key]?.auth) {
+          const decoded = Buffer.from(auths[key].auth, 'base64').toString('utf-8');
+          const colonIndex = decoded.indexOf(':');
+          if (colonIndex > 0) {
+            return {
+              username: decoded.substring(0, colonIndex),
+              password: decoded.substring(colonIndex + 1),
+            };
+          }
+        }
+      }
+
+      return null;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Returns the API base URL for a registry.
+   * Docker Hub uses registry-1.docker.io as API endpoint.
+   */
+  private getRegistryApiBase(registry: string): string {
+    if (registry === 'docker.io' || registry === 'index.docker.io') {
+      return 'https://registry-1.docker.io';
+    }
+    // Local registries (localhost) use HTTP
+    if (registry.startsWith('localhost') || registry.startsWith('127.0.0.1')) {
+      return `http://${registry}`;
+    }
+    return `https://${registry}`;
+  }
+
+  /**
+   * Obtains a Bearer token for registry operations.
+   * Follows the standard Docker auth flow:
+   *   GET /v2/ → 401 with Www-Authenticate → request token
+   */
+  private async getToken(
+    registry: string,
+    repo: string,
+    actions: string,
+    credentials?: IRegistryCredentials | null,
+  ): Promise<string | null> {
+    const scope = `repository:${repo}:${actions}`;
+    const cached = this.tokenCache[`${registry}/${scope}`];
+    if (cached && cached.expiry > Date.now()) {
+      return cached.token;
+    }
+
+    const apiBase = this.getRegistryApiBase(registry);
+
+    // Local registries typically don't need auth
+    if (registry.startsWith('localhost') || registry.startsWith('127.0.0.1')) {
+      return null;
+    }
+
+    try {
+      const checkResp = await fetch(`${apiBase}/v2/`, { method: 'GET' });
+      if (checkResp.ok) return null; // No auth needed
+
+      const wwwAuth = checkResp.headers.get('www-authenticate') || '';
+      const realmMatch = wwwAuth.match(/realm="([^"]+)"/);
+      const serviceMatch = wwwAuth.match(/service="([^"]+)"/);
+
+      if (!realmMatch) return null;
+
+      const realm = realmMatch[1];
+      const service = serviceMatch ? serviceMatch[1] : '';
+
+      const tokenUrl = new URL(realm);
+      tokenUrl.searchParams.set('scope', scope);
+      if (service) tokenUrl.searchParams.set('service', service);
+
+      const headers: Record<string, string> = {};
+      const creds = credentials || RegistryCopy.getDockerConfigCredentials(registry);
+      if (creds) {
+        headers['Authorization'] = 'Basic ' + Buffer.from(`${creds.username}:${creds.password}`).toString('base64');
+      }
+
+      const tokenResp = await fetch(tokenUrl.toString(), { headers });
+      if (!tokenResp.ok) {
+        const body = await tokenResp.text();
+        throw new Error(`Token request failed (${tokenResp.status}): ${body}`);
+      }
+
+      const tokenData = await tokenResp.json() as any;
+      const token = tokenData.token || tokenData.access_token;
+
+      if (token) {
+        // Cache for 5 minutes (conservative)
+        this.tokenCache[`${registry}/${scope}`] = {
+          token,
+          expiry: Date.now() + 5 * 60 * 1000,
+        };
+      }
+
+      return token;
+    } catch (err) {
+      logger.log('warn', `Auth for ${registry}: ${(err as Error).message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Makes an authenticated request to a registry.
+   */
+  private async registryFetch(
+    registry: string,
+    path: string,
+    options: {
+      method?: string;
+      headers?: Record<string, string>;
+      body?: Buffer | ReadableStream | null;
+      repo?: string;
+      actions?: string;
+      credentials?: IRegistryCredentials | null;
+    } = {},
+  ): Promise<Response> {
+    const apiBase = this.getRegistryApiBase(registry);
+    const method = options.method || 'GET';
+    const headers: Record<string, string> = { ...(options.headers || {}) };
+
+    const repo = options.repo || '';
+    const actions = options.actions || 'pull';
+    const token = await this.getToken(registry, repo, actions, options.credentials);
+
+    if (token) {
+      headers['Authorization'] = `Bearer ${token}`;
+    }
+
+    const url = `${apiBase}${path}`;
+    const fetchOptions: any = { method, headers };
+    if (options.body) {
+      fetchOptions.body = options.body;
+      fetchOptions.duplex = 'half'; // Required for streaming body in Node
+    }
+
+    return fetch(url, fetchOptions);
+  }
+
+  /**
+   * Gets a manifest from a registry (supports both manifest lists and single manifests).
+   */
+  private async getManifest(
+    registry: string,
+    repo: string,
+    reference: string,
+    credentials?: IRegistryCredentials | null,
+  ): Promise<{ contentType: string; body: any; digest: string; raw: Buffer }> {
+    const accept = [
+      'application/vnd.oci.image.index.v1+json',
+      'application/vnd.docker.distribution.manifest.list.v2+json',
+      'application/vnd.oci.image.manifest.v1+json',
+      'application/vnd.docker.distribution.manifest.v2+json',
+    ].join(', ');
+
+    const resp = await this.registryFetch(registry, `/v2/${repo}/manifests/${reference}`, {
+      headers: { 'Accept': accept },
+      repo,
+      actions: 'pull',
+      credentials,
+    });
+
+    if (!resp.ok) {
+      const body = await resp.text();
+      throw new Error(`Failed to get manifest ${registry}/${repo}:${reference} (${resp.status}): ${body}`);
+    }
+
+    const raw = Buffer.from(await resp.arrayBuffer());
+    const contentType = resp.headers.get('content-type') || '';
+    const digest = resp.headers.get('docker-content-digest') || this.computeDigest(raw);
+    const body = JSON.parse(raw.toString('utf-8'));
+
+    return { contentType, body, digest, raw };
+  }
+
+  /**
+   * Checks if a blob exists in the destination registry.
+   */
+  private async blobExists(
+    registry: string,
+    repo: string,
+    digest: string,
+    credentials?: IRegistryCredentials | null,
+  ): Promise<boolean> {
+    const resp = await this.registryFetch(registry, `/v2/${repo}/blobs/${digest}`, {
+      method: 'HEAD',
+      repo,
+      actions: 'pull,push',
+      credentials,
+    });
+    return resp.ok;
+  }
+
+  /**
+   * Copies a single blob from source to destination registry.
+   * Uses monolithic upload (POST initiate + PUT complete).
+   */
+  private async copyBlob(
+    srcRegistry: string,
+    srcRepo: string,
+    destRegistry: string,
+    destRepo: string,
+    digest: string,
+    srcCredentials?: IRegistryCredentials | null,
+    destCredentials?: IRegistryCredentials | null,
+  ): Promise<void> {
+    // Check if blob already exists at destination
+    const exists = await this.blobExists(destRegistry, destRepo, digest, destCredentials);
+    if (exists) {
+      logger.log('info', `  Blob ${digest.substring(0, 19)}... already exists, skipping`);
+      return;
+    }
+
+    // Download blob from source
+    const getResp = await this.registryFetch(srcRegistry, `/v2/${srcRepo}/blobs/${digest}`, {
+      repo: srcRepo,
+      actions: 'pull',
+      credentials: srcCredentials,
+    });
+
+    if (!getResp.ok) {
+      throw new Error(`Failed to get blob ${digest} from ${srcRegistry}/${srcRepo}: ${getResp.status}`);
+    }
+
+    const blobData = Buffer.from(await getResp.arrayBuffer());
+    const blobSize = blobData.length;
+
+    // Initiate upload at destination
+    const postResp = await this.registryFetch(destRegistry, `/v2/${destRepo}/blobs/uploads/`, {
+      method: 'POST',
+      headers: { 'Content-Length': '0' },
+      repo: destRepo,
+      actions: 'pull,push',
+      credentials: destCredentials,
+    });
+
+    if (!postResp.ok && postResp.status !== 202) {
+      const body = await postResp.text();
+      throw new Error(`Failed to initiate upload at ${destRegistry}/${destRepo}: ${postResp.status} ${body}`);
+    }
+
+    // Get upload URL from Location header
+    let uploadUrl = postResp.headers.get('location') || '';
+    if (!uploadUrl) {
+      throw new Error(`No upload location returned from ${destRegistry}/${destRepo}`);
+    }
+
+    // Make upload URL absolute if relative
+    if (uploadUrl.startsWith('/')) {
+      const apiBase = this.getRegistryApiBase(destRegistry);
+      uploadUrl = `${apiBase}${uploadUrl}`;
+    }
+
+    // Complete upload with PUT (monolithic)
+    const separator = uploadUrl.includes('?') ? '&' : '?';
+    const putUrl = `${uploadUrl}${separator}digest=${encodeURIComponent(digest)}`;
+
+    // For PUT to the upload URL, we need auth
+    const token = await this.getToken(destRegistry, destRepo, 'pull,push', destCredentials);
+    const putHeaders: Record<string, string> = {
+      'Content-Type': 'application/octet-stream',
+      'Content-Length': String(blobSize),
+    };
+    if (token) {
+      putHeaders['Authorization'] = `Bearer ${token}`;
+    }
+
+    const putResp = await fetch(putUrl, {
+      method: 'PUT',
+      headers: putHeaders,
+      body: blobData,
+    });
+
+    if (!putResp.ok) {
+      const body = await putResp.text();
+      throw new Error(`Failed to upload blob ${digest} to ${destRegistry}/${destRepo}: ${putResp.status} ${body}`);
+    }
+
+    const sizeStr = blobSize > 1048576
+      ? `${(blobSize / 1048576).toFixed(1)} MB`
+      : `${(blobSize / 1024).toFixed(1)} KB`;
+    logger.log('info', `  Copied blob ${digest.substring(0, 19)}... (${sizeStr})`);
+  }
+
+  /**
+   * Pushes a manifest to a registry.
+   */
+  private async putManifest(
+    registry: string,
+    repo: string,
+    reference: string,
+    manifest: Buffer,
+    contentType: string,
+    credentials?: IRegistryCredentials | null,
+  ): Promise<string> {
+    const resp = await this.registryFetch(registry, `/v2/${repo}/manifests/${reference}`, {
+      method: 'PUT',
+      headers: {
+        'Content-Type': contentType,
+        'Content-Length': String(manifest.length),
+      },
+      body: manifest,
+      repo,
+      actions: 'pull,push',
+      credentials,
+    });
+
+    if (!resp.ok) {
+      const body = await resp.text();
+      throw new Error(`Failed to put manifest ${registry}/${repo}:${reference} (${resp.status}): ${body}`);
+    }
+
+    const digest = resp.headers.get('docker-content-digest') || this.computeDigest(manifest);
+    return digest;
+  }
+
+  /**
+   * Copies a single-platform manifest and all its blobs from source to destination.
+   */
+  private async copySingleManifest(
+    srcRegistry: string,
+    srcRepo: string,
+    destRegistry: string,
+    destRepo: string,
+    manifestDigest: string,
+    srcCredentials?: IRegistryCredentials | null,
+    destCredentials?: IRegistryCredentials | null,
+  ): Promise<void> {
+    // Get the platform manifest
+    const { body: manifest, contentType, raw } = await this.getManifest(
+      srcRegistry, srcRepo, manifestDigest, srcCredentials,
+    );
+
+    // Copy config blob
+    if (manifest.config?.digest) {
+      logger.log('info', `  Copying config blob...`);
+      await this.copyBlob(
+        srcRegistry, srcRepo, destRegistry, destRepo,
+        manifest.config.digest, srcCredentials, destCredentials,
+      );
+    }
+
+    // Copy layer blobs
+    const layers = manifest.layers || [];
+    for (let i = 0; i < layers.length; i++) {
+      const layer = layers[i];
+      logger.log('info', `  Copying layer ${i + 1}/${layers.length}...`);
+      await this.copyBlob(
+        srcRegistry, srcRepo, destRegistry, destRepo,
+        layer.digest, srcCredentials, destCredentials,
+      );
+    }
+
+    // Push the platform manifest by digest
+    await this.putManifest(
+      destRegistry, destRepo, manifestDigest, raw, contentType, destCredentials,
+    );
+  }
+
+  /**
+   * Copies a complete image (single or multi-arch) from source to destination registry.
+   *
+   * @param srcRegistry - Source registry host (e.g., "localhost:5234")
+   * @param srcRepo - Source repository (e.g., "myapp")
+   * @param srcTag - Source tag (e.g., "v1.0.0")
+   * @param destRegistry - Destination registry host (e.g., "registry.gitlab.com")
+   * @param destRepo - Destination repository (e.g., "org/myapp")
+   * @param destTag - Destination tag (e.g., "v1.0.0" or "v1.0.0_arm64")
+   * @param credentials - Optional credentials for destination registry
+   */
+  public async copyImage(
+    srcRegistry: string,
+    srcRepo: string,
+    srcTag: string,
+    destRegistry: string,
+    destRepo: string,
+    destTag: string,
+    credentials?: IRegistryCredentials | null,
+  ): Promise<void> {
+    logger.log('info', `Copying ${srcRegistry}/${srcRepo}:${srcTag} -> ${destRegistry}/${destRepo}:${destTag}`);
+
+    // Source is always the local registry (no credentials needed)
+    const srcCredentials: IRegistryCredentials | null = null;
+    const destCredentials = credentials || RegistryCopy.getDockerConfigCredentials(destRegistry);
+
+    // Get the top-level manifest
+    const topManifest = await this.getManifest(srcRegistry, srcRepo, srcTag, srcCredentials);
+    const { body, contentType, raw } = topManifest;
+
+    const isManifestList =
+      contentType.includes('manifest.list') ||
+      contentType.includes('image.index') ||
+      body.manifests !== undefined;
+
+    if (isManifestList) {
+      // Multi-arch: copy each platform manifest + blobs, then push the manifest list
+      const platforms = (body.manifests || []) as any[];
+      logger.log('info', `Multi-arch manifest with ${platforms.length} platform(s)`);
+
+      for (const platformEntry of platforms) {
+        const platDesc = platformEntry.platform
+          ? `${platformEntry.platform.os}/${platformEntry.platform.architecture}`
+          : platformEntry.digest;
+        logger.log('info', `Copying platform: ${platDesc}`);
+
+        await this.copySingleManifest(
+          srcRegistry, srcRepo, destRegistry, destRepo,
+          platformEntry.digest, srcCredentials, destCredentials,
+        );
+      }
+
+      // Push the manifest list/index with the destination tag
+      const digest = await this.putManifest(
+        destRegistry, destRepo, destTag, raw, contentType, destCredentials,
+      );
+      logger.log('ok', `Pushed manifest list to ${destRegistry}/${destRepo}:${destTag} (${digest.substring(0, 19)}...)`);
+    } else {
+      // Single-platform manifest: copy blobs + push manifest
+      logger.log('info', 'Single-platform manifest');
+
+      // Copy config blob
+      if (body.config?.digest) {
+        logger.log('info', '  Copying config blob...');
+        await this.copyBlob(
+          srcRegistry, srcRepo, destRegistry, destRepo,
+          body.config.digest, srcCredentials, destCredentials,
+        );
+      }
+
+      // Copy layer blobs
+      const layers = body.layers || [];
+      for (let i = 0; i < layers.length; i++) {
+        logger.log('info', `  Copying layer ${i + 1}/${layers.length}...`);
+        await this.copyBlob(
+          srcRegistry, srcRepo, destRegistry, destRepo,
+          layers[i].digest, srcCredentials, destCredentials,
+        );
+      }
+
+      // Push the manifest with the destination tag
+      const digest = await this.putManifest(
+        destRegistry, destRepo, destTag, raw, contentType, destCredentials,
+      );
+      logger.log('ok', `Pushed manifest to ${destRegistry}/${destRepo}:${destTag} (${digest.substring(0, 19)}...)`);
+    }
+  }
+
+  /**
+   * Computes sha256 digest of a buffer.
+   */
+  private computeDigest(data: Buffer): string {
+    const crypto = require('crypto');
+    const hash = crypto.createHash('sha256').update(data).digest('hex');
+    return `sha256:${hash}`;
+  }
+}

package/ts/classes.tsdockermanager.ts

@@ -187,11 +187,7 @@ export class TsDockerManager {
 
       const total = toBuild.length;
       const overallStart = Date.now();
-      const useRegistry = Dockerfile.needsLocalRegistry(toBuild, options);
-
-      if (useRegistry) {
-        await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
-      }
+      await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
 
       try {
         if (options?.parallel) {
@@ -230,7 +226,7 @@ export class TsDockerManager {
 
             await Dockerfile.runWithConcurrency(tasks, concurrency);
 
-            // After the entire level completes, tag + push for dependency resolution
+            // After the entire level completes, push all to local registry + tag for deps
             for (const df of level) {
               const dependentBaseImages = new Set<string>();
               for (const other of toBuild) {
@@ -242,7 +238,8 @@ export class TsDockerManager {
                 logger.log('info', `Tagging ${df.buildTag} as ${fullTag} for local dependency resolution`);
                 await smartshellInstance.exec(`docker tag ${df.buildTag} ${fullTag}`);
               }
-              if (useRegistry && toBuild.some(other => other.localBaseDockerfile === df)) {
+              // Push ALL images to local registry (skip if already pushed via buildx)
+              if (!df.localRegistryTag) {
                 await Dockerfile.pushToLocalRegistry(df);
               }
             }
@@ -281,16 +278,14 @@ export class TsDockerManager {
               await smartshellInstance.exec(`docker tag ${dockerfileArg.buildTag} ${fullTag}`);
             }
 
-            // Push to local registry for buildx (even for cache hits — image exists but registry doesn't)
-            if (useRegistry && toBuild.some(other => other.localBaseDockerfile === dockerfileArg)) {
+            // Push ALL images to local registry (skip if already pushed via buildx)
+            if (!dockerfileArg.localRegistryTag) {
               await Dockerfile.pushToLocalRegistry(dockerfileArg);
             }
           }
         }
       } finally {
-        if (useRegistry) {
-          await Dockerfile.stopLocalRegistry();
-        }
+        await Dockerfile.stopLocalRegistry();
       }
 
       logger.log('info', `Total build time: ${formatDuration(Date.now() - overallStart)}`);
@@ -398,11 +393,17 @@ export class TsDockerManager {
       return;
     }
 
-    // Push each Dockerfile to each registry
-    for (const dockerfile of this.dockerfiles) {
-      for (const registry of registriesToPush) {
-        await dockerfile.push(registry);
+    // Start local registry (reads from persistent .nogit/docker-registry/)
+    await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
+    try {
+      // Push each Dockerfile to each registry via OCI copy
+      for (const dockerfile of this.dockerfiles) {
+        for (const registry of registriesToPush) {
+          await dockerfile.push(registry);
+        }
       }
+    } finally {
+      await Dockerfile.stopLocalRegistry();
     }
 
     logger.log('success', 'All images pushed successfully');
@@ -429,7 +430,8 @@ export class TsDockerManager {
   }
 
   /**
-   * Runs tests for all Dockerfiles
+   * Runs tests for all Dockerfiles.
+   * Starts the local registry so multi-platform images can be auto-pulled.
    */
   public async test(): Promise<void> {
     if (this.dockerfiles.length === 0) {
@@ -443,7 +445,14 @@ export class TsDockerManager {
 
     logger.log('info', '');
     logger.log('info', '=== TEST PHASE ===');
-    await Dockerfile.testDockerfiles(this.dockerfiles);
+
+    await Dockerfile.startLocalRegistry(this.dockerContext.contextInfo?.isRootless);
+    try {
+      await Dockerfile.testDockerfiles(this.dockerfiles);
+    } finally {
+      await Dockerfile.stopLocalRegistry();
+    }
+
     logger.log('success', 'All tests completed');
   }