@seanmozeik/s3up 0.3.1

package/src/lib/multipart.ts

@@ -0,0 +1,507 @@
+// src/lib/multipart.ts
+// S3 Multipart Upload orchestrator
+
+import {
+  createProgressState,
+  finishProgress,
+  type ProgressState,
+  updateProgress,
+  writeProgress
+} from './progress-bar';
+import type { S3Config } from './providers';
+import { getEndpoint } from './providers';
+import { type AwsCredentials, getRegionForSigning, signRequest } from './signing';
+import {
+  addCompletedPart,
+  type CompletedPart,
+  createInitialState,
+  deleteState,
+  hasFileChanged,
+  loadState,
+  saveState,
+  type UploadState
+} from './state';
+
+export interface MultipartOptions {
+  chunkSize: number;
+  connections: number;
+}
+
+interface MultipartUploadResult {
+  success: true;
+  publicUrl: string;
+}
+
+interface MultipartUploadError {
+  success: false;
+  error: string;
+}
+
+export type MultipartOutcome = MultipartUploadResult | MultipartUploadError;
+
+// Track abort state for graceful shutdown
+let abortController: AbortController | null = null;
+let isAborting = false;
+
+/**
+ * Get AWS credentials from S3Config
+ */
+function getCredentials(config: S3Config): AwsCredentials {
+  return {
+    accessKeyId: config.accessKeyId,
+    region: getRegionForSigning(config.provider, config.region),
+    secretAccessKey: config.secretAccessKey
+  };
+}
+
+/**
+ * Initiate a multipart upload
+ */
+async function initiateMultipartUpload(config: S3Config, key: string): Promise<string> {
+  const endpoint = getEndpoint(config);
+  const url = `${endpoint}/${config.bucket}/${encodeURIComponent(key)}?uploads`;
+  const credentials = getCredentials(config);
+
+  const signed = signRequest(
+    'POST',
+    url,
+    {
+      'content-type': 'application/octet-stream'
+    },
+    '',
+    credentials
+  );
+
+  const response = await fetch(signed.url, {
+    headers: signed.headers,
+    method: signed.method
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Failed to initiate multipart upload: ${response.status} ${text}`);
+  }
+
+  const xml = await response.text();
+  const match = xml.match(/<UploadId>([^<]+)<\/UploadId>/);
+  if (!match) {
+    throw new Error('No UploadId in response');
+  }
+
+  return match[1];
+}
+
+/**
+ * Upload a single part
+ */
+async function uploadPart(
+  config: S3Config,
+  key: string,
+  uploadId: string,
+  partNumber: number,
+  body: Uint8Array,
+  signal?: AbortSignal
+): Promise<CompletedPart> {
+  const endpoint = getEndpoint(config);
+  const url = `${endpoint}/${config.bucket}/${encodeURIComponent(key)}?partNumber=${partNumber}&uploadId=${encodeURIComponent(uploadId)}`;
+  const credentials = getCredentials(config);
+
+  // Include content-type for R2 compatibility
+  const signed = signRequest(
+    'PUT',
+    url,
+    { 'content-type': 'application/octet-stream' },
+    body,
+    credentials
+  );
+
+  const response = await fetch(signed.url, {
+    body: Buffer.from(body),
+    headers: signed.headers,
+    method: signed.method,
+    signal
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Failed to upload part ${partNumber}: ${response.status} ${text}`);
+  }
+
+  const etag = response.headers.get('etag');
+  if (!etag) {
+    throw new Error(`No ETag for part ${partNumber}`);
+  }
+
+  return { etag, partNumber };
+}
+
+/**
+ * Complete the multipart upload
+ */
+async function completeMultipartUpload(
+  config: S3Config,
+  key: string,
+  uploadId: string,
+  parts: CompletedPart[]
+): Promise<void> {
+  const endpoint = getEndpoint(config);
+  const url = `${endpoint}/${config.bucket}/${encodeURIComponent(key)}?uploadId=${encodeURIComponent(uploadId)}`;
+  const credentials = getCredentials(config);
+
+  // Build completion XML
+  const partsXml = parts
+    .sort((a, b) => a.partNumber - b.partNumber)
+    .map((p) => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${p.etag}</ETag></Part>`)
+    .join('');
+  const body = `<?xml version="1.0" encoding="UTF-8"?><CompleteMultipartUpload>${partsXml}</CompleteMultipartUpload>`;
+
+  const signed = signRequest(
+    'POST',
+    url,
+    {
+      'content-type': 'application/xml'
+    },
+    body,
+    credentials
+  );
+
+  const response = await fetch(signed.url, {
+    body,
+    headers: signed.headers,
+    method: signed.method
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Failed to complete multipart upload: ${response.status} ${text}`);
+  }
+}
+
+/**
+ * Abort a multipart upload
+ */
+async function abortMultipartUpload(
+  config: S3Config,
+  key: string,
+  uploadId: string
+): Promise<void> {
+  const endpoint = getEndpoint(config);
+  const url = `${endpoint}/${config.bucket}/${encodeURIComponent(key)}?uploadId=${encodeURIComponent(uploadId)}`;
+  const credentials = getCredentials(config);
+
+  const signed = signRequest('DELETE', url, {}, null, credentials);
+
+  try {
+    const response = await fetch(signed.url, {
+      headers: signed.headers,
+      method: signed.method
+    });
+
+    // 204 or 404 are both acceptable
+    if (!response.ok && response.status !== 404) {
+      console.warn(`Warning: Failed to abort upload: ${response.status}`);
+    }
+  } catch {
+    // Ignore abort errors
+  }
+}
+
+/**
+ * List parts for an existing upload (to verify state)
+ */
+async function listParts(
+  config: S3Config,
+  key: string,
+  uploadId: string
+): Promise<CompletedPart[]> {
+  const endpoint = getEndpoint(config);
+  const url = `${endpoint}/${config.bucket}/${encodeURIComponent(key)}?uploadId=${encodeURIComponent(uploadId)}`;
+  const credentials = getCredentials(config);
+
+  const signed = signRequest('GET', url, {}, null, credentials);
+
+  const response = await fetch(signed.url, {
+    headers: signed.headers,
+    method: signed.method
+  });
+
+  if (!response.ok) {
+    if (response.status === 404) {
+      return []; // Upload expired or doesn't exist
+    }
+    throw new Error(`Failed to list parts: ${response.status}`);
+  }
+
+  const xml = await response.text();
+  const parts: CompletedPart[] = [];
+
+  // Parse parts from XML
+  const partMatches = xml.matchAll(
+    /<Part>[\s\S]*?<PartNumber>(\d+)<\/PartNumber>[\s\S]*?<ETag>([^<]+)<\/ETag>[\s\S]*?<\/Part>/g
+  );
+  for (const match of partMatches) {
+    parts.push({
+      etag: match[2],
+      partNumber: parseInt(match[1], 10)
+    });
+  }
+
+  return parts;
+}
+
+/**
+ * Run parallel part uploads with concurrency limit
+ */
+async function uploadPartsInParallel(
+  config: S3Config,
+  filePath: string,
+  state: UploadState,
+  progress: ProgressState,
+  connections: number
+): Promise<CompletedPart[]> {
+  const file = Bun.file(filePath);
+  const completedPartNumbers = new Set(state.completedParts.map((p) => p.partNumber));
+  const pendingParts: number[] = [];
+
+  // Build list of parts to upload
+  for (let i = 1; i <= state.totalParts; i++) {
+    if (!completedPartNumbers.has(i)) {
+      pendingParts.push(i);
+    }
+  }
+
+  // Update progress with already completed parts
+  progress.completedParts = state.completedParts.length;
+  progress.bytesUploaded = state.completedParts.reduce((sum, p) => {
+    const partSize =
+      p.partNumber < state.totalParts
+        ? state.chunkSize
+        : state.fileSize - (state.totalParts - 1) * state.chunkSize;
+    return sum + partSize;
+  }, 0);
+
+  const allParts = [...state.completedParts];
+  const executing = new Set<Promise<void>>();
+
+  abortController = new AbortController();
+
+  for (const partNumber of pendingParts) {
+    if (isAborting) break;
+
+    const start = (partNumber - 1) * state.chunkSize;
+    const end = Math.min(start + state.chunkSize, state.fileSize);
+    const partSize = end - start;
+
+    const uploadPromise = (async () => {
+      try {
+        // Read chunk from file
+        const chunk = await file.slice(start, end).arrayBuffer();
+        const body = new Uint8Array(chunk);
+
+        // Upload part
+        const part = await uploadPart(
+          config,
+          state.key,
+          state.uploadId,
+          partNumber,
+          body,
+          abortController?.signal
+        );
+
+        // Save to state immediately
+        await addCompletedPart(filePath, state, part);
+        allParts.push(part);
+
+        // Update progress
+        updateProgress(progress, partSize);
+        writeProgress(progress);
+      } catch (err) {
+        if (!isAborting) {
+          throw err;
+        }
+      }
+    })();
+
+    executing.add(uploadPromise);
+    uploadPromise.finally(() => executing.delete(uploadPromise));
+
+    // Limit concurrency
+    if (executing.size >= connections) {
+      await Promise.race(executing);
+    }
+  }
+
+  // Wait for remaining uploads
+  await Promise.all(executing);
+
+  return allParts;
+}
+
+/**
+ * Setup SIGINT handler for graceful abort
+ */
+export function setupAbortHandler(_filePath: string, state: UploadState | null): void {
+  const handler = async () => {
+    if (isAborting) {
+      process.exit(1);
+    }
+
+    isAborting = true;
+    console.log('\n\nInterrupted! Saving progress...');
+
+    if (abortController) {
+      abortController.abort();
+    }
+
+    // Wait a moment for in-flight requests
+    await new Promise((r) => setTimeout(r, 1000));
+
+    if (state) {
+      console.log(`Upload paused at ${state.completedParts.length}/${state.totalParts} parts.`);
+      console.log('Run the same command to resume.');
+    }
+
+    process.exit(0);
+  };
+
+  process.on('SIGINT', handler);
+}
+
+/**
+ * Main multipart upload function
+ */
+export async function uploadMultipart(
+  filePath: string,
+  config: S3Config,
+  key: string,
+  options: MultipartOptions
+): Promise<MultipartOutcome> {
+  const file = Bun.file(filePath);
+  const stat = await file.stat();
+  const fileSize = file.size;
+  const fileModified = stat.mtime.getTime();
+  const endpoint = getEndpoint(config);
+
+  let state = await loadState(filePath);
+
+  // Check if we can resume
+  if (state) {
+    // Check if file changed
+    if (await hasFileChanged(filePath, state)) {
+      state = null; // Force fresh start
+    }
+  }
+
+  // Initialize progress
+  const totalParts = Math.ceil(fileSize / options.chunkSize);
+  const progress = createProgressState(key, totalParts, fileSize);
+
+  try {
+    if (!state) {
+      // Start new upload
+      const uploadId = await initiateMultipartUpload(config, key);
+      state = createInitialState(
+        uploadId,
+        config.bucket,
+        key,
+        fileSize,
+        fileModified,
+        options.chunkSize,
+        config.provider,
+        endpoint
+      );
+      await saveState(filePath, state);
+    } else {
+      // Resuming - verify parts still exist on S3
+      const remoteParts = await listParts(config, key, state.uploadId);
+      if (remoteParts.length === 0 && state.completedParts.length > 0) {
+        // Upload expired, start fresh
+        await deleteState(filePath);
+        const uploadId = await initiateMultipartUpload(config, key);
+        state = createInitialState(
+          uploadId,
+          config.bucket,
+          key,
+          fileSize,
+          fileModified,
+          options.chunkSize,
+          config.provider,
+          endpoint
+        );
+        await saveState(filePath, state);
+      }
+    }
+
+    // Setup abort handler
+    setupAbortHandler(filePath, state);
+
+    // Show initial progress bar
+    writeProgress(progress);
+
+    // Upload all parts
+    const allParts = await uploadPartsInParallel(
+      config,
+      filePath,
+      state,
+      progress,
+      options.connections
+    );
+
+    if (isAborting) {
+      return { error: 'Upload interrupted', success: false };
+    }
+
+    finishProgress();
+
+    // Complete the upload
+    await completeMultipartUpload(config, key, state.uploadId, allParts);
+
+    // Clean up state file
+    await deleteState(filePath);
+
+    const publicUrl = `${config.publicUrlBase}/${key}`;
+    return { publicUrl, success: true };
+  } catch (err) {
+    finishProgress();
+    return {
+      error: err instanceof Error ? err.message : String(err),
+      success: false
+    };
+  }
+}
+
+/**
+ * Check if there's a resumable upload and prompt user
+ */
+export async function checkResumableUpload(filePath: string): Promise<{
+  canResume: boolean;
+  state: UploadState | null;
+  percentComplete: number;
+}> {
+  const state = await loadState(filePath);
+
+  if (!state) {
+    return { canResume: false, percentComplete: 0, state: null };
+  }
+
+  // Check if file has changed
+  if (await hasFileChanged(filePath, state)) {
+    return { canResume: false, percentComplete: 0, state };
+  }
+
+  const percentComplete = Math.round((state.completedParts.length / state.totalParts) * 100);
+
+  return { canResume: true, percentComplete, state };
+}
+
+/**
+ * Abort and clean up an existing upload
+ */
+export async function cleanupExistingUpload(filePath: string, config: S3Config): Promise<void> {
+  const state = await loadState(filePath);
+
+  if (state) {
+    await abortMultipartUpload(config, state.key, state.uploadId);
+    await deleteState(filePath);
+  }
+}

package/src/lib/output.test.ts

@@ -0,0 +1,74 @@
+// src/lib/output.test.ts
+import { describe, expect, test } from 'bun:test';
+import { formatBytes, formatDeleteSummary, formatListItem, formatUploadSuccess } from './output';
+
+describe('formatBytes', () => {
+  test('formats bytes', () => {
+    expect(formatBytes(500)).toBe('500 B');
+  });
+
+  test('formats kilobytes', () => {
+    expect(formatBytes(1024)).toBe('1.0 KB');
+    expect(formatBytes(1536)).toBe('1.5 KB');
+  });
+
+  test('formats megabytes', () => {
+    expect(formatBytes(1024 * 1024)).toBe('1.0 MB');
+    expect(formatBytes(2.5 * 1024 * 1024)).toBe('2.5 MB');
+  });
+
+  test('formats gigabytes', () => {
+    expect(formatBytes(1024 * 1024 * 1024)).toBe('1.0 GB');
+  });
+});
+
+describe('formatUploadSuccess (quiet mode)', () => {
+  test('formats single file upload', () => {
+    const result = formatUploadSuccess(
+      'backup.tar.gz',
+      'https://cdn.example.com/backup.tar.gz',
+      2500000,
+      true
+    );
+    expect(result).toBe('backup.tar.gz → https://cdn.example.com/backup.tar.gz (2.4 MB)');
+  });
+});
+
+describe('formatListItem (quiet mode)', () => {
+  test('formats list item with tabs', () => {
+    const result = formatListItem(
+      'backups/file.tar.gz',
+      47395635,
+      new Date('2026-01-28T03:00:05Z'),
+      true
+    );
+    expect(result).toContain('backups/file.tar.gz');
+    expect(result).toContain('45.2 MB');
+    expect(result).toContain('2026-01-28');
+  });
+
+  test('formats list item as JSON', () => {
+    const result = formatListItem(
+      'backups/file.tar.gz',
+      47395635,
+      new Date('2026-01-28T03:00:05Z'),
+      true,
+      true
+    );
+    const parsed = JSON.parse(result);
+    expect(parsed.key).toBe('backups/file.tar.gz');
+    expect(parsed.size).toBe(47395635);
+  });
+});
+
+describe('formatDeleteSummary', () => {
+  test('formats deletion summary', () => {
+    const result = formatDeleteSummary(3, 138200000, false, true);
+    expect(result).toBe('Deleted 3 objects (131.8 MB)');
+  });
+
+  test('formats dry-run summary', () => {
+    const result = formatDeleteSummary(3, 138200000, true, true);
+    expect(result).toContain('Would delete 3 objects');
+  });
+});

package/src/lib/output.ts

@@ -0,0 +1,63 @@
+export function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+}
+
+export function formatUploadSuccess(
+  filename: string,
+  url: string,
+  size: number,
+  quiet: boolean
+): string {
+  if (quiet) {
+    return `${filename} → ${url} (${formatBytes(size)})`;
+  }
+  // Normal mode formatting handled by UI layer
+  return `${filename} → ${url} (${formatBytes(size)})`;
+}
+
+export function formatUploadError(filename: string, error: string): string {
+  return `Error: ${filename} - ${error}`;
+}
+
+export function formatListItem(
+  key: string,
+  size: number,
+  lastModified: Date,
+  quiet: boolean,
+  json = false
+): string {
+  if (json) {
+    return JSON.stringify({
+      key,
+      lastModified: lastModified.toISOString(),
+      size
+    });
+  }
+
+  if (quiet) {
+    const sizeStr = formatBytes(size).padStart(10);
+    const dateStr = lastModified.toISOString().split('T')[0];
+    return `${key}\t${sizeStr}\t${dateStr}`;
+  }
+
+  // Normal mode with colors handled by UI layer
+  return `${key}\t${formatBytes(size)}\t${lastModified.toISOString()}`;
+}
+
+export function formatDeleteSummary(
+  count: number,
+  totalSize: number,
+  dryRun: boolean,
+  _quiet: boolean
+): string {
+  const action = dryRun ? 'Would delete' : 'Deleted';
+  return `${action} ${count} objects (${formatBytes(totalSize)})`;
+}
+
+export function formatDryRunList(objects: Array<{ key: string; size: number }>): string {
+  const lines = objects.map((o) => `  ${o.key} (${formatBytes(o.size)})`);
+  return lines.join('\n');
+}