@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-08-api-reference.md

@@ -1,1055 +1,1055 @@
-# Module 8: API Reference
-
-**Level:** Reference
-**Estimated Time:** Reference material
-
-## Overview
-
-Complete API reference for all file operation methods in S3DataSource and SftpDataSource.
-
-## S3DataSource API
-
-### Constructor
-
-```typescript
-constructor(
-  config: {
-    s3Config: S3ClientConfig;
-    bucketName: string;
-  },
-  logger: Logger
-)
-```
-
-**Parameters:**
-
-- `s3Config`: AWS S3 client configuration
-  - `region`: AWS region (e.g., 'us-east-1')
-  - `credentials`: AWS credentials object
-    - `accessKeyId`: AWS access key ID
-    - `secretAccessKey`: AWS secret access key
-- `bucketName`: S3 bucket name
-- `logger`: Logger instance for logging
-
-**Example:**
-
-```typescript
-const s3 = new S3DataSource(
-  {
-    type: 'S3_CSV',
-    connectionId: 's3-api',
-    name: 'S3 API Example',
-    s3Config: {
-      region: 'us-east-1',
-      bucket: 'my-bucket',
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-    },
-  },
-  logger
-);
-```
-
-### listFiles()
-
-```typescript
-listFiles(params?: {
-  prefix?: string;
-  delimiter?: string;
-  maxKeys?: number;
-  continuationToken?: string;
-}): Promise<FileMetadata[]>
-```
-
-**Parameters:**
-
-- `params.prefix`: Filter files by key prefix (optional)
-- `params.delimiter`: Delimiter for grouping keys (optional, e.g., '/' for folder structure)
-- `params.maxKeys`: Maximum number of keys to return (optional, default: 1000)
-- `params.continuationToken`: Token for pagination (optional, from previous response)
-
-**Returns:** Array of FileMetadata objects (see [FileMetadata interface](#filemetadata))
-
-**Example:**
-
-```typescript
-// List all files
-const allFiles = await s3.listFiles();
-
-// List with prefix
-const csvFiles = await s3.listFiles({ prefix: 'incoming/', maxKeys: 100 });
-
-// List with pagination
-const firstPage = await s3.listFiles({ prefix: 'data/', maxKeys: 100 });
-// Use continuationToken from response metadata for next page
-```
-
-### downloadFile()
-
-```typescript
-downloadFile(
-  key: string,
-  options?: {
-    encoding?: 'utf8' | 'binary';
-    responseHeaders?: Record<string, string>;
-  }
-): Promise<string | Buffer>
-```
-
-**Parameters:**
-
-- `key`: S3 object key (no leading slash)
-- `options.encoding`: Return encoding ('utf8' for string, 'binary' for Buffer) (optional)
-- `options.responseHeaders`: Custom response headers (optional)
-
-**Returns:** File content as string (default) or Buffer (if encoding: 'binary')
-
-**Throws:**
-
-- `NoSuchKey`: If file doesn't exist
-- `AccessDenied`: If no read permission
-
-**Example:**
-
-```typescript
-const content = await s3.downloadFile('data/orders.csv');
-```
-
-### uploadFile()
-
-```typescript
-uploadFile(
-  key: string,
-  content: string | Buffer,
-  options?: {
-    contentType?: string;
-    metadata?: Record<string, string>;
-  }
-): Promise<void>
-```
-
-**Parameters:**
-
-- `key`: S3 object key (no leading slash)
-- `content`: Content to upload (string or Buffer)
-- `options.contentType`: MIME type of the content (optional, auto-detected if not provided)
-- `options.metadata`: Custom metadata key-value pairs (optional)
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- `AccessDenied`: If no write permission
-
-**Example:**
-
-```typescript
-await s3.uploadFile('output/results.json', JSON.stringify(data));
-```
-
-### moveFile()
-
-```typescript
-moveFile(
-  sourcePath: string,
-  destPath: string,
-  deleteSource: boolean = true
-): Promise<void>
-```
-
-**Parameters:**
-
-- `sourcePath`: Source S3 key (no leading slash)
-- `destPath`: Destination S3 key (no leading slash)
-- `deleteSource`: Delete source file after copy (default: **true**)
-
-**Returns:** Promise<void>
-
-**Implementation:** Copies file to destination, then optionally deletes source (2 operations if deleteSource=true, 1 if false)
-
-⚠️ **IMPORTANT:** `deleteSource` defaults to `true` - the source file WILL BE DELETED unless you explicitly pass `false`
-
-**Throws:**
-
-- `NoSuchKey`: If source file doesn't exist
-- `AccessDenied`: If no copy or delete permission
-
-**Example:**
-
-```typescript
-// Move (copy + delete source) - DEFAULT BEHAVIOR
-await s3.moveFile('incoming/data.csv', 'archive/data.csv');
-
-// Copy without deleting source
-await s3.moveFile('incoming/data.csv', 'archive/data.csv', false);
-```
-
-### copyFile()
-
-```typescript
-copyFile(
-  sourceKey: string,
-  destKey: string,
-  options?: {
-    sourceBucket?: string;
-    metadata?: Record<string, string>;
-  }
-): Promise<void>
-```
-
-**Parameters:**
-
-- `sourceKey`: Source S3 key (no leading slash)
-- `destKey`: Destination S3 key (no leading slash)
-- `options.sourceBucket`: Source bucket name (optional, defaults to same bucket)
-- `options.metadata`: Custom metadata to apply to destination (optional)
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- `NoSuchKey`: If source file doesn't exist
-- `AccessDenied`: If no copy permission
-
-**Example:**
-
-```typescript
-// Copy within same bucket
-await s3.copyFile('important/data.csv', 'backup/data.csv');
-
-// Copy with metadata
-await s3.copyFile('source/file.txt', 'dest/file.txt', {
-  metadata: { processedBy: 'sdk' }
-});
-
-// Copy from different bucket
-await s3.copyFile('source/file.txt', 'dest/file.txt', {
-  sourceBucket: 'other-bucket'
-});
-```
-
-### deleteFile()
-
-```typescript
-deleteFile(key: string): Promise<void>
-```
-
-**Parameters:**
-
-- `key`: S3 object key to delete
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- `AccessDenied`: If no delete permission
-
-**Note:** Does not throw if file doesn't exist (idempotent)
-
-**Example:**
-
-```typescript
-await s3.deleteFile('temp/old-file.csv');
-```
-
-### fileExists()
-
-```typescript
-fileExists(key: string): Promise<boolean>
-```
-
-**Parameters:**
-
-- `key`: S3 object key to check
-
-**Returns:** true if file exists, false otherwise
-
-**Example:**
-
-```typescript
-if (await s3.fileExists('config/settings.json')) {
-  const config = await s3.downloadFile('config/settings.json');
-}
-```
-
-### writeParquetContent()
-
-```typescript
-writeParquetContent(
-  records: ParquetDataRecord[],
-  options?: {
-    compression?: 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI';
-    schema?: Record<string, any>;
-    rowGroupSize?: number;
-  }
-): Promise<Buffer>
-```
-
-**Parameters:**
-
-- `records`: Array of data records
-- `options`: Optional Parquet generation options
-  - `compression`: Compression algorithm (default: `'UNCOMPRESSED'`)
-  - `schema`: Explicit schema definition (default: auto-detected)
-  - `rowGroupSize`: Number of rows per row group (default: 5000)
-
-**Returns:** Promise<Buffer> - Parquet file content as Buffer
-
-**Example:**
-
-```typescript
-const records = [
-  { sku: 'SKU-001', quantity: 10, price: 99.99 }
-];
-const parquetBuffer = await s3.writeParquetContent(records, {
-  compression: 'GZIP',
-  rowGroupSize: 5000
-});
-await s3.uploadFile('data/inventory.parquet', parquetBuffer);
-```
-
-### Handling Large Files
-
-For large files (>100MB), use `downloadFile()` with `encoding: 'binary'` and process in chunks:
-
-```typescript
-import { Buffer } from 'node:buffer';
-
-// Download as Buffer (use encoding: 'binary' for S3)
-const buffer = await s3.downloadFile('large-file.csv', { encoding: 'binary' });
-
-// Process in chunks
-const chunkSize = 1024 * 1024; // 1MB chunks
-for (let offset = 0; offset < buffer.length; offset += chunkSize) {
-  const chunk = buffer.slice(offset, offset + chunkSize);
-  processChunk(chunk.toString('utf8'));
-}
-```
-
-**Note:** The SDK does not provide built-in streaming. See [Module 5: Streaming & Performance](./file-operations-05-streaming-performance.md) for advanced patterns.
-
-## SftpDataSource API
-
-### Constructor
-
-```typescript
-constructor(
-  config: {
-    settings: SftpSettings;
-  },
-  logger: Logger
-)
-```
-
-**Parameters:**
-
-- `settings`: SFTP connection settings
-  - `host`: SFTP server hostname
-  - `port`: SFTP port (default: 22)
-  - `username`: SFTP username
-  - `password`: SFTP password (if password auth)
-  - `privateKey`: SSH private key (if key auth)
-  - `passphrase`: Private key passphrase (optional)
-  - `remotePath`: Base remote path
-  - `filePattern`: File pattern to match (e.g., '\*.csv')
-  - `connectTimeout`: Connection timeout in ms (optional)
-- `logger`: Logger instance
-
-**Example:**
-
-```typescript
-const sftp = new SftpDataSource(
-  {
-    type: 'SFTP_CSV',
-    connectionId: 'sftp-api',
-    name: 'SFTP API Example',
-    settings: {
-      host: 'sftp.example.com',
-      port: 22,
-      username: 'myuser',
-      password: process.env.SFTP_PASSWORD!,
-    },
-  },
-  logger
-);
-```
-
-### listFiles()
-
-```typescript
-listFiles(params?: {
-  lastProcessedTimestamp?: string;
-  fileName?: string;
-  remotePath?: string;
-  filePattern?: string;
-}): Promise<FileMetadata[]>
-```
-
-**Parameters:**
-
-- `params.lastProcessedTimestamp`: Filter files modified after this timestamp (optional)
-- `params.fileName`: Filter by specific file name (optional)
-- `params.remotePath`: Remote path to list (optional, uses constructor config if not provided)
-- `params.filePattern`: File pattern to match (optional, uses constructor config if not provided)
-
-**Returns:** Array of FileMetadata objects (see [FileMetadata interface](#filemetadata))
-
-**Example:**
-
-```typescript
-// List all files (uses constructor config)
-const files = await sftp.listFiles();
-
-// List files with custom path and pattern
-const csvFiles = await sftp.listFiles({
-  remotePath: '/data/incoming',
-  filePattern: '*.csv'
-});
-
-// List files modified after specific timestamp
-const recentFiles = await sftp.listFiles({
-  lastProcessedTimestamp: '2025-01-15T00:00:00Z'
-});
-
-files.forEach(f => console.log(f.path, f.source)); // path and source ('SFTP')
-```
-
-### downloadFile()
-
-```typescript
-downloadFile(path: string, options?: {
-  encoding?: string;
-  asBuffer?: boolean;
-  maxSize?: number;
-}): Promise<string | Buffer>
-```
-
-**Parameters:**
-
-- `path`: Absolute remote path (with leading slash)
-- `options.encoding`: Character encoding (default: 'utf8'). Supported: 'utf8', 'ascii', 'base64', 'hex', 'utf16le', 'latin1'
-- `options.asBuffer`: Return as Buffer instead of string (default: false)
-- `options.maxSize`: Maximum file size in bytes (optional, throws error if exceeded)
-
-**Returns:** File content as string (default) or Buffer (if `asBuffer: true`)
-
-**Throws:**
-
-- Error with "No such file" if file doesn't exist
-- Error with "Permission denied" if no read permission
-- Error if file size exceeds `maxSize`
-
-**Example:**
-
-```typescript
-// Download as string (default)
-const content = await sftp.downloadFile('/data/incoming/orders.csv');
-
-// Download as Buffer
-const buffer = await sftp.downloadFile('/data/incoming/orders.csv', { asBuffer: true });
-
-// Download with size limit
-const smallFile = await sftp.downloadFile('/config/settings.json', {
-  maxSize: 1024 * 1024 // 1MB max
-});
-
-// Download with specific encoding
-const latin1Content = await sftp.downloadFile('/data/older-system.txt', {
-  encoding: 'latin1'
-});
-```
-
-### uploadFile()
-
-```typescript
-uploadFile(
-  remotePath: string,
-  content: string | Buffer,
-  options?: {
-    overwrite?: boolean;
-    createDirectories?: boolean;
-    permissions?: string;
-    encoding?: string;
-  }
-): Promise<void>
-```
-
-**Parameters:**
-
-- `remotePath`: Absolute remote path (with leading slash)
-- `content`: Content to upload (string or Buffer)
-- `options`: Optional upload parameters (default: all false/undefined)
-  - `overwrite`: Overwrite existing file (default: `false`)
-    - `false`: Throws `DataSourceError` if file already exists
-    - `true`: Replaces existing file without error
-  - `createDirectories`: Create parent directories if needed (default: `false`)
-    - `false`: Throws error if parent directory doesn't exist
-    - `true`: Creates all parent directories recursively
-  - `permissions`: File permissions in Unix octal notation (default: server default)
-    - Common values: `'0644'` (rw-r--r--), `'0755'` (rwxr-xr-x), `'0600'` (rw-------)
-  - `encoding`: Content encoding when `content` is string (default: `'utf8'`)
-    - Supported: `'utf8'`, `'ascii'`, `'base64'`, `'hex'`, `'utf16le'`, `'latin1'`
-    - Ignored when `content` is a Buffer
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- `DataSourceError`: "Remote file already exists" when `overwrite: false` and file exists
-- `DataSourceError`: "Permission denied" if no write permission
-- `DataSourceError`: "No such file" if parent directory missing and `createDirectories: false`
-- `DataSourceError`: Network/connection failures (auto-retried per retry config)
-
-**Example:**
-
-```typescript
-// Simple upload
-await sftp.uploadFile('/data/outgoing/results.json', JSON.stringify(data));
-
-// With options
-await sftp.uploadFile('/data/outgoing/results.json', JSON.stringify(data), {
-  overwrite: true,
-  createDirectories: true,
-  permissions: '0644'
-});
-```
-
-### moveFile()
-
-```typescript
-moveFile(
-  oldPath: string,
-  newPath: string,
-  overwrite: boolean = false
-): Promise<void>
-```
-
-**Parameters:**
-
-- `oldPath`: Absolute source path (with leading slash)
-- `newPath`: Absolute destination path (with leading slash)
-- `overwrite`: Overwrite destination if it exists (default: false)
-
-**Returns:** Promise<void>
-
-**Implementation:** Native SFTP rename operation (1 operation)
-
-**Throws:**
-
-- Error with "No such file" if source doesn't exist
-- Error with "Permission denied" if no rename permission
-- Error if destination exists and `overwrite: false`
-
-**Example:**
-
-```typescript
-// Simple move
-await sftp.moveFile('/data/incoming/orders.csv', '/data/processed/orders.csv');
-
-// Move with overwrite
-await sftp.moveFile(
-  '/data/incoming/orders.csv',
-  '/archive/orders.csv',
-  true // Overwrite if exists
-);
-```
-
-### copyFile()
-
-```typescript
-copyFile(
-  sourcePath: string,
-  destPath: string,
-  overwrite: boolean = false
-): Promise<void>
-```
-
-**Parameters:**
-
-- `sourcePath`: Absolute source path (with leading slash)
-- `destPath`: Absolute destination path (with leading slash)
-- `overwrite`: Overwrite destination if it exists (default: false)
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- Error with "No such file" if source doesn't exist
-- Error with "Permission denied" if no read/write permission
-- Error if destination exists and `overwrite: false`
-
-**Example:**
-
-```typescript
-// Simple copy
-await sftp.copyFile('/data/important/config.json', '/backup/config.json');
-
-// Copy with overwrite
-await sftp.copyFile(
-  '/data/important/config.json',
-  '/backup/config.json',
-  true // Overwrite if exists
-);
-```
-
-### deleteFile()
-
-```typescript
-deleteFile(remotePath: string): Promise<void>
-```
-
-**Parameters:**
-
-- `remotePath`: Absolute remote path (with leading slash)
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- Error with "No such file" if file doesn't exist
-- Error with "Permission denied" if no delete permission
-
-**Example:**
-
-```typescript
-await sftp.deleteFile('/data/temp/old-file.csv');
-```
-
-### fileExists()
-
-```typescript
-fileExists(remotePath: string): Promise<boolean>
-```
-
-**Parameters:**
-
-- `remotePath`: Absolute remote path (with leading slash)
-
-**Returns:** true if file exists, false otherwise
-
-**Example:**
-
-```typescript
-if (await sftp.fileExists('/data/incoming/orders.csv')) {
-  const content = await sftp.downloadFile('/data/incoming/orders.csv');
-}
-```
-
-### createDirectory()
-
-```typescript
-createDirectory(
-  remotePath: string,
-  recursive: boolean = false,
-  permissions?: string
-): Promise<void>
-```
-
-**Parameters:**
-
-- `remotePath`: Absolute remote directory path (with leading slash)
-- `recursive`: Create parent directories if they don't exist (default: `false`)
-- `permissions`: Optional directory permissions in Unix octal notation (e.g., `'0755'`)
-
-**Returns:** Promise<void>
-
-**Throws:**
-
-- `DataSourceError`: "Permission denied" if no write permission
-- `DataSourceError`: "No such file" if parent directory missing and `recursive: false`
-
-**Example:**
-
-```typescript
-// Create single directory
-await sftp.createDirectory('/data/incoming');
-
-// Create recursively
-await sftp.createDirectory('/data/archive/2025/01/15', true);
-
-// Create with permissions
-await sftp.createDirectory('/data/public', true, '0755');
-```
-
-### directoryExists()
-
-```typescript
-directoryExists(remotePath: string): Promise<boolean>
-```
-
-**Parameters:**
-
-- `remotePath`: Absolute remote directory path (with leading slash)
-
-**Returns:** true if directory exists, false otherwise
-
-**Example:**
-
-```typescript
-if (!(await sftp.directoryExists('/data/incoming'))) {
-  await sftp.createDirectory('/data/incoming', true);
-}
-```
-
-### parseCsvContent()
-
-```typescript
-parseCsvContent(
-  csvContent: string,
-  fileMetadata: FileMetadata
-): CsvRecord[]
-```
-
-**Parameters:**
-
-- `csvContent`: CSV content as string
-- `fileMetadata`: File metadata for context (includes path, name, size, source)
-
-**Returns:** Array of parsed CSV records (objects with column names as keys)
-
-**Example:**
-
-```typescript
-const content = await sftp.downloadFile('/data/incoming/orders.csv');
-const records = sftp.parseCsvContent(content as string, {
-  path: '/data/incoming/orders.csv',
-  name: 'orders.csv',
-  size: content.length,
-  source: 'SFTP'
-});
-```
-
-### parseJsonContent()
-
-```typescript
-parseJsonContent(
-  jsonContent: string,
-  fileMetadata: FileMetadata,
-  options?: {
-    format?: 'json' | 'jsonl';
-    validate?: boolean;
-    maxDepth?: number;
-  }
-): Record<string, unknown>[]
-```
-
-**Parameters:**
-
-- `jsonContent`: JSON content as string
-- `fileMetadata`: File metadata for context
-- `options`: Optional parsing options
-  - `format`: `'json'` for JSON array or `'jsonl'` for JSON Lines (default: auto-detect)
-  - `validate`: Throw error on invalid JSON (default: `false`)
-  - `maxDepth`: Limit nesting depth (default: unlimited)
-
-**Returns:** Array of parsed JSON objects
-
-**Example:**
-
-```typescript
-const content = await sftp.downloadFile('/data/incoming/orders.json');
-const records = sftp.parseJsonContent(content as string, {
-  path: '/data/incoming/orders.json',
-  name: 'orders.json',
-  size: content.length,
-  source: 'SFTP'
-}, {
-  format: 'json',
-  validate: true,
-  maxDepth: 10
-});
-```
-
-### writeCsvContent()
-
-```typescript
-writeCsvContent(
-  records: Record<string, any>[],
-  options?: {
-    headers?: string[];
-    delimiter?: string;
-    includeHeaders?: boolean;
-  }
-): string
-```
-
-**Parameters:**
-
-- `records`: Array of objects to convert to CSV
-- `options`: Optional CSV generation options
-  - `headers`: Column names (default: inferred from first record)
-  - `delimiter`: Field delimiter (default: `','`)
-  - `includeHeaders`: Include header row (default: `true`)
-
-**Returns:** CSV content as string
-
-**Example:**
-
-```typescript
-const records = [
-  { sku: 'SKU-001', quantity: 10 },
-  { sku: 'SKU-002', quantity: 25 }
-];
-const csv = sftp.writeCsvContent(records, {
-  headers: ['sku', 'quantity'],
-  delimiter: ',',
-  includeHeaders: true
-});
-await sftp.uploadFile('/data/outgoing/inventory.csv', csv);
-```
-
-### writeJsonContent()
-
-```typescript
-writeJsonContent(
-  records: Record<string, any>[],
-  options?: {
-    format?: 'json' | 'jsonl';
-    pretty?: boolean;
-  }
-): string
-```
-
-**Parameters:**
-
-- `records`: Array of objects to convert to JSON
-- `options`: Optional JSON generation options
-  - `format`: `'json'` for JSON array or `'jsonl'` for JSON Lines (default: `'json'`)
-  - `pretty`: Pretty-print JSON (default: `false`)
-
-**Returns:** JSON content as string
-
-**Example:**
-
-```typescript
-const orders = [{ id: 'ORD-001', total: 99.99 }];
-const json = sftp.writeJsonContent(orders, {
-  format: 'json',
-  pretty: true
-});
-await sftp.uploadFile('/data/outgoing/orders.json', json);
-```
-
-### writeParquetContent()
-
-```typescript
-writeParquetContent(
-  records: ParquetDataRecord[],
-  options?: {
-    compression?: 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI';
-    schema?: Record<string, any>;
-    rowGroupSize?: number;
-  }
-): Promise<Buffer>
-```
-
-**Parameters:**
-
-- `records`: Array of data records
-- `options`: Optional Parquet generation options
-  - `compression`: Compression algorithm (default: `'UNCOMPRESSED'`)
-  - `schema`: Explicit schema definition (default: auto-detected)
-  - `rowGroupSize`: Number of rows per row group (default: 5000)
-
-**Returns:** Promise<Buffer> - Parquet file content as Buffer
-
-**Example:**
-
-```typescript
-const records = [
-  { sku: 'SKU-001', quantity: 10, price: 99.99 }
-];
-const parquetBuffer = await sftp.writeParquetContent(records, {
-  compression: 'GZIP',
-  rowGroupSize: 5000
-});
-await sftp.uploadFile('/data/outgoing/inventory.parquet', parquetBuffer);
-```
-
-### Handling Large Files
-
-For large files (>100MB), use `downloadFile()` with `asBuffer: true` and `maxSize` option:
-
-```typescript
-import { Buffer } from 'node:buffer';
-
-// Download as Buffer with size limit
-const buffer = await sftp.downloadFile('/data/large-file.csv', {
-  asBuffer: true,
-  maxSize: 100 * 1024 * 1024  // 100 MB limit
-});
-
-// Process in chunks
-const chunkSize = 1024 * 1024; // 1MB chunks
-for (let offset = 0; offset < buffer.length; offset += chunkSize) {
-  const chunk = buffer.slice(offset, offset + chunkSize);
-  processChunk(chunk.toString('utf8'));
-}
-```
-
-**Note:** The SDK does not provide built-in streaming. See [Module 5: Streaming & Performance](./file-operations-05-streaming-performance.md) for advanced patterns.
-
-### dispose()
-
-```typescript
-dispose(): Promise<void>
-```
-
-**Parameters:** None
-
-**Returns:** Promise<void>
-
-**Purpose:** Close SFTP connection and free resources
-
-**CRITICAL:** Always call in `finally` block
-
-**Example:**
-
-```typescript
-const sftp = new SftpDataSource(config, logger);
-try {
-  await sftp.listFiles();
-} finally {
-  await sftp.dispose(); // REQUIRED!
-}
-```
-
-## Common Types
-
-### FileMetadata
-
-```typescript
-interface FileMetadata {
-  path: string;              // REQUIRED: Full file path/key
-  name: string;              // REQUIRED: File name
-  size?: number;             // File size in bytes (optional)
-  lastModified?: string;     // ISO 8601 timestamp string (NOT Date object!)
-  contentType?: string;      // MIME type (optional, S3 only)
-  source: string;            // REQUIRED: Data source type ('S3' or 'SFTP')
-  etag?: string;             // S3 ETag (optional, S3 only)
-  bucket?: string;           // S3 bucket name (optional, S3 only)
-  region?: string;           // S3 region (optional, S3 only)
-}
-```
-
-**Key Points:**
-
-- ✅ `path` is REQUIRED (not optional!)
-- ✅ `lastModified` is an ISO string, NOT a Date object
-- ✅ `source` indicates the data source type ('S3' or 'SFTP')
-- ✅ S3-specific fields: `contentType`, `etag`, `bucket`, `region`
-
-**Example Usage:**
-
-```typescript
-const files: FileMetadata[] = await dataSource.listFiles();
-
-files.forEach(file => {
-  console.log(`File: ${file.name}`);
-  console.log(`Path: ${file.path}`);
-  console.log(`Source: ${file.source}`); // 'S3' or 'SFTP'
-  console.log(`Modified: ${file.lastModified}`); // ISO string
-  if (file.contentType) {
-    console.log(`Type: ${file.contentType}`); // S3 only
-  }
-});
-```
-
-### Logger
-
-```typescript
-interface Logger {
-  debug(message: string, meta?: any): void;
-  info(message: string, meta?: any): void;
-  warn(message: string, meta?: any): void;
-  error(message: string, error?: Error, meta?: any): void;
-}
-```
-
-**Note:** Use `createConsoleLogger()` or `toStructuredLogger()` from the SDK to create Logger instances. See [Logging Guide](../../../02-CORE-GUIDES/logging-guide.md) for details.
-
-## Key Differences: S3 vs SFTP
-
-| Feature                 | S3DataSource                          | SftpDataSource                                  |
-| ----------------------- | ------------------------------------- | ----------------------------------------------- |
-| **Path Format**         | No leading slash: `folder/file.csv`   | Leading slash: `/folder/file.csv`               |
-| **List Method**         | `listFiles({ prefix?, delimiter?, maxKeys?, continuationToken? })` | `listFiles({ lastProcessedTimestamp?, fileName?, remotePath?, filePattern? })` |
-| **Upload Signature**    | `uploadFile(key, content, options?)`  | `uploadFile(remotePath, content, options?)`     |
-| **Move Signature**      | `moveFile(sourcePath, destPath, deleteSource?)` | `moveFile(oldPath, newPath, overwrite?)` |
-| **Move Implementation** | Copy + Delete (2 ops)                 | Native rename (1 op)                            |
-| **Directory Creation**  | Automatic                             | Optional (`createDirectories` flag)             |
-| **Connection**          | Stateless (per-request)               | Stateful (must dispose)                         |
-| **Cleanup**             | Not needed                            | **Required** (`dispose()`)                      |
-
-## Best Practices
-
-### S3 Best Practices
-
-```typescript
-// ✅ Use meaningful prefixes
-await s3.listFiles({ prefix: 'incoming/orders/' });
-
-// ✅ Use maxKeys for pagination control
-await s3.listFiles({ prefix: 'data/', maxKeys: 100 });
-
-// ✅ Check existence before operations
-if (await s3.fileExists('file.csv')) {
-  await s3.downloadFile('file.csv');
-}
-
-// ✅ Use chunked processing for large files
-if (fileSize > 100 * 1024 * 1024) {
-  // >100MB: Download as Buffer and process in chunks
-  import { Buffer } from 'node:buffer';
-  const buffer = await s3.downloadFile('large-file.csv', { encoding: 'binary' });
-  // Process buffer in chunks (see Module 5 for patterns)
-}
-```
-
-### SFTP Best Practices
-
-```typescript
-// ✅ Always use try/finally with dispose
-try {
-  const files = await sftp.listFiles();
-  // ... process files
-} finally {
-  await sftp.dispose(); // CRITICAL!
-}
-
-// ✅ Use absolute paths
-await sftp.downloadFile('/data/file.csv'); // ✅
-// NOT: await sftp.downloadFile('data/file.csv'); // ❌
-
-// ✅ Use createDirectories flag when necessary
-await sftp.moveFile(
-  '/source/file.csv',
-  '/archive/2025/01/file.csv',
-  true // Create /archive/2025/01/ if it doesn't exist
-);
-
-// ✅ Reuse connection for multiple operations
-try {
-  for (const file of files) {
-    await sftp.downloadFile(file.path);
-  }
-} finally {
-  await sftp.dispose(); // Once at the end
-}
-```
-
-## See Also
-
-- [Module 1: Foundations](./file-operations-01-foundations.md) - Core concepts
-- [Module 2: Quick Start](./file-operations-02-quick-start.md) - Working examples
-- [Module 3: S3 Operations](./file-operations-03-s3-operations.md) - S3 deep dive
-- [Module 4: SFTP Operations](./file-operations-04-sftp-operations.md) - SFTP deep dive
-- [Quick Reference](../../../02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md) - Cheat sheet
-
----
-
-**Need Help?**
-
-- 📖 Browse [Examples](../examples/)
-- 🔍 Check [Troubleshooting](./file-operations-07-testing-troubleshooting.md)
-- 📧 Contact support@fluentcommerce.com
+# Module 8: API Reference
+
+**Level:** Reference
+**Estimated Time:** Reference material
+
+## Overview
+
+Complete API reference for all file operation methods in S3DataSource and SftpDataSource.
+
+## S3DataSource API
+
+### Constructor
+
+```typescript
+constructor(
+  config: {
+    s3Config: S3ClientConfig;
+    bucketName: string;
+  },
+  logger: Logger
+)
+```
+
+**Parameters:**
+
+- `s3Config`: AWS S3 client configuration
+  - `region`: AWS region (e.g., 'us-east-1')
+  - `credentials`: AWS credentials object
+    - `accessKeyId`: AWS access key ID
+    - `secretAccessKey`: AWS secret access key
+- `bucketName`: S3 bucket name
+- `logger`: Logger instance for logging
+
+**Example:**
+
+```typescript
+const s3 = new S3DataSource(
+  {
+    type: 'S3_CSV',
+    connectionId: 's3-api',
+    name: 'S3 API Example',
+    s3Config: {
+      region: 'us-east-1',
+      bucket: 'my-bucket',
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    },
+  },
+  logger
+);
+```
+
+### listFiles()
+
+```typescript
+listFiles(params?: {
+  prefix?: string;
+  delimiter?: string;
+  maxKeys?: number;
+  continuationToken?: string;
+}): Promise<FileMetadata[]>
+```
+
+**Parameters:**
+
+- `params.prefix`: Filter files by key prefix (optional)
+- `params.delimiter`: Delimiter for grouping keys (optional, e.g., '/' for folder structure)
+- `params.maxKeys`: Maximum number of keys to return (optional, default: 1000)
+- `params.continuationToken`: Token for pagination (optional, from previous response)
+
+**Returns:** Array of FileMetadata objects (see [FileMetadata interface](#filemetadata))
+
+**Example:**
+
+```typescript
+// List all files
+const allFiles = await s3.listFiles();
+
+// List with prefix
+const csvFiles = await s3.listFiles({ prefix: 'incoming/', maxKeys: 100 });
+
+// List with pagination
+const firstPage = await s3.listFiles({ prefix: 'data/', maxKeys: 100 });
+// Use continuationToken from response metadata for next page
+```
+
+### downloadFile()
+
+```typescript
+downloadFile(
+  key: string,
+  options?: {
+    encoding?: 'utf8' | 'binary';
+    responseHeaders?: Record<string, string>;
+  }
+): Promise<string | Buffer>
+```
+
+**Parameters:**
+
+- `key`: S3 object key (no leading slash)
+- `options.encoding`: Return encoding ('utf8' for string, 'binary' for Buffer) (optional)
+- `options.responseHeaders`: Custom response headers (optional)
+
+**Returns:** File content as string (default) or Buffer (if encoding: 'binary')
+
+**Throws:**
+
+- `NoSuchKey`: If file doesn't exist
+- `AccessDenied`: If no read permission
+
+**Example:**
+
+```typescript
+const content = await s3.downloadFile('data/orders.csv');
+```
+
+### uploadFile()
+
+```typescript
+uploadFile(
+  key: string,
+  content: string | Buffer,
+  options?: {
+    contentType?: string;
+    metadata?: Record<string, string>;
+  }
+): Promise<void>
+```
+
+**Parameters:**
+
+- `key`: S3 object key (no leading slash)
+- `content`: Content to upload (string or Buffer)
+- `options.contentType`: MIME type of the content (optional, auto-detected if not provided)
+- `options.metadata`: Custom metadata key-value pairs (optional)
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- `AccessDenied`: If no write permission
+
+**Example:**
+
+```typescript
+await s3.uploadFile('output/results.json', JSON.stringify(data));
+```
+
+### moveFile()
+
+```typescript
+moveFile(
+  sourcePath: string,
+  destPath: string,
+  deleteSource: boolean = true
+): Promise<void>
+```
+
+**Parameters:**
+
+- `sourcePath`: Source S3 key (no leading slash)
+- `destPath`: Destination S3 key (no leading slash)
+- `deleteSource`: Delete source file after copy (default: **true**)
+
+**Returns:** Promise<void>
+
+**Implementation:** Copies file to destination, then optionally deletes source (2 operations if deleteSource=true, 1 if false)
+
+⚠️ **IMPORTANT:** `deleteSource` defaults to `true` - the source file WILL BE DELETED unless you explicitly pass `false`
+
+**Throws:**
+
+- `NoSuchKey`: If source file doesn't exist
+- `AccessDenied`: If no copy or delete permission
+
+**Example:**
+
+```typescript
+// Move (copy + delete source) - DEFAULT BEHAVIOR
+await s3.moveFile('incoming/data.csv', 'archive/data.csv');
+
+// Copy without deleting source
+await s3.moveFile('incoming/data.csv', 'archive/data.csv', false);
+```
+
+### copyFile()
+
+```typescript
+copyFile(
+  sourceKey: string,
+  destKey: string,
+  options?: {
+    sourceBucket?: string;
+    metadata?: Record<string, string>;
+  }
+): Promise<void>
+```
+
+**Parameters:**
+
+- `sourceKey`: Source S3 key (no leading slash)
+- `destKey`: Destination S3 key (no leading slash)
+- `options.sourceBucket`: Source bucket name (optional, defaults to same bucket)
+- `options.metadata`: Custom metadata to apply to destination (optional)
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- `NoSuchKey`: If source file doesn't exist
+- `AccessDenied`: If no copy permission
+
+**Example:**
+
+```typescript
+// Copy within same bucket
+await s3.copyFile('important/data.csv', 'backup/data.csv');
+
+// Copy with metadata
+await s3.copyFile('source/file.txt', 'dest/file.txt', {
+  metadata: { processedBy: 'sdk' }
+});
+
+// Copy from different bucket
+await s3.copyFile('source/file.txt', 'dest/file.txt', {
+  sourceBucket: 'other-bucket'
+});
+```
+
+### deleteFile()
+
+```typescript
+deleteFile(key: string): Promise<void>
+```
+
+**Parameters:**
+
+- `key`: S3 object key to delete
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- `AccessDenied`: If no delete permission
+
+**Note:** Does not throw if file doesn't exist (idempotent)
+
+**Example:**
+
+```typescript
+await s3.deleteFile('temp/old-file.csv');
+```
+
+### fileExists()
+
+```typescript
+fileExists(key: string): Promise<boolean>
+```
+
+**Parameters:**
+
+- `key`: S3 object key to check
+
+**Returns:** true if file exists, false otherwise
+
+**Example:**
+
+```typescript
+if (await s3.fileExists('config/settings.json')) {
+  const config = await s3.downloadFile('config/settings.json');
+}
+```
+
+### writeParquetContent()
+
+```typescript
+writeParquetContent(
+  records: ParquetDataRecord[],
+  options?: {
+    compression?: 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI';
+    schema?: Record<string, any>;
+    rowGroupSize?: number;
+  }
+): Promise<Buffer>
+```
+
+**Parameters:**
+
+- `records`: Array of data records
+- `options`: Optional Parquet generation options
+  - `compression`: Compression algorithm (default: `'UNCOMPRESSED'`)
+  - `schema`: Explicit schema definition (default: auto-detected)
+  - `rowGroupSize`: Number of rows per row group (default: 5000)
+
+**Returns:** Promise<Buffer> - Parquet file content as Buffer
+
+**Example:**
+
+```typescript
+const records = [
+  { sku: 'SKU-001', quantity: 10, price: 99.99 }
+];
+const parquetBuffer = await s3.writeParquetContent(records, {
+  compression: 'GZIP',
+  rowGroupSize: 5000
+});
+await s3.uploadFile('data/inventory.parquet', parquetBuffer);
+```
+
+### Handling Large Files
+
+For large files (>100MB), use `downloadFile()` with `encoding: 'binary'` and process in chunks:
+
+```typescript
+import { Buffer } from 'node:buffer';
+
+// Download as Buffer (use encoding: 'binary' for S3)
+const buffer = await s3.downloadFile('large-file.csv', { encoding: 'binary' });
+
+// Process in chunks
+const chunkSize = 1024 * 1024; // 1MB chunks
+for (let offset = 0; offset < buffer.length; offset += chunkSize) {
+  const chunk = buffer.slice(offset, offset + chunkSize);
+  processChunk(chunk.toString('utf8'));
+}
+```
+
+**Note:** The SDK does not provide built-in streaming. See [Module 5: Streaming & Performance](./file-operations-05-streaming-performance.md) for advanced patterns.
+
+## SftpDataSource API
+
+### Constructor
+
+```typescript
+constructor(
+  config: {
+    settings: SftpSettings;
+  },
+  logger: Logger
+)
+```
+
+**Parameters:**
+
+- `settings`: SFTP connection settings
+  - `host`: SFTP server hostname
+  - `port`: SFTP port (default: 22)
+  - `username`: SFTP username
+  - `password`: SFTP password (if password auth)
+  - `privateKey`: SSH private key (if key auth)
+  - `passphrase`: Private key passphrase (optional)
+  - `remotePath`: Base remote path
+  - `filePattern`: File pattern to match (e.g., '\*.csv')
+  - `connectTimeout`: Connection timeout in ms (optional)
+- `logger`: Logger instance
+
+**Example:**
+
+```typescript
+const sftp = new SftpDataSource(
+  {
+    type: 'SFTP_CSV',
+    connectionId: 'sftp-api',
+    name: 'SFTP API Example',
+    settings: {
+      host: 'sftp.example.com',
+      port: 22,
+      username: 'myuser',
+      password: process.env.SFTP_PASSWORD!,
+    },
+  },
+  logger
+);
+```
+
+### listFiles()
+
+```typescript
+listFiles(params?: {
+  lastProcessedTimestamp?: string;
+  fileName?: string;
+  remotePath?: string;
+  filePattern?: string;
+}): Promise<FileMetadata[]>
+```
+
+**Parameters:**
+
+- `params.lastProcessedTimestamp`: Filter files modified after this timestamp (optional)
+- `params.fileName`: Filter by specific file name (optional)
+- `params.remotePath`: Remote path to list (optional, uses constructor config if not provided)
+- `params.filePattern`: File pattern to match (optional, uses constructor config if not provided)
+
+**Returns:** Array of FileMetadata objects (see [FileMetadata interface](#filemetadata))
+
+**Example:**
+
+```typescript
+// List all files (uses constructor config)
+const files = await sftp.listFiles();
+
+// List files with custom path and pattern
+const csvFiles = await sftp.listFiles({
+  remotePath: '/data/incoming',
+  filePattern: '*.csv'
+});
+
+// List files modified after specific timestamp
+const recentFiles = await sftp.listFiles({
+  lastProcessedTimestamp: '2025-01-15T00:00:00Z'
+});
+
+files.forEach(f => console.log(f.path, f.source)); // path and source ('SFTP')
+```
+
+### downloadFile()
+
+```typescript
+downloadFile(path: string, options?: {
+  encoding?: string;
+  asBuffer?: boolean;
+  maxSize?: number;
+}): Promise<string | Buffer>
+```
+
+**Parameters:**
+
+- `path`: Absolute remote path (with leading slash)
+- `options.encoding`: Character encoding (default: 'utf8'). Supported: 'utf8', 'ascii', 'base64', 'hex', 'utf16le', 'latin1'
+- `options.asBuffer`: Return as Buffer instead of string (default: false)
+- `options.maxSize`: Maximum file size in bytes (optional, throws error if exceeded)
+
+**Returns:** File content as string (default) or Buffer (if `asBuffer: true`)
+
+**Throws:**
+
+- Error with "No such file" if file doesn't exist
+- Error with "Permission denied" if no read permission
+- Error if file size exceeds `maxSize`
+
+**Example:**
+
+```typescript
+// Download as string (default)
+const content = await sftp.downloadFile('/data/incoming/orders.csv');
+
+// Download as Buffer
+const buffer = await sftp.downloadFile('/data/incoming/orders.csv', { asBuffer: true });
+
+// Download with size limit
+const smallFile = await sftp.downloadFile('/config/settings.json', {
+  maxSize: 1024 * 1024 // 1MB max
+});
+
+// Download with specific encoding
+const latin1Content = await sftp.downloadFile('/data/older-system.txt', {
+  encoding: 'latin1'
+});
+```
+
+### uploadFile()
+
+```typescript
+uploadFile(
+  remotePath: string,
+  content: string | Buffer,
+  options?: {
+    overwrite?: boolean;
+    createDirectories?: boolean;
+    permissions?: string;
+    encoding?: string;
+  }
+): Promise<void>
+```
+
+**Parameters:**
+
+- `remotePath`: Absolute remote path (with leading slash)
+- `content`: Content to upload (string or Buffer)
+- `options`: Optional upload parameters (default: all false/undefined)
+  - `overwrite`: Overwrite existing file (default: `false`)
+    - `false`: Throws `DataSourceError` if file already exists
+    - `true`: Replaces existing file without error
+  - `createDirectories`: Create parent directories if needed (default: `false`)
+    - `false`: Throws error if parent directory doesn't exist
+    - `true`: Creates all parent directories recursively
+  - `permissions`: File permissions in Unix octal notation (default: server default)
+    - Common values: `'0644'` (rw-r--r--), `'0755'` (rwxr-xr-x), `'0600'` (rw-------)
+  - `encoding`: Content encoding when `content` is string (default: `'utf8'`)
+    - Supported: `'utf8'`, `'ascii'`, `'base64'`, `'hex'`, `'utf16le'`, `'latin1'`
+    - Ignored when `content` is a Buffer
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- `DataSourceError`: "Remote file already exists" when `overwrite: false` and file exists
+- `DataSourceError`: "Permission denied" if no write permission
+- `DataSourceError`: "No such file" if parent directory missing and `createDirectories: false`
+- `DataSourceError`: Network/connection failures (auto-retried per retry config)
+
+**Example:**
+
+```typescript
+// Simple upload
+await sftp.uploadFile('/data/outgoing/results.json', JSON.stringify(data));
+
+// With options
+await sftp.uploadFile('/data/outgoing/results.json', JSON.stringify(data), {
+  overwrite: true,
+  createDirectories: true,
+  permissions: '0644'
+});
+```
+
+### moveFile()
+
+```typescript
+moveFile(
+  oldPath: string,
+  newPath: string,
+  overwrite: boolean = false
+): Promise<void>
+```
+
+**Parameters:**
+
+- `oldPath`: Absolute source path (with leading slash)
+- `newPath`: Absolute destination path (with leading slash)
+- `overwrite`: Overwrite destination if it exists (default: false)
+
+**Returns:** Promise<void>
+
+**Implementation:** Native SFTP rename operation (1 operation)
+
+**Throws:**
+
+- Error with "No such file" if source doesn't exist
+- Error with "Permission denied" if no rename permission
+- Error if destination exists and `overwrite: false`
+
+**Example:**
+
+```typescript
+// Simple move
+await sftp.moveFile('/data/incoming/orders.csv', '/data/processed/orders.csv');
+
+// Move with overwrite
+await sftp.moveFile(
+  '/data/incoming/orders.csv',
+  '/archive/orders.csv',
+  true // Overwrite if exists
+);
+```
+
+### copyFile()
+
+```typescript
+copyFile(
+  sourcePath: string,
+  destPath: string,
+  overwrite: boolean = false
+): Promise<void>
+```
+
+**Parameters:**
+
+- `sourcePath`: Absolute source path (with leading slash)
+- `destPath`: Absolute destination path (with leading slash)
+- `overwrite`: Overwrite destination if it exists (default: false)
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- Error with "No such file" if source doesn't exist
+- Error with "Permission denied" if no read/write permission
+- Error if destination exists and `overwrite: false`
+
+**Example:**
+
+```typescript
+// Simple copy
+await sftp.copyFile('/data/important/config.json', '/backup/config.json');
+
+// Copy with overwrite
+await sftp.copyFile(
+  '/data/important/config.json',
+  '/backup/config.json',
+  true // Overwrite if exists
+);
+```
+
+### deleteFile()
+
+```typescript
+deleteFile(remotePath: string): Promise<void>
+```
+
+**Parameters:**
+
+- `remotePath`: Absolute remote path (with leading slash)
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- Error with "No such file" if file doesn't exist
+- Error with "Permission denied" if no delete permission
+
+**Example:**
+
+```typescript
+await sftp.deleteFile('/data/temp/old-file.csv');
+```
+
+### fileExists()
+
+```typescript
+fileExists(remotePath: string): Promise<boolean>
+```
+
+**Parameters:**
+
+- `remotePath`: Absolute remote path (with leading slash)
+
+**Returns:** true if file exists, false otherwise
+
+**Example:**
+
+```typescript
+if (await sftp.fileExists('/data/incoming/orders.csv')) {
+  const content = await sftp.downloadFile('/data/incoming/orders.csv');
+}
+```
+
+### createDirectory()
+
+```typescript
+createDirectory(
+  remotePath: string,
+  recursive: boolean = false,
+  permissions?: string
+): Promise<void>
+```
+
+**Parameters:**
+
+- `remotePath`: Absolute remote directory path (with leading slash)
+- `recursive`: Create parent directories if they don't exist (default: `false`)
+- `permissions`: Optional directory permissions in Unix octal notation (e.g., `'0755'`)
+
+**Returns:** Promise<void>
+
+**Throws:**
+
+- `DataSourceError`: "Permission denied" if no write permission
+- `DataSourceError`: "No such file" if parent directory missing and `recursive: false`
+
+**Example:**
+
+```typescript
+// Create single directory
+await sftp.createDirectory('/data/incoming');
+
+// Create recursively
+await sftp.createDirectory('/data/archive/2025/01/15', true);
+
+// Create with permissions
+await sftp.createDirectory('/data/public', true, '0755');
+```
+
+### directoryExists()
+
+```typescript
+directoryExists(remotePath: string): Promise<boolean>
+```
+
+**Parameters:**
+
+- `remotePath`: Absolute remote directory path (with leading slash)
+
+**Returns:** true if directory exists, false otherwise
+
+**Example:**
+
+```typescript
+if (!(await sftp.directoryExists('/data/incoming'))) {
+  await sftp.createDirectory('/data/incoming', true);
+}
+```
+
+### parseCsvContent()
+
+```typescript
+parseCsvContent(
+  csvContent: string,
+  fileMetadata: FileMetadata
+): CsvRecord[]
+```
+
+**Parameters:**
+
+- `csvContent`: CSV content as string
+- `fileMetadata`: File metadata for context (includes path, name, size, source)
+
+**Returns:** Array of parsed CSV records (objects with column names as keys)
+
+**Example:**
+
+```typescript
+const content = await sftp.downloadFile('/data/incoming/orders.csv');
+const records = sftp.parseCsvContent(content as string, {
+  path: '/data/incoming/orders.csv',
+  name: 'orders.csv',
+  size: content.length,
+  source: 'SFTP'
+});
+```
+
+### parseJsonContent()
+
+```typescript
+parseJsonContent(
+  jsonContent: string,
+  fileMetadata: FileMetadata,
+  options?: {
+    format?: 'json' | 'jsonl';
+    validate?: boolean;
+    maxDepth?: number;
+  }
+): Record<string, unknown>[]
+```
+
+**Parameters:**
+
+- `jsonContent`: JSON content as string
+- `fileMetadata`: File metadata for context
+- `options`: Optional parsing options
+  - `format`: `'json'` for JSON array or `'jsonl'` for JSON Lines (default: auto-detect)
+  - `validate`: Throw error on invalid JSON (default: `false`)
+  - `maxDepth`: Limit nesting depth (default: unlimited)
+
+**Returns:** Array of parsed JSON objects
+
+**Example:**
+
+```typescript
+const content = await sftp.downloadFile('/data/incoming/orders.json');
+const records = sftp.parseJsonContent(content as string, {
+  path: '/data/incoming/orders.json',
+  name: 'orders.json',
+  size: content.length,
+  source: 'SFTP'
+}, {
+  format: 'json',
+  validate: true,
+  maxDepth: 10
+});
+```
+
+### writeCsvContent()
+
+```typescript
+writeCsvContent(
+  records: Record<string, any>[],
+  options?: {
+    headers?: string[];
+    delimiter?: string;
+    includeHeaders?: boolean;
+  }
+): string
+```
+
+**Parameters:**
+
+- `records`: Array of objects to convert to CSV
+- `options`: Optional CSV generation options
+  - `headers`: Column names (default: inferred from first record)
+  - `delimiter`: Field delimiter (default: `','`)
+  - `includeHeaders`: Include header row (default: `true`)
+
+**Returns:** CSV content as string
+
+**Example:**
+
+```typescript
+const records = [
+  { sku: 'SKU-001', quantity: 10 },
+  { sku: 'SKU-002', quantity: 25 }
+];
+const csv = sftp.writeCsvContent(records, {
+  headers: ['sku', 'quantity'],
+  delimiter: ',',
+  includeHeaders: true
+});
+await sftp.uploadFile('/data/outgoing/inventory.csv', csv);
+```
+
+### writeJsonContent()
+
+```typescript
+writeJsonContent(
+  records: Record<string, any>[],
+  options?: {
+    format?: 'json' | 'jsonl';
+    pretty?: boolean;
+  }
+): string
+```
+
+**Parameters:**
+
+- `records`: Array of objects to convert to JSON
+- `options`: Optional JSON generation options
+  - `format`: `'json'` for JSON array or `'jsonl'` for JSON Lines (default: `'json'`)
+  - `pretty`: Pretty-print JSON (default: `false`)
+
+**Returns:** JSON content as string
+
+**Example:**
+
+```typescript
+const orders = [{ id: 'ORD-001', total: 99.99 }];
+const json = sftp.writeJsonContent(orders, {
+  format: 'json',
+  pretty: true
+});
+await sftp.uploadFile('/data/outgoing/orders.json', json);
+```
+
+### writeParquetContent()
+
+```typescript
+writeParquetContent(
+  records: ParquetDataRecord[],
+  options?: {
+    compression?: 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI';
+    schema?: Record<string, any>;
+    rowGroupSize?: number;
+  }
+): Promise<Buffer>
+```
+
+**Parameters:**
+
+- `records`: Array of data records
+- `options`: Optional Parquet generation options
+  - `compression`: Compression algorithm (default: `'UNCOMPRESSED'`)
+  - `schema`: Explicit schema definition (default: auto-detected)
+  - `rowGroupSize`: Number of rows per row group (default: 5000)
+
+**Returns:** Promise<Buffer> - Parquet file content as Buffer
+
+**Example:**
+
+```typescript
+const records = [
+  { sku: 'SKU-001', quantity: 10, price: 99.99 }
+];
+const parquetBuffer = await sftp.writeParquetContent(records, {
+  compression: 'GZIP',
+  rowGroupSize: 5000
+});
+await sftp.uploadFile('/data/outgoing/inventory.parquet', parquetBuffer);
+```
+
+### Handling Large Files
+
+For large files (>100MB), use `downloadFile()` with `asBuffer: true` and `maxSize` option:
+
+```typescript
+import { Buffer } from 'node:buffer';
+
+// Download as Buffer with size limit
+const buffer = await sftp.downloadFile('/data/large-file.csv', {
+  asBuffer: true,
+  maxSize: 100 * 1024 * 1024  // 100 MB limit
+});
+
+// Process in chunks
+const chunkSize = 1024 * 1024; // 1MB chunks
+for (let offset = 0; offset < buffer.length; offset += chunkSize) {
+  const chunk = buffer.slice(offset, offset + chunkSize);
+  processChunk(chunk.toString('utf8'));
+}
+```
+
+**Note:** The SDK does not provide built-in streaming. See [Module 5: Streaming & Performance](./file-operations-05-streaming-performance.md) for advanced patterns.
+
+### dispose()
+
+```typescript
+dispose(): Promise<void>
+```
+
+**Parameters:** None
+
+**Returns:** Promise<void>
+
+**Purpose:** Close SFTP connection and free resources
+
+**CRITICAL:** Always call in `finally` block
+
+**Example:**
+
+```typescript
+const sftp = new SftpDataSource(config, logger);
+try {
+  await sftp.listFiles();
+} finally {
+  await sftp.dispose(); // REQUIRED!
+}
+```
+
+## Common Types
+
+### FileMetadata
+
+```typescript
+interface FileMetadata {
+  path: string;              // REQUIRED: Full file path/key
+  name: string;              // REQUIRED: File name
+  size?: number;             // File size in bytes (optional)
+  lastModified?: string;     // ISO 8601 timestamp string (NOT Date object!)
+  contentType?: string;      // MIME type (optional, S3 only)
+  source: string;            // REQUIRED: Data source type ('S3' or 'SFTP')
+  etag?: string;             // S3 ETag (optional, S3 only)
+  bucket?: string;           // S3 bucket name (optional, S3 only)
+  region?: string;           // S3 region (optional, S3 only)
+}
+```
+
+**Key Points:**
+
+- ✅ `path` is REQUIRED (not optional!)
+- ✅ `lastModified` is an ISO string, NOT a Date object
+- ✅ `source` indicates the data source type ('S3' or 'SFTP')
+- ✅ S3-specific fields: `contentType`, `etag`, `bucket`, `region`
+
+**Example Usage:**
+
+```typescript
+const files: FileMetadata[] = await dataSource.listFiles();
+
+files.forEach(file => {
+  console.log(`File: ${file.name}`);
+  console.log(`Path: ${file.path}`);
+  console.log(`Source: ${file.source}`); // 'S3' or 'SFTP'
+  console.log(`Modified: ${file.lastModified}`); // ISO string
+  if (file.contentType) {
+    console.log(`Type: ${file.contentType}`); // S3 only
+  }
+});
+```
+
+### Logger
+
+```typescript
+interface Logger {
+  debug(message: string, meta?: any): void;
+  info(message: string, meta?: any): void;
+  warn(message: string, meta?: any): void;
+  error(message: string, error?: Error, meta?: any): void;
+}
+```
+
+**Note:** Use `createConsoleLogger()` or `toStructuredLogger()` from the SDK to create Logger instances. See [Logging Guide](../../../02-CORE-GUIDES/logging-guide.md) for details.
+
+## Key Differences: S3 vs SFTP
+
+| Feature                 | S3DataSource                          | SftpDataSource                                  |
+| ----------------------- | ------------------------------------- | ----------------------------------------------- |
+| **Path Format**         | No leading slash: `folder/file.csv`   | Leading slash: `/folder/file.csv`               |
+| **List Method**         | `listFiles({ prefix?, delimiter?, maxKeys?, continuationToken? })` | `listFiles({ lastProcessedTimestamp?, fileName?, remotePath?, filePattern? })` |
+| **Upload Signature**    | `uploadFile(key, content, options?)`  | `uploadFile(remotePath, content, options?)`     |
+| **Move Signature**      | `moveFile(sourcePath, destPath, deleteSource?)` | `moveFile(oldPath, newPath, overwrite?)` |
+| **Move Implementation** | Copy + Delete (2 ops)                 | Native rename (1 op)                            |
+| **Directory Creation**  | Automatic                             | Optional (`createDirectories` flag)             |
+| **Connection**          | Stateless (per-request)               | Stateful (must dispose)                         |
+| **Cleanup**             | Not needed                            | **Required** (`dispose()`)                      |
+
+## Best Practices
+
+### S3 Best Practices
+
+```typescript
+// ✅ Use meaningful prefixes
+await s3.listFiles({ prefix: 'incoming/orders/' });
+
+// ✅ Use maxKeys for pagination control
+await s3.listFiles({ prefix: 'data/', maxKeys: 100 });
+
+// ✅ Check existence before operations
+if (await s3.fileExists('file.csv')) {
+  await s3.downloadFile('file.csv');
+}
+
+// ✅ Use chunked processing for large files
+if (fileSize > 100 * 1024 * 1024) {
+  // >100MB: Download as Buffer and process in chunks
+  import { Buffer } from 'node:buffer';
+  const buffer = await s3.downloadFile('large-file.csv', { encoding: 'binary' });
+  // Process buffer in chunks (see Module 5 for patterns)
+}
+```
+
+### SFTP Best Practices
+
+```typescript
+// ✅ Always use try/finally with dispose
+try {
+  const files = await sftp.listFiles();
+  // ... process files
+} finally {
+  await sftp.dispose(); // CRITICAL!
+}
+
+// ✅ Use absolute paths
+await sftp.downloadFile('/data/file.csv'); // ✅
+// NOT: await sftp.downloadFile('data/file.csv'); // ❌
+
+// ✅ Use createDirectories flag when necessary
+await sftp.moveFile(
+  '/source/file.csv',
+  '/archive/2025/01/file.csv',
+  true // Create /archive/2025/01/ if it doesn't exist
+);
+
+// ✅ Reuse connection for multiple operations
+try {
+  for (const file of files) {
+    await sftp.downloadFile(file.path);
+  }
+} finally {
+  await sftp.dispose(); // Once at the end
+}
+```
+
+## See Also
+
+- [Module 1: Foundations](./file-operations-01-foundations.md) - Core concepts
+- [Module 2: Quick Start](./file-operations-02-quick-start.md) - Working examples
+- [Module 3: S3 Operations](./file-operations-03-s3-operations.md) - S3 deep dive
+- [Module 4: SFTP Operations](./file-operations-04-sftp-operations.md) - SFTP deep dive
+- [Quick Reference](../../../02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md) - Cheat sheet
+
+---
+
+**Need Help?**
+
+- 📖 Browse [Examples](../examples/)
+- 🔍 Check [Troubleshooting](./file-operations-07-testing-troubleshooting.md)
+- 📧 Contact support@fluentcommerce.com