@fluentcommerce/fc-connect-sdk 0.1.53 → 0.1.55

package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-06-s3-integration.md

@@ -1,511 +1,511 @@
-# Module 6: S3 Integration
-
-[← Back to Parquet Guide](../parquet-readme.md)
-
-**Module 6 of 8** | **Level**: Advanced | **Time**: 20 minutes
-
----
-
-## Overview
-
-Integrate Parquet processing with S3 for data lake operations, including reading from S3, writing to S3, and using presigned URLs.
-
-## Learning Objectives
-
-By the end of this module, you will:
-
-- ✅ Read Parquet files from S3
-- ✅ Write extracted Parquet files to S3
-- ✅ Use streaming for large S3 files
-- ✅ Implement presigned URL workflows
-- ✅ Build complete ETL pipelines
-
----
-
-## Reading Parquet from S3
-
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori runtime
-import {
-  S3DataSource,
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-class S3ParquetProcessor {
-  private s3: S3DataSource;
-  private parser: ParquetParserService;
-  private logger: Logger;
-
-  constructor(s3Config: any) {
-    this.logger = toStructuredLogger(createConsoleLogger(), {
-      logLevel: 'info'
-    });
-    this.s3 = new S3DataSource(
-      {
-        type: 'S3_CSV',
-        s3Config: {
-          bucket: process.env.S3_BUCKET!,
-          accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-          region: process.env.AWS_REGION!,
-        },
-      },
-      this.logger
-    );
-    this.parser = new ParquetParserService(this.logger);
-  }
-
-  async processParquetFromS3(key: string): Promise<any[]> {
-    console.log(`Processing s3://${this.s3.s3Config.bucket}/${key}`);
-
-    // 1. Download from S3
-    const fileBuffer = await this.s3.downloadFile(key, { encoding: 'binary' });
-
-    // 2. Convert to ArrayBuffer
-    const arrayBuffer = fileBuffer.buffer.slice(
-      fileBuffer.byteOffset,
-      fileBuffer.byteOffset + fileBuffer.byteLength
-    );
-
-    // 3. Parse Parquet file
-    const { records, stats } = await this.parser.parse(arrayBuffer, key);
-
-    console.log(`✅ Processed ${stats.totalRecords} records from S3`);
-    console.log(`   Processing time: ${stats.processingTimeMs}ms`);
-    console.log(`   File size: ${(stats.fileSizeBytes / 1024 / 1024).toFixed(2)} MB`);
-
-    return records;
-  }
-}
-```
-
----
-
-## Writing Parquet to S3
-
-### Method 1: Direct Write (Recommended)
-
-Use `S3DataSource.writeParquetContent()` for streamlined direct writes:
-
-```typescript
-import { S3DataSource, createClient, createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
-
-/**
- * Extract from Fluent Commerce and write directly to S3
- * Business use case: Daily inventory export to data lake
- */
-async function extractToS3Direct(s3Key: string): Promise<void> {
-  const logger = toStructuredLogger(createConsoleLogger(), { service: 'Extraction' });
-
-  // 1. Setup client
-  const client = await createClient({
-    config: {
-      baseUrl: process.env.FLUENT_BASE_URL!,
-      clientId: process.env.FLUENT_CLIENT_ID!,
-      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-      retailerId: process.env.FLUENT_RETAILER_ID!,
-    },
-  });
-
-  // 2. Setup S3
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      s3Config: {
-        bucket: process.env.S3_BUCKET!,
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-      },
-    },
-    logger
-  );
-
-  // 3. Extract data (pagination loop)
-  const allRecords = [];
-  let hasNextPage = true;
-  let cursor: string | null = null;
-
-  while (hasNextPage) {
-    const result = await client.graphql({
-      query: `
-        query ExtractInventory($after: String, $first: Int!) {
-          inventoryPositions(after: $after, first: $first) {
-            pageInfo { hasNextPage }
-            edges {
-              cursor
-              node {
-                ref
-                productRef
-                locationRef
-                qty
-              }
-            }
-          }
-        }
-      `,
-      variables: { first: 1000, after: cursor },
-    });
-
-    const { edges, pageInfo } = result.data.inventoryPositions;
-
-    allRecords.push(...edges.map((e) => ({
-      ...e.node,
-      extractedAt: new Date().toISOString(),
-    })));
-
-    hasNextPage = pageInfo.hasNextPage;
-    cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
-  }
-
-  console.log(`✅ Extracted ${allRecords.length} records`);
-
-  // 4. Generate Parquet Buffer and upload to S3
-  const parquetBuffer = await s3.writeParquetContent(allRecords, {
-    schema: {
-      ref: { type: 'UTF8' },
-      productRef: { type: 'UTF8' },
-      locationRef: { type: 'UTF8' },
-      qty: { type: 'INT32' },
-      extractedAt: { type: 'UTF8' },
-    },
-    compression: 'SNAPPY',
-  });
-
-  await s3.uploadFile(s3Key, parquetBuffer);
-
-  console.log(`✅ Uploaded to s3://${process.env.S3_BUCKET}/${s3Key}`);
-}
-```
-
-**Advantages:**
-- ✅ No temporary files needed
-- ✅ Generates Buffer directly in memory
-- ✅ Upload separately for flexibility
-- ✅ Can upload to S3, SFTP, or save locally
-
-### Method 2: Temp File Approach
-
-For advanced use cases requiring fine-grained control:
-
-```typescript
-import * as parquet from 'parquetjs';
-import * as fs from 'fs/promises';
-
-/**
- * Extract from Fluent Commerce and upload to S3
- * Business use case: Daily inventory export to data lake
- */
-async function extractToS3(s3Key: string): Promise<void> {
-  const tempFile = `/tmp/inventory_${Date.now()}.parquet`;
-  const logger = toStructuredLogger(createConsoleLogger(), {
-    logLevel: 'info'
-  });
-
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      s3Config: {
-        bucket: process.env.S3_BUCKET!,
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-      },
-    },
-    logger
-  );
-
-  try {
-    // 1. Extract to local Parquet file
-    const schema = new parquet.ParquetSchema({
-      ref: { type: 'UTF8' },
-      productRef: { type: 'UTF8' },
-      locationRef: { type: 'UTF8' },
-      qty: { type: 'INT32' },
-      extractedAt: { type: 'TIMESTAMP_MILLIS' },
-    });
-
-    const writer = await parquet.ParquetWriter.openFile(schema, tempFile, {
-      compression: 'SNAPPY',
-    });
-
-    // 2. Extract data (pagination loop)
-    let hasNextPage = true;
-    let cursor: string | null = null;
-    let totalRecords = 0;
-
-    while (hasNextPage) {
-      const result = await client.graphql({
-        query: `
-          query ExtractInventory($after: String, $first: Int!) {
-            inventoryPositions(after: $after, first: $first) {
-              pageInfo { hasNextPage }
-              edges {
-                cursor
-                node {
-                  ref
-                  productRef
-                  locationRef
-                  qty
-                }
-              }
-            }
-          }
-        `,
-        variables: { first: 1000, after: cursor },
-      });
-
-      const { edges, pageInfo } = result.data.inventoryPositions;
-
-      for (const edge of edges) {
-        await writer.appendRow({
-          ...edge.node,
-          extractedAt: new Date(),
-        });
-        totalRecords++;
-      }
-
-      hasNextPage = pageInfo.hasNextPage;
-      cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
-    }
-
-    await writer.close();
-    console.log(`✅ Extracted ${totalRecords} records to ${tempFile}`);
-
-    // 3. Upload to S3
-    const fileBuffer = await fs.readFile(tempFile);
-    await s3.uploadFile(s3Key, fileBuffer, {
-      contentType: 'application/octet-stream',
-    });
-
-    console.log(`✅ Uploaded to s3://${process.env.S3_BUCKET}/${s3Key}`);
-
-    // 4. Cleanup temp file
-    await fs.unlink(tempFile);
-  } catch (error) {
-    // Cleanup on error
-    try {
-      await fs.unlink(tempFile);
-    } catch {}
-    throw error;
-  }
-}
-```
-
----
-
-## Streaming S3 to S3
-
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori
-
-/**
- * Stream Parquet from S3, transform, write to target S3
- * Business use case: Data lake transformation pipeline
- */
-class S3ParquetStreamer {
-  constructor(
-    private sourceS3: S3DataSource,
-    private targetS3: S3DataSource,
-    private parser: ParquetParserService
-  ) {}
-
-  async streamParquetToJSON(
-    sourceBucket: string,
-    sourceKey: string,
-    targetBucket: string,
-    targetKey: string
-  ): Promise<void> {
-    console.log(`Streaming ${sourceKey}...`);
-
-    // 1. Download from source S3
-    const buffer = await this.sourceS3.downloadFile(sourceKey, { encoding: 'binary' });
-    const arrayBuffer = buffer.buffer.slice(
-      buffer.byteOffset,
-      buffer.byteOffset + buffer.byteLength
-    );
-
-    // 2. Stream parse
-    const streamResult = await this.parser.parseStreaming(arrayBuffer, sourceKey, {
-      batchSize: 1000,
-    });
-
-    const jsonLines: string[] = [];
-    let totalRecords = 0;
-
-    // 3. Process batches
-    for await (const batch of streamResult.records) {
-      for (const record of batch) {
-        jsonLines.push(JSON.stringify(record));
-        totalRecords++;
-
-        // Write chunks to S3 every 10K records
-        if (jsonLines.length >= 10000) {
-          await this.writeChunk(targetKey, jsonLines.join('\n'), totalRecords);
-          jsonLines.length = 0; // Clear array
-        }
-      }
-    }
-
-    // 4. Write remaining records
-    if (jsonLines.length > 0) {
-      await this.writeChunk(targetKey, jsonLines.join('\n'), totalRecords);
-    }
-
-    const stats = await streamResult.stats;
-    console.log(`✅ Streaming complete:`);
-    console.log(`   Total records: ${totalRecords}`);
-    console.log(`   Processing time: ${stats.processingTimeMs}ms`);
-  }
-
-  private async writeChunk(
-    targetKeyBase: string,
-    content: string,
-    recordNumber: number
-  ): Promise<void> {
-    const chunkKey = `${targetKeyBase}_chunk_${Math.floor(recordNumber / 10000)}.jsonl`;
-
-    await this.targetS3.uploadFile(chunkKey, Buffer.from(content), {
-      contentType: 'application/x-ndjson',
-    });
-
-    console.log(`   Wrote chunk: ${chunkKey}`);
-  }
-}
-```
-
----
-
-## Multiple File Processing
-
-```typescript
-/**
- * Process all Parquet files in S3 prefix
- * Business use case: Batch processing of daily extracts
- */
-async function processAllParquetFiles(prefix: string): Promise<void> {
-  const logger = toStructuredLogger(createConsoleLogger(), {
-    logLevel: 'info'
-  });
-
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      s3Config: {
-        bucket: process.env.S3_BUCKET!,
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-      },
-    },
-    logger
-  );
-
-  const parser = new ParquetParserService(logger);
-
-  // 1. List all Parquet files
-  const files = await s3.listFiles({ prefix });
-  const parquetFiles = files.filter(f => f.path.endsWith('.parquet'));
-
-  console.log(`Found ${parquetFiles.length} Parquet files with prefix: ${prefix}`);
-
-  // 2. Process each file
-  for (const file of parquetFiles) {
-    try {
-      console.log(`\nProcessing ${file.path}...`);
-
-      const buffer = await s3.downloadFile(file.path, { encoding: 'binary' });
-      const arrayBuffer = buffer.buffer.slice(
-        buffer.byteOffset,
-        buffer.byteOffset + buffer.byteLength
-      );
-
-      const { records, stats } = await parser.parse(arrayBuffer, file.path);
-
-      console.log(`✅ ${file.path}: ${stats.totalRecords} records`);
-
-      // Your processing logic
-      await processRecords(records, file.key);
-    } catch (error) {
-      console.error(`❌ Failed to process ${file.path}:`, (error as any).message);
-      // Continue processing other files
-    }
-  }
-
-  console.log(`\n✅ Batch processing complete`);
-}
-```
-
----
-
-## Presigned URL Workflows
-
-```typescript
-import { S3PresignService } from '@fluentcommerce/fc-connect-sdk';
-
-/**
- * Generate presigned URLs for Parquet downloads
- * Business use case: Secure file sharing with analytics team
- */
-class ParquetPresignService {
-  private presign: S3PresignService;
-
-  constructor(s3Config: any) {
-    this.presign = new S3PresignService(s3Config);
-  }
-
-  async generateDownloadUrl(
-    bucket: string,
-    key: string,
-    expiresIn: number = 3600 // 1 hour
-  ): Promise<string> {
-    const url = await this.presign.getPresignedUrl({
-      bucket,
-      key,
-      operation: 'getObject',
-      expiresIn,
-    });
-
-    console.log(`✅ Generated presigned URL for ${key}`);
-    console.log(`   Expires in: ${expiresIn} seconds`);
-
-    return url;
-  }
-
-  async shareParquetFile(bucket: string, key: string): Promise<void> {
-    const url = await this.generateDownloadUrl(bucket, key, 86400); // 24 hours
-
-    console.log(`\nShare this URL with analytics team:`);
-    console.log(url);
-    console.log(`\nValid for 24 hours`);
-  }
-}
-```
-
----
-
-## Key Takeaways
-
-- 🎯 **S3DataSource.writeParquetContent()** - Generates Parquet Buffer (then upload separately)
-- 🎯 **S3DataSource integration** - Seamless S3 operations with SDK
-- 🎯 **Stream large files** - Avoid loading entire file into memory
-- 🎯 **Batch processing** - Handle multiple files efficiently
-- 🎯 **Presigned URLs** - Secure file sharing
-- 🎯 **Temp file cleanup** - Only needed for advanced temp file approach
-
----
-
-## Next Steps
-
-Continue to [Module 7: Performance Optimization](./03-pattern-guides-parquet-07-performance-optimization.md) to learn advanced performance tuning techniques.
-
----
-
-[← Previous: Module 5 - GraphQL Extraction](./03-pattern-guides-parquet-05-graphql-extraction.md/) | [Back to Guide](../parquet-readme.md) | [Next: Module 7 - Performance →](./03-pattern-guides-parquet-07-performance-optimization.md)
-
-## Related Documentation
-
-- [S3 Complete Guide](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - Complete S3 patterns
-- [S3 Presigned URLs](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - Security and expiration
+# Module 6: S3 Integration
+
+[← Back to Parquet Guide](../parquet-readme.md)
+
+**Module 6 of 8** | **Level**: Advanced | **Time**: 20 minutes
+
+---
+
+## Overview
+
+Integrate Parquet processing with S3 for data lake operations, including reading from S3, writing to S3, and using presigned URLs.
+
+## Learning Objectives
+
+By the end of this module, you will:
+
+- ✅ Read Parquet files from S3
+- ✅ Write extracted Parquet files to S3
+- ✅ Use streaming for large S3 files
+- ✅ Implement presigned URL workflows
+- ✅ Build complete ETL pipelines
+
+---
+
+## Reading Parquet from S3
+
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori runtime
+import {
+  S3DataSource,
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+class S3ParquetProcessor {
+  private s3: S3DataSource;
+  private parser: ParquetParserService;
+  private logger: Logger;
+
+  constructor(s3Config: any) {
+    this.logger = toStructuredLogger(createConsoleLogger(), {
+      logLevel: 'info'
+    });
+    this.s3 = new S3DataSource(
+      {
+        type: 'S3_CSV',
+        s3Config: {
+          bucket: process.env.S3_BUCKET!,
+          accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+          region: process.env.AWS_REGION!,
+        },
+      },
+      this.logger
+    );
+    this.parser = new ParquetParserService(this.logger);
+  }
+
+  async processParquetFromS3(key: string): Promise<any[]> {
+    console.log(`Processing s3://${this.s3.s3Config.bucket}/${key}`);
+
+    // 1. Download from S3
+    const fileBuffer = await this.s3.downloadFile(key, { encoding: 'binary' });
+
+    // 2. Convert to ArrayBuffer
+    const arrayBuffer = fileBuffer.buffer.slice(
+      fileBuffer.byteOffset,
+      fileBuffer.byteOffset + fileBuffer.byteLength
+    );
+
+    // 3. Parse Parquet file
+    const { records, stats } = await this.parser.parse(arrayBuffer, key);
+
+    console.log(`✅ Processed ${stats.totalRecords} records from S3`);
+    console.log(`   Processing time: ${stats.processingTimeMs}ms`);
+    console.log(`   File size: ${(stats.fileSizeBytes / 1024 / 1024).toFixed(2)} MB`);
+
+    return records;
+  }
+}
+```
+
+---
+
+## Writing Parquet to S3
+
+### Method 1: Direct Write (Recommended)
+
+Use `S3DataSource.writeParquetContent()` for streamlined direct writes:
+
+```typescript
+import { S3DataSource, createClient, createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
+
+/**
+ * Extract from Fluent Commerce and write directly to S3
+ * Business use case: Daily inventory export to data lake
+ */
+async function extractToS3Direct(s3Key: string): Promise<void> {
+  const logger = toStructuredLogger(createConsoleLogger(), { service: 'Extraction' });
+
+  // 1. Setup client
+  const client = await createClient({
+    config: {
+      baseUrl: process.env.FLUENT_BASE_URL!,
+      clientId: process.env.FLUENT_CLIENT_ID!,
+      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+      retailerId: process.env.FLUENT_RETAILER_ID!,
+    },
+  });
+
+  // 2. Setup S3
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      s3Config: {
+        bucket: process.env.S3_BUCKET!,
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+      },
+    },
+    logger
+  );
+
+  // 3. Extract data (pagination loop)
+  const allRecords = [];
+  let hasNextPage = true;
+  let cursor: string | null = null;
+
+  while (hasNextPage) {
+    const result = await client.graphql({
+      query: `
+        query ExtractInventory($after: String, $first: Int!) {
+          inventoryPositions(after: $after, first: $first) {
+            pageInfo { hasNextPage }
+            edges {
+              cursor
+              node {
+                ref
+                productRef
+                locationRef
+                qty
+              }
+            }
+          }
+        }
+      `,
+      variables: { first: 1000, after: cursor },
+    });
+
+    const { edges, pageInfo } = result.data.inventoryPositions;
+
+    allRecords.push(...edges.map((e) => ({
+      ...e.node,
+      extractedAt: new Date().toISOString(),
+    })));
+
+    hasNextPage = pageInfo.hasNextPage;
+    cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
+  }
+
+  console.log(`✅ Extracted ${allRecords.length} records`);
+
+  // 4. Generate Parquet Buffer and upload to S3
+  const parquetBuffer = await s3.writeParquetContent(allRecords, {
+    schema: {
+      ref: { type: 'UTF8' },
+      productRef: { type: 'UTF8' },
+      locationRef: { type: 'UTF8' },
+      qty: { type: 'INT32' },
+      extractedAt: { type: 'UTF8' },
+    },
+    compression: 'SNAPPY',
+  });
+
+  await s3.uploadFile(s3Key, parquetBuffer);
+
+  console.log(`✅ Uploaded to s3://${process.env.S3_BUCKET}/${s3Key}`);
+}
+```
+
+**Advantages:**
+- ✅ No temporary files needed
+- ✅ Generates Buffer directly in memory
+- ✅ Upload separately for flexibility
+- ✅ Can upload to S3, SFTP, or save locally
+
+### Method 2: Temp File Approach
+
+For advanced use cases requiring fine-grained control:
+
+```typescript
+import * as parquet from 'parquetjs';
+import * as fs from 'fs/promises';
+
+/**
+ * Extract from Fluent Commerce and upload to S3
+ * Business use case: Daily inventory export to data lake
+ */
+async function extractToS3(s3Key: string): Promise<void> {
+  const tempFile = `/tmp/inventory_${Date.now()}.parquet`;
+  const logger = toStructuredLogger(createConsoleLogger(), {
+    logLevel: 'info'
+  });
+
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      s3Config: {
+        bucket: process.env.S3_BUCKET!,
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+      },
+    },
+    logger
+  );
+
+  try {
+    // 1. Extract to local Parquet file
+    const schema = new parquet.ParquetSchema({
+      ref: { type: 'UTF8' },
+      productRef: { type: 'UTF8' },
+      locationRef: { type: 'UTF8' },
+      qty: { type: 'INT32' },
+      extractedAt: { type: 'TIMESTAMP_MILLIS' },
+    });
+
+    const writer = await parquet.ParquetWriter.openFile(schema, tempFile, {
+      compression: 'SNAPPY',
+    });
+
+    // 2. Extract data (pagination loop)
+    let hasNextPage = true;
+    let cursor: string | null = null;
+    let totalRecords = 0;
+
+    while (hasNextPage) {
+      const result = await client.graphql({
+        query: `
+          query ExtractInventory($after: String, $first: Int!) {
+            inventoryPositions(after: $after, first: $first) {
+              pageInfo { hasNextPage }
+              edges {
+                cursor
+                node {
+                  ref
+                  productRef
+                  locationRef
+                  qty
+                }
+              }
+            }
+          }
+        `,
+        variables: { first: 1000, after: cursor },
+      });
+
+      const { edges, pageInfo } = result.data.inventoryPositions;
+
+      for (const edge of edges) {
+        await writer.appendRow({
+          ...edge.node,
+          extractedAt: new Date(),
+        });
+        totalRecords++;
+      }
+
+      hasNextPage = pageInfo.hasNextPage;
+      cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
+    }
+
+    await writer.close();
+    console.log(`✅ Extracted ${totalRecords} records to ${tempFile}`);
+
+    // 3. Upload to S3
+    const fileBuffer = await fs.readFile(tempFile);
+    await s3.uploadFile(s3Key, fileBuffer, {
+      contentType: 'application/octet-stream',
+    });
+
+    console.log(`✅ Uploaded to s3://${process.env.S3_BUCKET}/${s3Key}`);
+
+    // 4. Cleanup temp file
+    await fs.unlink(tempFile);
+  } catch (error) {
+    // Cleanup on error
+    try {
+      await fs.unlink(tempFile);
+    } catch {}
+    throw error;
+  }
+}
+```
+
+---
+
+## Streaming S3 to S3
+
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori
+
+/**
+ * Stream Parquet from S3, transform, write to target S3
+ * Business use case: Data lake transformation pipeline
+ */
+class S3ParquetStreamer {
+  constructor(
+    private sourceS3: S3DataSource,
+    private targetS3: S3DataSource,
+    private parser: ParquetParserService
+  ) {}
+
+  async streamParquetToJSON(
+    sourceBucket: string,
+    sourceKey: string,
+    targetBucket: string,
+    targetKey: string
+  ): Promise<void> {
+    console.log(`Streaming ${sourceKey}...`);
+
+    // 1. Download from source S3
+    const buffer = await this.sourceS3.downloadFile(sourceKey, { encoding: 'binary' });
+    const arrayBuffer = buffer.buffer.slice(
+      buffer.byteOffset,
+      buffer.byteOffset + buffer.byteLength
+    );
+
+    // 2. Stream parse
+    const streamResult = await this.parser.parseStreaming(arrayBuffer, sourceKey, {
+      batchSize: 1000,
+    });
+
+    const jsonLines: string[] = [];
+    let totalRecords = 0;
+
+    // 3. Process batches
+    for await (const batch of streamResult.records) {
+      for (const record of batch) {
+        jsonLines.push(JSON.stringify(record));
+        totalRecords++;
+
+        // Write chunks to S3 every 10K records
+        if (jsonLines.length >= 10000) {
+          await this.writeChunk(targetKey, jsonLines.join('\n'), totalRecords);
+          jsonLines.length = 0; // Clear array
+        }
+      }
+    }
+
+    // 4. Write remaining records
+    if (jsonLines.length > 0) {
+      await this.writeChunk(targetKey, jsonLines.join('\n'), totalRecords);
+    }
+
+    const stats = await streamResult.stats;
+    console.log(`✅ Streaming complete:`);
+    console.log(`   Total records: ${totalRecords}`);
+    console.log(`   Processing time: ${stats.processingTimeMs}ms`);
+  }
+
+  private async writeChunk(
+    targetKeyBase: string,
+    content: string,
+    recordNumber: number
+  ): Promise<void> {
+    const chunkKey = `${targetKeyBase}_chunk_${Math.floor(recordNumber / 10000)}.jsonl`;
+
+    await this.targetS3.uploadFile(chunkKey, Buffer.from(content), {
+      contentType: 'application/x-ndjson',
+    });
+
+    console.log(`   Wrote chunk: ${chunkKey}`);
+  }
+}
+```
+
+---
+
+## Multiple File Processing
+
+```typescript
+/**
+ * Process all Parquet files in S3 prefix
+ * Business use case: Batch processing of daily extracts
+ */
+async function processAllParquetFiles(prefix: string): Promise<void> {
+  const logger = toStructuredLogger(createConsoleLogger(), {
+    logLevel: 'info'
+  });
+
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      s3Config: {
+        bucket: process.env.S3_BUCKET!,
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+      },
+    },
+    logger
+  );
+
+  const parser = new ParquetParserService(logger);
+
+  // 1. List all Parquet files
+  const files = await s3.listFiles({ prefix });
+  const parquetFiles = files.filter(f => f.path.endsWith('.parquet'));
+
+  console.log(`Found ${parquetFiles.length} Parquet files with prefix: ${prefix}`);
+
+  // 2. Process each file
+  for (const file of parquetFiles) {
+    try {
+      console.log(`\nProcessing ${file.path}...`);
+
+      const buffer = await s3.downloadFile(file.path, { encoding: 'binary' });
+      const arrayBuffer = buffer.buffer.slice(
+        buffer.byteOffset,
+        buffer.byteOffset + buffer.byteLength
+      );
+
+      const { records, stats } = await parser.parse(arrayBuffer, file.path);
+
+      console.log(`✅ ${file.path}: ${stats.totalRecords} records`);
+
+      // Your processing logic
+      await processRecords(records, file.key);
+    } catch (error) {
+      console.error(`❌ Failed to process ${file.path}:`, (error as any).message);
+      // Continue processing other files
+    }
+  }
+
+  console.log(`\n✅ Batch processing complete`);
+}
+```
+
+---
+
+## Presigned URL Workflows
+
+```typescript
+import { S3PresignService } from '@fluentcommerce/fc-connect-sdk';
+
+/**
+ * Generate presigned URLs for Parquet downloads
+ * Business use case: Secure file sharing with analytics team
+ */
+class ParquetPresignService {
+  private presign: S3PresignService;
+
+  constructor(s3Config: any) {
+    this.presign = new S3PresignService(s3Config);
+  }
+
+  async generateDownloadUrl(
+    bucket: string,
+    key: string,
+    expiresIn: number = 3600 // 1 hour
+  ): Promise<string> {
+    const url = await this.presign.getPresignedUrl({
+      bucket,
+      key,
+      operation: 'getObject',
+      expiresIn,
+    });
+
+    console.log(`✅ Generated presigned URL for ${key}`);
+    console.log(`   Expires in: ${expiresIn} seconds`);
+
+    return url;
+  }
+
+  async shareParquetFile(bucket: string, key: string): Promise<void> {
+    const url = await this.generateDownloadUrl(bucket, key, 86400); // 24 hours
+
+    console.log(`\nShare this URL with analytics team:`);
+    console.log(url);
+    console.log(`\nValid for 24 hours`);
+  }
+}
+```
+
+---
+
+## Key Takeaways
+
+- 🎯 **S3DataSource.writeParquetContent()** - Generates Parquet Buffer (then upload separately)
+- 🎯 **S3DataSource integration** - Seamless S3 operations with SDK
+- 🎯 **Stream large files** - Avoid loading entire file into memory
+- 🎯 **Batch processing** - Handle multiple files efficiently
+- 🎯 **Presigned URLs** - Secure file sharing
+- 🎯 **Temp file cleanup** - Only needed for advanced temp file approach
+
+---
+
+## Next Steps
+
+Continue to [Module 7: Performance Optimization](./03-pattern-guides-parquet-07-performance-optimization.md) to learn advanced performance tuning techniques.
+
+---
+
+[← Previous: Module 5 - GraphQL Extraction](./03-pattern-guides-parquet-05-graphql-extraction.md/) | [Back to Guide](../parquet-readme.md) | [Next: Module 7 - Performance →](./03-pattern-guides-parquet-07-performance-optimization.md)
+
+## Related Documentation
+
+- [S3 Complete Guide](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - Complete S3 patterns
+- [S3 Presigned URLs](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - Security and expiration