@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-03-parquet-processing.md

@@ -1,507 +1,507 @@
-# Module 3: Parquet Processing
-
-**Level:** Intermediate
-**Estimated Time:** 25 minutes
-
-## Overview
-
-This module covers reading and writing Parquet files using the SDK's `ParquetParserService`, including S3 integration, streaming for large files, and validation.
-
-## Learning Objectives
-
-By the end of this module, you will:
-
-- ✅ Read Parquet files from S3 using `ParquetParserService`
-- ✅ Stream large Parquet files with batching
-- ✅ Validate Parquet data and handle statistics
-- ✅ Combine Parquet data with GraphQL enrichment
-- ✅ Write transformed data back to Parquet format
-
-## What is Parquet?
-
-**Apache Parquet** is a columnar storage format optimized for:
-
-- **Analytics workloads**: Efficient column scanning
-- **Compression**: Smaller file sizes
-- **Schema evolution**: Add/remove columns over time
-- **Interoperability**: Supported by Spark, Athena, BigQuery, etc.
-
-**Common use cases:**
-
-- Data lake storage
-- ETL pipelines
-- Analytics exports
-- Backup archives
-
-## Reading Parquet Files
-
-### Basic Parquet Reading
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-const parser = new ParquetParserService(logger);
-
-// Parse Parquet file
-const arrayBuffer = ... // Load file into ArrayBuffer
-const { records, stats } = await parser.parse(
-  arrayBuffer,
-  'inventory.parquet'
-);
-
-console.log(`Parsed ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
-console.log(`Columns: ${Object.keys(stats.columnStats).join(', ')}`);
-
-// Process records
-for (const record of records) {
-  console.log(record);
-}
-```
-
-### Reading from S3
-
-```typescript
-import {
-  S3DataSource,
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function readParquetFromS3(bucket: string, key: string) {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_PARQUET',
-      connectionId: 's3-parquet-read',
-      name: 'S3 Parquet Read',
-      s3Config: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-        bucket: process.env.S3_BUCKET!,
-      },
-    },
-    logger
-  );
-  const parser = new ParquetParserService(logger);
-
-  // Download from S3
-  const buffer = await s3.downloadFile(key, { encoding: 'binary' });
-
-  // Convert Buffer to ArrayBuffer
-  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-  // Parse Parquet
-  const { records, stats } = await parser.parse(arrayBuffer, key);
-
-  console.log(`Loaded ${records.length} records from s3://${bucket}/${key}`);
-
-  return records;
-}
-
-// Usage
-const records = await readParquetFromS3('my-bucket', 'exports/inventory.parquet');
-```
-
-### Parquet Validation
-
-```typescript
-const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet', {
-  validation: {
-    enabled: true,
-    requiredColumns: ['id', 'ref', 'productRef', 'qty'],
-    allowExtraColumns: true,
-  },
-});
-
-// Check for validation errors
-if (stats.validationErrors && stats.validationErrors.length > 0) {
-  console.error('Validation errors:', stats.validationErrors);
-  throw new Error('Parquet validation failed');
-}
-```
-
-### Understanding Parquet Statistics
-
-```typescript
-const { records, stats } = await parser.parse(arrayBuffer, 'data.parquet');
-
-console.log('=== Parquet Statistics ===');
-console.log(`Total records: ${stats.totalRecords}`);
-console.log(`Processing time: ${stats.processingTimeMs}ms`);
-console.log(`Records/sec: ${stats.recordsPerSecond}`);
-
-// Column-level statistics
-for (const [columnName, columnStats] of Object.entries(stats.columnStats)) {
-  console.log(`\nColumn: ${columnName}`);
-  console.log(`  Type: ${columnStats.type}`);
-  console.log(`  Nulls: ${columnStats.nullCount}`);
-  console.log(`  Unique: ${columnStats.uniqueValues}`);
-  console.log(`  Min: ${columnStats.minValue}`);
-  console.log(`  Max: ${columnStats.maxValue}`);
-}
-```
-
-## Streaming Large Parquet Files
-
-### Streaming API
-
-For very large Parquet files, use streaming to avoid loading all data into memory:
-
-```typescript
-async function streamLargeParquet(arrayBuffer: ArrayBuffer) {
-  const parser = new ParquetParserService(logger, {
-    batchSize: 5000,
-    enableStreaming: true,
-  });
-
-  const streamResult = await parser.parseStreaming(arrayBuffer, 'large_file.parquet', {
-    batchSize: 5000,
-  });
-
-  let totalProcessed = 0;
-
-  for await (const batch of streamResult.records) {
-    // Process each batch
-    for (const record of batch) {
-      await processRecord(record);
-      totalProcessed++;
-    }
-
-    console.log(`Processed ${totalProcessed} records so far...`);
-  }
-
-  // Get final statistics
-  const stats = await streamResult.stats;
-  console.log(`\nCompleted: ${stats.totalRecords} total records`);
-}
-
-async function processRecord(record: any) {
-  // Apply transformations, validations, etc.
-  return {
-    ...record,
-    processedAt: new Date().toISOString(),
-  };
-}
-```
-
-### Batch Processing
-
-```typescript
-async function batchProcessParquet(arrayBuffer: ArrayBuffer) {
-  const parser = new ParquetParserService(logger);
-  const streamResult = await parser.parseStreaming(arrayBuffer, 'data.parquet', {
-    batchSize: 1000,
-  });
-
-  const processedBatches = [];
-
-  for await (const batch of streamResult.records) {
-    // Transform entire batch at once
-    const transformed = transformBatch(batch);
-
-    // Validate batch
-    const valid = validateBatch(transformed);
-
-    if (valid) {
-      processedBatches.push(transformed);
-
-      // Write batch to destination
-      await writeBatchToS3(transformed);
-    }
-  }
-
-  return processedBatches;
-}
-
-function transformBatch(batch: any[]): any[] {
-  return batch.map(record => ({
-    sku: record.productRef,
-    warehouse: record.locationRef,
-    quantity: parseInt(record.qty) || 0,
-    status: record.status?.toLowerCase(),
-  }));
-}
-
-function validateBatch(batch: any[]): boolean {
-  return batch.every(record => record.sku && record.warehouse && record.quantity >= 0);
-}
-```
-
-## Combining Parquet with GraphQL Enrichment
-
-### Extract from Parquet, Enrich with GraphQL
-
-```typescript
-import {
-  createClient,
-  S3DataSource,
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function extractAndEnrichFromParquet() {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const client = await createClient({ config });
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_PARQUET',
-      connectionId: 's3-parquet-extract',
-      name: 'S3 Parquet Extract',
-      s3Config: s3Config,
-    },
-    logger
-  );
-  const parser = new ParquetParserService(logger);
-
-  // Step 1: Load Parquet file from S3
-  const buffer = await s3.downloadFile('inventory.parquet', { encoding: 'binary' });
-  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-  const { records } = await parser.parse(arrayBuffer, 'inventory.parquet');
-  logger.info(`Loaded ${records.length} records from Parquet`);
-
-  // Step 2: Enrich with GraphQL data
-  const enriched = await enrichWithProductDetails(client, records);
-  logger.info(`Enriched ${enriched.length} records`);
-
-  return enriched;
-}
-
-async function enrichWithProductDetails(client: any, records: any[]) {
-  const batchSize = 50;
-  const enriched = [];
-
-  for (let i = 0; i < records.length; i += batchSize) {
-    const batch = records.slice(i, i + batchSize);
-    const productRefs = batch.map(r => r.productRef);
-
-    // Batch query for product details
-    const result = await client.graphql({
-      query: `
-        query GetProducts($refs: [String!]!) {
-          products(filter: { ref: { in: $refs } }, first: ${batchSize}) {
-            edges {
-              node {
-                ref
-                name
-                gtin
-                catalogue { ref name }
-              }
-            }
-          }
-        }
-      `,
-      variables: { refs: productRefs },
-    });
-
-    // Map products to records
-    const productMap = new Map(result.data.products.edges.map(e => [e.node.ref, e.node]));
-
-    for (const record of batch) {
-      enriched.push({
-        ...record,
-        product: productMap.get(record.productRef) || null,
-      });
-    }
-  }
-
-  return enriched;
-}
-```
-
-### Complete Workflow Example
-
-```typescript
-async function parquetToJsonWorkflow() {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const client = await createClient({ config });
-  const sourceS3 = new S3DataSource(
-    {
-      type: 'S3_PARQUET',
-      connectionId: 's3-source-parquet',
-      name: 'S3 Source Parquet',
-      s3Config: sourceConfig,
-    },
-    logger
-  );
-  const targetS3 = new S3DataSource(
-    {
-      type: 'S3_JSON',
-      connectionId: 's3-target-json',
-      name: 'S3 Target JSON',
-      s3Config: targetConfig,
-    },
-    logger
-  );
-  const parser = new ParquetParserService(logger);
-
-  // Load all Parquet files
-  const files = await sourceS3.listFiles({ prefix: 'exports/' });
-  logger.info(`Found ${files.length} Parquet files`);
-
-  for (const file of files) {
-    if (!file.path.endsWith('.parquet')) continue;
-
-    logger.info(`Processing ${file.path}...`);
-
-    // Load and parse
-    const buffer = await sourceS3.downloadFile(file.path, { encoding: 'binary' }) as Buffer;
-    const arrayBuffer = buffer.buffer.slice(
-      buffer.byteOffset,
-      buffer.byteOffset + buffer.byteLength
-    );
-    const { records } = await parser.parse(arrayBuffer, file.path);
-
-    // Enrich
-    const enriched = await enrichWithProductDetails(client, records);
-
-    // Transform (flatten structure)
-    const transformed = enriched.map(record => ({
-      sku: record.productRef,
-      productName: record.product?.name || 'Unknown',
-      warehouse: record.locationRef,
-      quantity: parseInt(record.qty) || 0,
-      status: record.status?.toLowerCase(),
-      lastUpdated: record.updatedOn,
-    }));
-
-    // Export as JSON
-    const outputPath = file.path.replace('.parquet', '.json');
-    await targetS3.uploadFile(outputPath, JSON.stringify(transformed, null, 2), {
-      contentType: 'application/json',
-    });
-
-    logger.info(`Exported ${transformed.length} records to ${outputPath}`);
-  }
-}
-```
-
-## Writing Parquet Files
-
-While the SDK focuses on reading Parquet, you can write Parquet using the `parquetjs` library:
-
-```typescript
-import { ParquetWriter, ParquetSchema } from 'parquetjs';
-
-async function writeParquetFile(records: any[], outputPath: string) {
-  // Define schema
-  const schema = new ParquetSchema({
-    sku: { type: 'UTF8' },
-    warehouse: { type: 'UTF8' },
-    quantity: { type: 'INT32' },
-    status: { type: 'UTF8' },
-    lastUpdated: { type: 'UTF8' },
-  });
-
-  // Create writer
-  const writer = await ParquetWriter.openFile(schema, outputPath);
-
-  // Write records
-  for (const record of records) {
-    await writer.appendRow(record);
-  }
-
-  // Close file
-  await writer.close();
-
-  console.log(`Wrote ${records.length} records to ${outputPath}`);
-}
-```
-
-## Practice Exercise
-
-**Goal**: Load inventory from Parquet, enrich with location details, export to JSON
-
-**Requirements:**
-
-1. Read Parquet file from S3
-2. Validate required columns: `id`, `productRef`, `locationRef`, `qty`
-3. Enrich with location details via GraphQL
-4. Transform to external format
-5. Write to S3 as JSON
-
-**Solution:**
-
-```typescript
-async function enrichInventoryFromParquet() {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const client = await createClient({ config });
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_PARQUET',
-      connectionId: 's3-parquet-enrich',
-      name: 'S3 Parquet Enrich',
-      s3Config: s3Config,
-    },
-    logger
-  );
-
-  // Load Parquet
-  const buffer = await s3.downloadFile('inventory.parquet', { encoding: 'binary' });
-  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-  const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet', {
-    validation: {
-      enabled: true,
-      requiredColumns: ['id', 'productRef', 'locationRef', 'qty'],
-    },
-  });
-
-  logger.info(`Loaded ${records.length} records`);
-
-  // Enrich with locations (batch query)
-  const locationRefs = [...new Set(records.map(r => r.locationRef))];
-  const locationResult = await client.graphql({
-    query: `
-      query GetLocations($refs: [String!]!) {
-        locations(filter: { ref: { in: $refs } }, first: 1000) {
-          edges {
-            node { ref name type }
-          }
-        }
-      }
-    `,
-    variables: { refs: locationRefs },
-  });
-
-  const locationMap = new Map(locationResult.data.locations.edges.map(e => [e.node.ref, e.node]));
-
-  // Transform
-  const transformed = records.map(record => ({
-    sku: record.productRef,
-    warehouse: record.locationRef,
-    warehouseName: locationMap.get(record.locationRef)?.name || 'Unknown',
-    warehouseType: locationMap.get(record.locationRef)?.type || 'Unknown',
-    quantity: parseInt(record.qty) || 0,
-  }));
-
-  // Export
-  await s3.uploadFile('enriched_inventory.json', JSON.stringify(transformed, null, 2), {
-    contentType: 'application/json',
-  });
-
-  logger.info('Export complete');
-}
-```
-
-## Key Takeaways
-
-- 🎯 Use `ParquetParserService` for reading Parquet files
-- 🎯 Convert S3 Buffer to ArrayBuffer before parsing
-- 🎯 Use streaming for large files to avoid memory issues
-- 🎯 Validate Parquet structure with required columns
-- 🎯 Combine Parquet base data with GraphQL enrichment
-- 🎯 Check statistics for data quality insights
-- 🎯 Use batch queries for efficient enrichment
-
-## Next Steps
-
-Continue to [Module 4: Data Enrichment](./02-core-guides-extraction-04-data-enrichment.md) to learn advanced GraphQL enrichment patterns.
+# Module 3: Parquet Processing
+
+**Level:** Intermediate
+**Estimated Time:** 25 minutes
+
+## Overview
+
+This module covers reading and writing Parquet files using the SDK's `ParquetParserService`, including S3 integration, streaming for large files, and validation.
+
+## Learning Objectives
+
+By the end of this module, you will:
+
+- ✅ Read Parquet files from S3 using `ParquetParserService`
+- ✅ Stream large Parquet files with batching
+- ✅ Validate Parquet data and handle statistics
+- ✅ Combine Parquet data with GraphQL enrichment
+- ✅ Write transformed data back to Parquet format
+
+## What is Parquet?
+
+**Apache Parquet** is a columnar storage format optimized for:
+
+- **Analytics workloads**: Efficient column scanning
+- **Compression**: Smaller file sizes
+- **Schema evolution**: Add/remove columns over time
+- **Interoperability**: Supported by Spark, Athena, BigQuery, etc.
+
+**Common use cases:**
+
+- Data lake storage
+- ETL pipelines
+- Analytics exports
+- Backup archives
+
+## Reading Parquet Files
+
+### Basic Parquet Reading
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+const parser = new ParquetParserService(logger);
+
+// Parse Parquet file
+const arrayBuffer = ... // Load file into ArrayBuffer
+const { records, stats } = await parser.parse(
+  arrayBuffer,
+  'inventory.parquet'
+);
+
+console.log(`Parsed ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
+console.log(`Columns: ${Object.keys(stats.columnStats).join(', ')}`);
+
+// Process records
+for (const record of records) {
+  console.log(record);
+}
+```
+
+### Reading from S3
+
+```typescript
+import {
+  S3DataSource,
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function readParquetFromS3(bucket: string, key: string) {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_PARQUET',
+      connectionId: 's3-parquet-read',
+      name: 'S3 Parquet Read',
+      s3Config: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+        bucket: process.env.S3_BUCKET!,
+      },
+    },
+    logger
+  );
+  const parser = new ParquetParserService(logger);
+
+  // Download from S3
+  const buffer = await s3.downloadFile(key, { encoding: 'binary' });
+
+  // Convert Buffer to ArrayBuffer
+  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+  // Parse Parquet
+  const { records, stats } = await parser.parse(arrayBuffer, key);
+
+  console.log(`Loaded ${records.length} records from s3://${bucket}/${key}`);
+
+  return records;
+}
+
+// Usage
+const records = await readParquetFromS3('my-bucket', 'exports/inventory.parquet');
+```
+
+### Parquet Validation
+
+```typescript
+const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet', {
+  validation: {
+    enabled: true,
+    requiredColumns: ['id', 'ref', 'productRef', 'qty'],
+    allowExtraColumns: true,
+  },
+});
+
+// Check for validation errors
+if (stats.validationErrors && stats.validationErrors.length > 0) {
+  console.error('Validation errors:', stats.validationErrors);
+  throw new Error('Parquet validation failed');
+}
+```
+
+### Understanding Parquet Statistics
+
+```typescript
+const { records, stats } = await parser.parse(arrayBuffer, 'data.parquet');
+
+console.log('=== Parquet Statistics ===');
+console.log(`Total records: ${stats.totalRecords}`);
+console.log(`Processing time: ${stats.processingTimeMs}ms`);
+console.log(`Records/sec: ${stats.recordsPerSecond}`);
+
+// Column-level statistics
+for (const [columnName, columnStats] of Object.entries(stats.columnStats)) {
+  console.log(`\nColumn: ${columnName}`);
+  console.log(`  Type: ${columnStats.type}`);
+  console.log(`  Nulls: ${columnStats.nullCount}`);
+  console.log(`  Unique: ${columnStats.uniqueValues}`);
+  console.log(`  Min: ${columnStats.minValue}`);
+  console.log(`  Max: ${columnStats.maxValue}`);
+}
+```
+
+## Streaming Large Parquet Files
+
+### Streaming API
+
+For very large Parquet files, use streaming to avoid loading all data into memory:
+
+```typescript
+async function streamLargeParquet(arrayBuffer: ArrayBuffer) {
+  const parser = new ParquetParserService(logger, {
+    batchSize: 5000,
+    enableStreaming: true,
+  });
+
+  const streamResult = await parser.parseStreaming(arrayBuffer, 'large_file.parquet', {
+    batchSize: 5000,
+  });
+
+  let totalProcessed = 0;
+
+  for await (const batch of streamResult.records) {
+    // Process each batch
+    for (const record of batch) {
+      await processRecord(record);
+      totalProcessed++;
+    }
+
+    console.log(`Processed ${totalProcessed} records so far...`);
+  }
+
+  // Get final statistics
+  const stats = await streamResult.stats;
+  console.log(`\nCompleted: ${stats.totalRecords} total records`);
+}
+
+async function processRecord(record: any) {
+  // Apply transformations, validations, etc.
+  return {
+    ...record,
+    processedAt: new Date().toISOString(),
+  };
+}
+```
+
+### Batch Processing
+
+```typescript
+async function batchProcessParquet(arrayBuffer: ArrayBuffer) {
+  const parser = new ParquetParserService(logger);
+  const streamResult = await parser.parseStreaming(arrayBuffer, 'data.parquet', {
+    batchSize: 1000,
+  });
+
+  const processedBatches = [];
+
+  for await (const batch of streamResult.records) {
+    // Transform entire batch at once
+    const transformed = transformBatch(batch);
+
+    // Validate batch
+    const valid = validateBatch(transformed);
+
+    if (valid) {
+      processedBatches.push(transformed);
+
+      // Write batch to destination
+      await writeBatchToS3(transformed);
+    }
+  }
+
+  return processedBatches;
+}
+
+function transformBatch(batch: any[]): any[] {
+  return batch.map(record => ({
+    sku: record.productRef,
+    warehouse: record.locationRef,
+    quantity: parseInt(record.qty) || 0,
+    status: record.status?.toLowerCase(),
+  }));
+}
+
+function validateBatch(batch: any[]): boolean {
+  return batch.every(record => record.sku && record.warehouse && record.quantity >= 0);
+}
+```
+
+## Combining Parquet with GraphQL Enrichment
+
+### Extract from Parquet, Enrich with GraphQL
+
+```typescript
+import {
+  createClient,
+  S3DataSource,
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function extractAndEnrichFromParquet() {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const client = await createClient({ config });
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_PARQUET',
+      connectionId: 's3-parquet-extract',
+      name: 'S3 Parquet Extract',
+      s3Config: s3Config,
+    },
+    logger
+  );
+  const parser = new ParquetParserService(logger);
+
+  // Step 1: Load Parquet file from S3
+  const buffer = await s3.downloadFile('inventory.parquet', { encoding: 'binary' });
+  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+  const { records } = await parser.parse(arrayBuffer, 'inventory.parquet');
+  logger.info(`Loaded ${records.length} records from Parquet`);
+
+  // Step 2: Enrich with GraphQL data
+  const enriched = await enrichWithProductDetails(client, records);
+  logger.info(`Enriched ${enriched.length} records`);
+
+  return enriched;
+}
+
+async function enrichWithProductDetails(client: any, records: any[]) {
+  const batchSize = 50;
+  const enriched = [];
+
+  for (let i = 0; i < records.length; i += batchSize) {
+    const batch = records.slice(i, i + batchSize);
+    const productRefs = batch.map(r => r.productRef);
+
+    // Batch query for product details
+    const result = await client.graphql({
+      query: `
+        query GetProducts($refs: [String!]!) {
+          products(filter: { ref: { in: $refs } }, first: ${batchSize}) {
+            edges {
+              node {
+                ref
+                name
+                gtin
+                catalogue { ref name }
+              }
+            }
+          }
+        }
+      `,
+      variables: { refs: productRefs },
+    });
+
+    // Map products to records
+    const productMap = new Map(result.data.products.edges.map(e => [e.node.ref, e.node]));
+
+    for (const record of batch) {
+      enriched.push({
+        ...record,
+        product: productMap.get(record.productRef) || null,
+      });
+    }
+  }
+
+  return enriched;
+}
+```
+
+### Complete Workflow Example
+
+```typescript
+async function parquetToJsonWorkflow() {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const client = await createClient({ config });
+  const sourceS3 = new S3DataSource(
+    {
+      type: 'S3_PARQUET',
+      connectionId: 's3-source-parquet',
+      name: 'S3 Source Parquet',
+      s3Config: sourceConfig,
+    },
+    logger
+  );
+  const targetS3 = new S3DataSource(
+    {
+      type: 'S3_JSON',
+      connectionId: 's3-target-json',
+      name: 'S3 Target JSON',
+      s3Config: targetConfig,
+    },
+    logger
+  );
+  const parser = new ParquetParserService(logger);
+
+  // Load all Parquet files
+  const files = await sourceS3.listFiles({ prefix: 'exports/' });
+  logger.info(`Found ${files.length} Parquet files`);
+
+  for (const file of files) {
+    if (!file.path.endsWith('.parquet')) continue;
+
+    logger.info(`Processing ${file.path}...`);
+
+    // Load and parse
+    const buffer = await sourceS3.downloadFile(file.path, { encoding: 'binary' }) as Buffer;
+    const arrayBuffer = buffer.buffer.slice(
+      buffer.byteOffset,
+      buffer.byteOffset + buffer.byteLength
+    );
+    const { records } = await parser.parse(arrayBuffer, file.path);
+
+    // Enrich
+    const enriched = await enrichWithProductDetails(client, records);
+
+    // Transform (flatten structure)
+    const transformed = enriched.map(record => ({
+      sku: record.productRef,
+      productName: record.product?.name || 'Unknown',
+      warehouse: record.locationRef,
+      quantity: parseInt(record.qty) || 0,
+      status: record.status?.toLowerCase(),
+      lastUpdated: record.updatedOn,
+    }));
+
+    // Export as JSON
+    const outputPath = file.path.replace('.parquet', '.json');
+    await targetS3.uploadFile(outputPath, JSON.stringify(transformed, null, 2), {
+      contentType: 'application/json',
+    });
+
+    logger.info(`Exported ${transformed.length} records to ${outputPath}`);
+  }
+}
+```
+
+## Writing Parquet Files
+
+While the SDK focuses on reading Parquet, you can write Parquet using the `parquetjs` library:
+
+```typescript
+import { ParquetWriter, ParquetSchema } from 'parquetjs';
+
+async function writeParquetFile(records: any[], outputPath: string) {
+  // Define schema
+  const schema = new ParquetSchema({
+    sku: { type: 'UTF8' },
+    warehouse: { type: 'UTF8' },
+    quantity: { type: 'INT32' },
+    status: { type: 'UTF8' },
+    lastUpdated: { type: 'UTF8' },
+  });
+
+  // Create writer
+  const writer = await ParquetWriter.openFile(schema, outputPath);
+
+  // Write records
+  for (const record of records) {
+    await writer.appendRow(record);
+  }
+
+  // Close file
+  await writer.close();
+
+  console.log(`Wrote ${records.length} records to ${outputPath}`);
+}
+```
+
+## Practice Exercise
+
+**Goal**: Load inventory from Parquet, enrich with location details, export to JSON
+
+**Requirements:**
+
+1. Read Parquet file from S3
+2. Validate required columns: `id`, `productRef`, `locationRef`, `qty`
+3. Enrich with location details via GraphQL
+4. Transform to external format
+5. Write to S3 as JSON
+
+**Solution:**
+
+```typescript
+async function enrichInventoryFromParquet() {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const client = await createClient({ config });
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_PARQUET',
+      connectionId: 's3-parquet-enrich',
+      name: 'S3 Parquet Enrich',
+      s3Config: s3Config,
+    },
+    logger
+  );
+
+  // Load Parquet
+  const buffer = await s3.downloadFile('inventory.parquet', { encoding: 'binary' });
+  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+  const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet', {
+    validation: {
+      enabled: true,
+      requiredColumns: ['id', 'productRef', 'locationRef', 'qty'],
+    },
+  });
+
+  logger.info(`Loaded ${records.length} records`);
+
+  // Enrich with locations (batch query)
+  const locationRefs = [...new Set(records.map(r => r.locationRef))];
+  const locationResult = await client.graphql({
+    query: `
+      query GetLocations($refs: [String!]!) {
+        locations(filter: { ref: { in: $refs } }, first: 1000) {
+          edges {
+            node { ref name type }
+          }
+        }
+      }
+    `,
+    variables: { refs: locationRefs },
+  });
+
+  const locationMap = new Map(locationResult.data.locations.edges.map(e => [e.node.ref, e.node]));
+
+  // Transform
+  const transformed = records.map(record => ({
+    sku: record.productRef,
+    warehouse: record.locationRef,
+    warehouseName: locationMap.get(record.locationRef)?.name || 'Unknown',
+    warehouseType: locationMap.get(record.locationRef)?.type || 'Unknown',
+    quantity: parseInt(record.qty) || 0,
+  }));
+
+  // Export
+  await s3.uploadFile('enriched_inventory.json', JSON.stringify(transformed, null, 2), {
+    contentType: 'application/json',
+  });
+
+  logger.info('Export complete');
+}
+```
+
+## Key Takeaways
+
+- 🎯 Use `ParquetParserService` for reading Parquet files
+- 🎯 Convert S3 Buffer to ArrayBuffer before parsing
+- 🎯 Use streaming for large files to avoid memory issues
+- 🎯 Validate Parquet structure with required columns
+- 🎯 Combine Parquet base data with GraphQL enrichment
+- 🎯 Check statistics for data quality insights
+- 🎯 Use batch queries for efficient enrichment
+
+## Next Steps
+
+Continue to [Module 4: Data Enrichment](./02-core-guides-extraction-04-data-enrichment.md) to learn advanced GraphQL enrichment patterns.