@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/02-CORE-GUIDES/ingestion/modules/02-core-guides-ingestion-05-advanced-parsers.md

@@ -1,676 +1,676 @@
-# Module 5: Advanced Parsers
-
-**Level:** Intermediate
-**Estimated Time:** 25 minutes
-
-## Overview
-
-This module covers advanced file parsing capabilities beyond CSV, including Parquet for large datasets, XML for B2B systems, and JSON for API integrations.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Parse Parquet files for efficient large dataset processing
-- ✅ Parse XML files from B2B systems and feeds
-- ✅ Parse JSON from APIs and web services
-- ✅ Choose the optimal parser for your use case
-- ✅ Handle streaming for memory-efficient processing
-
-## Parquet Parser
-
-### Why Parquet?
-
-Parquet is a columnar storage format ideal for:
-
-| Feature | Benefit |
-|---------|---------|
-| **Compression** | 5-10x smaller than CSV |
-| **Speed** | Faster reads for analytics |
-| **Schema** | Built-in type validation |
-| **Large Files** | Efficient memory usage |
-
-### Basic Parquet Parsing
-
-```typescript
-import {
-  ParquetParserService,
-  S3DataSource,
-  UniversalMapper,
-  createConsoleLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function parseParquetFile() {
-  // Initialize services
-  const logger = createConsoleLogger();
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_PARQUET',
-      connectionId: 's3-parquet',
-      name: 'S3 Parquet',
-      s3Config: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-        bucket: 'inventory-bucket'
-      }
-    },
-    logger
-  );
-
-  const parquetParser = new ParquetParserService(logger);
-
-  // Read Parquet file from S3
-  const buffer = await s3.downloadFile('data/inventory-positions.parquet', { encoding: 'binary' }) as Buffer;
-
-  // Parse Parquet
-  const records = await parquetParser.parse(buffer);
-  console.log(`Parsed ${records.length} records from Parquet file`);
-
-  // Records are already typed based on Parquet schema
-  return records;
-}
-```
-
-### Streaming Large Parquet Files
-
-For files larger than available memory:
-
-```typescript
-import { ParquetParserService, S3DataSource, UniversalMapper, createConsoleLogger } from '@fluentcommerce/fc-connect-sdk';
-
-async function streamParquetFile() {
-  const logger = createConsoleLogger();
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_PARQUET',
-      connectionId: 's3-parquet-stream',
-      name: 'S3 Parquet Stream',
-      s3Config: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-        bucket: 'inventory-bucket'
-      }
-    },
-    logger
-  );
-  const parquetParser = new ParquetParserService(logger);
-
-  // Download Parquet file from S3 (full file download)
-  const buffer = await s3.downloadFile('data/large-inventory.parquet', { encoding: 'binary' }) as Buffer;
-
-  // Parse Parquet (parser handles large files efficiently)
-  const records = await parquetParser.parse(buffer);
-
-  // Process records in chunks
-  const batchSize = 1000;
-  for (let i = 0; i < records.length; i += batchSize) {
-    const batch = records.slice(i, i + batchSize);
-    await processBatch(batch);
-  }
-}
-
-async function processBatch(records: any[]) {
-  // Map fields
-  const mapper = new UniversalMapper(mappingConfig);
-  const result = await mapper.map(records);
-
-  if (result.success) {
-    // Send to Fluent
-    await sendToFluent(result.data);
-  }
-}
-```
-
-### Parquet Schema Validation
-
-```typescript
-import { ParquetParserService, createConsoleLogger } from '@fluentcommerce/fc-connect-sdk';
-
-async function validateParquetSchema(file: Buffer) {
-  const logger = createConsoleLogger();
-  const parquetParser = new ParquetParserService(logger);
-
-  // Extract schema information
-  const metadata = await parquetParser.getMetadata(file);
-
-  console.log('Parquet Schema:', {
-    numRows: metadata.numRows,
-    columns: metadata.schema.fields.map(f => ({
-      name: f.name,
-      type: f.type,
-      optional: f.optional
-    })),
-    compression: metadata.compression,
-    createdBy: metadata.createdBy
-  });
-
-  // Validate expected columns exist
-  const requiredColumns = ['sku', 'location', 'quantity'];
-  const schemaColumns = metadata.schema.fields.map(f => f.name);
-
-  const missingColumns = requiredColumns.filter(
-    col => !schemaColumns.includes(col)
-  );
-
-  if (missingColumns.length > 0) {
-    throw new Error(`Missing required columns: ${missingColumns.join(', ')}`);
-  }
-
-  return metadata;
-}
-```
-
-## XML Parser
-
-### Why XML?
-
-XML parsing is essential for:
-
-| Use Case | Example |
-|----------|---------|
-| **Enterprise Systems** | ERP exports, warehouse management systems |
-| **Standard Formats** | EDI, SOAP, RSS feeds |
-| **Nested Data** | Complex hierarchical structures |
-| **SFCC Orders** | Salesforce Commerce Cloud integrations |
-
-### Basic XML Parsing
-
-```typescript
-import { XMLParserService } from '@fluentcommerce/fc-connect-sdk';
-
-async function parseXMLFile() {
-  const xmlParser = new XMLParserService();
-
-  const xmlData = `<?xml version="1.0" encoding="UTF-8"?>
-<inventory>
-  <position>
-    <sku>SKU-WM-001</sku>
-    <location>WH-001</location>
-    <quantity>500</quantity>
-    <status>available</status>
-  </position>
-  <position>
-    <sku>SKU-WM-002</sku>
-    <location>WH-001</location>
-    <quantity>150</quantity>
-    <status>reserved</status>
-  </position>
-</inventory>`;
-
-  // Parse XML to JavaScript object
-  const parsed = await xmlParser.parse(xmlData);
-
-  // Access nested data
-  const positions = parsed.inventory.position;
-
-  console.log(`Parsed ${positions.length} positions from XML`);
-  return positions;
-}
-```
-
-### XML with Path Resolution
-
-For complex XML structures, use path-based field mapping:
-
-```typescript
-async function parseComplexXML() {
-  const xmlParser = new XMLParserService();
-
-  const complexXML = `<?xml version="1.0" encoding="UTF-8"?>
-<warehouse-export>
-  <metadata>
-    <export-date>2024-01-15</export-date>
-    <warehouse>WH-001</warehouse>
-  </metadata>
-  <inventory-positions>
-    <position id="1">
-      <product>
-        <sku>SKU-WM-001</sku>
-        <name>Widget Master 3000</name>
-      </product>
-      <location>
-        <zone>A</zone>
-        <aisle>12</aisle>
-        <bin>45</bin>
-      </location>
-      <quantities>
-        <on-hand>500</on-hand>
-        <available>450</available>
-        <reserved>50</reserved>
-      </quantities>
-      <status>AVAILABLE</status>
-    </position>
-  </inventory-positions>
-</warehouse-export>`;
-
-  const parsed = await xmlParser.parse(complexXML);
-
-  // Extract with path notation
-  const mapper = new UniversalMapper({
-    fields: {
-      ref: {
-        source: 'product.sku',
-        required: true
-      },
-      productRef: {
-        source: 'product.sku',
-        required: true
-      },
-      locationRef: {
-        source: 'warehouse', // From root metadata
-        resolver: (value, data) => {
-          // Combine location parts
-          return `${data.location.zone}-${data.location.aisle}-${data.location.bin}`;
-        },
-        required: true
-      },
-      qty: {
-        source: 'quantities.on-hand',
-        resolver: 'sdk.parseInt',
-        required: true
-      },
-      status: {
-        source: 'status',
-        resolver: 'sdk.uppercase',
-        required: true
-      }
-    }
-  });
-
-  // Map each position
-  const positions = parsed['warehouse-export']['inventory-positions'].position;
-  const result = await mapper.map(positions);
-
-  return result.data;
-}
-```
-
-### XML Namespace Handling
-
-```typescript
-async function parseXMLWithNamespaces() {
-  const xmlParser = new XMLParserService({
-    ignoreAttributes: false,
-    attributeNamePrefix: '@_',
-    parseAttributeValue: true
-  });
-
-  const nsXML = `<?xml version="1.0" encoding="UTF-8"?>
-<inv:inventory xmlns:inv="http://example.com/inventory" xmlns:prod="http://example.com/product">
-  <inv:position inv:id="1">
-    <prod:sku>SKU-WM-001</prod:sku>
-    <inv:location>WH-001</inv:location>
-    <inv:quantity>500</inv:quantity>
-  </inv:position>
-</inv:inventory>`;
-
-  const parsed = await xmlParser.parse(nsXML);
-
-  // Access namespaced elements
-  console.log('Parsed with namespaces:', parsed);
-  return parsed;
-}
-```
-
-## JSON Parser
-
-### Why JSON?
-
-JSON parsing is ideal for:
-
-| Use Case | Example |
-|----------|---------|
-| **REST APIs** | Third-party inventory services |
-| **Web Services** | Cloud-based systems |
-| **Modern Exports** | Contemporary systems |
-| **Nested Objects** | Complex data structures |
-
-### Basic JSON Parsing
-
-```typescript
-import { JSONParserService } from '@fluentcommerce/fc-connect-sdk';
-
-async function parseJSONFile() {
-  const jsonParser = new JSONParserService();
-
-  const jsonData = {
-    "metadata": {
-      "export_date": "2024-01-15",
-      "total_records": 2
-    },
-    "inventory_positions": [
-      {
-        "sku": "SKU-WM-001",
-        "location": "WH-001",
-        "quantity": 500,
-        "status": "available"
-      },
-      {
-        "sku": "SKU-WM-002",
-        "location": "WH-001",
-        "quantity": 150,
-        "status": "reserved"
-      }
-    ]
-  };
-
-  // Parse JSON (validates structure)
-  const parsed = await jsonParser.parse(JSON.stringify(jsonData));
-
-  const positions = parsed.inventory_positions;
-  console.log(`Parsed ${positions.length} positions from JSON`);
-
-  return positions;
-}
-```
-
-### JSON Array Streaming
-
-For large JSON arrays, use streaming to avoid memory issues:
-
-```typescript
-import { S3DataSource, JSONParserService, createConsoleLogger } from '@fluentcommerce/fc-connect-sdk';
-
-async function streamLargeJSONArray() {
-  const logger = createConsoleLogger();
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_JSON',
-      connectionId: 's3-json-stream',
-      name: 'S3 JSON Stream',
-      s3Config: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-        bucket: 'inventory-bucket'
-      }
-    },
-    logger
-  );
-  const jsonParser = new JSONParserService();
-
-  // Download JSON file from S3
-  const jsonContent = await s3.downloadFile('data/large-inventory.json') as string;
-  
-  // Parse JSON array
-  const parsed = await jsonParser.parse(jsonContent);
-  const positions = parsed.inventory_positions || [];
-
-  // Process records in batches
-  const batchSize = 1000;
-  let count = 0;
-  for (let i = 0; i < positions.length; i += batchSize) {
-    const batch = positions.slice(i, i + batchSize);
-    for (const record of batch) {
-      await processRecord(record);
-      count++;
-    }
-
-    if (count % 1000 === 0) {
-      console.log(`Processed ${count} records...`);
-    }
-  }
-
-  console.log(`Total records processed: ${count}`);
-}
-```
-
-### JSON Schema Validation
-
-```typescript
-async function validateJSONSchema(data: any) {
-  const schema = {
-    type: 'object',
-    required: ['inventory_positions'],
-    properties: {
-      metadata: {
-        type: 'object',
-        properties: {
-          export_date: { type: 'string' },
-          total_records: { type: 'number' }
-        }
-      },
-      inventory_positions: {
-        type: 'array',
-        items: {
-          type: 'object',
-          required: ['sku', 'location', 'quantity'],
-          properties: {
-            sku: { type: 'string' },
-            location: { type: 'string' },
-            quantity: { type: 'number' },
-            status: { type: 'string' }
-          }
-        }
-      }
-    }
-  };
-
-  const jsonParser = new JSONParserService({ schema });
-
-  // Validate and parse
-  const result = await jsonParser.parseWithValidation(data);
-
-  if (!result.valid) {
-    throw new Error(`JSON validation failed: ${result.errors.join(', ')}`);
-  }
-
-  return result.data;
-}
-```
-
-## Parser Comparison
-
-### When to Use Which Parser
-
-| Parser | Best For | Pros | Cons | File Size Limit |
-|--------|----------|------|------|-----------------|
-| **CSV** | Simple tabular data, exports, integrations | Universal, easy to inspect | No schema, flat structure | < 1 GB with streaming |
-| **Parquet** | Large datasets, analytics, data lakes | Fast, compressed, typed | Binary, needs tools | Unlimited with streaming |
-| **XML** | Enterprise systems, SOAP, hierarchical data | Structured, validated, nested | Verbose, complex parsing | < 500 MB |
-| **JSON** | REST APIs, modern systems, web services | Flexible, nested, readable | Can be large, no schema | < 500 MB with streaming |
-
-### Performance Comparison
-
-```typescript
-async function benchmarkParsers() {
-  const testData = generateTestData(10000); // 10K records
-
-  // CSV Performance
-  const csvStart = Date.now();
-  const csvParser = new CSVParserService();
-  const csvData = await csvParser.parse(toCSV(testData), { columns: true });
-  console.log(`CSV: ${Date.now() - csvStart}ms`);
-
-  // Parquet Performance
-  const parquetStart = Date.now();
-  const logger = createConsoleLogger();
-  const parquetParser = new ParquetParserService(logger);
-  const parquetData = await parquetParser.parse(toParquet(testData));
-  console.log(`Parquet: ${Date.now() - parquetStart}ms`);
-
-  // XML Performance
-  const xmlStart = Date.now();
-  const xmlParser = new XMLParserService();
-  const xmlData = await xmlParser.parse(toXML(testData));
-  console.log(`XML: ${Date.now() - xmlStart}ms`);
-
-  // JSON Performance
-  const jsonStart = Date.now();
-  const jsonParser = new JSONParserService();
-  const jsonData = await jsonParser.parse(toJSON(testData));
-  console.log(`JSON: ${Date.now() - jsonStart}ms`);
-}
-```
-
-## Complete Example: Multi-Format Ingestion
-
-```typescript
-import {
-  createClient,
-  S3DataSource,
-  CSVParserService,
-  ParquetParserService,
-  XMLParserService,
-  JSONParserService,
-  UniversalMapper,
-  createConsoleLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function ingestMultiFormat() {
-  const logger = createConsoleLogger();
-  
-  const client = await createClient({
-    config: {
-      baseUrl: process.env.FLUENT_BASE_URL!,
-      clientId: process.env.FLUENT_CLIENT_ID!,
-      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-      retailerId: process.env.FLUENT_RETAILER_ID!
-    }
-  });
-  
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      connectionId: 's3-multi-format',
-      name: 'S3 Multi-Format',
-      s3Config: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-        region: process.env.AWS_REGION!,
-        bucket: 'inventory-bucket'
-      }
-    },
-    logger
-  );
-  const mapper = new UniversalMapper(mappingConfig);
-
-  // List all files
-  const files = await s3.listFiles({ prefix: 'data/' });
-
-  for (const file of files) {
-    let records: any[];
-
-    // Choose parser based on file extension
-    if (file.name.endsWith('.csv')) {
-      const csvParser = new CSVParserService();
-      const data = await s3.downloadFile(file.path) as string;
-      records = await csvParser.parse(data);
-
-    } else if (file.name.endsWith('.parquet')) {
-      const parquetParser = new ParquetParserService(logger);
-      const data = await s3.downloadFile(file.path, { encoding: 'binary' }) as Buffer;
-      records = await parquetParser.parse(data);
-
-    } else if (file.name.endsWith('.xml')) {
-      const xmlParser = new XMLParserService(logger);
-      const data = await s3.downloadFile(file.path) as string;
-      const parsed = await xmlParser.parse(data);
-      records = extractInventoryFromXML(parsed);
-
-    } else if (file.name.endsWith('.json')) {
-      const jsonParser = new JSONParserService();
-      const data = await s3.downloadFile(file.path) as string;
-      const parsed = await jsonParser.parse(data);
-      records = parsed.inventory_positions || [];
-
-    } else {
-      console.log(`Skipping unsupported file: ${file.name}`);
-      continue;
-    }
-
-    // Transform with UniversalMapper
-    const result = await mapper.map(records);
-
-    if (!result.success) {
-      console.error(`Mapping failed for ${file.name}:`, result.errors);
-      continue;
-    }
-
-    // Send to Fluent
-    const job = await client.createJob({
-      name: `Import - ${file.name}`,
-      retailerId: process.env.FLUENT_RETAILER_ID!
-    });
-
-    await client.sendBatch(job.id, {
-      action: 'UPSERT',
-      entityType: 'INVENTORY',
-      source: 'S3_CSV',
-      event: 'INVENTORY_SYNC',
-      entities: result.data
-    });
-
-    console.log(`Processed ${result.data.length} records from ${file.name}`);
-  }
-}
-
-function extractInventoryFromXML(parsed: any): any[] {
-  // Extract inventory positions from XML structure
-  const positions = parsed.inventory?.position || [];
-  return Array.isArray(positions) ? positions : [positions];
-}
-```
-
-## Practice Exercise
-
-### Challenge
-
-Parse a multi-format dataset and ingest into Fluent Commerce:
-
-1. Download sample files (CSV, Parquet, XML, JSON) from S3
-2. Parse each format using the appropriate parser
-3. Validate data structure and required fields
-4. Transform using UniversalMapper
-5. Send to Fluent via Batch API
-6. Track processing metrics (time, records, errors)
-
-### Solution Template
-
-```typescript
-async function multiFormatIngestionChallenge() {
-  const metrics = {
-    totalFiles: 0,
-    totalRecords: 0,
-    successfulFiles: 0,
-    failedFiles: 0,
-    processingTime: 0
-  };
-
-  const startTime = Date.now();
-
-  try {
-    // Your implementation here
-    // 1. List files
-    // 2. Parse by format
-    // 3. Transform data
-    // 4. Send to Fluent
-    // 5. Track metrics
-
-  } finally {
-    metrics.processingTime = Date.now() - startTime;
-    console.log('Ingestion Metrics:', metrics);
-  }
-}
-```
-
-## Key Takeaways
-
-- 🎯 **Parquet** - Best for large datasets, compressed, typed, fastest
-- 🎯 **XML** - Required for B2B systems, hierarchical data, namespaces
-- 🎯 **JSON** - Modern APIs, flexible, nested structures
-- 🎯 **Streaming** - Essential for memory-efficient processing of large files
-- 🎯 **UniversalMapper** - One mapping config works across ALL formats
-
-## Next Steps
-
-Continue to [Module 6: Batch API →](./02-core-guides-ingestion-06-batch-api.md) to learn about job creation, batch sending, and status monitoring.
-
----
-
-**Need help choosing a parser?**
-- 📊 Large datasets (> 1 GB) → Use Parquet with streaming
-- 🏢 B2B systems → Use XML with namespace handling
-- 🌐 REST APIs → Use JSON with schema validation
-- 📈 Simple exports → Use CSV (Module 3)
+# Module 5: Advanced Parsers
+
+**Level:** Intermediate
+**Estimated Time:** 25 minutes
+
+## Overview
+
+This module covers advanced file parsing capabilities beyond CSV, including Parquet for large datasets, XML for B2B systems, and JSON for API integrations.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Parse Parquet files for efficient large dataset processing
+- ✅ Parse XML files from B2B systems and feeds
+- ✅ Parse JSON from APIs and web services
+- ✅ Choose the optimal parser for your use case
+- ✅ Handle streaming for memory-efficient processing
+
+## Parquet Parser
+
+### Why Parquet?
+
+Parquet is a columnar storage format ideal for:
+
+| Feature | Benefit |
+|---------|---------|
+| **Compression** | 5-10x smaller than CSV |
+| **Speed** | Faster reads for analytics |
+| **Schema** | Built-in type validation |
+| **Large Files** | Efficient memory usage |
+
+### Basic Parquet Parsing
+
+```typescript
+import {
+  ParquetParserService,
+  S3DataSource,
+  UniversalMapper,
+  createConsoleLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function parseParquetFile() {
+  // Initialize services
+  const logger = createConsoleLogger();
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_PARQUET',
+      connectionId: 's3-parquet',
+      name: 'S3 Parquet',
+      s3Config: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+        bucket: 'inventory-bucket'
+      }
+    },
+    logger
+  );
+
+  const parquetParser = new ParquetParserService(logger);
+
+  // Read Parquet file from S3
+  const buffer = await s3.downloadFile('data/inventory-positions.parquet', { encoding: 'binary' }) as Buffer;
+
+  // Parse Parquet
+  const records = await parquetParser.parse(buffer);
+  console.log(`Parsed ${records.length} records from Parquet file`);
+
+  // Records are already typed based on Parquet schema
+  return records;
+}
+```
+
+### Streaming Large Parquet Files
+
+For files larger than available memory:
+
+```typescript
+import { ParquetParserService, S3DataSource, UniversalMapper, createConsoleLogger } from '@fluentcommerce/fc-connect-sdk';
+
+async function streamParquetFile() {
+  const logger = createConsoleLogger();
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_PARQUET',
+      connectionId: 's3-parquet-stream',
+      name: 'S3 Parquet Stream',
+      s3Config: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+        bucket: 'inventory-bucket'
+      }
+    },
+    logger
+  );
+  const parquetParser = new ParquetParserService(logger);
+
+  // Download Parquet file from S3 (full file download)
+  const buffer = await s3.downloadFile('data/large-inventory.parquet', { encoding: 'binary' }) as Buffer;
+
+  // Parse Parquet (parser handles large files efficiently)
+  const records = await parquetParser.parse(buffer);
+
+  // Process records in chunks
+  const batchSize = 1000;
+  for (let i = 0; i < records.length; i += batchSize) {
+    const batch = records.slice(i, i + batchSize);
+    await processBatch(batch);
+  }
+}
+
+async function processBatch(records: any[]) {
+  // Map fields
+  const mapper = new UniversalMapper(mappingConfig);
+  const result = await mapper.map(records);
+
+  if (result.success) {
+    // Send to Fluent
+    await sendToFluent(result.data);
+  }
+}
+```
+
+### Parquet Schema Validation
+
+```typescript
+import { ParquetParserService, createConsoleLogger } from '@fluentcommerce/fc-connect-sdk';
+
+async function validateParquetSchema(file: Buffer) {
+  const logger = createConsoleLogger();
+  const parquetParser = new ParquetParserService(logger);
+
+  // Extract schema information
+  const metadata = await parquetParser.getMetadata(file);
+
+  console.log('Parquet Schema:', {
+    numRows: metadata.numRows,
+    columns: metadata.schema.fields.map(f => ({
+      name: f.name,
+      type: f.type,
+      optional: f.optional
+    })),
+    compression: metadata.compression,
+    createdBy: metadata.createdBy
+  });
+
+  // Validate expected columns exist
+  const requiredColumns = ['sku', 'location', 'quantity'];
+  const schemaColumns = metadata.schema.fields.map(f => f.name);
+
+  const missingColumns = requiredColumns.filter(
+    col => !schemaColumns.includes(col)
+  );
+
+  if (missingColumns.length > 0) {
+    throw new Error(`Missing required columns: ${missingColumns.join(', ')}`);
+  }
+
+  return metadata;
+}
+```
+
+## XML Parser
+
+### Why XML?
+
+XML parsing is essential for:
+
+| Use Case | Example |
+|----------|---------|
+| **Enterprise Systems** | ERP exports, warehouse management systems |
+| **Standard Formats** | EDI, SOAP, RSS feeds |
+| **Nested Data** | Complex hierarchical structures |
+| **SFCC Orders** | Salesforce Commerce Cloud integrations |
+
+### Basic XML Parsing
+
+```typescript
+import { XMLParserService } from '@fluentcommerce/fc-connect-sdk';
+
+async function parseXMLFile() {
+  const xmlParser = new XMLParserService();
+
+  const xmlData = `<?xml version="1.0" encoding="UTF-8"?>
+<inventory>
+  <position>
+    <sku>SKU-WM-001</sku>
+    <location>WH-001</location>
+    <quantity>500</quantity>
+    <status>available</status>
+  </position>
+  <position>
+    <sku>SKU-WM-002</sku>
+    <location>WH-001</location>
+    <quantity>150</quantity>
+    <status>reserved</status>
+  </position>
+</inventory>`;
+
+  // Parse XML to JavaScript object
+  const parsed = await xmlParser.parse(xmlData);
+
+  // Access nested data
+  const positions = parsed.inventory.position;
+
+  console.log(`Parsed ${positions.length} positions from XML`);
+  return positions;
+}
+```
+
+### XML with Path Resolution
+
+For complex XML structures, use path-based field mapping:
+
+```typescript
+async function parseComplexXML() {
+  const xmlParser = new XMLParserService();
+
+  const complexXML = `<?xml version="1.0" encoding="UTF-8"?>
+<warehouse-export>
+  <metadata>
+    <export-date>2024-01-15</export-date>
+    <warehouse>WH-001</warehouse>
+  </metadata>
+  <inventory-positions>
+    <position id="1">
+      <product>
+        <sku>SKU-WM-001</sku>
+        <name>Widget Master 3000</name>
+      </product>
+      <location>
+        <zone>A</zone>
+        <aisle>12</aisle>
+        <bin>45</bin>
+      </location>
+      <quantities>
+        <on-hand>500</on-hand>
+        <available>450</available>
+        <reserved>50</reserved>
+      </quantities>
+      <status>AVAILABLE</status>
+    </position>
+  </inventory-positions>
+</warehouse-export>`;
+
+  const parsed = await xmlParser.parse(complexXML);
+
+  // Extract with path notation
+  const mapper = new UniversalMapper({
+    fields: {
+      ref: {
+        source: 'product.sku',
+        required: true
+      },
+      productRef: {
+        source: 'product.sku',
+        required: true
+      },
+      locationRef: {
+        source: 'warehouse', // From root metadata
+        resolver: (value, data) => {
+          // Combine location parts
+          return `${data.location.zone}-${data.location.aisle}-${data.location.bin}`;
+        },
+        required: true
+      },
+      qty: {
+        source: 'quantities.on-hand',
+        resolver: 'sdk.parseInt',
+        required: true
+      },
+      status: {
+        source: 'status',
+        resolver: 'sdk.uppercase',
+        required: true
+      }
+    }
+  });
+
+  // Map each position
+  const positions = parsed['warehouse-export']['inventory-positions'].position;
+  const result = await mapper.map(positions);
+
+  return result.data;
+}
+```
+
+### XML Namespace Handling
+
+```typescript
+async function parseXMLWithNamespaces() {
+  const xmlParser = new XMLParserService({
+    ignoreAttributes: false,
+    attributeNamePrefix: '@_',
+    parseAttributeValue: true
+  });
+
+  const nsXML = `<?xml version="1.0" encoding="UTF-8"?>
+<inv:inventory xmlns:inv="http://example.com/inventory" xmlns:prod="http://example.com/product">
+  <inv:position inv:id="1">
+    <prod:sku>SKU-WM-001</prod:sku>
+    <inv:location>WH-001</inv:location>
+    <inv:quantity>500</inv:quantity>
+  </inv:position>
+</inv:inventory>`;
+
+  const parsed = await xmlParser.parse(nsXML);
+
+  // Access namespaced elements
+  console.log('Parsed with namespaces:', parsed);
+  return parsed;
+}
+```
+
+## JSON Parser
+
+### Why JSON?
+
+JSON parsing is ideal for:
+
+| Use Case | Example |
+|----------|---------|
+| **REST APIs** | Third-party inventory services |
+| **Web Services** | Cloud-based systems |
+| **Modern Exports** | Contemporary systems |
+| **Nested Objects** | Complex data structures |
+
+### Basic JSON Parsing
+
+```typescript
+import { JSONParserService } from '@fluentcommerce/fc-connect-sdk';
+
+async function parseJSONFile() {
+  const jsonParser = new JSONParserService();
+
+  const jsonData = {
+    "metadata": {
+      "export_date": "2024-01-15",
+      "total_records": 2
+    },
+    "inventory_positions": [
+      {
+        "sku": "SKU-WM-001",
+        "location": "WH-001",
+        "quantity": 500,
+        "status": "available"
+      },
+      {
+        "sku": "SKU-WM-002",
+        "location": "WH-001",
+        "quantity": 150,
+        "status": "reserved"
+      }
+    ]
+  };
+
+  // Parse JSON (validates structure)
+  const parsed = await jsonParser.parse(JSON.stringify(jsonData));
+
+  const positions = parsed.inventory_positions;
+  console.log(`Parsed ${positions.length} positions from JSON`);
+
+  return positions;
+}
+```
+
+### JSON Array Streaming
+
+For large JSON arrays, use streaming to avoid memory issues:
+
+```typescript
+import { S3DataSource, JSONParserService, createConsoleLogger } from '@fluentcommerce/fc-connect-sdk';
+
+async function streamLargeJSONArray() {
+  const logger = createConsoleLogger();
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_JSON',
+      connectionId: 's3-json-stream',
+      name: 'S3 JSON Stream',
+      s3Config: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+        bucket: 'inventory-bucket'
+      }
+    },
+    logger
+  );
+  const jsonParser = new JSONParserService();
+
+  // Download JSON file from S3
+  const jsonContent = await s3.downloadFile('data/large-inventory.json') as string;
+  
+  // Parse JSON array
+  const parsed = await jsonParser.parse(jsonContent);
+  const positions = parsed.inventory_positions || [];
+
+  // Process records in batches
+  const batchSize = 1000;
+  let count = 0;
+  for (let i = 0; i < positions.length; i += batchSize) {
+    const batch = positions.slice(i, i + batchSize);
+    for (const record of batch) {
+      await processRecord(record);
+      count++;
+    }
+
+    if (count % 1000 === 0) {
+      console.log(`Processed ${count} records...`);
+    }
+  }
+
+  console.log(`Total records processed: ${count}`);
+}
+```
+
+### JSON Schema Validation
+
+```typescript
+async function validateJSONSchema(data: any) {
+  const schema = {
+    type: 'object',
+    required: ['inventory_positions'],
+    properties: {
+      metadata: {
+        type: 'object',
+        properties: {
+          export_date: { type: 'string' },
+          total_records: { type: 'number' }
+        }
+      },
+      inventory_positions: {
+        type: 'array',
+        items: {
+          type: 'object',
+          required: ['sku', 'location', 'quantity'],
+          properties: {
+            sku: { type: 'string' },
+            location: { type: 'string' },
+            quantity: { type: 'number' },
+            status: { type: 'string' }
+          }
+        }
+      }
+    }
+  };
+
+  const jsonParser = new JSONParserService({ schema });
+
+  // Validate and parse
+  const result = await jsonParser.parseWithValidation(data);
+
+  if (!result.valid) {
+    throw new Error(`JSON validation failed: ${result.errors.join(', ')}`);
+  }
+
+  return result.data;
+}
+```
+
+## Parser Comparison
+
+### When to Use Which Parser
+
+| Parser | Best For | Pros | Cons | File Size Limit |
+|--------|----------|------|------|-----------------|
+| **CSV** | Simple tabular data, exports, integrations | Universal, easy to inspect | No schema, flat structure | < 1 GB with streaming |
+| **Parquet** | Large datasets, analytics, data lakes | Fast, compressed, typed | Binary, needs tools | Unlimited with streaming |
+| **XML** | Enterprise systems, SOAP, hierarchical data | Structured, validated, nested | Verbose, complex parsing | < 500 MB |
+| **JSON** | REST APIs, modern systems, web services | Flexible, nested, readable | Can be large, no schema | < 500 MB with streaming |
+
+### Performance Comparison
+
+```typescript
+async function benchmarkParsers() {
+  const testData = generateTestData(10000); // 10K records
+
+  // CSV Performance
+  const csvStart = Date.now();
+  const csvParser = new CSVParserService();
+  const csvData = await csvParser.parse(toCSV(testData), { columns: true });
+  console.log(`CSV: ${Date.now() - csvStart}ms`);
+
+  // Parquet Performance
+  const parquetStart = Date.now();
+  const logger = createConsoleLogger();
+  const parquetParser = new ParquetParserService(logger);
+  const parquetData = await parquetParser.parse(toParquet(testData));
+  console.log(`Parquet: ${Date.now() - parquetStart}ms`);
+
+  // XML Performance
+  const xmlStart = Date.now();
+  const xmlParser = new XMLParserService();
+  const xmlData = await xmlParser.parse(toXML(testData));
+  console.log(`XML: ${Date.now() - xmlStart}ms`);
+
+  // JSON Performance
+  const jsonStart = Date.now();
+  const jsonParser = new JSONParserService();
+  const jsonData = await jsonParser.parse(toJSON(testData));
+  console.log(`JSON: ${Date.now() - jsonStart}ms`);
+}
+```
+
+## Complete Example: Multi-Format Ingestion
+
+```typescript
+import {
+  createClient,
+  S3DataSource,
+  CSVParserService,
+  ParquetParserService,
+  XMLParserService,
+  JSONParserService,
+  UniversalMapper,
+  createConsoleLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function ingestMultiFormat() {
+  const logger = createConsoleLogger();
+  
+  const client = await createClient({
+    config: {
+      baseUrl: process.env.FLUENT_BASE_URL!,
+      clientId: process.env.FLUENT_CLIENT_ID!,
+      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+      retailerId: process.env.FLUENT_RETAILER_ID!
+    }
+  });
+  
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      connectionId: 's3-multi-format',
+      name: 'S3 Multi-Format',
+      s3Config: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+        region: process.env.AWS_REGION!,
+        bucket: 'inventory-bucket'
+      }
+    },
+    logger
+  );
+  const mapper = new UniversalMapper(mappingConfig);
+
+  // List all files
+  const files = await s3.listFiles({ prefix: 'data/' });
+
+  for (const file of files) {
+    let records: any[];
+
+    // Choose parser based on file extension
+    if (file.name.endsWith('.csv')) {
+      const csvParser = new CSVParserService();
+      const data = await s3.downloadFile(file.path) as string;
+      records = await csvParser.parse(data);
+
+    } else if (file.name.endsWith('.parquet')) {
+      const parquetParser = new ParquetParserService(logger);
+      const data = await s3.downloadFile(file.path, { encoding: 'binary' }) as Buffer;
+      records = await parquetParser.parse(data);
+
+    } else if (file.name.endsWith('.xml')) {
+      const xmlParser = new XMLParserService(logger);
+      const data = await s3.downloadFile(file.path) as string;
+      const parsed = await xmlParser.parse(data);
+      records = extractInventoryFromXML(parsed);
+
+    } else if (file.name.endsWith('.json')) {
+      const jsonParser = new JSONParserService();
+      const data = await s3.downloadFile(file.path) as string;
+      const parsed = await jsonParser.parse(data);
+      records = parsed.inventory_positions || [];
+
+    } else {
+      console.log(`Skipping unsupported file: ${file.name}`);
+      continue;
+    }
+
+    // Transform with UniversalMapper
+    const result = await mapper.map(records);
+
+    if (!result.success) {
+      console.error(`Mapping failed for ${file.name}:`, result.errors);
+      continue;
+    }
+
+    // Send to Fluent
+    const job = await client.createJob({
+      name: `Import - ${file.name}`,
+      retailerId: process.env.FLUENT_RETAILER_ID!
+    });
+
+    await client.sendBatch(job.id, {
+      action: 'UPSERT',
+      entityType: 'INVENTORY',
+      source: 'S3_CSV',
+      event: 'INVENTORY_SYNC',
+      entities: result.data
+    });
+
+    console.log(`Processed ${result.data.length} records from ${file.name}`);
+  }
+}
+
+function extractInventoryFromXML(parsed: any): any[] {
+  // Extract inventory positions from XML structure
+  const positions = parsed.inventory?.position || [];
+  return Array.isArray(positions) ? positions : [positions];
+}
+```
+
+## Practice Exercise
+
+### Challenge
+
+Parse a multi-format dataset and ingest into Fluent Commerce:
+
+1. Download sample files (CSV, Parquet, XML, JSON) from S3
+2. Parse each format using the appropriate parser
+3. Validate data structure and required fields
+4. Transform using UniversalMapper
+5. Send to Fluent via Batch API
+6. Track processing metrics (time, records, errors)
+
+### Solution Template
+
+```typescript
+async function multiFormatIngestionChallenge() {
+  const metrics = {
+    totalFiles: 0,
+    totalRecords: 0,
+    successfulFiles: 0,
+    failedFiles: 0,
+    processingTime: 0
+  };
+
+  const startTime = Date.now();
+
+  try {
+    // Your implementation here
+    // 1. List files
+    // 2. Parse by format
+    // 3. Transform data
+    // 4. Send to Fluent
+    // 5. Track metrics
+
+  } finally {
+    metrics.processingTime = Date.now() - startTime;
+    console.log('Ingestion Metrics:', metrics);
+  }
+}
+```
+
+## Key Takeaways
+
+- 🎯 **Parquet** - Best for large datasets, compressed, typed, fastest
+- 🎯 **XML** - Required for B2B systems, hierarchical data, namespaces
+- 🎯 **JSON** - Modern APIs, flexible, nested structures
+- 🎯 **Streaming** - Essential for memory-efficient processing of large files
+- 🎯 **UniversalMapper** - One mapping config works across ALL formats
+
+## Next Steps
+
+Continue to [Module 6: Batch API →](./02-core-guides-ingestion-06-batch-api.md) to learn about job creation, batch sending, and status monitoring.
+
+---
+
+**Need help choosing a parser?**
+- 📊 Large datasets (> 1 GB) → Use Parquet with streaming
+- 🏢 B2B systems → Use XML with namespace handling
+- 🌐 REST APIs → Use JSON with schema validation
+- 📈 Simple exports → Use CSV (Module 3)