@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/04-REFERENCE/testing/modules/04-reference-testing-04-integration-testing.md

@@ -1,699 +1,699 @@
-# Module 4: Integration Testing
-
-**Level:** Intermediate
-**Estimated Time:** 35 minutes
-
-## Overview
-
-This module covers end-to-end integration testing, combining multiple testers to validate complete data pipelines. You'll learn how to test ingestion workflows, measure performance, and ensure reliability.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Understand how to build end-to-end integration tests
-- ✅ Know how to test complete ingestion pipelines
-- ✅ Be able to measure and optimize performance
-- ✅ Understand load testing patterns
-
-## End-to-End Ingestion Test
-
-Test a complete ingestion pipeline from S3 to Fluent Commerce:
-
-```typescript
-import {
-  S3SDKTester,
-  FluentConnectionTester,
-  createClient,
-  S3DataSource,
-  CSVParserService,
-  UniversalMapper
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function testIngestionPipeline() {
-  console.log('Starting end-to-end ingestion test...');
-
-  // Step 1: Test S3 connectivity
-  console.log('Step 1: Testing S3 connectivity...');
-  const s3Tester = new S3SDKTester({
-    region: 'us-east-1',
-    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
-  });
-
-  const s3Result = await s3Tester.testAll('test-data');
-  if (!s3Result.summary.passed) {
-    throw new Error('S3 connectivity test failed');
-  }
-  console.log('✅ S3 connectivity validated');
-
-  // Step 2: Test Fluent connectivity
-  console.log('Step 2: Testing Fluent API connectivity...');
-  const client = await createClient({
-    config: {
-      baseUrl: process.env.FLUENT_BASE_URL!,
-      clientId: process.env.FLUENT_CLIENT_ID!,
-      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-      username: process.env.FLUENT_USERNAME!,
-      password: process.env.FLUENT_PASSWORD!,
-      retailerId: '1'
-    }
-  });
-
-  const fluentTester = new FluentConnectionTester(client, {
-    logger: console
-  });
-
-  const fluentResult = await fluentTester.testConnection();
-  if (!fluentResult.success) {
-    throw new Error('Fluent connectivity test failed');
-  }
-  console.log('✅ Fluent API connectivity validated');
-
-  // Step 3: Upload test data to S3 (done via S3DataSource below)
-  console.log('Step 3: Uploading test inventory data...');
-  const testData = 'SKU,QTY,LOCATION\nTEST001,100,DC01\nTEST002,50,DC02\nTEST003,75,DC01';
-
-  // Step 4: Run ingestion using building blocks
-  console.log('Step 4: Running ingestion with SDK building blocks...');
-  const startTime = Date.now();
-
-  // Initialize components
-  const s3 = new S3DataSource({
-    type: 'S3_CSV',
-    connectionId: 'test-s3',
-    name: 'Test S3',
-    s3Config: {
-      bucket: 'test-data',
-      region: 'us-east-1',
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
-    }
-  });
-
-  const parser = new CSVParserService();
-  const mapper = new UniversalMapper({
-    fields: {
-      ref: { source: 'SKU', required: true },
-      qty: { source: 'QTY', resolver: 'sdk.parseInt' },
-      locationRef: { source: 'LOCATION', required: true }
-    }
-  });
-
-  // Download and parse
-  const csvContent = await s3.downloadFile('test-inventory.csv');
-  const records = await parser.parse(csvContent);
-
-  // Map records
-  const inventory = [];
-  const errors = [];
-  for (const record of records) {
-    const result = await mapper.map(record);
-    if (result.success) {
-      inventory.push(result.data);
-    } else {
-      errors.push({ record, errors: result.errors });
-    }
-  }
-
-  // Create job and send batch
-  const job = await client.createJob({
-    name: 'Test Inventory Ingestion',
-    retailerId: '1'
-  });
-
-  await client.sendBatch(job.id, {
-    action: 'UPSERT',
-    entityType: 'INVENTORY',
-    entities: inventory
-  });
-
-  const duration = Date.now() - startTime;
-
-  // Step 5: Verify results
-  console.log('Step 5: Verifying ingestion results...');
-  const ingestionResult = {
-    success: errors.length === 0,
-    processedCount: inventory.length,
-    errors,
-    duration
-  };
-
-  if (ingestionResult.success) {
-    console.log('✅ End-to-end test passed!');
-    console.log(`   Records processed: ${ingestionResult.processedCount}`);
-    console.log(`   Duration: ${ingestionResult.duration}ms`);
-  } else {
-    console.error('❌ Ingestion had errors:', ingestionResult.errors);
-  }
-
-  // Step 6: Cleanup (handled by S3 service)
-  console.log('Step 6: Test complete');
-
-  return ingestionResult;
-}
-
-// Run test
-testIngestionPipeline()
-  .then(result => {
-    console.log('Integration test completed:', result.success ? '✅' : ');
-    process.exit(result.success ? 0 : 1);
-  })
-  .catch(error => {
-    console.error('Integration test failed:', error);
-    process.exit(1);
-  });
-```
-
-## Testing Different Data Flows
-
-### Test CSV Ingestion
-
-```typescript
-async function testCSVIngestion() {
-  // Create test CSV data
-  const csvData = `
-sku,quantity,location,status
-SKU001,100,WH001,available
-SKU002,50,WH001,available
-SKU003,75,WH002,available
-  `.trim();
-
-  // Ingest using building blocks
-  const s3 = new S3DataSource({
-    type: 'S3_CSV',
-    connectionId: 'test-s3',
-    name: 'Test S3',
-    s3Config: { bucket: 'test-bucket', ...s3Credentials }
-  });
-
-  const parser = new CSVParserService();
-  const mapper = new UniversalMapper({
-    fields: {
-      ref: { source: 'sku', required: true },
-      qty: { source: 'quantity', resolver: 'sdk.parseInt' },
-      locationRef: { source: 'location', required: true },
-      status: { source: 'status' }
-    }
-  });
-
-  const content = await s3.downloadFile('inventory.csv');
-  const records = await parser.parse(content);
-
-  const inventory = [];
-  for (const record of records) {
-    const result = await mapper.map(record);
-    if (result.success) inventory.push(result.data);
-  }
-
-  const job = await client.createJob({ name: 'Test CSV', retailerId: '1' });
-  await client.sendBatch(job.id, {
-    action: 'UPSERT',
-    entityType: 'INVENTORY',
-    entities: inventory
-  });
-
-  // Verify
-  const result = {
-    success: inventory.length > 0,
-    processedCount: inventory.length
-  };
-  console.log(`CSV Ingestion: ${result.success ? '✅' : '}`);
-  console.log(`Records: ${result.processedCount}`);
-
-  return result;
-}
-```
-
-### Test Parquet Extraction
-
-```typescript
-async function testParquetExtraction() {
-  // Query inventory from Fluent
-  const query = `
-    query GetInventory($first: Int!, $after: String) {
-      inventoryPositions(first: $first, after: $after) {
-        edges {
-          node {
-            id
-            ref
-            productRef
-            locationRef
-            qty
-          }
-          cursor
-        }
-        pageInfo {
-          hasNextPage
-        }
-      }
-    }
-  `;
-
-  const result = await client.graphql({
-    query,
-    variables: { first: 100 },
-    pagination: { maxRecords: 1000 }
-  });
-
-  // Transform to Parquet
-  const logger = createConsoleLogger();
-  const parquetService = new ParquetParserService(logger);
-  const parquetBuffer = await parquetService.arrayToParquet(
-    result.data.inventoryPositions.edges.map(e => e.node)
-  );
-
-  // Upload to S3
-  const s3 = new S3DataSource(s3Config);
-  await s3.putObject('output-bucket', 'inventory.parquet', parquetBuffer);
-
-  // Verify result
-  console.log(`Parquet Extraction: ${parquetBuffer ? '✅' : '}`);
-
-  return !!parquetBuffer;
-}
-```
-
-## Performance Testing
-
-### Load Testing S3 Operations
-
-Test S3 performance under concurrent load:
-
-```typescript
-async function loadTestS3(concurrency: number = 10) {
-  const s3 = new S3DataSource({
-    type: 'S3_CSV',
-    connectionId: 'test-s3',
-    name: 'Load Test S3',
-    s3Config: {
-      region: 'us-east-1',
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
-    }
-  });
-
-  const startTime = Date.now();
-  const promises = [];
-
-  // Run concurrent write operations
-  for (let i = 0; i < concurrency; i++) {
-    promises.push(
-      s3.putObject('load-test', `test-${i}.txt`, Buffer.from(`Content ${i}`))
-    );
-  }
-
-  const results = await Promise.allSettled(promises);
-  const duration = Date.now() - startTime;
-
-  const successCount = results.filter(r => r.status === 'fulfilled').length;
-  const failureCount = results.filter(r => r.status === 'rejected').length;
-
-  console.log('Load Test Results:');
-  console.log('  Concurrency:', concurrency);
-  console.log('  Duration:', duration, 'ms');
-  console.log('  Success:', successCount);
-  console.log('  Failures:', failureCount);
-  console.log('  Avg time per operation:', duration / concurrency, 'ms');
-  console.log('  Operations per second:', (concurrency / (duration / 1000)).toFixed(2));
-
-  return {
-    concurrency,
-    duration,
-    successCount,
-    failureCount,
-    avgTimePerOperation: duration / concurrency,
-    operationsPerSecond: concurrency / (duration / 1000)
-  };
-}
-
-// Test with increasing load
-async function performanceTest() {
-  const loads = [1, 5, 10, 20, 50];
-  const results = [];
-
-  for (const load of loads) {
-    console.log(`\nTesting with ${load} concurrent operations...`);
-    const result = await loadTestS3(load);
-    results.push(result);
-
-    // Wait between tests
-    await new Promise(resolve => setTimeout(resolve, 2000));
-  }
-
-  // Analyze results
-  console.log('\nPerformance Analysis:');
-  results.forEach(r => {
-    console.log(`Load ${r.concurrency}:`);
-    console.log(`  Avg time: ${r.avgTimePerOperation.toFixed(2)}ms`);
-    console.log(`  Throughput: ${r.operationsPerSecond.toFixed(2)} ops/sec`);
-  });
-
-  // Identify performance degradation
-  const baseline = results[0];
-  results.forEach((r, i) => {
-    if (i === 0) return;
-    const degradation = ((r.avgTimePerOperation - baseline.avgTimePerOperation) / baseline.avgTimePerOperation * 100).toFixed(2);
-    console.log(`Load ${r.concurrency} degradation: ${degradation}% slower than baseline`);
-  });
-}
-```
-
-### Load Testing Fluent API
-
-```typescript
-async function loadTestFluentAPI(concurrency: number = 10) {
-  const client = await createClient({ config: fluentConfig });
-  const startTime = Date.now();
-  const promises = [];
-
-  // Run concurrent GraphQL queries
-  const query = `
-    query GetInventoryPosition($ref: String!) {
-      inventoryPosition(ref: $ref) {
-        id
-        ref
-        qty
-      }
-    }
-  `;
-
-  for (let i = 0; i < concurrency; i++) {
-    promises.push(
-      client.graphql({
-        query,
-        variables: { ref: `TEST${i}` }
-      })
-    );
-  }
-
-  const results = await Promise.all(promises.map(p =>
-    p.then(() => ({ success: true }))
-     .catch(error => ({ success: false, error }))
-  ));
-
-  const duration = Date.now() - startTime;
-  const successCount = results.filter(r => r.success).length;
-  const failureCount = results.filter(r => !r.success).length;
-
-  console.log('Fluent API Load Test:');
-  console.log('  Concurrency:', concurrency);
-  console.log('  Duration:', duration, 'ms');
-  console.log('  Success:', successCount);
-  console.log('  Failures:', failureCount);
-  console.log('  Avg response time:', duration / concurrency, 'ms');
-
-  return {
-    concurrency,
-    duration,
-    successCount,
-    failureCount,
-    avgResponseTime: duration / concurrency
-  };
-}
-```
-
-## Reliability Testing
-
-### Retry Logic Testing
-
-Test error recovery and retry mechanisms:
-
-```typescript
-async function testRetryLogic() {
-  let attemptCount = 0;
-  const maxAttempts = 3;
-
-  async function operationWithRetry() {
-    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-      try {
-        attemptCount++;
-        console.log(`Attempt ${attempt}/${maxAttempts}...`);
-
-        // Simulate operation
-        const result = await client.graphql({ query });
-
-        console.log(`✅ Succeeded on attempt ${attempt}`);
-        return result;
-      } catch (error) {
-        if (attempt === maxAttempts) {
-          console.error(`❌ Failed after ${maxAttempts} attempts`);
-          throw error;
-        }
-
-        const backoff = Math.pow(2, attempt) * 1000;
-        console.log(`⚠️ Attempt ${attempt} failed, retrying in ${backoff}ms...`);
-        await new Promise(resolve => setTimeout(resolve, backoff));
-      }
-    }
-  }
-
-  try {
-    await operationWithRetry();
-    console.log(`Total attempts: ${attemptCount}`);
-    return true;
-  } catch (error) {
-    console.error('Operation failed after retries:', error);
-    return false;
-  }
-}
-```
-
-### Circuit Breaker Testing
-
-Test circuit breaker patterns:
-
-```typescript
-class CircuitBreaker {
-  private failures = 0;
-  private lastFailureTime = 0;
-  private state: 'CLOSED' | 'OPEN' | 'HALF_OPEN' = 'CLOSED';
-
-  constructor(
-    private threshold: number = 5,
-    private timeout: number = 60000
-  ) {}
-
-  async execute<T>(operation: () => Promise<T>): Promise<T> {
-    if (this.state === 'OPEN') {
-      if (Date.now() - this.lastFailureTime > this.timeout) {
-        this.state = 'HALF_OPEN';
-        console.log('Circuit breaker: HALF_OPEN');
-      } else {
-        throw new Error('Circuit breaker is OPEN');
-      }
-    }
-
-    try {
-      const result = await operation();
-      this.onSuccess();
-      return result;
-    } catch (error) {
-      this.onFailure();
-      throw error;
-    }
-  }
-
-  private onSuccess() {
-    this.failures = 0;
-    if (this.state === 'HALF_OPEN') {
-      this.state = 'CLOSED';
-      console.log('Circuit breaker: CLOSED');
-    }
-  }
-
-  private onFailure() {
-    this.failures++;
-    this.lastFailureTime = Date.now();
-
-    if (this.failures >= this.threshold) {
-      this.state = 'OPEN';
-      console.log('Circuit breaker: OPEN');
-    }
-  }
-
-  getState() {
-    return this.state;
-  }
-}
-
-// Test circuit breaker
-async function testCircuitBreaker() {
-  const breaker = new CircuitBreaker(3, 5000);
-
-  // Simulate failures
-  for (let i = 0; i < 5; i++) {
-    try {
-      await breaker.execute(async () => {
-        throw new Error('Simulated failure');
-      });
-    } catch (error) {
-      console.log(`Request ${i + 1}:`, error.message);
-    }
-  }
-
-  console.log('Circuit breaker state:', breaker.getState());
-}
-```
-
-## Complete Integration Test Suite
-
-```typescript
-import {
-  S3SDKTester,
-  FluentConnectionTester,
-  createClient
-} from '@fluentcommerce/fc-connect-sdk';
-
-class IntegrationTestSuite {
-  private s3Tester: S3SDKTester;
-  private fluentTester: FluentConnectionTester;
-  private client: any;
-
-  async setup() {
-    console.log('Setting up integration test suite...');
-
-    // Initialize S3 tester
-    this.s3Tester = new S3SDKTester({
-      region: process.env.AWS_REGION!,
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
-    });
-
-    // Initialize Fluent client
-    this.client = await createClient({
-      config: {
-        baseUrl: process.env.FLUENT_BASE_URL!,
-        clientId: process.env.FLUENT_CLIENT_ID!,
-        clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-        username: process.env.FLUENT_USERNAME!,
-        password: process.env.FLUENT_PASSWORD!,
-        retailerId: process.env.FLUENT_RETAILER_ID!
-      }
-    });
-
-    // Initialize Fluent tester
-    this.fluentTester = new FluentConnectionTester(this.client, {
-      logger: console
-    });
-
-    console.log('✅ Setup complete');
-  }
-
-  async testConnectivity() {
-    console.log('\n--- Testing Connectivity ---');
-
-    const s3Result = await this.s3Tester.testAll(process.env.TEST_BUCKET!);
-    const fluentResult = await this.fluentTester.testConnection();
-
-    return {
-      s3: s3Result.summary.passed,
-      fluent: fluentResult.success,
-      overall: s3Result.summary.passed && fluentResult.success
-    };
-  }
-
-  async testIngestion() {
-    console.log('\n--- Testing Ingestion ---');
-
-    // TODO: Implement ingestion test with S3DataSource
-    // Use S3DataSource to upload, parse, and ingest test data
-    // Similar to testIngestionPipeline example above
-
-    return { success: true };
-  }
-
-  async testPerformance() {
-    console.log('\n--- Testing Performance ---');
-
-    const s3Performance = await loadTestS3(10);
-    const fluentPerformance = await loadTestFluentAPI(10);
-
-    return {
-      s3: s3Performance,
-      fluent: fluentPerformance
-    };
-  }
-
-  async cleanup() {
-    console.log('\n--- Cleanup ---');
-    // Cleanup handled by S3 service when operations complete
-    console.log('✅ Cleanup complete');
-  }
-
-  async run() {
-    try {
-      await this.setup();
-
-      const connectivity = await this.testConnectivity();
-      console.log('Connectivity:', connectivity.overall ? '✅' : ');
-
-      const ingestion = await this.testIngestion();
-      console.log('Ingestion:', ingestion.success ? '✅' : ');
-
-      const performance = await this.testPerformance();
-      console.log('Performance tests complete');
-
-      await this.cleanup();
-
-      return {
-        connectivity,
-        ingestion,
-        performance
-      };
-    } catch (error) {
-      console.error('Test suite failed:', error);
-      await this.cleanup();
-      throw error;
-    }
-  }
-}
-
-// Run test suite
-const suite = new IntegrationTestSuite();
-suite.run()
-  .then(results => {
-    console.log('\n=== Test Suite Results ===');
-    console.log(JSON.stringify(results, null, 2));
-  })
-  .catch(error => {
-    console.error('Test suite error:', error);
-    process.exit(1);
-  });
-```
-
-## Key Takeaways
-
-- 🎯 Integration tests validate complete data pipelines
-- 🎯 Always test connectivity before running integration tests
-- 🎯 Performance testing helps identify bottlenecks
-- 🎯 Load testing reveals scalability limits
-- 🎯 Reliability testing ensures error recovery works
-
-## Practice Exercise
-
-Build an integration test for your specific use case:
-
-```typescript
-async function myIntegrationTest() {
-  // TODO: Test S3 connectivity
-  // TODO: Test Fluent connectivity
-  // TODO: Upload test data
-  // TODO: Run ingestion
-  // TODO: Verify results
-  // TODO: Cleanup
-}
-```
-
-**Solution:** See [examples/integration-test.ts](../examples/integration-test.ts)
-
-## Next Steps
-
-Continue to [Module 5: Debugging →](./04-reference-testing-05-debugging.md) to learn debugging techniques.
-
----
-
-**Module Navigation:**
-- **Previous:** [Module 3 - Fluent Testing](./04-reference-testing-03-fluent-testing.md)
-- **Current:** Module 4 - Integration Testing
-- **Next:** [Module 5 - Debugging](./04-reference-testing-05-debugging.md)
-- **Back to:** [Testing Guide Index](../testing-readme.md)
+# Module 4: Integration Testing
+
+**Level:** Intermediate
+**Estimated Time:** 35 minutes
+
+## Overview
+
+This module covers end-to-end integration testing, combining multiple testers to validate complete data pipelines. You'll learn how to test ingestion workflows, measure performance, and ensure reliability.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Understand how to build end-to-end integration tests
+- ✅ Know how to test complete ingestion pipelines
+- ✅ Be able to measure and optimize performance
+- ✅ Understand load testing patterns
+
+## End-to-End Ingestion Test
+
+Test a complete ingestion pipeline from S3 to Fluent Commerce:
+
+```typescript
+import {
+  S3SDKTester,
+  FluentConnectionTester,
+  createClient,
+  S3DataSource,
+  CSVParserService,
+  UniversalMapper
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function testIngestionPipeline() {
+  console.log('Starting end-to-end ingestion test...');
+
+  // Step 1: Test S3 connectivity
+  console.log('Step 1: Testing S3 connectivity...');
+  const s3Tester = new S3SDKTester({
+    region: 'us-east-1',
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
+  });
+
+  const s3Result = await s3Tester.testAll('test-data');
+  if (!s3Result.summary.passed) {
+    throw new Error('S3 connectivity test failed');
+  }
+  console.log('✅ S3 connectivity validated');
+
+  // Step 2: Test Fluent connectivity
+  console.log('Step 2: Testing Fluent API connectivity...');
+  const client = await createClient({
+    config: {
+      baseUrl: process.env.FLUENT_BASE_URL!,
+      clientId: process.env.FLUENT_CLIENT_ID!,
+      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+      username: process.env.FLUENT_USERNAME!,
+      password: process.env.FLUENT_PASSWORD!,
+      retailerId: '1'
+    }
+  });
+
+  const fluentTester = new FluentConnectionTester(client, {
+    logger: console
+  });
+
+  const fluentResult = await fluentTester.testConnection();
+  if (!fluentResult.success) {
+    throw new Error('Fluent connectivity test failed');
+  }
+  console.log('✅ Fluent API connectivity validated');
+
+  // Step 3: Upload test data to S3 (done via S3DataSource below)
+  console.log('Step 3: Uploading test inventory data...');
+  const testData = 'SKU,QTY,LOCATION\nTEST001,100,DC01\nTEST002,50,DC02\nTEST003,75,DC01';
+
+  // Step 4: Run ingestion using building blocks
+  console.log('Step 4: Running ingestion with SDK building blocks...');
+  const startTime = Date.now();
+
+  // Initialize components
+  const s3 = new S3DataSource({
+    type: 'S3_CSV',
+    connectionId: 'test-s3',
+    name: 'Test S3',
+    s3Config: {
+      bucket: 'test-data',
+      region: 'us-east-1',
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
+    }
+  });
+
+  const parser = new CSVParserService();
+  const mapper = new UniversalMapper({
+    fields: {
+      ref: { source: 'SKU', required: true },
+      qty: { source: 'QTY', resolver: 'sdk.parseInt' },
+      locationRef: { source: 'LOCATION', required: true }
+    }
+  });
+
+  // Download and parse
+  const csvContent = await s3.downloadFile('test-inventory.csv');
+  const records = await parser.parse(csvContent);
+
+  // Map records
+  const inventory = [];
+  const errors = [];
+  for (const record of records) {
+    const result = await mapper.map(record);
+    if (result.success) {
+      inventory.push(result.data);
+    } else {
+      errors.push({ record, errors: result.errors });
+    }
+  }
+
+  // Create job and send batch
+  const job = await client.createJob({
+    name: 'Test Inventory Ingestion',
+    retailerId: '1'
+  });
+
+  await client.sendBatch(job.id, {
+    action: 'UPSERT',
+    entityType: 'INVENTORY',
+    entities: inventory
+  });
+
+  const duration = Date.now() - startTime;
+
+  // Step 5: Verify results
+  console.log('Step 5: Verifying ingestion results...');
+  const ingestionResult = {
+    success: errors.length === 0,
+    processedCount: inventory.length,
+    errors,
+    duration
+  };
+
+  if (ingestionResult.success) {
+    console.log('✅ End-to-end test passed!');
+    console.log(`   Records processed: ${ingestionResult.processedCount}`);
+    console.log(`   Duration: ${ingestionResult.duration}ms`);
+  } else {
+    console.error('❌ Ingestion had errors:', ingestionResult.errors);
+  }
+
+  // Step 6: Cleanup (handled by S3 service)
+  console.log('Step 6: Test complete');
+
+  return ingestionResult;
+}
+
+// Run test
+testIngestionPipeline()
+  .then(result => {
+    console.log('Integration test completed:', result.success ? '✅' : ');
+    process.exit(result.success ? 0 : 1);
+  })
+  .catch(error => {
+    console.error('Integration test failed:', error);
+    process.exit(1);
+  });
+```
+
+## Testing Different Data Flows
+
+### Test CSV Ingestion
+
+```typescript
+async function testCSVIngestion() {
+  // Create test CSV data
+  const csvData = `
+sku,quantity,location,status
+SKU001,100,WH001,available
+SKU002,50,WH001,available
+SKU003,75,WH002,available
+  `.trim();
+
+  // Ingest using building blocks
+  const s3 = new S3DataSource({
+    type: 'S3_CSV',
+    connectionId: 'test-s3',
+    name: 'Test S3',
+    s3Config: { bucket: 'test-bucket', ...s3Credentials }
+  });
+
+  const parser = new CSVParserService();
+  const mapper = new UniversalMapper({
+    fields: {
+      ref: { source: 'sku', required: true },
+      qty: { source: 'quantity', resolver: 'sdk.parseInt' },
+      locationRef: { source: 'location', required: true },
+      status: { source: 'status' }
+    }
+  });
+
+  const content = await s3.downloadFile('inventory.csv');
+  const records = await parser.parse(content);
+
+  const inventory = [];
+  for (const record of records) {
+    const result = await mapper.map(record);
+    if (result.success) inventory.push(result.data);
+  }
+
+  const job = await client.createJob({ name: 'Test CSV', retailerId: '1' });
+  await client.sendBatch(job.id, {
+    action: 'UPSERT',
+    entityType: 'INVENTORY',
+    entities: inventory
+  });
+
+  // Verify
+  const result = {
+    success: inventory.length > 0,
+    processedCount: inventory.length
+  };
+  console.log(`CSV Ingestion: ${result.success ? '✅' : '}`);
+  console.log(`Records: ${result.processedCount}`);
+
+  return result;
+}
+```
+
+### Test Parquet Extraction
+
+```typescript
+async function testParquetExtraction() {
+  // Query inventory from Fluent
+  const query = `
+    query GetInventory($first: Int!, $after: String) {
+      inventoryPositions(first: $first, after: $after) {
+        edges {
+          node {
+            id
+            ref
+            productRef
+            locationRef
+            qty
+          }
+          cursor
+        }
+        pageInfo {
+          hasNextPage
+        }
+      }
+    }
+  `;
+
+  const result = await client.graphql({
+    query,
+    variables: { first: 100 },
+    pagination: { maxRecords: 1000 }
+  });
+
+  // Transform to Parquet
+  const logger = createConsoleLogger();
+  const parquetService = new ParquetParserService(logger);
+  const parquetBuffer = await parquetService.arrayToParquet(
+    result.data.inventoryPositions.edges.map(e => e.node)
+  );
+
+  // Upload to S3
+  const s3 = new S3DataSource(s3Config);
+  await s3.putObject('output-bucket', 'inventory.parquet', parquetBuffer);
+
+  // Verify result
+  console.log(`Parquet Extraction: ${parquetBuffer ? '✅' : '}`);
+
+  return !!parquetBuffer;
+}
+```
+
+## Performance Testing
+
+### Load Testing S3 Operations
+
+Test S3 performance under concurrent load:
+
+```typescript
+async function loadTestS3(concurrency: number = 10) {
+  const s3 = new S3DataSource({
+    type: 'S3_CSV',
+    connectionId: 'test-s3',
+    name: 'Load Test S3',
+    s3Config: {
+      region: 'us-east-1',
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
+    }
+  });
+
+  const startTime = Date.now();
+  const promises = [];
+
+  // Run concurrent write operations
+  for (let i = 0; i < concurrency; i++) {
+    promises.push(
+      s3.putObject('load-test', `test-${i}.txt`, Buffer.from(`Content ${i}`))
+    );
+  }
+
+  const results = await Promise.allSettled(promises);
+  const duration = Date.now() - startTime;
+
+  const successCount = results.filter(r => r.status === 'fulfilled').length;
+  const failureCount = results.filter(r => r.status === 'rejected').length;
+
+  console.log('Load Test Results:');
+  console.log('  Concurrency:', concurrency);
+  console.log('  Duration:', duration, 'ms');
+  console.log('  Success:', successCount);
+  console.log('  Failures:', failureCount);
+  console.log('  Avg time per operation:', duration / concurrency, 'ms');
+  console.log('  Operations per second:', (concurrency / (duration / 1000)).toFixed(2));
+
+  return {
+    concurrency,
+    duration,
+    successCount,
+    failureCount,
+    avgTimePerOperation: duration / concurrency,
+    operationsPerSecond: concurrency / (duration / 1000)
+  };
+}
+
+// Test with increasing load
+async function performanceTest() {
+  const loads = [1, 5, 10, 20, 50];
+  const results = [];
+
+  for (const load of loads) {
+    console.log(`\nTesting with ${load} concurrent operations...`);
+    const result = await loadTestS3(load);
+    results.push(result);
+
+    // Wait between tests
+    await new Promise(resolve => setTimeout(resolve, 2000));
+  }
+
+  // Analyze results
+  console.log('\nPerformance Analysis:');
+  results.forEach(r => {
+    console.log(`Load ${r.concurrency}:`);
+    console.log(`  Avg time: ${r.avgTimePerOperation.toFixed(2)}ms`);
+    console.log(`  Throughput: ${r.operationsPerSecond.toFixed(2)} ops/sec`);
+  });
+
+  // Identify performance degradation
+  const baseline = results[0];
+  results.forEach((r, i) => {
+    if (i === 0) return;
+    const degradation = ((r.avgTimePerOperation - baseline.avgTimePerOperation) / baseline.avgTimePerOperation * 100).toFixed(2);
+    console.log(`Load ${r.concurrency} degradation: ${degradation}% slower than baseline`);
+  });
+}
+```
+
+### Load Testing Fluent API
+
+```typescript
+async function loadTestFluentAPI(concurrency: number = 10) {
+  const client = await createClient({ config: fluentConfig });
+  const startTime = Date.now();
+  const promises = [];
+
+  // Run concurrent GraphQL queries
+  const query = `
+    query GetInventoryPosition($ref: String!) {
+      inventoryPosition(ref: $ref) {
+        id
+        ref
+        qty
+      }
+    }
+  `;
+
+  for (let i = 0; i < concurrency; i++) {
+    promises.push(
+      client.graphql({
+        query,
+        variables: { ref: `TEST${i}` }
+      })
+    );
+  }
+
+  const results = await Promise.all(promises.map(p =>
+    p.then(() => ({ success: true }))
+     .catch(error => ({ success: false, error }))
+  ));
+
+  const duration = Date.now() - startTime;
+  const successCount = results.filter(r => r.success).length;
+  const failureCount = results.filter(r => !r.success).length;
+
+  console.log('Fluent API Load Test:');
+  console.log('  Concurrency:', concurrency);
+  console.log('  Duration:', duration, 'ms');
+  console.log('  Success:', successCount);
+  console.log('  Failures:', failureCount);
+  console.log('  Avg response time:', duration / concurrency, 'ms');
+
+  return {
+    concurrency,
+    duration,
+    successCount,
+    failureCount,
+    avgResponseTime: duration / concurrency
+  };
+}
+```
+
+## Reliability Testing
+
+### Retry Logic Testing
+
+Test error recovery and retry mechanisms:
+
+```typescript
+async function testRetryLogic() {
+  let attemptCount = 0;
+  const maxAttempts = 3;
+
+  async function operationWithRetry() {
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        attemptCount++;
+        console.log(`Attempt ${attempt}/${maxAttempts}...`);
+
+        // Simulate operation
+        const result = await client.graphql({ query });
+
+        console.log(`✅ Succeeded on attempt ${attempt}`);
+        return result;
+      } catch (error) {
+        if (attempt === maxAttempts) {
+          console.error(`❌ Failed after ${maxAttempts} attempts`);
+          throw error;
+        }
+
+        const backoff = Math.pow(2, attempt) * 1000;
+        console.log(`⚠️ Attempt ${attempt} failed, retrying in ${backoff}ms...`);
+        await new Promise(resolve => setTimeout(resolve, backoff));
+      }
+    }
+  }
+
+  try {
+    await operationWithRetry();
+    console.log(`Total attempts: ${attemptCount}`);
+    return true;
+  } catch (error) {
+    console.error('Operation failed after retries:', error);
+    return false;
+  }
+}
+```
+
+### Circuit Breaker Testing
+
+Test circuit breaker patterns:
+
+```typescript
+class CircuitBreaker {
+  private failures = 0;
+  private lastFailureTime = 0;
+  private state: 'CLOSED' | 'OPEN' | 'HALF_OPEN' = 'CLOSED';
+
+  constructor(
+    private threshold: number = 5,
+    private timeout: number = 60000
+  ) {}
+
+  async execute<T>(operation: () => Promise<T>): Promise<T> {
+    if (this.state === 'OPEN') {
+      if (Date.now() - this.lastFailureTime > this.timeout) {
+        this.state = 'HALF_OPEN';
+        console.log('Circuit breaker: HALF_OPEN');
+      } else {
+        throw new Error('Circuit breaker is OPEN');
+      }
+    }
+
+    try {
+      const result = await operation();
+      this.onSuccess();
+      return result;
+    } catch (error) {
+      this.onFailure();
+      throw error;
+    }
+  }
+
+  private onSuccess() {
+    this.failures = 0;
+    if (this.state === 'HALF_OPEN') {
+      this.state = 'CLOSED';
+      console.log('Circuit breaker: CLOSED');
+    }
+  }
+
+  private onFailure() {
+    this.failures++;
+    this.lastFailureTime = Date.now();
+
+    if (this.failures >= this.threshold) {
+      this.state = 'OPEN';
+      console.log('Circuit breaker: OPEN');
+    }
+  }
+
+  getState() {
+    return this.state;
+  }
+}
+
+// Test circuit breaker
+async function testCircuitBreaker() {
+  const breaker = new CircuitBreaker(3, 5000);
+
+  // Simulate failures
+  for (let i = 0; i < 5; i++) {
+    try {
+      await breaker.execute(async () => {
+        throw new Error('Simulated failure');
+      });
+    } catch (error) {
+      console.log(`Request ${i + 1}:`, error.message);
+    }
+  }
+
+  console.log('Circuit breaker state:', breaker.getState());
+}
+```
+
+## Complete Integration Test Suite
+
+```typescript
+import {
+  S3SDKTester,
+  FluentConnectionTester,
+  createClient
+} from '@fluentcommerce/fc-connect-sdk';
+
+class IntegrationTestSuite {
+  private s3Tester: S3SDKTester;
+  private fluentTester: FluentConnectionTester;
+  private client: any;
+
+  async setup() {
+    console.log('Setting up integration test suite...');
+
+    // Initialize S3 tester
+    this.s3Tester = new S3SDKTester({
+      region: process.env.AWS_REGION!,
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!
+    });
+
+    // Initialize Fluent client
+    this.client = await createClient({
+      config: {
+        baseUrl: process.env.FLUENT_BASE_URL!,
+        clientId: process.env.FLUENT_CLIENT_ID!,
+        clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+        username: process.env.FLUENT_USERNAME!,
+        password: process.env.FLUENT_PASSWORD!,
+        retailerId: process.env.FLUENT_RETAILER_ID!
+      }
+    });
+
+    // Initialize Fluent tester
+    this.fluentTester = new FluentConnectionTester(this.client, {
+      logger: console
+    });
+
+    console.log('✅ Setup complete');
+  }
+
+  async testConnectivity() {
+    console.log('\n--- Testing Connectivity ---');
+
+    const s3Result = await this.s3Tester.testAll(process.env.TEST_BUCKET!);
+    const fluentResult = await this.fluentTester.testConnection();
+
+    return {
+      s3: s3Result.summary.passed,
+      fluent: fluentResult.success,
+      overall: s3Result.summary.passed && fluentResult.success
+    };
+  }
+
+  async testIngestion() {
+    console.log('\n--- Testing Ingestion ---');
+
+    // TODO: Implement ingestion test with S3DataSource
+    // Use S3DataSource to upload, parse, and ingest test data
+    // Similar to testIngestionPipeline example above
+
+    return { success: true };
+  }
+
+  async testPerformance() {
+    console.log('\n--- Testing Performance ---');
+
+    const s3Performance = await loadTestS3(10);
+    const fluentPerformance = await loadTestFluentAPI(10);
+
+    return {
+      s3: s3Performance,
+      fluent: fluentPerformance
+    };
+  }
+
+  async cleanup() {
+    console.log('\n--- Cleanup ---');
+    // Cleanup handled by S3 service when operations complete
+    console.log('✅ Cleanup complete');
+  }
+
+  async run() {
+    try {
+      await this.setup();
+
+      const connectivity = await this.testConnectivity();
+      console.log('Connectivity:', connectivity.overall ? '✅' : ');
+
+      const ingestion = await this.testIngestion();
+      console.log('Ingestion:', ingestion.success ? '✅' : ');
+
+      const performance = await this.testPerformance();
+      console.log('Performance tests complete');
+
+      await this.cleanup();
+
+      return {
+        connectivity,
+        ingestion,
+        performance
+      };
+    } catch (error) {
+      console.error('Test suite failed:', error);
+      await this.cleanup();
+      throw error;
+    }
+  }
+}
+
+// Run test suite
+const suite = new IntegrationTestSuite();
+suite.run()
+  .then(results => {
+    console.log('\n=== Test Suite Results ===');
+    console.log(JSON.stringify(results, null, 2));
+  })
+  .catch(error => {
+    console.error('Test suite error:', error);
+    process.exit(1);
+  });
+```
+
+## Key Takeaways
+
+- 🎯 Integration tests validate complete data pipelines
+- 🎯 Always test connectivity before running integration tests
+- 🎯 Performance testing helps identify bottlenecks
+- 🎯 Load testing reveals scalability limits
+- 🎯 Reliability testing ensures error recovery works
+
+## Practice Exercise
+
+Build an integration test for your specific use case:
+
+```typescript
+async function myIntegrationTest() {
+  // TODO: Test S3 connectivity
+  // TODO: Test Fluent connectivity
+  // TODO: Upload test data
+  // TODO: Run ingestion
+  // TODO: Verify results
+  // TODO: Cleanup
+}
+```
+
+**Solution:** See [examples/integration-test.ts](../examples/integration-test.ts)
+
+## Next Steps
+
+Continue to [Module 5: Debugging →](./04-reference-testing-05-debugging.md) to learn debugging techniques.
+
+---
+
+**Module Navigation:**
+- **Previous:** [Module 3 - Fluent Testing](./04-reference-testing-03-fluent-testing.md)
+- **Current:** Module 4 - Integration Testing
+- **Next:** [Module 5 - Debugging](./04-reference-testing-05-debugging.md)
+- **Back to:** [Testing Guide Index](../testing-readme.md)