@fluentcommerce/fc-connect-sdk 0.1.53 → 0.1.55

package/docs/02-CORE-GUIDES/extraction/modules/02-core-guides-extraction-07-optimization.md

@@ -1,692 +1,692 @@
-# Module 7: Optimization & Best Practices
-
-**Level:** Advanced
-**Estimated Time:** 25 minutes
-
-## Overview
-
-This module covers performance optimization, caching strategies, error handling, monitoring, and production best practices for data extraction workflows.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Optimize GraphQL queries for performance
-- ✅ Implement caching strategies
-- ✅ Handle errors and implement retry logic
-- ✅ Monitor extraction performance
-- ✅ Implement delta (incremental) extraction
-- ✅ Apply production best practices
-
-## Query Optimization
-
-### Field Selection
-
-**❌ BAD: Fetch all fields**
-```typescript
-query GetProducts {
-  products(first: 100) {
-    edges {
-      node {
-        id ref name description gtin price
-        catalogue { id ref name description }
-        attributes { name value type }
-        categories { edges { node { name path parent { name } } } }
-        inventoryPositions { edges { node { qty locationRef } } }
-      }
-    }
-  }
-}
-```
-
-**✅ GOOD: Fetch only needed fields**
-```typescript
-query GetProducts {
-  products(first: 100) {
-    edges {
-      node {
-        ref
-        name
-        gtin
-      }
-    }
-  }
-}
-```
-
-### Page Size Tuning
-
-| Entity Complexity | Recommended Page Size | Rationale |
-|------------------|----------------------|-----------|
-| Simple (5-10 fields) | 100-200 | Small payload per record |
-| Medium (10-20 fields) | 50-100 | Moderate payload |
-| Complex (nested relations) | 20-50 | Large payload, avoid timeouts |
-| Very complex (deep nesting) | 10-20 | Prevent server timeouts |
-
-### GraphQL Fragments
-
-Reuse field selections:
-
-```typescript
-const fragments = `
-  fragment ProductFields on Product {
-    ref
-    name
-    gtin
-  }
-
-  fragment LocationFields on Location {
-    ref
-    name
-    type
-  }
-`;
-
-const query = `
-  ${fragments}
-
-  query ExtractInventory($first: Int!, $after: String) {
-    inventoryPositions(first: $first, after: $after) {
-      edges {
-        node {
-          ref
-          productRef
-          locationRef
-          qty
-          product { ...ProductFields }
-          location { ...LocationFields }
-        }
-        cursor
-      }
-      pageInfo { hasNextPage }
-    }
-  }
-`;
-```
-
-## Caching Strategies
-
-### Simple In-Memory Cache
-
-```typescript
-class ExtractionCache {
-  private cache = new Map<string, { data: any; expiry: number }>();
-  private ttl: number;
-
-  constructor(ttlMs = 3600000) { // 1 hour default
-    this.ttl = ttlMs;
-  }
-
-  get(key: string): any | null {
-    const cached = this.cache.get(key);
-
-    if (!cached) return null;
-
-    if (cached.expiry < Date.now()) {
-      this.cache.delete(key);
-      return null;
-    }
-
-    return cached.data;
-  }
-
-  set(key: string, data: any): void {
-    this.cache.set(key, {
-      data,
-      expiry: Date.now() + this.ttl
-    });
-  }
-
-  invalidate(pattern?: string): void {
-    if (pattern) {
-      for (const key of this.cache.keys()) {
-        if (key.includes(pattern)) {
-          this.cache.delete(key);
-        }
-      }
-    } else {
-      this.cache.clear();
-    }
-  }
-}
-
-// Usage
-const cache = new ExtractionCache(3600000);
-
-async function getProductWithCache(ref: string) {
-  const cacheKey = `product:${ref}`;
-  let product = cache.get(cacheKey);
-
-  if (!product) {
-    const result = await client.graphql({
-      query: `query { product(ref: "${ref}") { ref name gtin } }`
-    });
-    product = result.data.product;
-    cache.set(cacheKey, product);
-  }
-
-  return product;
-}
-```
-
-### Redis-Based Distributed Cache
-
-```typescript
-import { createClient as createRedisClient } from 'redis';
-
-class DistributedCache {
-  private redis: any;
-
-  async connect() {
-    this.redis = createRedisClient({
-      url: process.env.REDIS_URL
-    });
-    await this.redis.connect();
-  }
-
-  async get(key: string): Promise<any | null> {
-    const cached = await this.redis.get(key);
-    return cached ? JSON.parse(cached) : null;
-  }
-
-  async set(key: string, data: any, ttlSeconds = 3600): Promise<void> {
-    await this.redis.setEx(key, ttlSeconds, JSON.stringify(data));
-  }
-
-  async invalidate(pattern: string): Promise<void> {
-    const keys = await this.redis.keys(pattern);
-    if (keys.length > 0) {
-      await this.redis.del(keys);
-    }
-  }
-}
-```
-
-## Delta (Incremental) Extraction
-
-### State Management
-
-```typescript
-import { StateService, VersoriKVAdapter } from '@fluentcommerce/fc-connect-sdk';
-
-class DeltaExtractor {
-  constructor(
-    private client: any,
-    private stateService: StateService
-  ) {}
-
-  async extractDelta(entityType: string): Promise<any[]> {
-    const stateKey = `lastExtraction_${entityType}`;
-
-    // Get last extraction timestamp
-    const lastExtraction = await this.stateService.get(stateKey);
-    const fromDate = lastExtraction || this.getDefaultFromDate();
-
-    console.log(`Extracting ${entityType} changed since ${fromDate}`);
-
-    // Query only changed records
-    const result = await this.client.graphql({
-      query: this.buildDeltaQuery(entityType),
-      variables: {
-        fromDate,
-        toDate: new Date().toISOString()
-      },
-      pagination: { maxRecords: 50000 }
-    });
-
-    const records = this.extractRecords(result, entityType);
-
-    // Update state
-    await this.stateService.set(stateKey, new Date().toISOString());
-
-    return records;
-  }
-
-  private buildDeltaQuery(entityType: string): string {
-    const queries: Record<string, string> = {
-      inventory: `
-        query DeltaInventory($fromDate: DateTime!, $toDate: DateTime!) {
-          inventoryPositions(
-            updatedOn: { from: $fromDate, to: $toDate }
-            first: 100
-          ) {
-            edges {
-              node { ref productRef locationRef qty status updatedOn }
-              cursor
-            }
-            pageInfo { hasNextPage }
-          }
-        }
-      `,
-      products: `
-        query DeltaProducts($fromDate: DateTime!, $toDate: DateTime!) {
-          products(
-            updatedOn: { from: $fromDate, to: $toDate }
-            first: 100
-          ) {
-            edges {
-              node { ref name gtin updatedOn }
-              cursor
-            }
-            pageInfo { hasNextPage }
-          }
-        }
-      `
-    };
-
-    return queries[entityType] || queries.inventory;
-  }
-
-  private extractRecords(result: any, entityType: string): any[] {
-    const paths: Record<string, string> = {
-      inventory: 'inventoryPositions',
-      products: 'products'
-    };
-
-    const path = paths[entityType];
-    return result.data[path]?.edges?.map((e: any) => e.node) || [];
-  }
-
-  private getDefaultFromDate(): string {
-    // Default to last 24 hours
-    const date = new Date();
-    date.setDate(date.getDate() - 1);
-    return date.toISOString();
-  }
-}
-```
-
-## Error Handling & Retry Logic
-
-### Exponential Backoff Retry
-
-```typescript
-class RetryStrategy {
-  async executeWithRetry<T>(
-    operation: () => Promise<T>,
-    maxRetries = 3,
-    baseDelay = 1000
-  ): Promise<T> {
-    let lastError: any;
-
-    for (let attempt = 0; attempt < maxRetries; attempt++) {
-      try {
-        return await operation();
-      } catch (error) {
-        lastError = error;
-        console.warn(`Attempt ${attempt + 1} failed:`, error.message);
-
-        if (attempt < maxRetries - 1) {
-          const delay = baseDelay * Math.pow(2, attempt);
-          console.log(`Retrying in ${delay}ms...`);
-          await this.sleep(delay);
-        }
-      }
-    }
-
-    throw new Error(`Operation failed after ${maxRetries} attempts: ${lastError.message}`);
-  }
-
-  private sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-}
-
-// Usage
-const retry = new RetryStrategy();
-
-const result = await retry.executeWithRetry(async () => {
-  return await client.graphql({ query, variables });
-}, 3, 1000);
-```
-
-### Circuit Breaker Pattern
-
-```typescript
-class CircuitBreaker {
-  private failures = 0;
-  private state: 'CLOSED' | 'OPEN' | 'HALF_OPEN' = 'CLOSED';
-  private lastFailTime = 0;
-
-  constructor(
-    private threshold = 5,
-    private timeout = 60000 // 1 minute
-  ) {}
-
-  async execute<T>(operation: () => Promise<T>): Promise<T> {
-    if (this.state === 'OPEN') {
-      if (Date.now() - this.lastFailTime > this.timeout) {
-        this.state = 'HALF_OPEN';
-      } else {
-        throw new Error('Circuit breaker is OPEN');
-      }
-    }
-
-    try {
-      const result = await operation();
-      this.onSuccess();
-      return result;
-    } catch (error) {
-      this.onFailure();
-      throw error;
-    }
-  }
-
-  private onSuccess(): void {
-    this.failures = 0;
-    this.state = 'CLOSED';
-  }
-
-  private onFailure(): void {
-    this.failures++;
-    this.lastFailTime = Date.now();
-
-    if (this.failures >= this.threshold) {
-      this.state = 'OPEN';
-      console.error('Circuit breaker opened');
-    }
-  }
-}
-```
-
-## Performance Monitoring
-
-### Extraction Metrics
-
-```typescript
-interface ExtractionMetrics {
-  startTime: number;
-  endTime: number;
-  totalRecords: number;
-  totalPages: number;
-  errors: number;
-  dataSize: number;
-}
-
-class ExtractionMonitor {
-  private metrics: ExtractionMetrics = {
-    startTime: 0,
-    endTime: 0,
-    totalRecords: 0,
-    totalPages: 0,
-    errors: 0,
-    dataSize: 0
-  };
-
-  start(): void {
-    this.metrics.startTime = Date.now();
-  }
-
-  recordPage(records: number): void {
-    this.metrics.totalRecords += records;
-    this.metrics.totalPages++;
-  }
-
-  recordError(): void {
-    this.metrics.errors++;
-  }
-
-  finish(dataSize: number): void {
-    this.metrics.endTime = Date.now();
-    this.metrics.dataSize = dataSize;
-    this.logMetrics();
-  }
-
-  private logMetrics(): void {
-    const duration = this.metrics.endTime - this.metrics.startTime;
-    const recordsPerSecond = (this.metrics.totalRecords / duration) * 1000;
-    const mbPerSecond = (this.metrics.dataSize / duration / 1024 / 1024) * 1000;
-
-    console.log('=== Extraction Metrics ===');
-    console.log(`Duration: ${duration}ms`);
-    console.log(`Records: ${this.metrics.totalRecords}`);
-    console.log(`Pages: ${this.metrics.totalPages}`);
-    console.log(`Errors: ${this.metrics.errors}`);
-    console.log(`Data size: ${(this.metrics.dataSize / 1024 / 1024).toFixed(2)} MB`);
-    console.log(`Throughput: ${recordsPerSecond.toFixed(0)} records/sec`);
-    console.log(`Bandwidth: ${mbPerSecond.toFixed(2)} MB/sec`);
-  }
-}
-```
-
-## Memory Management
-
-### Streaming for Large Datasets
-
-```typescript
-async function* streamExtraction(query: string, pageSize = 100) {
-  let hasNextPage = true;
-  let cursor = null;
-  let pageCount = 0;
-
-  while (hasNextPage) {
-    const result = await client.graphql({
-      query,
-      variables: { first: pageSize, after: cursor }
-    });
-
-    const edges = result.data.inventoryPositions.edges;
-    yield edges.map(e => e.node);
-
-    hasNextPage = result.data.inventoryPositions.pageInfo.hasNextPage;
-    cursor = edges[edges.length - 1]?.cursor;
-    pageCount++;
-
-    // Force GC every 100 pages
-    if (pageCount % 100 === 0 && global.gc) {
-      global.gc();
-    }
-  }
-}
-
-// Process in batches
-for await (const batch of streamExtraction(query)) {
-  await processBatch(batch);
-  // Batch is garbage collected after processing
-}
-```
-
-## Production Best Practices
-
-### 1. Comprehensive Logging
-
-```typescript
-import {
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-
-async function productionExtraction() {
-  logger.info('Starting extraction', { entityType: 'inventory' });
-
-  try {
-    const result = await client.graphql({ query, variables, pagination });
-
-    logger.info('Extraction complete', {
-      records: result.extensions.autoPagination.totalRecords,
-      pages: result.extensions.autoPagination.totalPages,
-      duration: result.extensions.autoPagination.timeElapsed
-    });
-
-    return result;
-  } catch (error) {
-    logger.error('Extraction failed', error as Error, { entityType: 'inventory' });
-    throw error;
-  }
-}
-```
-
-### 2. Configuration Management
-
-```typescript
-interface ExtractionConfig {
-  pageSize: number;
-  maxRecords: number;
-  maxPages: number;
-  timeoutMs: number;
-  retryAttempts: number;
-  cacheEnabled: boolean;
-  cacheTtl: number;
-}
-
-const config: ExtractionConfig = {
-  pageSize: parseInt(process.env.PAGE_SIZE || '100'),
-  maxRecords: parseInt(process.env.MAX_RECORDS || '50000'),
-  maxPages: parseInt(process.env.MAX_PAGES || '500'),
-  timeoutMs: parseInt(process.env.TIMEOUT_MS || '300000'),
-  retryAttempts: parseInt(process.env.RETRY_ATTEMPTS || '3'),
-  cacheEnabled: process.env.CACHE_ENABLED === 'true',
-  cacheTtl: parseInt(process.env.CACHE_TTL || '3600000')
-};
-```
-
-### 3. Health Checks
-
-```typescript
-async function healthCheck(): Promise<boolean> {
-  try {
-    // Simple query to verify connectivity
-    const result = await client.graphql({
-      query: `query { products(first: 1) { edges { node { ref } } } }`
-    });
-
-    return !!result.data;
-  } catch (error) {
-    console.error('Health check failed:', error);
-    return false;
-  }
-}
-```
-
-### 4. Rate Limiting
-
-```typescript
-class RateLimiter {
-  private requests = 0;
-  private resetTime = Date.now();
-
-  constructor(
-    private maxRequests = 100,
-    private windowMs = 60000 // 1 minute
-  ) {}
-
-  async acquire(): Promise<void> {
-    if (Date.now() > this.resetTime) {
-      this.requests = 0;
-      this.resetTime = Date.now() + this.windowMs;
-    }
-
-    if (this.requests >= this.maxRequests) {
-      const waitTime = this.resetTime - Date.now();
-      console.log(`Rate limit reached, waiting ${waitTime}ms`);
-      await this.sleep(waitTime);
-      this.requests = 0;
-      this.resetTime = Date.now() + this.windowMs;
-    }
-
-    this.requests++;
-  }
-
-  private sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-}
-```
-
-## Complete Production Example
-
-```typescript
-import {
-  createClient,
-  UniversalMapper,
-  S3DataSource,
-  StateService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function productionExtractionWorkflow() {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const monitor = new ExtractionMonitor();
-  const retry = new RetryStrategy();
-  const cache = new ExtractionCache();
-
-  monitor.start();
-
-  try {
-    // Health check
-    if (!await healthCheck()) {
-      throw new Error('Health check failed');
-    }
-
-    // Initialize services
-    const client = await createClient({ config });
-    const s3 = new S3DataSource(s3Config);
-    const state = new StateService(kvAdapter);
-
-    // Delta extraction
-    const deltaExtractor = new DeltaExtractor(client, state);
-    const records = await retry.executeWithRetry(async () => {
-      return await deltaExtractor.extractDelta('inventory');
-    });
-
-    logger.info(`Extracted ${records.length} changed records`);
-    monitor.recordPage(records.length);
-
-    // Transform
-    const mapper = new UniversalMapper(mappingConfig);
-    const result = await mapper.map(records);
-
-    if (!result.success) {
-      const transformError = new Error('Transformation failed');
-      logger.error('Transformation errors', transformError, { errors: result.errors });
-      throw transformError;
-    }
-
-    // Export
-    const output = JSON.stringify(result.data, null, 2);
-    await s3.uploadFile(`inventory_delta_${new Date().toISOString()}.json`, output, {
-      contentType: 'application/json',
-    });
-
-    monitor.finish(output.length);
-    logger.info('Workflow complete');
-
-  } catch (error) {
-    monitor.recordError();
-    logger.error('Workflow failed', error as Error);
-    throw error;
-  }
-}
-```
-
-## Key Takeaways
-
-- 🎯 Select only needed fields to minimize payload size
-- 🎯 Tune page sizes based on entity complexity
-- 🎯 Implement caching for frequently accessed reference data
-- 🎯 Use delta extraction for incremental updates
-- 🎯 Add retry logic with exponential backoff
-- 🎯 Monitor performance metrics (throughput, errors, duration)
-- 🎯 Stream large datasets to avoid memory issues
-- 🎯 Implement comprehensive logging for production
-- 🎯 Use configuration management for flexibility
-- 🎯 Add health checks and rate limiting
-
-## Summary
-
-You've completed the Data Extraction Learning Path! You now know how to:
-- Extract data from Fluent Commerce using validated GraphQL queries
-- Process Parquet files and enrich with GraphQL data
-- Transform data using UniversalMapper and custom logic
-- Export to multiple formats (JSON, CSV, XML, Parquet)
-- Optimize for performance with caching and streaming
-- Build production-ready extraction workflows
-
-## Next Steps
-
-- Explore [Auto-Pagination Guide](../../auto-pagination/) for advanced pagination patterns
-- See [Universal Mapping Guide](../../mapping/mapping-readme.md) for complex transformations
-- Review [Use Case Examples](../../../01-TEMPLATES/) for real-world scenarios
-- Check [SDK Architecture](../../../04-REFERENCE/architecture/) for deeper understanding
+# Module 7: Optimization & Best Practices
+
+**Level:** Advanced
+**Estimated Time:** 25 minutes
+
+## Overview
+
+This module covers performance optimization, caching strategies, error handling, monitoring, and production best practices for data extraction workflows.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Optimize GraphQL queries for performance
+- ✅ Implement caching strategies
+- ✅ Handle errors and implement retry logic
+- ✅ Monitor extraction performance
+- ✅ Implement delta (incremental) extraction
+- ✅ Apply production best practices
+
+## Query Optimization
+
+### Field Selection
+
+**❌ BAD: Fetch all fields**
+```typescript
+query GetProducts {
+  products(first: 100) {
+    edges {
+      node {
+        id ref name description gtin price
+        catalogue { id ref name description }
+        attributes { name value type }
+        categories { edges { node { name path parent { name } } } }
+        inventoryPositions { edges { node { qty locationRef } } }
+      }
+    }
+  }
+}
+```
+
+**✅ GOOD: Fetch only needed fields**
+```typescript
+query GetProducts {
+  products(first: 100) {
+    edges {
+      node {
+        ref
+        name
+        gtin
+      }
+    }
+  }
+}
+```
+
+### Page Size Tuning
+
+| Entity Complexity | Recommended Page Size | Rationale |
+|------------------|----------------------|-----------|
+| Simple (5-10 fields) | 100-200 | Small payload per record |
+| Medium (10-20 fields) | 50-100 | Moderate payload |
+| Complex (nested relations) | 20-50 | Large payload, avoid timeouts |
+| Very complex (deep nesting) | 10-20 | Prevent server timeouts |
+
+### GraphQL Fragments
+
+Reuse field selections:
+
+```typescript
+const fragments = `
+  fragment ProductFields on Product {
+    ref
+    name
+    gtin
+  }
+
+  fragment LocationFields on Location {
+    ref
+    name
+    type
+  }
+`;
+
+const query = `
+  ${fragments}
+
+  query ExtractInventory($first: Int!, $after: String) {
+    inventoryPositions(first: $first, after: $after) {
+      edges {
+        node {
+          ref
+          productRef
+          locationRef
+          qty
+          product { ...ProductFields }
+          location { ...LocationFields }
+        }
+        cursor
+      }
+      pageInfo { hasNextPage }
+    }
+  }
+`;
+```
+
+## Caching Strategies
+
+### Simple In-Memory Cache
+
+```typescript
+class ExtractionCache {
+  private cache = new Map<string, { data: any; expiry: number }>();
+  private ttl: number;
+
+  constructor(ttlMs = 3600000) { // 1 hour default
+    this.ttl = ttlMs;
+  }
+
+  get(key: string): any | null {
+    const cached = this.cache.get(key);
+
+    if (!cached) return null;
+
+    if (cached.expiry < Date.now()) {
+      this.cache.delete(key);
+      return null;
+    }
+
+    return cached.data;
+  }
+
+  set(key: string, data: any): void {
+    this.cache.set(key, {
+      data,
+      expiry: Date.now() + this.ttl
+    });
+  }
+
+  invalidate(pattern?: string): void {
+    if (pattern) {
+      for (const key of this.cache.keys()) {
+        if (key.includes(pattern)) {
+          this.cache.delete(key);
+        }
+      }
+    } else {
+      this.cache.clear();
+    }
+  }
+}
+
+// Usage
+const cache = new ExtractionCache(3600000);
+
+async function getProductWithCache(ref: string) {
+  const cacheKey = `product:${ref}`;
+  let product = cache.get(cacheKey);
+
+  if (!product) {
+    const result = await client.graphql({
+      query: `query { product(ref: "${ref}") { ref name gtin } }`
+    });
+    product = result.data.product;
+    cache.set(cacheKey, product);
+  }
+
+  return product;
+}
+```
+
+### Redis-Based Distributed Cache
+
+```typescript
+import { createClient as createRedisClient } from 'redis';
+
+class DistributedCache {
+  private redis: any;
+
+  async connect() {
+    this.redis = createRedisClient({
+      url: process.env.REDIS_URL
+    });
+    await this.redis.connect();
+  }
+
+  async get(key: string): Promise<any | null> {
+    const cached = await this.redis.get(key);
+    return cached ? JSON.parse(cached) : null;
+  }
+
+  async set(key: string, data: any, ttlSeconds = 3600): Promise<void> {
+    await this.redis.setEx(key, ttlSeconds, JSON.stringify(data));
+  }
+
+  async invalidate(pattern: string): Promise<void> {
+    const keys = await this.redis.keys(pattern);
+    if (keys.length > 0) {
+      await this.redis.del(keys);
+    }
+  }
+}
+```
+
+## Delta (Incremental) Extraction
+
+### State Management
+
+```typescript
+import { StateService, VersoriKVAdapter } from '@fluentcommerce/fc-connect-sdk';
+
+class DeltaExtractor {
+  constructor(
+    private client: any,
+    private stateService: StateService
+  ) {}
+
+  async extractDelta(entityType: string): Promise<any[]> {
+    const stateKey = `lastExtraction_${entityType}`;
+
+    // Get last extraction timestamp
+    const lastExtraction = await this.stateService.get(stateKey);
+    const fromDate = lastExtraction || this.getDefaultFromDate();
+
+    console.log(`Extracting ${entityType} changed since ${fromDate}`);
+
+    // Query only changed records
+    const result = await this.client.graphql({
+      query: this.buildDeltaQuery(entityType),
+      variables: {
+        fromDate,
+        toDate: new Date().toISOString()
+      },
+      pagination: { maxRecords: 50000 }
+    });
+
+    const records = this.extractRecords(result, entityType);
+
+    // Update state
+    await this.stateService.set(stateKey, new Date().toISOString());
+
+    return records;
+  }
+
+  private buildDeltaQuery(entityType: string): string {
+    const queries: Record<string, string> = {
+      inventory: `
+        query DeltaInventory($fromDate: DateTime!, $toDate: DateTime!) {
+          inventoryPositions(
+            updatedOn: { from: $fromDate, to: $toDate }
+            first: 100
+          ) {
+            edges {
+              node { ref productRef locationRef qty status updatedOn }
+              cursor
+            }
+            pageInfo { hasNextPage }
+          }
+        }
+      `,
+      products: `
+        query DeltaProducts($fromDate: DateTime!, $toDate: DateTime!) {
+          products(
+            updatedOn: { from: $fromDate, to: $toDate }
+            first: 100
+          ) {
+            edges {
+              node { ref name gtin updatedOn }
+              cursor
+            }
+            pageInfo { hasNextPage }
+          }
+        }
+      `
+    };
+
+    return queries[entityType] || queries.inventory;
+  }
+
+  private extractRecords(result: any, entityType: string): any[] {
+    const paths: Record<string, string> = {
+      inventory: 'inventoryPositions',
+      products: 'products'
+    };
+
+    const path = paths[entityType];
+    return result.data[path]?.edges?.map((e: any) => e.node) || [];
+  }
+
+  private getDefaultFromDate(): string {
+    // Default to last 24 hours
+    const date = new Date();
+    date.setDate(date.getDate() - 1);
+    return date.toISOString();
+  }
+}
+```
+
+## Error Handling & Retry Logic
+
+### Exponential Backoff Retry
+
+```typescript
+class RetryStrategy {
+  async executeWithRetry<T>(
+    operation: () => Promise<T>,
+    maxRetries = 3,
+    baseDelay = 1000
+  ): Promise<T> {
+    let lastError: any;
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        return await operation();
+      } catch (error) {
+        lastError = error;
+        console.warn(`Attempt ${attempt + 1} failed:`, error.message);
+
+        if (attempt < maxRetries - 1) {
+          const delay = baseDelay * Math.pow(2, attempt);
+          console.log(`Retrying in ${delay}ms...`);
+          await this.sleep(delay);
+        }
+      }
+    }
+
+    throw new Error(`Operation failed after ${maxRetries} attempts: ${lastError.message}`);
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+// Usage
+const retry = new RetryStrategy();
+
+const result = await retry.executeWithRetry(async () => {
+  return await client.graphql({ query, variables });
+}, 3, 1000);
+```
+
+### Circuit Breaker Pattern
+
+```typescript
+class CircuitBreaker {
+  private failures = 0;
+  private state: 'CLOSED' | 'OPEN' | 'HALF_OPEN' = 'CLOSED';
+  private lastFailTime = 0;
+
+  constructor(
+    private threshold = 5,
+    private timeout = 60000 // 1 minute
+  ) {}
+
+  async execute<T>(operation: () => Promise<T>): Promise<T> {
+    if (this.state === 'OPEN') {
+      if (Date.now() - this.lastFailTime > this.timeout) {
+        this.state = 'HALF_OPEN';
+      } else {
+        throw new Error('Circuit breaker is OPEN');
+      }
+    }
+
+    try {
+      const result = await operation();
+      this.onSuccess();
+      return result;
+    } catch (error) {
+      this.onFailure();
+      throw error;
+    }
+  }
+
+  private onSuccess(): void {
+    this.failures = 0;
+    this.state = 'CLOSED';
+  }
+
+  private onFailure(): void {
+    this.failures++;
+    this.lastFailTime = Date.now();
+
+    if (this.failures >= this.threshold) {
+      this.state = 'OPEN';
+      console.error('Circuit breaker opened');
+    }
+  }
+}
+```
+
+## Performance Monitoring
+
+### Extraction Metrics
+
+```typescript
+interface ExtractionMetrics {
+  startTime: number;
+  endTime: number;
+  totalRecords: number;
+  totalPages: number;
+  errors: number;
+  dataSize: number;
+}
+
+class ExtractionMonitor {
+  private metrics: ExtractionMetrics = {
+    startTime: 0,
+    endTime: 0,
+    totalRecords: 0,
+    totalPages: 0,
+    errors: 0,
+    dataSize: 0
+  };
+
+  start(): void {
+    this.metrics.startTime = Date.now();
+  }
+
+  recordPage(records: number): void {
+    this.metrics.totalRecords += records;
+    this.metrics.totalPages++;
+  }
+
+  recordError(): void {
+    this.metrics.errors++;
+  }
+
+  finish(dataSize: number): void {
+    this.metrics.endTime = Date.now();
+    this.metrics.dataSize = dataSize;
+    this.logMetrics();
+  }
+
+  private logMetrics(): void {
+    const duration = this.metrics.endTime - this.metrics.startTime;
+    const recordsPerSecond = (this.metrics.totalRecords / duration) * 1000;
+    const mbPerSecond = (this.metrics.dataSize / duration / 1024 / 1024) * 1000;
+
+    console.log('=== Extraction Metrics ===');
+    console.log(`Duration: ${duration}ms`);
+    console.log(`Records: ${this.metrics.totalRecords}`);
+    console.log(`Pages: ${this.metrics.totalPages}`);
+    console.log(`Errors: ${this.metrics.errors}`);
+    console.log(`Data size: ${(this.metrics.dataSize / 1024 / 1024).toFixed(2)} MB`);
+    console.log(`Throughput: ${recordsPerSecond.toFixed(0)} records/sec`);
+    console.log(`Bandwidth: ${mbPerSecond.toFixed(2)} MB/sec`);
+  }
+}
+```
+
+## Memory Management
+
+### Streaming for Large Datasets
+
+```typescript
+async function* streamExtraction(query: string, pageSize = 100) {
+  let hasNextPage = true;
+  let cursor = null;
+  let pageCount = 0;
+
+  while (hasNextPage) {
+    const result = await client.graphql({
+      query,
+      variables: { first: pageSize, after: cursor }
+    });
+
+    const edges = result.data.inventoryPositions.edges;
+    yield edges.map(e => e.node);
+
+    hasNextPage = result.data.inventoryPositions.pageInfo.hasNextPage;
+    cursor = edges[edges.length - 1]?.cursor;
+    pageCount++;
+
+    // Force GC every 100 pages
+    if (pageCount % 100 === 0 && global.gc) {
+      global.gc();
+    }
+  }
+}
+
+// Process in batches
+for await (const batch of streamExtraction(query)) {
+  await processBatch(batch);
+  // Batch is garbage collected after processing
+}
+```
+
+## Production Best Practices
+
+### 1. Comprehensive Logging
+
+```typescript
+import {
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+
+async function productionExtraction() {
+  logger.info('Starting extraction', { entityType: 'inventory' });
+
+  try {
+    const result = await client.graphql({ query, variables, pagination });
+
+    logger.info('Extraction complete', {
+      records: result.extensions.autoPagination.totalRecords,
+      pages: result.extensions.autoPagination.totalPages,
+      duration: result.extensions.autoPagination.timeElapsed
+    });
+
+    return result;
+  } catch (error) {
+    logger.error('Extraction failed', error as Error, { entityType: 'inventory' });
+    throw error;
+  }
+}
+```
+
+### 2. Configuration Management
+
+```typescript
+interface ExtractionConfig {
+  pageSize: number;
+  maxRecords: number;
+  maxPages: number;
+  timeoutMs: number;
+  retryAttempts: number;
+  cacheEnabled: boolean;
+  cacheTtl: number;
+}
+
+const config: ExtractionConfig = {
+  pageSize: parseInt(process.env.PAGE_SIZE || '100'),
+  maxRecords: parseInt(process.env.MAX_RECORDS || '50000'),
+  maxPages: parseInt(process.env.MAX_PAGES || '500'),
+  timeoutMs: parseInt(process.env.TIMEOUT_MS || '300000'),
+  retryAttempts: parseInt(process.env.RETRY_ATTEMPTS || '3'),
+  cacheEnabled: process.env.CACHE_ENABLED === 'true',
+  cacheTtl: parseInt(process.env.CACHE_TTL || '3600000')
+};
+```
+
+### 3. Health Checks
+
+```typescript
+async function healthCheck(): Promise<boolean> {
+  try {
+    // Simple query to verify connectivity
+    const result = await client.graphql({
+      query: `query { products(first: 1) { edges { node { ref } } } }`
+    });
+
+    return !!result.data;
+  } catch (error) {
+    console.error('Health check failed:', error);
+    return false;
+  }
+}
+```
+
+### 4. Rate Limiting
+
+```typescript
+class RateLimiter {
+  private requests = 0;
+  private resetTime = Date.now();
+
+  constructor(
+    private maxRequests = 100,
+    private windowMs = 60000 // 1 minute
+  ) {}
+
+  async acquire(): Promise<void> {
+    if (Date.now() > this.resetTime) {
+      this.requests = 0;
+      this.resetTime = Date.now() + this.windowMs;
+    }
+
+    if (this.requests >= this.maxRequests) {
+      const waitTime = this.resetTime - Date.now();
+      console.log(`Rate limit reached, waiting ${waitTime}ms`);
+      await this.sleep(waitTime);
+      this.requests = 0;
+      this.resetTime = Date.now() + this.windowMs;
+    }
+
+    this.requests++;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+```
+
+## Complete Production Example
+
+```typescript
+import {
+  createClient,
+  UniversalMapper,
+  S3DataSource,
+  StateService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function productionExtractionWorkflow() {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const monitor = new ExtractionMonitor();
+  const retry = new RetryStrategy();
+  const cache = new ExtractionCache();
+
+  monitor.start();
+
+  try {
+    // Health check
+    if (!await healthCheck()) {
+      throw new Error('Health check failed');
+    }
+
+    // Initialize services
+    const client = await createClient({ config });
+    const s3 = new S3DataSource(s3Config);
+    const state = new StateService(kvAdapter);
+
+    // Delta extraction
+    const deltaExtractor = new DeltaExtractor(client, state);
+    const records = await retry.executeWithRetry(async () => {
+      return await deltaExtractor.extractDelta('inventory');
+    });
+
+    logger.info(`Extracted ${records.length} changed records`);
+    monitor.recordPage(records.length);
+
+    // Transform
+    const mapper = new UniversalMapper(mappingConfig);
+    const result = await mapper.map(records);
+
+    if (!result.success) {
+      const transformError = new Error('Transformation failed');
+      logger.error('Transformation errors', transformError, { errors: result.errors });
+      throw transformError;
+    }
+
+    // Export
+    const output = JSON.stringify(result.data, null, 2);
+    await s3.uploadFile(`inventory_delta_${new Date().toISOString()}.json`, output, {
+      contentType: 'application/json',
+    });
+
+    monitor.finish(output.length);
+    logger.info('Workflow complete');
+
+  } catch (error) {
+    monitor.recordError();
+    logger.error('Workflow failed', error as Error);
+    throw error;
+  }
+}
+```
+
+## Key Takeaways
+
+- 🎯 Select only needed fields to minimize payload size
+- 🎯 Tune page sizes based on entity complexity
+- 🎯 Implement caching for frequently accessed reference data
+- 🎯 Use delta extraction for incremental updates
+- 🎯 Add retry logic with exponential backoff
+- 🎯 Monitor performance metrics (throughput, errors, duration)
+- 🎯 Stream large datasets to avoid memory issues
+- 🎯 Implement comprehensive logging for production
+- 🎯 Use configuration management for flexibility
+- 🎯 Add health checks and rate limiting
+
+## Summary
+
+You've completed the Data Extraction Learning Path! You now know how to:
+- Extract data from Fluent Commerce using validated GraphQL queries
+- Process Parquet files and enrich with GraphQL data
+- Transform data using UniversalMapper and custom logic
+- Export to multiple formats (JSON, CSV, XML, Parquet)
+- Optimize for performance with caching and streaming
+- Build production-ready extraction workflows
+
+## Next Steps
+
+- Explore [Auto-Pagination Guide](../../auto-pagination/) for advanced pagination patterns
+- See [Universal Mapping Guide](../../mapping/mapping-readme.md) for complex transformations
+- Review [Use Case Examples](../../../01-TEMPLATES/) for real-world scenarios
+- Check [SDK Architecture](../../../04-REFERENCE/architecture/) for deeper understanding