@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/02-CORE-GUIDES/ingestion/ingestion-quick-reference.md

@@ -1,546 +1,546 @@
-# Ingestion Quick Reference
-
-One-page cheat sheet for FC Connect SDK data ingestion.
-
-## Import Statements
-
-```typescript
-import {
-  // Client
-  createClient, FluentClient,
-
-  // Data Sources
-  S3DataSource, SftpDataSource,
-
-  // Parsers
-  CSVParserService, ParquetParserService, XMLParserService, JSONParserService,
-
-  // Mapping & Transformation
-  UniversalMapper,
-
-  // State Management
-  StateService, VersoriKVAdapter,
-
-  // Types
-  FluentConfig, BatchJob, FieldMapping
-} from '@fluentcommerce/fc-connect-sdk';
-
-// Note: For standalone/testing, implement your own KVStore interface:
-// interface KVStore { get, set, delete, atomic }
-```
-
-## Client Setup
-
-```typescript
-// Standalone (Node.js/Deno)
-const client = await createClient({
-  config: {
-    baseUrl: 'https://api.fluentcommerce.com',
-    clientId: process.env.FLUENT_CLIENT_ID!,
-    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-    retailerId: process.env.FLUENT_RETAILER_ID!
-  }
-});
-
-// Versori Platform
-const client = await createClient(ctx); // Auto-detects Versori context
-```
-
-## Data Sources
-
-### S3
-
-```typescript
-const s3 = new S3DataSource(
-  {
-    type: 'S3_CSV',
-    connectionId: 's3-inventory',
-    name: 'S3 Inventory',
-    s3Config: {
-      bucket: 'bucket',
-      region: process.env.AWS_REGION!,
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-    },
-  },
-  logger
-);
-
-// List files
-const files = await s3.listFiles({ prefix: 'prefix/' });
-
-// Read file
-const data = await s3.downloadFile('key');
-```
-
-### SFTP
-
-```typescript
-const sftp = new SftpDataSource({
-  host: 'sftp.example.com',
-  port: 22,
-  username: 'user',
-  password: 'pass'
-});
-
-// No connect/disconnect needed - handled internally
-const files = await sftp.listFiles('/path/');
-const data = await sftp.downloadFile('/path/file.csv');
-```
-
-## Parsers
-
-### CSV
-
-```typescript
-const parser = new CSVParserService();
-const records = await parser.parse(csvData);
-
-// Streaming
-const data = await s3.downloadFile('file.csv');
-for await (const record of parser.parseStreaming(data)) {
-  // Process record
-}
-```
-
-### Parquet
-
-```typescript
-const logger = createConsoleLogger();
-const parser = new ParquetParserService(logger);
-const records = await parser.parse(parquetBuffer);
-
-// Get metadata
-const metadata = await parser.getMetadata(parquetBuffer);
-console.log(\`Rows: \${metadata.numRows}\`);
-```
-
-### XML
-
-```typescript
-const parser = new XMLParserService();
-const parsed = await parser.parse(xmlString);
-
-// Access nested data
-const positions = parsed.inventory.position;
-```
-
-### JSON
-
-```typescript
-const parser = new JSONParserService();
-const parsed = await parser.parse(jsonString);
-
-// Streaming JSON array
-const data = await s3.downloadFile('file.json');
-// Note: JSONParserService doesn't have parseArrayStream - use parse() then iterate
-```
-
-## Field Mapping
-
-### Basic Mapping
-
-```typescript
-const mapper = new UniversalMapper({
-  fields: {
-    ref: { source: 'sku', required: true },
-    productRef: { source: 'sku', required: true },
-    locationRef: { source: 'location', required: true },
-    qty: { source: 'quantity', resolver: 'sdk.parseInt', required: true },
-    status: { source: 'status', resolver: 'sdk.uppercase', defaultValue: 'AVAILABLE' }
-  }
-});
-
-const result = await mapper.map(sourceData);
-
-if (result.success) {
-  // result.data is transformed InventoryPositionInput[]
-} else {
-  console.error('Errors:', result.errors);
-}
-```
-
-### SDK Resolvers
-
-| Resolver | Description |
-|----------|-------------|
-| \`sdk.parseInt\` | Parse string to integer |
-| \`sdk.parseFloat\` | Parse string to float |
-| \`sdk.uppercase\` | Convert to uppercase |
-| \`sdk.lowercase\` | Convert to lowercase |
-| \`sdk.trim\` | Trim whitespace |
-| \`sdk.formatDate\` | Format as ISO date string |
-| \`sdk.boolean\` | Convert to boolean |
-
-### Custom Resolvers
-
-```typescript
-const mapper = new UniversalMapper(config, {
-  customResolvers: {
-    'custom.mapStatus': (value, data, config, helpers) => {
-      const map = { 'in_stock': 'AVAILABLE', 'out': 'ON_HOLD' };
-      return map[value] || 'AVAILABLE';
-    }
-  }
-});
-```
-
-## Batch API
-
-### Create Job
-
-```typescript
-const job = await client.createJob({
-  name: 'Inventory Import',
-  retailerId: process.env.FLUENT_RETAILER_ID!,
-  metadata: { source: 's3', file: 'inventory.csv' } // Optional
-});
-```
-
-### Send Batch
-
-```typescript
-await client.sendBatch(job.id, {
-  action: 'UPSERT',           // String, not enum
-  entityType: 'INVENTORY',    // String, not enum
-  source: 'S3_CSV_IMPORT',    // REQUIRED - Source system identifier
-  event: 'INVENTORY_UPDATE',  // REQUIRED - Event type identifier
-  entities: transformedData   // InventoryPositionInput[]
-});
-```
-
-### Get Job Status
-
-```typescript
-const status = await client.getJobStatus(jobId);
-console.log(\`Status: \${status.status}\`); // PENDING, PROCESSING, COMPLETED, FAILED
-```
-
-## State Management
-
-### Setup
-
-```typescript
-// Node.js standalone
-import { createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'inventory-ingestion',
-  correlationId: generateCorrelationId()
-});
-const stateService = new StateService(logger);
-
-// Versori platform
-const logger = toStructuredLogger(ctx.log, {
-  service: 'inventory-ingestion'
-});
-const stateService = new StateService(logger);
-
-// KV adapter setup
-// ✅ CORRECT: Access openKv from Versori context
-const { openKv } = ctx;
-const kvAdapter = new VersoriKVAdapter(openKv(':project:'));
-```
-
-### File Tracking
-
-```typescript
-// Check if processed
-const fileKey = ['file', 'default', fileName];
-const alreadyProcessed = await kv.get(fileKey);
-
-if (alreadyProcessed) {
-  console.log('Already processed');
-  return;
-}
-
-// Mark as processed
-await state.updateSyncState(kv, [{
-  fileName: fileName,
-  lastModified: new Date().toISOString(),
-  recordCount: 1000
-}]);
-
-// Get sync state
-const syncState = await state.getSyncState(kv);
-console.log('Last processed:', syncState.lastProcessedFile);
-```
-
-### Distributed Locking
-
-```typescript
-const lockAcquired = await state.acquireLock(fileKey, kv, 15); // 15 min TTL
-
-if (!lockAcquired) {
-  console.log('Locked by another process');
-  return;
-}
-
-try {
-  // Process file (exclusive access)
-  await processFile(fileKey);
-} finally {
-  // Always release lock
-  await state.releaseLock(fileKey, kv);
-}
-```
-
-## Complete Workflow
-
-```typescript
-async function ingestInventory(kv: any) {
-  // 1. Initialize
-  const client = await createClient({ config });
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      connectionId: 's3-inventory',
-      name: 'S3 Inventory',
-      s3Config: s3Config,
-    },
-    logger
-  );
-  const parser = new CSVParserService(logger);
-  const mapper = new UniversalMapper(mappingConfig);
-
-  // Initialize state management
-  const logger = toStructuredLogger(createConsoleLogger(), {
-    service: 'inventory-ingestion',
-    correlationId: generateCorrelationId()
-  });
-  const stateService = new StateService(logger);
-  const kvAdapter = new VersoriKVAdapter(kv);
-
-  // 2. List files
-  const files = await s3.listFiles({ prefix: 'data/' });
-
-  for (const file of files) {
-    // 3. Check state
-    const fileKey = ['file', 'default', file.path];
-    const alreadyProcessed = await kvAdapter.get(fileKey);
-    if (alreadyProcessed) continue;
-
-    // 4. Acquire lock
-    const lockAcquired = await state.acquireLock(file.path, kvAdapter, 15);
-    if (!lockAcquired) continue;
-
-    try {
-      // 5. Read & parse
-      const data = await s3.downloadFile(file.path);
-      const records = await parser.parse(data);
-
-      // 6. Transform
-      const result = await mapper.map(records);
-      if (!result.success) throw new Error('Mapping failed');
-
-      // 7. Get or create job
-      let job = await state.getDailyJob(kvAdapter, 'daily-import');
-      if (!job) {
-        const newJob = await client.createJob({
-          name: \`Daily Import - \${new Date().toISOString().split('T')[0]}\`,
-          retailerId: process.env.FLUENT_RETAILER_ID!
-        });
-        await state.setDailyJob(kvAdapter, 'daily-import', newJob.id, 24);
-        job = { jobId: newJob.id };
-      }
-
-      // 8. Send batch
-      await client.sendBatch(job.jobId, {
-        action: 'UPSERT',
-        entityType: 'INVENTORY',
-        source: 'S3_CSV_IMPORT',
-        event: 'INVENTORY_UPDATE',
-        entities: result.data
-      });
-
-      // 9. Mark processed
-      await state.updateSyncState(kvAdapter, [{
-        fileName: file.path,
-        lastModified: new Date().toISOString(),
-        recordCount: result.data.length
-      }]);
-
-    } finally {
-      await state.releaseLock(file.path, kvAdapter);
-    }
-  }
-}
-```
-
-## Performance Tips
-
-### Batch Sizing
-
-```typescript
-// Small files (< 10K records)
-const BATCH_SIZE = 1000;
-
-// Large files (> 100K records)
-const BATCH_SIZE = 5000;
-
-// Split into batches
-const batches = [];
-for (let i = 0; i < data.length; i += BATCH_SIZE) {
-  batches.push(data.slice(i, i + BATCH_SIZE));
-}
-```
-
-### Parallel Processing
-
-```typescript
-async function parallelProcess(files: string[], concurrency: number = 5) {
-  const queue = [...files];
-
-  const workers = Array(concurrency).fill(null).map(async () => {
-    while (queue.length > 0) {
-      const file = queue.shift();
-      if (file) await processFile(file);
-    }
-  });
-
-  await Promise.all(workers);
-}
-```
-
-### Streaming for Large Files
-
-```typescript
-const stream = await s3.streamObject('bucket', 'large-file.csv');
-const recordIterator = parser.parseStream(stream);
-
-let batch = [];
-for await (const record of recordIterator) {
-  batch.push(record);
-
-  if (batch.length >= 1000) {
-    await processBatch(batch);
-    batch = [];
-  }
-}
-```
-
-## Job Strategies
-
-### DAILY (Reuse Same Job)
-
-```typescript
-const dateKey = new Date().toISOString().split('T')[0];
-let jobId = await state.get(\`daily-job:\${dateKey}\`);
-
-if (!jobId) {
-  const job = await client.createJob({
-    name: \`Daily Inventory - \${dateKey}\`,
-    retailerId: process.env.FLUENT_RETAILER_ID!
-  });
-  jobId = job.id;
-  await state.set(\`daily-job:\${dateKey}\`, jobId, 24 * 60 * 60 * 1000);
-}
-
-// Use same job for all files
-```
-
-### PER_FILE (Dedicated Job per File)
-
-```typescript
-for (const file of files) {
-  const job = await client.createJob({
-    name: \`Import - \${file.key}\`,
-    retailerId: process.env.FLUENT_RETAILER_ID!
-  });
-
-  await client.sendBatch(job.id, {
-    action: 'UPSERT',
-    entityType: 'INVENTORY',
-    source: 'PER_FILE_STRATEGY',
-    event: 'INVENTORY_FILE_IMPORT',
-    entities: data
-  });
-}
-```
-
-## Error Handling
-
-### Retry with Backoff
-
-```typescript
-async function retryWithBackoff<T>(
-  operation: () => Promise<T>,
-  maxRetries: number = 3
-): Promise<T> {
-  let delay = 1000;
-
-  for (let attempt = 1; attempt <= maxRetries; attempt++) {
-    try {
-      return await operation();
-    } catch (error) {
-      if (attempt === maxRetries) throw error;
-
-      await new Promise(resolve => setTimeout(resolve, delay));
-      delay *= 2;
-    }
-  }
-}
-```
-
-### Validation
-
-```typescript
-function validateRecord(record: any): boolean {
-  return (
-    record.ref &&
-    record.productRef &&
-    record.locationRef &&
-    typeof record.qty === 'number' &&
-    record.qty >= 0
-  );
-}
-
-const validRecords = records.filter(validateRecord);
-```
-
-## Common Errors
-
-| Error | Cause | Solution |
-|-------|-------|----------|
-| \`JOB_EXPIRED\` | Job > 24 hours old | Create new job |
-| \`VALIDATION_ERROR\` | Invalid field values | Check schema, filter invalid records |
-| \`RATE_LIMIT_ERROR\` | Too many requests | Add delay, reduce concurrency |
-| \`NETWORK_ERROR\` | Connection issues | Retry with backoff |
-
-## Environment Variables
-
-```bash
-# Fluent Commerce
-FLUENT_BASE_URL=https://api.fluentcommerce.com
-FLUENT_CLIENT_ID=your-client-id
-FLUENT_CLIENT_SECRET=your-client-secret
-FLUENT_RETAILER_ID=1
-
-# AWS S3
-AWS_ACCESS_KEY_ID=your-access-key
-AWS_SECRET_ACCESS_KEY=your-secret-key
-AWS_REGION=us-east-1
-
-# SFTP
-SFTP_HOST=sftp.example.com
-SFTP_PORT=22
-SFTP_USERNAME=your-username
-SFTP_PASSWORD=your-password
-```
-
-## CLI Tools
-
-```bash
-# Introspect schema
-npx @fluentcommerce/fc-connect-sdk introspect-schema \
-  --url https://api.fluentcommerce.com/graphql \
-  --output schema.json
-
-# Validate mapping
-npx @fluentcommerce/fc-connect-sdk validate-schema \
-  --mapping config/field-mappings.json \
-  --schema schema.json
-```
-
----
-
-**See complete documentation:** [Ingestion Learning Path](./ingestion-readme.md)
+# Ingestion Quick Reference
+
+One-page cheat sheet for FC Connect SDK data ingestion.
+
+## Import Statements
+
+```typescript
+import {
+  // Client
+  createClient, FluentClient,
+
+  // Data Sources
+  S3DataSource, SftpDataSource,
+
+  // Parsers
+  CSVParserService, ParquetParserService, XMLParserService, JSONParserService,
+
+  // Mapping & Transformation
+  UniversalMapper,
+
+  // State Management
+  StateService, VersoriKVAdapter,
+
+  // Types
+  FluentConfig, BatchJob, FieldMapping
+} from '@fluentcommerce/fc-connect-sdk';
+
+// Note: For standalone/testing, implement your own KVStore interface:
+// interface KVStore { get, set, delete, atomic }
+```
+
+## Client Setup
+
+```typescript
+// Standalone (Node.js/Deno)
+const client = await createClient({
+  config: {
+    baseUrl: 'https://api.fluentcommerce.com',
+    clientId: process.env.FLUENT_CLIENT_ID!,
+    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+    retailerId: process.env.FLUENT_RETAILER_ID!
+  }
+});
+
+// Versori Platform
+const client = await createClient(ctx); // Auto-detects Versori context
+```
+
+## Data Sources
+
+### S3
+
+```typescript
+const s3 = new S3DataSource(
+  {
+    type: 'S3_CSV',
+    connectionId: 's3-inventory',
+    name: 'S3 Inventory',
+    s3Config: {
+      bucket: 'bucket',
+      region: process.env.AWS_REGION!,
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    },
+  },
+  logger
+);
+
+// List files
+const files = await s3.listFiles({ prefix: 'prefix/' });
+
+// Read file
+const data = await s3.downloadFile('key');
+```
+
+### SFTP
+
+```typescript
+const sftp = new SftpDataSource({
+  host: 'sftp.example.com',
+  port: 22,
+  username: 'user',
+  password: 'pass'
+});
+
+// No connect/disconnect needed - handled internally
+const files = await sftp.listFiles('/path/');
+const data = await sftp.downloadFile('/path/file.csv');
+```
+
+## Parsers
+
+### CSV
+
+```typescript
+const parser = new CSVParserService();
+const records = await parser.parse(csvData);
+
+// Streaming
+const data = await s3.downloadFile('file.csv');
+for await (const record of parser.parseStreaming(data)) {
+  // Process record
+}
+```
+
+### Parquet
+
+```typescript
+const logger = createConsoleLogger();
+const parser = new ParquetParserService(logger);
+const records = await parser.parse(parquetBuffer);
+
+// Get metadata
+const metadata = await parser.getMetadata(parquetBuffer);
+console.log(\`Rows: \${metadata.numRows}\`);
+```
+
+### XML
+
+```typescript
+const parser = new XMLParserService();
+const parsed = await parser.parse(xmlString);
+
+// Access nested data
+const positions = parsed.inventory.position;
+```
+
+### JSON
+
+```typescript
+const parser = new JSONParserService();
+const parsed = await parser.parse(jsonString);
+
+// Streaming JSON array
+const data = await s3.downloadFile('file.json');
+// Note: JSONParserService doesn't have parseArrayStream - use parse() then iterate
+```
+
+## Field Mapping
+
+### Basic Mapping
+
+```typescript
+const mapper = new UniversalMapper({
+  fields: {
+    ref: { source: 'sku', required: true },
+    productRef: { source: 'sku', required: true },
+    locationRef: { source: 'location', required: true },
+    qty: { source: 'quantity', resolver: 'sdk.parseInt', required: true },
+    status: { source: 'status', resolver: 'sdk.uppercase', defaultValue: 'AVAILABLE' }
+  }
+});
+
+const result = await mapper.map(sourceData);
+
+if (result.success) {
+  // result.data is transformed InventoryPositionInput[]
+} else {
+  console.error('Errors:', result.errors);
+}
+```
+
+### SDK Resolvers
+
+| Resolver | Description |
+|----------|-------------|
+| \`sdk.parseInt\` | Parse string to integer |
+| \`sdk.parseFloat\` | Parse string to float |
+| \`sdk.uppercase\` | Convert to uppercase |
+| \`sdk.lowercase\` | Convert to lowercase |
+| \`sdk.trim\` | Trim whitespace |
+| \`sdk.formatDate\` | Format as ISO date string |
+| \`sdk.boolean\` | Convert to boolean |
+
+### Custom Resolvers
+
+```typescript
+const mapper = new UniversalMapper(config, {
+  customResolvers: {
+    'custom.mapStatus': (value, data, config, helpers) => {
+      const map = { 'in_stock': 'AVAILABLE', 'out': 'ON_HOLD' };
+      return map[value] || 'AVAILABLE';
+    }
+  }
+});
+```
+
+## Batch API
+
+### Create Job
+
+```typescript
+const job = await client.createJob({
+  name: 'Inventory Import',
+  retailerId: process.env.FLUENT_RETAILER_ID!,
+  metadata: { source: 's3', file: 'inventory.csv' } // Optional
+});
+```
+
+### Send Batch
+
+```typescript
+await client.sendBatch(job.id, {
+  action: 'UPSERT',           // String, not enum
+  entityType: 'INVENTORY',    // String, not enum
+  source: 'S3_CSV_IMPORT',    // REQUIRED - Source system identifier
+  event: 'INVENTORY_UPDATE',  // REQUIRED - Event type identifier
+  entities: transformedData   // InventoryPositionInput[]
+});
+```
+
+### Get Job Status
+
+```typescript
+const status = await client.getJobStatus(jobId);
+console.log(\`Status: \${status.status}\`); // PENDING, PROCESSING, COMPLETED, FAILED
+```
+
+## State Management
+
+### Setup
+
+```typescript
+// Node.js standalone
+import { createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'inventory-ingestion',
+  correlationId: generateCorrelationId()
+});
+const stateService = new StateService(logger);
+
+// Versori platform
+const logger = toStructuredLogger(ctx.log, {
+  service: 'inventory-ingestion'
+});
+const stateService = new StateService(logger);
+
+// KV adapter setup
+// ✅ CORRECT: Access openKv from Versori context
+const { openKv } = ctx;
+const kvAdapter = new VersoriKVAdapter(openKv(':project:'));
+```
+
+### File Tracking
+
+```typescript
+// Check if processed
+const fileKey = ['file', 'default', fileName];
+const alreadyProcessed = await kv.get(fileKey);
+
+if (alreadyProcessed) {
+  console.log('Already processed');
+  return;
+}
+
+// Mark as processed
+await state.updateSyncState(kv, [{
+  fileName: fileName,
+  lastModified: new Date().toISOString(),
+  recordCount: 1000
+}]);
+
+// Get sync state
+const syncState = await state.getSyncState(kv);
+console.log('Last processed:', syncState.lastProcessedFile);
+```
+
+### Distributed Locking
+
+```typescript
+const lockAcquired = await state.acquireLock(fileKey, kv, 15); // 15 min TTL
+
+if (!lockAcquired) {
+  console.log('Locked by another process');
+  return;
+}
+
+try {
+  // Process file (exclusive access)
+  await processFile(fileKey);
+} finally {
+  // Always release lock
+  await state.releaseLock(fileKey, kv);
+}
+```
+
+## Complete Workflow
+
+```typescript
+async function ingestInventory(kv: any) {
+  // 1. Initialize
+  const client = await createClient({ config });
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      connectionId: 's3-inventory',
+      name: 'S3 Inventory',
+      s3Config: s3Config,
+    },
+    logger
+  );
+  const parser = new CSVParserService(logger);
+  const mapper = new UniversalMapper(mappingConfig);
+
+  // Initialize state management
+  const logger = toStructuredLogger(createConsoleLogger(), {
+    service: 'inventory-ingestion',
+    correlationId: generateCorrelationId()
+  });
+  const stateService = new StateService(logger);
+  const kvAdapter = new VersoriKVAdapter(kv);
+
+  // 2. List files
+  const files = await s3.listFiles({ prefix: 'data/' });
+
+  for (const file of files) {
+    // 3. Check state
+    const fileKey = ['file', 'default', file.path];
+    const alreadyProcessed = await kvAdapter.get(fileKey);
+    if (alreadyProcessed) continue;
+
+    // 4. Acquire lock
+    const lockAcquired = await state.acquireLock(file.path, kvAdapter, 15);
+    if (!lockAcquired) continue;
+
+    try {
+      // 5. Read & parse
+      const data = await s3.downloadFile(file.path);
+      const records = await parser.parse(data);
+
+      // 6. Transform
+      const result = await mapper.map(records);
+      if (!result.success) throw new Error('Mapping failed');
+
+      // 7. Get or create job
+      let job = await state.getDailyJob(kvAdapter, 'daily-import');
+      if (!job) {
+        const newJob = await client.createJob({
+          name: \`Daily Import - \${new Date().toISOString().split('T')[0]}\`,
+          retailerId: process.env.FLUENT_RETAILER_ID!
+        });
+        await state.setDailyJob(kvAdapter, 'daily-import', newJob.id, 24);
+        job = { jobId: newJob.id };
+      }
+
+      // 8. Send batch
+      await client.sendBatch(job.jobId, {
+        action: 'UPSERT',
+        entityType: 'INVENTORY',
+        source: 'S3_CSV_IMPORT',
+        event: 'INVENTORY_UPDATE',
+        entities: result.data
+      });
+
+      // 9. Mark processed
+      await state.updateSyncState(kvAdapter, [{
+        fileName: file.path,
+        lastModified: new Date().toISOString(),
+        recordCount: result.data.length
+      }]);
+
+    } finally {
+      await state.releaseLock(file.path, kvAdapter);
+    }
+  }
+}
+```
+
+## Performance Tips
+
+### Batch Sizing
+
+```typescript
+// Small files (< 10K records)
+const BATCH_SIZE = 1000;
+
+// Large files (> 100K records)
+const BATCH_SIZE = 5000;
+
+// Split into batches
+const batches = [];
+for (let i = 0; i < data.length; i += BATCH_SIZE) {
+  batches.push(data.slice(i, i + BATCH_SIZE));
+}
+```
+
+### Parallel Processing
+
+```typescript
+async function parallelProcess(files: string[], concurrency: number = 5) {
+  const queue = [...files];
+
+  const workers = Array(concurrency).fill(null).map(async () => {
+    while (queue.length > 0) {
+      const file = queue.shift();
+      if (file) await processFile(file);
+    }
+  });
+
+  await Promise.all(workers);
+}
+```
+
+### Streaming for Large Files
+
+```typescript
+const stream = await s3.streamObject('bucket', 'large-file.csv');
+const recordIterator = parser.parseStream(stream);
+
+let batch = [];
+for await (const record of recordIterator) {
+  batch.push(record);
+
+  if (batch.length >= 1000) {
+    await processBatch(batch);
+    batch = [];
+  }
+}
+```
+
+## Job Strategies
+
+### DAILY (Reuse Same Job)
+
+```typescript
+const dateKey = new Date().toISOString().split('T')[0];
+let jobId = await state.get(\`daily-job:\${dateKey}\`);
+
+if (!jobId) {
+  const job = await client.createJob({
+    name: \`Daily Inventory - \${dateKey}\`,
+    retailerId: process.env.FLUENT_RETAILER_ID!
+  });
+  jobId = job.id;
+  await state.set(\`daily-job:\${dateKey}\`, jobId, 24 * 60 * 60 * 1000);
+}
+
+// Use same job for all files
+```
+
+### PER_FILE (Dedicated Job per File)
+
+```typescript
+for (const file of files) {
+  const job = await client.createJob({
+    name: \`Import - \${file.key}\`,
+    retailerId: process.env.FLUENT_RETAILER_ID!
+  });
+
+  await client.sendBatch(job.id, {
+    action: 'UPSERT',
+    entityType: 'INVENTORY',
+    source: 'PER_FILE_STRATEGY',
+    event: 'INVENTORY_FILE_IMPORT',
+    entities: data
+  });
+}
+```
+
+## Error Handling
+
+### Retry with Backoff
+
+```typescript
+async function retryWithBackoff<T>(
+  operation: () => Promise<T>,
+  maxRetries: number = 3
+): Promise<T> {
+  let delay = 1000;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      return await operation();
+    } catch (error) {
+      if (attempt === maxRetries) throw error;
+
+      await new Promise(resolve => setTimeout(resolve, delay));
+      delay *= 2;
+    }
+  }
+}
+```
+
+### Validation
+
+```typescript
+function validateRecord(record: any): boolean {
+  return (
+    record.ref &&
+    record.productRef &&
+    record.locationRef &&
+    typeof record.qty === 'number' &&
+    record.qty >= 0
+  );
+}
+
+const validRecords = records.filter(validateRecord);
+```
+
+## Common Errors
+
+| Error | Cause | Solution |
+|-------|-------|----------|
+| \`JOB_EXPIRED\` | Job > 24 hours old | Create new job |
+| \`VALIDATION_ERROR\` | Invalid field values | Check schema, filter invalid records |
+| \`RATE_LIMIT_ERROR\` | Too many requests | Add delay, reduce concurrency |
+| \`NETWORK_ERROR\` | Connection issues | Retry with backoff |
+
+## Environment Variables
+
+```bash
+# Fluent Commerce
+FLUENT_BASE_URL=https://api.fluentcommerce.com
+FLUENT_CLIENT_ID=your-client-id
+FLUENT_CLIENT_SECRET=your-client-secret
+FLUENT_RETAILER_ID=1
+
+# AWS S3
+AWS_ACCESS_KEY_ID=your-access-key
+AWS_SECRET_ACCESS_KEY=your-secret-key
+AWS_REGION=us-east-1
+
+# SFTP
+SFTP_HOST=sftp.example.com
+SFTP_PORT=22
+SFTP_USERNAME=your-username
+SFTP_PASSWORD=your-password
+```
+
+## CLI Tools
+
+```bash
+# Introspect schema
+npx @fluentcommerce/fc-connect-sdk introspect-schema \
+  --url https://api.fluentcommerce.com/graphql \
+  --output schema.json
+
+# Validate mapping
+npx @fluentcommerce/fc-connect-sdk validate-schema \
+  --mapping config/field-mappings.json \
+  --schema schema.json
+```
+
+---
+
+**See complete documentation:** [Ingestion Learning Path](./ingestion-readme.md)