@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.55

package/docs/01-TEMPLATES/standalone/standalone-graphql-to-parquet-partitioned-s3.md

@@ -1,432 +1,432 @@
-# Standalone: GraphQL Extraction → Partitioned Parquet on S3
-
-**FC Connect SDK Use Case Guide**
-
-> **SDK**: [@fluentcommerce/fc-connect-sdk](https://www.npmjs.com/package/@fluentcommerce/fc-connect-sdk)
-> **Version**: Use latest - `npm install @fluentcommerce/fc-connect-sdk@latest`
-
-**Context**: Extract records from Fluent Commerce via GraphQL with auto-pagination, transform with UniversalMapper, and write multiple Parquet files partitioned by date/entity to S3.
-
-**Type**: Extraction
-
-**Complexity**: Medium
-
-**Runtime**: Node.js ≥18 / Deno
-
-**Estimated Lines**: ~550
-
-## What You'll Build
-
-- OAuth2-authenticated Fluent client
-- GraphQL query with auto-pagination and progress callbacks
-- UniversalMapper transform for export schema
-- Partition planner (by date, entity type, or size)
-- Parquet writer (multiple files) and S3 uploads
-- Incremental extraction using last-export state
-- Metrics, logging, and retries
-
-## SDK Methods Used
-
-```typescript
-import {
-  createClient,
-  UniversalMapper,
-  S3DataSource,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-await createClient({ config: { baseUrl, clientId, clientSecret, retailerId } });
-await client.graphql({ query, variables, pagination });
-new UniversalMapper(mapping, { customResolvers });
-await s3.writeParquetContent(data, { compression: 'SNAPPY' });
-await s3.uploadFile(key, buffer, { contentType: 'application/octet-stream' });
-```
-
-### Method selection (SDK)
-
-- Read data with `query` for simple cases.
-- Use `graphql` with pagination config for large extractions and progress
-
-```typescript
-// Simple read
-const data = await client.graphql({ query, variables });
-
-// Auto-pagination with progress
-const res = await client.graphql({
-  query,
-  variables,
-  pagination: { maxPages: 100 },
-});
-```
-
-## Project Setup
-
-```bash
-mkdir graphql-to-parquet && cd $_
-npm init -y
-npm install @fluentcommerce/fc-connect-sdk dotenv
-mkdir -p src config state logs
-```
-
-### Package Configuration (package.json)
-
-```json
-{
-  "name": "graphql-to-parquet",
-  "version": "1.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "tsx src/export.ts",
-    "build": "tsc",
-    "start": "node dist/export.js"
-  },
-  "dependencies": {
-    "@fluentcommerce/fc-connect-sdk": "^0.1.39",
-    "dotenv": "^16.0.0"
-  },
-  "devDependencies": {
-    "tsx": "^4.0.0",
-    "typescript": "^5.0.0",
-    "@types/node": "^20.0.0"
-  }
-}
-```
-
-### TypeScript Configuration (tsconfig.json)
-
-```json
-{
-  "compilerOptions": {
-    "module": "ES2022",
-    "target": "ES2024",
-    "moduleResolution": "node"
-  }
-}
-```
-
-## Environment (.env)
-
-```bash
-# Fluent Commerce OAuth2
-FLUENT_BASE_URL=https://api.fluentcommerce.com
-FLUENT_CLIENT_ID=xxx
-FLUENT_CLIENT_SECRET=yyy
-FLUENT_RETAILER_ID=zzz
-
-# Target S3
-TARGET_AWS_ACCESS_KEY_ID=...
-TARGET_AWS_SECRET_ACCESS_KEY=...
-TARGET_AWS_REGION=us-east-1
-TARGET_S3_BUCKET=fluent-exports
-TARGET_S3_PREFIX=exports/
-
-# Extraction
-ENTITY_TYPE=inventory   # inventory | orders (examples provided)
-PAGE_SIZE=200
-MAX_RECORDS=20000
-INCREMENTAL=true
-STATE_FILE=./state/last-export.json
-PARTITION_BY=date       # date | entity | none
-PARTITION_SIZE=10000    # records per file (cap)
-```
-
-## Config: Queries (config/queries.json)
-
-```json
-{
-  "inventory": {
-    "connectionPath": "inventoryPositions",
-    "query": "query GetInventory($retailerId: ID!, $first: Int!, $after: String, $updatedFrom: DateTime) {\n  inventoryPositions(retailerId: $retailerId, first: $first, after: $after, updatedOn: { from: $updatedFrom }) {\n    edges { node { id ref productRef locationRef qty availableQty reservedQty status updatedOn } cursor }\n    pageInfo { hasNextPage }\n  }\n}"
-  },
-  "orders": {
-    "connectionPath": "orders",
-    "query": "query GetOrders($retailerId: ID!, $first: Int!, $after: String, $createdFrom: DateTime) {\n  orders(retailerId: $retailerId, first: $first, after: $after, createdOn: { from: $createdFrom }) {\n    edges { node { id ref status totalPrice totalTaxPrice createdOn customer { ref email firstName lastName } } cursor }\n    pageInfo { hasNextPage }\n  }\n}"
-  }
-}
-```
-
-## Config: Mapping (config/extraction-mapping.json)
-
-```json
-{
-  "inventory": {
-    "fields": {
-      "sku": { "source": "productRef", "resolver": "sdk.uppercase" },
-      "location": { "source": "locationRef" },
-      "quantity": { "source": "qty", "resolver": "sdk.parseInt" },
-      "available": { "source": "availableQty", "resolver": "sdk.parseInt" },
-      "status": { "source": "status", "resolver": "sdk.lowercase" },
-      "updated_at": { "source": "updatedOn", "resolver": "sdk.formatDate" }
-    }
-  },
-  "orders": {
-    "fields": {
-      "order_id": { "source": "ref" },
-      "status": { "source": "status", "resolver": "sdk.lowercase" },
-      "total": { "source": "totalPrice", "resolver": "sdk.parseFloat" },
-      "created_at": { "source": "createdOn", "resolver": "sdk.formatDate" },
-      "customer_email": { "source": "customer.email" },
-      "customer_name": {
-        "source": null,
-        "resolver": "custom.fullName"
-      }
-    }
-  }
-}
-```
-
-## Complete Script (src/export.ts)
-
-```typescript
-import 'dotenv/config';
-import * as fs from 'node:fs/promises';
-import * as path from 'node:path';
-import {
-  createClient,
-  UniversalMapper,
-  S3DataSource,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  logLevel: process.env.LOG_LEVEL || 'info'
-});
-
-interface ExportState {
-  lastExportIso?: string;
-  totalExported?: number;
-}
-
-async function loadState(filePath: string): Promise<ExportState> {
-  try {
-    return JSON.parse(await fs.readFile(filePath, 'utf-8'));
-  } catch {
-    return {};
-  }
-}
-
-async function saveState(filePath: string, state: ExportState): Promise<void> {
-  await fs.mkdir(path.dirname(filePath), { recursive: true });
-  await fs.writeFile(filePath, JSON.stringify(state, null, 2));
-}
-
-function getEnv(name: string, fallback?: string): string {
-  const v = process.env[name] || fallback;
-  if (!v) throw new Error(`Missing env ${name}`);
-  return v;
-}
-
-async function main() {
-  // Setup
-  const entityType = (process.env.ENTITY_TYPE || 'inventory').toLowerCase();
-  const pageSize = parseInt(process.env.PAGE_SIZE || '200', 10);
-  const maxRecords = parseInt(process.env.MAX_RECORDS || '20000', 10);
-  const incremental = process.env.INCREMENTAL === 'true';
-  const stateFile = process.env.STATE_FILE || './state/last-export.json';
-  const partitionBy = (process.env.PARTITION_BY || 'date').toLowerCase();
-  const partitionSize = parseInt(process.env.PARTITION_SIZE || '10000', 10);
-
-  const client = await createClient({
-    config: {
-      baseUrl: getEnv('FLUENT_BASE_URL'),
-      clientId: getEnv('FLUENT_CLIENT_ID'),
-      clientSecret: getEnv('FLUENT_CLIENT_SECRET'),
-      retailerId: getEnv('FLUENT_RETAILER_ID'),
-    },
-  });
-
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      connectionId: 'target-s3',
-      name: 'Target S3',
-      s3Config: {
-        bucket: getEnv('TARGET_S3_BUCKET'),
-        region: getEnv('TARGET_AWS_REGION'),
-        credentials: {
-          accessKeyId: getEnv('TARGET_AWS_ACCESS_KEY_ID'),
-          secretAccessKey: getEnv('TARGET_AWS_SECRET_ACCESS_KEY'),
-        },
-      },
-    },
-    logger
-  );
-
-  // Load configs
-  const queries = JSON.parse(await fs.readFile('config/queries.json', 'utf-8'));
-  const mappings = JSON.parse(await fs.readFile('config/extraction-mapping.json', 'utf-8'));
-
-  const queryCfg = queries[entityType];
-  if (!queryCfg) throw new Error(`Unknown entityType ${entityType}`);
-
-  const variables: any = { retailerId: getEnv('FLUENT_RETAILER_ID'), first: pageSize };
-
-  if (incremental) {
-    const state = await loadState(stateFile);
-    if (state.lastExportIso) {
-      if (entityType === 'inventory') variables.updatedFrom = state.lastExportIso;
-      if (entityType === 'orders') variables.createdFrom = state.lastExportIso;
-    }
-  }
-
-  // Query with auto-pagination
-  logger.info('Executing GraphQL extraction...', { entityType, pageSize, maxRecords });
-  const startQuery = Date.now();
-  const gqlResult = await (client as any).graphql({
-    query: queryCfg.query,
-    variables,
-    pagination: {
-      maxRecords,
-    },
-  });
-  const queryMs = Date.now() - startQuery;
-
-  // Flatten nodes
-  const connection = gqlResult.data[queryCfg.connectionPath];
-  const nodes = Array.isArray(connection?.edges) ? connection.edges.map((e: any) => e.node) : [];
-
-  logger.info('Extracted', { count: nodes.length, queryMs });
-
-  if (nodes.length === 0) return;
-
-  // Transform
-  const mapper = new UniversalMapper(mappings[entityType] || { fields: {} });
-  const transformed: any[] = [];
-  for (const n of nodes) {
-    const res = await mapper.map(n);
-    if (res.success) transformed.push(res.data);
-  }
-
-  logger.info('Transformed', { count: transformed.length });
-
-  // Partitioning
-  const groups: any[][] = [];
-  if (partitionBy === 'none') {
-    for (let i = 0; i < transformed.length; i += partitionSize) {
-      groups.push(transformed.slice(i, i + partitionSize));
-    }
-  } else if (partitionBy === 'entity') {
-    // Example: split by status/category field when available
-    const byKey = new Map<string, any[]>();
-    for (const rec of transformed) {
-      const key = String(rec.status || 'default');
-      const arr = byKey.get(key) || [];
-      arr.push(rec);
-      byKey.set(key, arr);
-    }
-    for (const arr of byKey.values()) {
-      for (let i = 0; i < arr.length; i += partitionSize)
-        groups.push(arr.slice(i, i + partitionSize));
-    }
-  } else {
-    // date partition (YYYY/MM/DD)
-    const byDate = new Map<string, any[]>();
-    for (const rec of transformed) {
-      const iso = String(rec.updated_at || rec.created_at || new Date().toISOString());
-      const d = new Date(iso);
-      const key = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`;
-      const arr = byDate.get(key) || [];
-      arr.push(rec);
-      byDate.set(key, arr);
-    }
-    for (const [dateKey, arr] of byDate.entries()) {
-      for (let i = 0; i < arr.length; i += partitionSize)
-        groups.push(Object.assign(arr.slice(i, i + partitionSize), { __dateKey: dateKey }));
-    }
-  }
-
-  // Write multiple Parquet files
-  const prefix = process.env.TARGET_S3_PREFIX || 'exports/';
-  const uploaded: string[] = [];
-  let idx = 0;
-
-  for (const group of groups) {
-    const dateKey = (group as any).__dateKey as string | undefined;
-    const ts = new Date().toISOString().replace(/[:.]/g, '-');
-    const folder =
-      partitionBy === 'date' && dateKey
-        ? `${prefix}${entityType}/${dateKey}/`
-        : `${prefix}${entityType}/`;
-    const key = `${folder}${entityType}_${ts}_part-${idx++}.parquet`;
-
-    const buffer = await (s3 as any).writeParquetContent(group as any[], {
-      compression: 'SNAPPY',
-      rowGroupSize: 10000,
-    });
-
-    await s3.uploadFile(key, buffer, {
-      contentType: 'application/octet-stream',
-      metadata: {
-        'entity-type': entityType,
-        'record-count': String(group.length),
-        'exported-at': new Date().toISOString(),
-      },
-    });
-
-    uploaded.push(key);
-    logger.info('Uploaded', { key, count: group.length });
-  }
-
-  // Update state (incremental)
-  if (incremental) {
-    await saveState(stateFile, {
-      lastExportIso: new Date().toISOString(),
-      totalExported: transformed.length,
-    });
-  }
-
-  logger.info('Export complete', { files: uploaded.length, totalRecords: transformed.length });
-}
-
-main().catch(e => {
-  console.error(e);
-  process.exit(1);
-});
-```
-
-## Testing
-
-```bash
-# Configure .env and configs, then run a small test
-PAGE_SIZE=50 MAX_RECORDS=200 INCREMENTAL=false npm run dev
-
-# Verify S3 uploads
-aws s3 ls s3://$TARGET_S3_BUCKET/$TARGET_S3_PREFIX --recursive | tail -n 5
-```
-
-## Deployment Options
-
-- Local/cron: run on schedule with Task Scheduler/cron
-- Docker: package script, mount .env, run on ECS/EKS/VM
-- Lambda: ensure memory/time for Parquet; consider splitting by pages
-
-## Common Issues & Solutions
-
-- Auto-pagination not triggered
-  - Ensure query uses $first and $after variables; pass first via variables
-- Very large exports cause memory issues
-  - Reduce MAX_RECORDS/PAGE_SIZE; write in smaller partitions
-- Time-based gaps in incremental mode
-  - Add overlap window (export last 5–10 minutes again) and dedupe downstream
-- S3 upload slow
-  - Use SNAPPY compression, smaller row groups, parallelize uploads if needed
-
-## Production Checklist
-
-- OAuth2 credentials managed securely
-- S3 bucket permissions least-privilege
-- Query and mapping validated against target analytics needs
-- Partition plan documented (date/entity/size) with consumers
-- Incremental strategy defined; overlap to prevent gaps
-- Observability: logs, metrics, counters
-- Backoff/retries for transient failures
-- Cost controls: partition sizes and schedule cadence
-
-## Related Guides
-
-- Auto-pagination: `../../02-CORE-GUIDES/auto-pagination/readme.md`
-- Universal Mapping: `../../02-CORE-GUIDES/mapping/readme.md`
-- Parquet writing: `../../02-CORE-GUIDES/parsers/readme.md`
-- Error handling pattern: `docs/use-cases/patterns/error-handling-retry.md`
+# Standalone: GraphQL Extraction → Partitioned Parquet on S3
+
+**FC Connect SDK Use Case Guide**
+
+> **SDK**: [@fluentcommerce/fc-connect-sdk](https://www.npmjs.com/package/@fluentcommerce/fc-connect-sdk)
+> **Version**: Use latest - `npm install @fluentcommerce/fc-connect-sdk@latest`
+
+**Context**: Extract records from Fluent Commerce via GraphQL with auto-pagination, transform with UniversalMapper, and write multiple Parquet files partitioned by date/entity to S3.
+
+**Type**: Extraction
+
+**Complexity**: Medium
+
+**Runtime**: Node.js ≥18 / Deno
+
+**Estimated Lines**: ~550
+
+## What You'll Build
+
+- OAuth2-authenticated Fluent client
+- GraphQL query with auto-pagination and progress callbacks
+- UniversalMapper transform for export schema
+- Partition planner (by date, entity type, or size)
+- Parquet writer (multiple files) and S3 uploads
+- Incremental extraction using last-export state
+- Metrics, logging, and retries
+
+## SDK Methods Used
+
+```typescript
+import {
+  createClient,
+  UniversalMapper,
+  S3DataSource,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+await createClient({ config: { baseUrl, clientId, clientSecret, retailerId } });
+await client.graphql({ query, variables, pagination });
+new UniversalMapper(mapping, { customResolvers });
+await s3.writeParquetContent(data, { compression: 'SNAPPY' });
+await s3.uploadFile(key, buffer, { contentType: 'application/octet-stream' });
+```
+
+### Method selection (SDK)
+
+- Read data with `query` for simple cases.
+- Use `graphql` with pagination config for large extractions and progress
+
+```typescript
+// Simple read
+const data = await client.graphql({ query, variables });
+
+// Auto-pagination with progress
+const res = await client.graphql({
+  query,
+  variables,
+  pagination: { maxPages: 100 },
+});
+```
+
+## Project Setup
+
+```bash
+mkdir graphql-to-parquet && cd $_
+npm init -y
+npm install @fluentcommerce/fc-connect-sdk dotenv
+mkdir -p src config state logs
+```
+
+### Package Configuration (package.json)
+
+```json
+{
+  "name": "graphql-to-parquet",
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "tsx src/export.ts",
+    "build": "tsc",
+    "start": "node dist/export.js"
+  },
+  "dependencies": {
+    "@fluentcommerce/fc-connect-sdk": "^0.1.39",
+    "dotenv": "^16.0.0"
+  },
+  "devDependencies": {
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0",
+    "@types/node": "^20.0.0"
+  }
+}
+```
+
+### TypeScript Configuration (tsconfig.json)
+
+```json
+{
+  "compilerOptions": {
+    "module": "ES2022",
+    "target": "ES2024",
+    "moduleResolution": "node"
+  }
+}
+```
+
+## Environment (.env)
+
+```bash
+# Fluent Commerce OAuth2
+FLUENT_BASE_URL=https://api.fluentcommerce.com
+FLUENT_CLIENT_ID=xxx
+FLUENT_CLIENT_SECRET=yyy
+FLUENT_RETAILER_ID=zzz
+
+# Target S3
+TARGET_AWS_ACCESS_KEY_ID=...
+TARGET_AWS_SECRET_ACCESS_KEY=...
+TARGET_AWS_REGION=us-east-1
+TARGET_S3_BUCKET=fluent-exports
+TARGET_S3_PREFIX=exports/
+
+# Extraction
+ENTITY_TYPE=inventory   # inventory | orders (examples provided)
+PAGE_SIZE=200
+MAX_RECORDS=20000
+INCREMENTAL=true
+STATE_FILE=./state/last-export.json
+PARTITION_BY=date       # date | entity | none
+PARTITION_SIZE=10000    # records per file (cap)
+```
+
+## Config: Queries (config/queries.json)
+
+```json
+{
+  "inventory": {
+    "connectionPath": "inventoryPositions",
+    "query": "query GetInventory($retailerId: ID!, $first: Int!, $after: String, $updatedFrom: DateTime) {\n  inventoryPositions(retailerId: $retailerId, first: $first, after: $after, updatedOn: { from: $updatedFrom }) {\n    edges { node { id ref productRef locationRef qty availableQty reservedQty status updatedOn } cursor }\n    pageInfo { hasNextPage }\n  }\n}"
+  },
+  "orders": {
+    "connectionPath": "orders",
+    "query": "query GetOrders($retailerId: ID!, $first: Int!, $after: String, $createdFrom: DateTime) {\n  orders(retailerId: $retailerId, first: $first, after: $after, createdOn: { from: $createdFrom }) {\n    edges { node { id ref status totalPrice totalTaxPrice createdOn customer { ref email firstName lastName } } cursor }\n    pageInfo { hasNextPage }\n  }\n}"
+  }
+}
+```
+
+## Config: Mapping (config/extraction-mapping.json)
+
+```json
+{
+  "inventory": {
+    "fields": {
+      "sku": { "source": "productRef", "resolver": "sdk.uppercase" },
+      "location": { "source": "locationRef" },
+      "quantity": { "source": "qty", "resolver": "sdk.parseInt" },
+      "available": { "source": "availableQty", "resolver": "sdk.parseInt" },
+      "status": { "source": "status", "resolver": "sdk.lowercase" },
+      "updated_at": { "source": "updatedOn", "resolver": "sdk.formatDate" }
+    }
+  },
+  "orders": {
+    "fields": {
+      "order_id": { "source": "ref" },
+      "status": { "source": "status", "resolver": "sdk.lowercase" },
+      "total": { "source": "totalPrice", "resolver": "sdk.parseFloat" },
+      "created_at": { "source": "createdOn", "resolver": "sdk.formatDate" },
+      "customer_email": { "source": "customer.email" },
+      "customer_name": {
+        "source": null,
+        "resolver": "custom.fullName"
+      }
+    }
+  }
+}
+```
+
+## Complete Script (src/export.ts)
+
+```typescript
+import 'dotenv/config';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import {
+  createClient,
+  UniversalMapper,
+  S3DataSource,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  logLevel: process.env.LOG_LEVEL || 'info'
+});
+
+interface ExportState {
+  lastExportIso?: string;
+  totalExported?: number;
+}
+
+async function loadState(filePath: string): Promise<ExportState> {
+  try {
+    return JSON.parse(await fs.readFile(filePath, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+async function saveState(filePath: string, state: ExportState): Promise<void> {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, JSON.stringify(state, null, 2));
+}
+
+function getEnv(name: string, fallback?: string): string {
+  const v = process.env[name] || fallback;
+  if (!v) throw new Error(`Missing env ${name}`);
+  return v;
+}
+
+async function main() {
+  // Setup
+  const entityType = (process.env.ENTITY_TYPE || 'inventory').toLowerCase();
+  const pageSize = parseInt(process.env.PAGE_SIZE || '200', 10);
+  const maxRecords = parseInt(process.env.MAX_RECORDS || '20000', 10);
+  const incremental = process.env.INCREMENTAL === 'true';
+  const stateFile = process.env.STATE_FILE || './state/last-export.json';
+  const partitionBy = (process.env.PARTITION_BY || 'date').toLowerCase();
+  const partitionSize = parseInt(process.env.PARTITION_SIZE || '10000', 10);
+
+  const client = await createClient({
+    config: {
+      baseUrl: getEnv('FLUENT_BASE_URL'),
+      clientId: getEnv('FLUENT_CLIENT_ID'),
+      clientSecret: getEnv('FLUENT_CLIENT_SECRET'),
+      retailerId: getEnv('FLUENT_RETAILER_ID'),
+    },
+  });
+
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      connectionId: 'target-s3',
+      name: 'Target S3',
+      s3Config: {
+        bucket: getEnv('TARGET_S3_BUCKET'),
+        region: getEnv('TARGET_AWS_REGION'),
+        credentials: {
+          accessKeyId: getEnv('TARGET_AWS_ACCESS_KEY_ID'),
+          secretAccessKey: getEnv('TARGET_AWS_SECRET_ACCESS_KEY'),
+        },
+      },
+    },
+    logger
+  );
+
+  // Load configs
+  const queries = JSON.parse(await fs.readFile('config/queries.json', 'utf-8'));
+  const mappings = JSON.parse(await fs.readFile('config/extraction-mapping.json', 'utf-8'));
+
+  const queryCfg = queries[entityType];
+  if (!queryCfg) throw new Error(`Unknown entityType ${entityType}`);
+
+  const variables: any = { retailerId: getEnv('FLUENT_RETAILER_ID'), first: pageSize };
+
+  if (incremental) {
+    const state = await loadState(stateFile);
+    if (state.lastExportIso) {
+      if (entityType === 'inventory') variables.updatedFrom = state.lastExportIso;
+      if (entityType === 'orders') variables.createdFrom = state.lastExportIso;
+    }
+  }
+
+  // Query with auto-pagination
+  logger.info('Executing GraphQL extraction...', { entityType, pageSize, maxRecords });
+  const startQuery = Date.now();
+  const gqlResult = await (client as any).graphql({
+    query: queryCfg.query,
+    variables,
+    pagination: {
+      maxRecords,
+    },
+  });
+  const queryMs = Date.now() - startQuery;
+
+  // Flatten nodes
+  const connection = gqlResult.data[queryCfg.connectionPath];
+  const nodes = Array.isArray(connection?.edges) ? connection.edges.map((e: any) => e.node) : [];
+
+  logger.info('Extracted', { count: nodes.length, queryMs });
+
+  if (nodes.length === 0) return;
+
+  // Transform
+  const mapper = new UniversalMapper(mappings[entityType] || { fields: {} });
+  const transformed: any[] = [];
+  for (const n of nodes) {
+    const res = await mapper.map(n);
+    if (res.success) transformed.push(res.data);
+  }
+
+  logger.info('Transformed', { count: transformed.length });
+
+  // Partitioning
+  const groups: any[][] = [];
+  if (partitionBy === 'none') {
+    for (let i = 0; i < transformed.length; i += partitionSize) {
+      groups.push(transformed.slice(i, i + partitionSize));
+    }
+  } else if (partitionBy === 'entity') {
+    // Example: split by status/category field when available
+    const byKey = new Map<string, any[]>();
+    for (const rec of transformed) {
+      const key = String(rec.status || 'default');
+      const arr = byKey.get(key) || [];
+      arr.push(rec);
+      byKey.set(key, arr);
+    }
+    for (const arr of byKey.values()) {
+      for (let i = 0; i < arr.length; i += partitionSize)
+        groups.push(arr.slice(i, i + partitionSize));
+    }
+  } else {
+    // date partition (YYYY/MM/DD)
+    const byDate = new Map<string, any[]>();
+    for (const rec of transformed) {
+      const iso = String(rec.updated_at || rec.created_at || new Date().toISOString());
+      const d = new Date(iso);
+      const key = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`;
+      const arr = byDate.get(key) || [];
+      arr.push(rec);
+      byDate.set(key, arr);
+    }
+    for (const [dateKey, arr] of byDate.entries()) {
+      for (let i = 0; i < arr.length; i += partitionSize)
+        groups.push(Object.assign(arr.slice(i, i + partitionSize), { __dateKey: dateKey }));
+    }
+  }
+
+  // Write multiple Parquet files
+  const prefix = process.env.TARGET_S3_PREFIX || 'exports/';
+  const uploaded: string[] = [];
+  let idx = 0;
+
+  for (const group of groups) {
+    const dateKey = (group as any).__dateKey as string | undefined;
+    const ts = new Date().toISOString().replace(/[:.]/g, '-');
+    const folder =
+      partitionBy === 'date' && dateKey
+        ? `${prefix}${entityType}/${dateKey}/`
+        : `${prefix}${entityType}/`;
+    const key = `${folder}${entityType}_${ts}_part-${idx++}.parquet`;
+
+    const buffer = await (s3 as any).writeParquetContent(group as any[], {
+      compression: 'SNAPPY',
+      rowGroupSize: 10000,
+    });
+
+    await s3.uploadFile(key, buffer, {
+      contentType: 'application/octet-stream',
+      metadata: {
+        'entity-type': entityType,
+        'record-count': String(group.length),
+        'exported-at': new Date().toISOString(),
+      },
+    });
+
+    uploaded.push(key);
+    logger.info('Uploaded', { key, count: group.length });
+  }
+
+  // Update state (incremental)
+  if (incremental) {
+    await saveState(stateFile, {
+      lastExportIso: new Date().toISOString(),
+      totalExported: transformed.length,
+    });
+  }
+
+  logger.info('Export complete', { files: uploaded.length, totalRecords: transformed.length });
+}
+
+main().catch(e => {
+  console.error(e);
+  process.exit(1);
+});
+```
+
+## Testing
+
+```bash
+# Configure .env and configs, then run a small test
+PAGE_SIZE=50 MAX_RECORDS=200 INCREMENTAL=false npm run dev
+
+# Verify S3 uploads
+aws s3 ls s3://$TARGET_S3_BUCKET/$TARGET_S3_PREFIX --recursive | tail -n 5
+```
+
+## Deployment Options
+
+- Local/cron: run on schedule with Task Scheduler/cron
+- Docker: package script, mount .env, run on ECS/EKS/VM
+- Lambda: ensure memory/time for Parquet; consider splitting by pages
+
+## Common Issues & Solutions
+
+- Auto-pagination not triggered
+  - Ensure query uses $first and $after variables; pass first via variables
+- Very large exports cause memory issues
+  - Reduce MAX_RECORDS/PAGE_SIZE; write in smaller partitions
+- Time-based gaps in incremental mode
+  - Add overlap window (export last 5–10 minutes again) and dedupe downstream
+- S3 upload slow
+  - Use SNAPPY compression, smaller row groups, parallelize uploads if needed
+
+## Production Checklist
+
+- OAuth2 credentials managed securely
+- S3 bucket permissions least-privilege
+- Query and mapping validated against target analytics needs
+- Partition plan documented (date/entity/size) with consumers
+- Incremental strategy defined; overlap to prevent gaps
+- Observability: logs, metrics, counters
+- Backoff/retries for transient failures
+- Cost controls: partition sizes and schedule cadence
+
+## Related Guides
+
+- Auto-pagination: `../../02-CORE-GUIDES/auto-pagination/readme.md`
+- Universal Mapping: `../../02-CORE-GUIDES/mapping/readme.md`
+- Parquet writing: `../../02-CORE-GUIDES/parsers/readme.md`
+- Error handling pattern: `docs/use-cases/patterns/error-handling-retry.md`