@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.55

package/docs/03-PATTERN-GUIDES/parquet/parquet-quick-reference.md

@@ -1,683 +1,683 @@
-# Parquet Quick Reference
-
-> One-page cheat sheet for Parquet file operations with FC Connect SDK
-
-## Import Statements
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-import * as parquet from 'parquetjs'; // For writing
-```
-
-## Reading Parquet Files
-
-### Basic Read
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-import * as fs from 'fs/promises';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'ParquetParser'
-});
-const parser = new ParquetParserService(logger);
-
-// Read file
-const buffer = await fs.readFile('inventory.parquet');
-const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-// Parse
-const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet');
-
-console.log(`Parsed ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
-```
-
-### Read from S3
-
-```typescript
-import {
-  S3DataSource,
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'S3ParquetReader'
-});
-
-const s3 = new S3DataSource(
-  {
-    type: 'S3_CSV',
-    s3Config: {
-      bucket: process.env.S3_BUCKET!,
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-      region: process.env.AWS_REGION!,
-    },
-  },
-  logger
-);
-
-const parser = new ParquetParserService(logger);
-
-// Download and parse
-const buffer = await s3.downloadFile('data/inventory.parquet', { encoding: 'binary' });
-const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet');
-```
-
-### Streaming Read (Large Files)
-
-```typescript
-const streamResult = await parser.parseStreaming(arrayBuffer, 'large-file.parquet', {
-  batchSize: 1000,
-});
-
-for await (const batch of streamResult.records) {
-  console.log(`Processing batch of ${batch.length} records`);
-  await processBatch(batch);
-}
-
-const stats = await streamResult.stats;
-console.log(`Total: ${stats.totalRecords} records`);
-```
-
-## Writing Parquet Files
-
-### Basic Write
-
-```typescript
-import * as parquet from 'parquetjs';
-
-// Define schema
-const schema = new parquet.ParquetSchema({
-  ref: { type: 'UTF8', optional: false },
-  productRef: { type: 'UTF8', optional: false },
-  locationRef: { type: 'UTF8', optional: false },
-  qty: { type: 'INT32', optional: false },
-  status: { type: 'UTF8', optional: true },
-});
-
-// Create writer
-const writer = await parquet.ParquetWriter.openFile(schema, 'output.parquet', {
-  compression: 'SNAPPY',
-});
-
-// Write records
-const records = [
-  { ref: 'IP-001', productRef: 'SKU-001', locationRef: 'WH-001', qty: 100, status: 'AVAILABLE' },
-  { ref: 'IP-002', productRef: 'SKU-002', locationRef: 'WH-001', qty: 50, status: 'RESERVED' },
-];
-
-for (const record of records) {
-  await writer.appendRow(record);
-}
-
-await writer.close();
-```
-
-### Write to S3
-
-**Method 1: Using writeParquetContent() (Recommended)**
-
-```typescript
-import { S3DataSource, createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'ParquetWriter'
-});
-
-const s3 = new S3DataSource(
-  {
-    type: 'S3_CSV',
-    s3Config: {
-      bucket: process.env.S3_BUCKET!,
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-      region: process.env.AWS_REGION!,
-    },
-  },
-  logger
-);
-
-const records = [
-  { id: 1, name: 'Product A', qty: 100 },
-  { id: 2, name: 'Product B', qty: 200 },
-];
-
-// Generate Parquet Buffer
-const parquetBuffer = await s3.writeParquetContent(records, {
-  schema: {
-    id: { type: 'INT32' },
-    name: { type: 'UTF8' },
-    qty: { type: 'INT32' },
-  },
-  compression: 'SNAPPY',
-});
-
-// Upload to S3
-await s3.uploadFile('data/output.parquet', parquetBuffer);
-```
-
-**Method 2: Temp File Approach**
-
-```typescript
-import * as parquet from 'parquetjs';
-import * as fs from 'fs/promises';
-import { S3DataSource, createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'ParquetWriter'
-});
-
-const s3 = new S3DataSource(s3Config, logger);
-
-// Define schema
-const schema = new parquet.ParquetSchema({
-  id: { type: 'INT32' },
-  name: { type: 'UTF8' },
-  qty: { type: 'INT32' },
-});
-
-// Write to local file first
-const writer = await parquet.ParquetWriter.openFile(schema, 'temp.parquet', {
-  compression: 'SNAPPY',
-});
-
-for (const record of records) {
-  await writer.appendRow(record);
-}
-
-await writer.close();
-
-// Upload to S3
-const buffer = await fs.readFile('temp.parquet');
-await s3.uploadFile('data/output.parquet', buffer);
-
-// Cleanup
-await fs.unlink('temp.parquet');
-```
-
-## Parquet Data Types
-
-| Parquet Type       | TypeScript Type | Use Case                  |
-| ------------------ | --------------- | ------------------------- |
-| `UTF8`             | string          | Text, SKUs, references    |
-| `INT32`            | number          | Quantities, small numbers |
-| `INT64`            | number          | Large numbers, IDs        |
-| `FLOAT`            | number          | Decimals (less precision) |
-| `DOUBLE`           | number          | Decimals (more precision) |
-| `BOOLEAN`          | boolean         | Flags, true/false         |
-| `DATE`             | Date            | Date only (no time)       |
-| `TIMESTAMP_MILLIS` | Date            | Date and time             |
-
-### Schema Examples
-
-```typescript
-// Inventory Position Schema
-const inventorySchema = new parquet.ParquetSchema({
-  ref: { type: 'UTF8', optional: false },
-  productRef: { type: 'UTF8', optional: false },
-  locationRef: { type: 'UTF8', optional: false },
-  qty: { type: 'INT32', optional: false },
-  status: { type: 'UTF8', optional: true },
-  updatedOn: { type: 'TIMESTAMP_MILLIS', optional: true },
-});
-
-// Product Schema
-const productSchema = new parquet.ParquetSchema({
-  sku: { type: 'UTF8', optional: false },
-  name: { type: 'UTF8', optional: false },
-  price: { type: 'DOUBLE', optional: false },
-  inStock: { type: 'BOOLEAN', optional: false },
-  lastUpdated: { type: 'TIMESTAMP_MILLIS', optional: true },
-});
-
-// Order Schema
-const orderSchema = new parquet.ParquetSchema({
-  orderNumber: { type: 'UTF8', optional: false },
-  customerId: { type: 'UTF8', optional: false },
-  totalAmount: { type: 'DOUBLE', optional: false },
-  itemCount: { type: 'INT32', optional: false },
-  orderDate: { type: 'TIMESTAMP_MILLIS', optional: false },
-  status: { type: 'UTF8', optional: true },
-});
-```
-
-## Compression Options
-
-| Algorithm      | Ratio | Speed     | Use When                |
-| -------------- | ----- | --------- | ----------------------- |
-| `UNCOMPRESSED` | 1x    | Fastest   | Local testing only      |
-| `SNAPPY`       | 2-3x  | Very Fast | **Recommended default** |
-| `GZIP`         | 3-4x  | Moderate  | Storage optimization    |
-| `BROTLI`       | 4-5x  | Slow      | Maximum compression     |
-
-```typescript
-// SNAPPY (recommended)
-const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
-  compression: 'SNAPPY',
-  rowGroupSize: 10000,
-});
-
-// GZIP (better compression)
-const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
-  compression: 'GZIP',
-  compressionLevel: 6, // 1-9
-});
-
-// BROTLI (max compression)
-const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
-  compression: 'BROTLI',
-  compressionLevel: 4, // 1-11
-});
-```
-
-## Parser Configuration
-
-```typescript
-const parser = new ParquetParserService(logger, {
-  batchSize: 1000, // Records per batch
-  maxFileSize: 100 * 1024 * 1024, // 100 MB max
-  enableStreaming: true, // Enable streaming
-});
-
-// Parse with validation
-const { records, stats } = await parser.parse(arrayBuffer, 'file.parquet', {
-  batchSize: 500,
-  validation: {
-    enabled: true,
-    requiredColumns: ['productRef', 'locationRef', 'qty'],
-    columnTypes: {
-      productRef: 'string',
-      locationRef: 'string',
-      qty: 'int32',
-    },
-    allowAdditionalColumns: true,
-  },
-});
-```
-
-## GraphQL to Parquet Extraction
-
-### Complete Workflow
-
-```typescript
-import { createClient, ParquetParserService } from '@fluentcommerce/fc-connect-sdk';
-import * as parquet from 'parquetjs';
-
-async function extractInventoryToParquet(outputFile: string) {
-  // 1. Create Fluent client
-  const client = await createClient({
-    config: {
-      baseUrl: process.env.FLUENT_BASE_URL!,
-      clientId: process.env.FLUENT_CLIENT_ID!,
-      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-      retailerId: process.env.FLUENT_RETAILER_ID!,
-    },
-  });
-
-  // 2. Define Parquet schema
-  const schema = new parquet.ParquetSchema({
-    ref: { type: 'UTF8' },
-    productRef: { type: 'UTF8' },
-    locationRef: { type: 'UTF8' },
-    qty: { type: 'INT32' },
-    status: { type: 'UTF8', optional: true },
-    extractedAt: { type: 'TIMESTAMP_MILLIS' },
-  });
-
-  // 3. Create writer
-  const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
-    compression: 'SNAPPY',
-    rowGroupSize: 10000,
-  });
-
-  // 4. Extract with pagination
-  let hasNextPage = true;
-  let cursor: string | null = null;
-  let totalRecords = 0;
-
-  while (hasNextPage) {
-    const result = await client.graphql({
-      query: `
-        query ExtractInventory($after: String, $first: Int!) {
-          inventoryPositions(after: $after, first: $first) {
-            pageInfo { hasNextPage }
-            edges {
-              cursor
-              node {
-                ref
-                productRef
-                locationRef
-                qty
-                status
-              }
-            }
-          }
-        }
-      `,
-      variables: { first: 1000, after: cursor },
-    });
-
-    const { edges, pageInfo } = result.data.inventoryPositions;
-
-    // 5. Write batch
-    for (const edge of edges) {
-      await writer.appendRow({
-        ...edge.node,
-        extractedAt: new Date(),
-      });
-      totalRecords++;
-    }
-
-    console.log(`Written ${totalRecords} records...`);
-
-    hasNextPage = pageInfo.hasNextPage;
-    cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
-  }
-
-  // 6. Close writer
-  await writer.close();
-  console.log(`Complete: ${totalRecords} records`);
-}
-```
-
-## S3 Integration
-
-### Download → Parse → Process
-
-```typescript
-import {
-  S3DataSource,
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'S3ParquetProcessor'
-});
-
-const s3 = new S3DataSource(
-  {
-    type: 'S3_CSV',
-    s3Config: {
-      bucket: process.env.S3_BUCKET!,
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-      region: process.env.AWS_REGION!,
-    },
-  },
-  logger
-);
-
-const parser = new ParquetParserService(logger);
-
-// List files
-const files = await s3.listFiles({ prefix: 'parquet/' });
-
-for (const file of files) {
-  // Download
-  const buffer = await s3.downloadFile(file.path, { encoding: 'binary' });
-
-  // Parse
-  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-  const { records, stats } = await parser.parse(arrayBuffer, file.path);
-
-  // Process
-  console.log(`${file.path}: ${stats.totalRecords} records`);
-  await processRecords(records);
-}
-```
-
-### Extract → Write → Upload
-
-**Complete Workflow**
-
-```typescript
-import { 
-  createClient, 
-  S3DataSource,
-  createConsoleLogger,
-  toStructuredLogger 
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'Extraction'
-});
-
-const s3 = new S3DataSource(s3Config, logger);
-const client = await createClient({ config });
-
-// Extract from Fluent
-const result = await client.graphql({ query, pagination: { maxPages: 10 } });
-
-const data = result.data.inventoryPositions.edges.map((e) => ({
-  ...e.node,
-  extractedAt: new Date().toISOString(),
-}));
-
-// Generate Parquet Buffer
-const parquetBuffer = await s3.writeParquetContent(data, {
-  schema: {
-    ref: { type: 'UTF8' },
-    productRef: { type: 'UTF8' },
-    locationRef: { type: 'UTF8' },
-    qty: { type: 'INT32' },
-    extractedAt: { type: 'UTF8' },
-  },
-  compression: 'SNAPPY',
-});
-
-// Upload to S3
-const s3Key = `extracts/inventory_${new Date().toISOString().split('T')[0]}.parquet`;
-await s3.uploadFile(s3Key, parquetBuffer);
-
-console.log(`Uploaded to s3://my-bucket/${s3Key}`);
-```
-
-## Parse Statistics
-
-```typescript
-const { records, stats } = await parser.parse(arrayBuffer, 'file.parquet');
-
-// Performance metrics
-console.log(`Total Records: ${stats.totalRecords}`);
-console.log(`Processing Time: ${stats.processingTimeMs}ms`);
-console.log(`Throughput: ${stats.recordsPerSecond} records/sec`);
-console.log(`Peak Memory: ${(stats.peakMemoryUsage / 1024 / 1024).toFixed(2)} MB`);
-
-// Column statistics
-Object.entries(stats.columnStats).forEach(([name, col]) => {
-  console.log(`${name}: ${col.type}, ${col.nullCount} nulls, ${col.uniqueValues} unique`);
-  if (col.minValue !== undefined) {
-    console.log(`  Range: ${col.minValue} - ${col.maxValue}`);
-  }
-});
-
-// Validation results (if enabled)
-if (stats.validation) {
-  if (!stats.validation.isValid) {
-    console.error('Validation errors:', stats.validation.errors);
-  }
-}
-```
-
-## Performance Tips
-
-### Batch Size Selection
-
-```typescript
-// Small files (< 10 MB) - large batches
-const parser = new ParquetParserService(logger, { batchSize: 5000 });
-
-// Medium files (10-50 MB) - medium batches
-const parser = new ParquetParserService(logger, { batchSize: 1000 });
-
-// Large files (> 50 MB) - small batches + streaming
-const parser = new ParquetParserService(logger, { batchSize: 500 });
-const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-```
-
-### Memory Management
-
-```typescript
-// For files > 50 MB, use streaming
-if (fileSize > 50 * 1024 * 1024) {
-  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-
-  for await (const batch of streamResult.records) {
-    await processBatch(batch);
-    // Batch is garbage collected after processing
-  }
-}
-```
-
-### Row Group Optimization
-
-```typescript
-// Small row groups (5K-10K) - better for selective queries
-const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
-  rowGroupSize: 5000,
-});
-
-// Large row groups (50K-100K) - better for full scans
-const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
-  rowGroupSize: 50000,
-});
-```
-
-## Common Patterns
-
-### Pattern 1: Read and Validate
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-import * as fs from 'fs/promises';
-
-async function readAndValidate(filePath: string): Promise<any[]> {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const parser = new ParquetParserService(logger);
-
-  // Read file
-  const buffer = await fs.readFile(filePath);
-  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-  // Parse with validation
-  const { records, stats } = await parser.parse(arrayBuffer, filePath, {
-    validation: {
-      enabled: true,
-      requiredColumns: ['productRef', 'locationRef', 'qty'],
-      columnTypes: {
-        productRef: 'string',
-        locationRef: 'string',
-        qty: 'int32'
-      }
-    }
-  });
-
-  // Check validation
-  if (stats.validation && !stats.validation.isValid) {
-    throw new Error(`Validation failed: ${JSON.stringify(stats.validation.errors)}`);
-  }
-
-  return records;
-}
-```
-
-### Pattern 2: Stream Large Files
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-import * as fs from 'fs/promises';
-
-async function streamLargeFile(filePath: string): Promise<void> {
-  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-  const parser = new ParquetParserService(logger, { batchSize: 500, enableStreaming: true });
-
-  const buffer = await fs.readFile(filePath);
-  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-
-  const streamResult = await parser.parseStreaming(arrayBuffer, filePath);
-
-  for await (const batch of streamResult.records) {
-    console.log(`Processing batch of ${batch.length} records`);
-    await processBatch(batch);
-  }
-
-  const stats = await streamResult.stats;
-  console.log(`Total: ${stats.totalRecords} records`);
-}
-```
-
-### Pattern 3: Extract GraphQL to Parquet
-
-```typescript
-import { createClient } from '@fluentcommerce/fc-connect-sdk';
-import * as parquet from 'parquetjs';
-
-async function extractToParquet(outputFile: string): Promise<void> {
-  const client = await createClient({ config: fluentConfig });
-
-  const schema = new parquet.ParquetSchema({
-    ref: { type: 'UTF8' },
-    productRef: { type: 'UTF8' },
-    locationRef: { type: 'UTF8' },
-    qty: { type: 'INT32' },
-    extractedAt: { type: 'TIMESTAMP_MILLIS' }
-  });
-
-  const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
-    compression: 'SNAPPY'
-  });
-
-  let hasNextPage = true;
-  let cursor: string | null = null;
-
-  while (hasNextPage) {
-    const result = await client.graphql({
-      query: `query($after: String, $first: Int!) {
-        inventoryPositions(after: $after, first: $first) {
-          pageInfo { hasNextPage }
-          edges {
-            cursor
-            node { ref productRef locationRef qty }
-          }
-        }
-      }`,
-      variables: { first: 1000, after: cursor }
-    });
-
-    const { edges, pageInfo } = result.data.inventoryPositions;
-
-    for (const edge of edges) {
-      await writer.appendRow({ ...edge.node, extractedAt: new Date() });
-    }
-
-    hasNextPage = pageInfo.hasNextPage;
-    cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
-  }
-
-  await writer.close();
-}
-```
+# Parquet Quick Reference
+
+> One-page cheat sheet for Parquet file operations with FC Connect SDK
+
+## Import Statements
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+import * as parquet from 'parquetjs'; // For writing
+```
+
+## Reading Parquet Files
+
+### Basic Read
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+import * as fs from 'fs/promises';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'ParquetParser'
+});
+const parser = new ParquetParserService(logger);
+
+// Read file
+const buffer = await fs.readFile('inventory.parquet');
+const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+// Parse
+const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet');
+
+console.log(`Parsed ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
+```
+
+### Read from S3
+
+```typescript
+import {
+  S3DataSource,
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'S3ParquetReader'
+});
+
+const s3 = new S3DataSource(
+  {
+    type: 'S3_CSV',
+    s3Config: {
+      bucket: process.env.S3_BUCKET!,
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+      region: process.env.AWS_REGION!,
+    },
+  },
+  logger
+);
+
+const parser = new ParquetParserService(logger);
+
+// Download and parse
+const buffer = await s3.downloadFile('data/inventory.parquet', { encoding: 'binary' });
+const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+const { records, stats } = await parser.parse(arrayBuffer, 'inventory.parquet');
+```
+
+### Streaming Read (Large Files)
+
+```typescript
+const streamResult = await parser.parseStreaming(arrayBuffer, 'large-file.parquet', {
+  batchSize: 1000,
+});
+
+for await (const batch of streamResult.records) {
+  console.log(`Processing batch of ${batch.length} records`);
+  await processBatch(batch);
+}
+
+const stats = await streamResult.stats;
+console.log(`Total: ${stats.totalRecords} records`);
+```
+
+## Writing Parquet Files
+
+### Basic Write
+
+```typescript
+import * as parquet from 'parquetjs';
+
+// Define schema
+const schema = new parquet.ParquetSchema({
+  ref: { type: 'UTF8', optional: false },
+  productRef: { type: 'UTF8', optional: false },
+  locationRef: { type: 'UTF8', optional: false },
+  qty: { type: 'INT32', optional: false },
+  status: { type: 'UTF8', optional: true },
+});
+
+// Create writer
+const writer = await parquet.ParquetWriter.openFile(schema, 'output.parquet', {
+  compression: 'SNAPPY',
+});
+
+// Write records
+const records = [
+  { ref: 'IP-001', productRef: 'SKU-001', locationRef: 'WH-001', qty: 100, status: 'AVAILABLE' },
+  { ref: 'IP-002', productRef: 'SKU-002', locationRef: 'WH-001', qty: 50, status: 'RESERVED' },
+];
+
+for (const record of records) {
+  await writer.appendRow(record);
+}
+
+await writer.close();
+```
+
+### Write to S3
+
+**Method 1: Using writeParquetContent() (Recommended)**
+
+```typescript
+import { S3DataSource, createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'ParquetWriter'
+});
+
+const s3 = new S3DataSource(
+  {
+    type: 'S3_CSV',
+    s3Config: {
+      bucket: process.env.S3_BUCKET!,
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+      region: process.env.AWS_REGION!,
+    },
+  },
+  logger
+);
+
+const records = [
+  { id: 1, name: 'Product A', qty: 100 },
+  { id: 2, name: 'Product B', qty: 200 },
+];
+
+// Generate Parquet Buffer
+const parquetBuffer = await s3.writeParquetContent(records, {
+  schema: {
+    id: { type: 'INT32' },
+    name: { type: 'UTF8' },
+    qty: { type: 'INT32' },
+  },
+  compression: 'SNAPPY',
+});
+
+// Upload to S3
+await s3.uploadFile('data/output.parquet', parquetBuffer);
+```
+
+**Method 2: Temp File Approach**
+
+```typescript
+import * as parquet from 'parquetjs';
+import * as fs from 'fs/promises';
+import { S3DataSource, createConsoleLogger, toStructuredLogger } from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'ParquetWriter'
+});
+
+const s3 = new S3DataSource(s3Config, logger);
+
+// Define schema
+const schema = new parquet.ParquetSchema({
+  id: { type: 'INT32' },
+  name: { type: 'UTF8' },
+  qty: { type: 'INT32' },
+});
+
+// Write to local file first
+const writer = await parquet.ParquetWriter.openFile(schema, 'temp.parquet', {
+  compression: 'SNAPPY',
+});
+
+for (const record of records) {
+  await writer.appendRow(record);
+}
+
+await writer.close();
+
+// Upload to S3
+const buffer = await fs.readFile('temp.parquet');
+await s3.uploadFile('data/output.parquet', buffer);
+
+// Cleanup
+await fs.unlink('temp.parquet');
+```
+
+## Parquet Data Types
+
+| Parquet Type       | TypeScript Type | Use Case                  |
+| ------------------ | --------------- | ------------------------- |
+| `UTF8`             | string          | Text, SKUs, references    |
+| `INT32`            | number          | Quantities, small numbers |
+| `INT64`            | number          | Large numbers, IDs        |
+| `FLOAT`            | number          | Decimals (less precision) |
+| `DOUBLE`           | number          | Decimals (more precision) |
+| `BOOLEAN`          | boolean         | Flags, true/false         |
+| `DATE`             | Date            | Date only (no time)       |
+| `TIMESTAMP_MILLIS` | Date            | Date and time             |
+
+### Schema Examples
+
+```typescript
+// Inventory Position Schema
+const inventorySchema = new parquet.ParquetSchema({
+  ref: { type: 'UTF8', optional: false },
+  productRef: { type: 'UTF8', optional: false },
+  locationRef: { type: 'UTF8', optional: false },
+  qty: { type: 'INT32', optional: false },
+  status: { type: 'UTF8', optional: true },
+  updatedOn: { type: 'TIMESTAMP_MILLIS', optional: true },
+});
+
+// Product Schema
+const productSchema = new parquet.ParquetSchema({
+  sku: { type: 'UTF8', optional: false },
+  name: { type: 'UTF8', optional: false },
+  price: { type: 'DOUBLE', optional: false },
+  inStock: { type: 'BOOLEAN', optional: false },
+  lastUpdated: { type: 'TIMESTAMP_MILLIS', optional: true },
+});
+
+// Order Schema
+const orderSchema = new parquet.ParquetSchema({
+  orderNumber: { type: 'UTF8', optional: false },
+  customerId: { type: 'UTF8', optional: false },
+  totalAmount: { type: 'DOUBLE', optional: false },
+  itemCount: { type: 'INT32', optional: false },
+  orderDate: { type: 'TIMESTAMP_MILLIS', optional: false },
+  status: { type: 'UTF8', optional: true },
+});
+```
+
+## Compression Options
+
+| Algorithm      | Ratio | Speed     | Use When                |
+| -------------- | ----- | --------- | ----------------------- |
+| `UNCOMPRESSED` | 1x    | Fastest   | Local testing only      |
+| `SNAPPY`       | 2-3x  | Very Fast | **Recommended default** |
+| `GZIP`         | 3-4x  | Moderate  | Storage optimization    |
+| `BROTLI`       | 4-5x  | Slow      | Maximum compression     |
+
+```typescript
+// SNAPPY (recommended)
+const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
+  compression: 'SNAPPY',
+  rowGroupSize: 10000,
+});
+
+// GZIP (better compression)
+const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
+  compression: 'GZIP',
+  compressionLevel: 6, // 1-9
+});
+
+// BROTLI (max compression)
+const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
+  compression: 'BROTLI',
+  compressionLevel: 4, // 1-11
+});
+```
+
+## Parser Configuration
+
+```typescript
+const parser = new ParquetParserService(logger, {
+  batchSize: 1000, // Records per batch
+  maxFileSize: 100 * 1024 * 1024, // 100 MB max
+  enableStreaming: true, // Enable streaming
+});
+
+// Parse with validation
+const { records, stats } = await parser.parse(arrayBuffer, 'file.parquet', {
+  batchSize: 500,
+  validation: {
+    enabled: true,
+    requiredColumns: ['productRef', 'locationRef', 'qty'],
+    columnTypes: {
+      productRef: 'string',
+      locationRef: 'string',
+      qty: 'int32',
+    },
+    allowAdditionalColumns: true,
+  },
+});
+```
+
+## GraphQL to Parquet Extraction
+
+### Complete Workflow
+
+```typescript
+import { createClient, ParquetParserService } from '@fluentcommerce/fc-connect-sdk';
+import * as parquet from 'parquetjs';
+
+async function extractInventoryToParquet(outputFile: string) {
+  // 1. Create Fluent client
+  const client = await createClient({
+    config: {
+      baseUrl: process.env.FLUENT_BASE_URL!,
+      clientId: process.env.FLUENT_CLIENT_ID!,
+      clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+      retailerId: process.env.FLUENT_RETAILER_ID!,
+    },
+  });
+
+  // 2. Define Parquet schema
+  const schema = new parquet.ParquetSchema({
+    ref: { type: 'UTF8' },
+    productRef: { type: 'UTF8' },
+    locationRef: { type: 'UTF8' },
+    qty: { type: 'INT32' },
+    status: { type: 'UTF8', optional: true },
+    extractedAt: { type: 'TIMESTAMP_MILLIS' },
+  });
+
+  // 3. Create writer
+  const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
+    compression: 'SNAPPY',
+    rowGroupSize: 10000,
+  });
+
+  // 4. Extract with pagination
+  let hasNextPage = true;
+  let cursor: string | null = null;
+  let totalRecords = 0;
+
+  while (hasNextPage) {
+    const result = await client.graphql({
+      query: `
+        query ExtractInventory($after: String, $first: Int!) {
+          inventoryPositions(after: $after, first: $first) {
+            pageInfo { hasNextPage }
+            edges {
+              cursor
+              node {
+                ref
+                productRef
+                locationRef
+                qty
+                status
+              }
+            }
+          }
+        }
+      `,
+      variables: { first: 1000, after: cursor },
+    });
+
+    const { edges, pageInfo } = result.data.inventoryPositions;
+
+    // 5. Write batch
+    for (const edge of edges) {
+      await writer.appendRow({
+        ...edge.node,
+        extractedAt: new Date(),
+      });
+      totalRecords++;
+    }
+
+    console.log(`Written ${totalRecords} records...`);
+
+    hasNextPage = pageInfo.hasNextPage;
+    cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
+  }
+
+  // 6. Close writer
+  await writer.close();
+  console.log(`Complete: ${totalRecords} records`);
+}
+```
+
+## S3 Integration
+
+### Download → Parse → Process
+
+```typescript
+import {
+  S3DataSource,
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'S3ParquetProcessor'
+});
+
+const s3 = new S3DataSource(
+  {
+    type: 'S3_CSV',
+    s3Config: {
+      bucket: process.env.S3_BUCKET!,
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+      region: process.env.AWS_REGION!,
+    },
+  },
+  logger
+);
+
+const parser = new ParquetParserService(logger);
+
+// List files
+const files = await s3.listFiles({ prefix: 'parquet/' });
+
+for (const file of files) {
+  // Download
+  const buffer = await s3.downloadFile(file.path, { encoding: 'binary' });
+
+  // Parse
+  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+  const { records, stats } = await parser.parse(arrayBuffer, file.path);
+
+  // Process
+  console.log(`${file.path}: ${stats.totalRecords} records`);
+  await processRecords(records);
+}
+```
+
+### Extract → Write → Upload
+
+**Complete Workflow**
+
+```typescript
+import { 
+  createClient, 
+  S3DataSource,
+  createConsoleLogger,
+  toStructuredLogger 
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'Extraction'
+});
+
+const s3 = new S3DataSource(s3Config, logger);
+const client = await createClient({ config });
+
+// Extract from Fluent
+const result = await client.graphql({ query, pagination: { maxPages: 10 } });
+
+const data = result.data.inventoryPositions.edges.map((e) => ({
+  ...e.node,
+  extractedAt: new Date().toISOString(),
+}));
+
+// Generate Parquet Buffer
+const parquetBuffer = await s3.writeParquetContent(data, {
+  schema: {
+    ref: { type: 'UTF8' },
+    productRef: { type: 'UTF8' },
+    locationRef: { type: 'UTF8' },
+    qty: { type: 'INT32' },
+    extractedAt: { type: 'UTF8' },
+  },
+  compression: 'SNAPPY',
+});
+
+// Upload to S3
+const s3Key = `extracts/inventory_${new Date().toISOString().split('T')[0]}.parquet`;
+await s3.uploadFile(s3Key, parquetBuffer);
+
+console.log(`Uploaded to s3://my-bucket/${s3Key}`);
+```
+
+## Parse Statistics
+
+```typescript
+const { records, stats } = await parser.parse(arrayBuffer, 'file.parquet');
+
+// Performance metrics
+console.log(`Total Records: ${stats.totalRecords}`);
+console.log(`Processing Time: ${stats.processingTimeMs}ms`);
+console.log(`Throughput: ${stats.recordsPerSecond} records/sec`);
+console.log(`Peak Memory: ${(stats.peakMemoryUsage / 1024 / 1024).toFixed(2)} MB`);
+
+// Column statistics
+Object.entries(stats.columnStats).forEach(([name, col]) => {
+  console.log(`${name}: ${col.type}, ${col.nullCount} nulls, ${col.uniqueValues} unique`);
+  if (col.minValue !== undefined) {
+    console.log(`  Range: ${col.minValue} - ${col.maxValue}`);
+  }
+});
+
+// Validation results (if enabled)
+if (stats.validation) {
+  if (!stats.validation.isValid) {
+    console.error('Validation errors:', stats.validation.errors);
+  }
+}
+```
+
+## Performance Tips
+
+### Batch Size Selection
+
+```typescript
+// Small files (< 10 MB) - large batches
+const parser = new ParquetParserService(logger, { batchSize: 5000 });
+
+// Medium files (10-50 MB) - medium batches
+const parser = new ParquetParserService(logger, { batchSize: 1000 });
+
+// Large files (> 50 MB) - small batches + streaming
+const parser = new ParquetParserService(logger, { batchSize: 500 });
+const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+```
+
+### Memory Management
+
+```typescript
+// For files > 50 MB, use streaming
+if (fileSize > 50 * 1024 * 1024) {
+  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+
+  for await (const batch of streamResult.records) {
+    await processBatch(batch);
+    // Batch is garbage collected after processing
+  }
+}
+```
+
+### Row Group Optimization
+
+```typescript
+// Small row groups (5K-10K) - better for selective queries
+const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
+  rowGroupSize: 5000,
+});
+
+// Large row groups (50K-100K) - better for full scans
+const writer = await parquet.ParquetWriter.openFile(schema, 'file.parquet', {
+  rowGroupSize: 50000,
+});
+```
+
+## Common Patterns
+
+### Pattern 1: Read and Validate
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+import * as fs from 'fs/promises';
+
+async function readAndValidate(filePath: string): Promise<any[]> {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const parser = new ParquetParserService(logger);
+
+  // Read file
+  const buffer = await fs.readFile(filePath);
+  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+  // Parse with validation
+  const { records, stats } = await parser.parse(arrayBuffer, filePath, {
+    validation: {
+      enabled: true,
+      requiredColumns: ['productRef', 'locationRef', 'qty'],
+      columnTypes: {
+        productRef: 'string',
+        locationRef: 'string',
+        qty: 'int32'
+      }
+    }
+  });
+
+  // Check validation
+  if (stats.validation && !stats.validation.isValid) {
+    throw new Error(`Validation failed: ${JSON.stringify(stats.validation.errors)}`);
+  }
+
+  return records;
+}
+```
+
+### Pattern 2: Stream Large Files
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+import * as fs from 'fs/promises';
+
+async function streamLargeFile(filePath: string): Promise<void> {
+  const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+  const parser = new ParquetParserService(logger, { batchSize: 500, enableStreaming: true });
+
+  const buffer = await fs.readFile(filePath);
+  const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+
+  const streamResult = await parser.parseStreaming(arrayBuffer, filePath);
+
+  for await (const batch of streamResult.records) {
+    console.log(`Processing batch of ${batch.length} records`);
+    await processBatch(batch);
+  }
+
+  const stats = await streamResult.stats;
+  console.log(`Total: ${stats.totalRecords} records`);
+}
+```
+
+### Pattern 3: Extract GraphQL to Parquet
+
+```typescript
+import { createClient } from '@fluentcommerce/fc-connect-sdk';
+import * as parquet from 'parquetjs';
+
+async function extractToParquet(outputFile: string): Promise<void> {
+  const client = await createClient({ config: fluentConfig });
+
+  const schema = new parquet.ParquetSchema({
+    ref: { type: 'UTF8' },
+    productRef: { type: 'UTF8' },
+    locationRef: { type: 'UTF8' },
+    qty: { type: 'INT32' },
+    extractedAt: { type: 'TIMESTAMP_MILLIS' }
+  });
+
+  const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
+    compression: 'SNAPPY'
+  });
+
+  let hasNextPage = true;
+  let cursor: string | null = null;
+
+  while (hasNextPage) {
+    const result = await client.graphql({
+      query: `query($after: String, $first: Int!) {
+        inventoryPositions(after: $after, first: $first) {
+          pageInfo { hasNextPage }
+          edges {
+            cursor
+            node { ref productRef locationRef qty }
+          }
+        }
+      }`,
+      variables: { first: 1000, after: cursor }
+    });
+
+    const { edges, pageInfo } = result.data.inventoryPositions;
+
+    for (const edge of edges) {
+      await writer.appendRow({ ...edge.node, extractedAt: new Date() });
+    }
+
+    hasNextPage = pageInfo.hasNextPage;
+    cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
+  }
+
+  await writer.close();
+}
+```