jtcsv 2.2.7 → 3.0.0

package/examples/express-api.ts

@@ -0,0 +1,164 @@
+// Express API example for jtcsv
+// Run: node examples/express-api.ts
+// Then visit: http://localhost:3000/export/users
+
+import express from 'express';
+import { jsonToCsv } from '../index';
+import fs from 'fs/promises';
+import path from 'path';
+
+const app = express();
+const PORT = 3000;
+
+// Mock database
+const mockUsers = Array.from({ length: 100 }, (_, i) => ({
+  id: i + 1,
+  name: `User ${i + 1}`,
+  email: `user${i + 1}@example.com`,
+  role: i % 3 === 0 ? 'admin' : i % 3 === 1 ? 'moderator' : 'user',
+  status: i % 5 === 0 ? 'inactive' : 'active',
+  createdAt: new Date(Date.now() - Math.random() * 31536000000).toISOString(),
+  profile: {
+    age: Math.floor(Math.random() * 50) + 18,
+    country: ['USA', 'UK', 'Germany', 'France', 'Japan'][Math.floor(Math.random() * 5)],
+    bio: `Bio for user ${i + 1}`
+  },
+  tags: ['customer', `tier${Math.floor(Math.random() * 3) + 1}`]
+}));
+
+// Middleware
+app.use(express.json());
+
+// Health check
+app.get('/', (req, res) => {
+  res.json({
+    message: 'jtcsv Express API Example',
+    endpoints: [
+      'GET /export/users - Export users as CSV',
+      'GET /export/users/download - Download users CSV file',
+      'POST /export/custom - Convert custom JSON to CSV'
+    ]
+  });
+});
+
+// Export users as CSV (inline)
+app.get('/export/users', (req, res) => {
+  try {
+    const csv = jsonToCsv(mockUsers, {
+      delimiter: ',',
+      renameMap: {
+        id: 'ID',
+        name: 'Full Name',
+        email: 'Email Address',
+        role: 'Role',
+        status: 'Status',
+        'profile.age': 'Age',
+        'profile.country': 'Country',
+        'profile.bio': 'Bio',
+        'createdAt': 'Created At'
+      }
+    });
+    
+    res.set('Content-Type', 'text/plain');
+    res.send(csv);
+  } catch (error: any) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Download users CSV file
+app.get('/export/users/download', async (req, res) => {
+  try {
+    const csv = jsonToCsv(mockUsers, {
+      delimiter: ',',
+      renameMap: {
+        id: 'ID',
+        name: 'Full Name',
+        email: 'Email Address'
+      }
+    });
+    
+    const filename = `users-export-${Date.now()}.csv`;
+    const filePath = path.join(__dirname, 'temp', filename);
+    
+    // Ensure temp directory exists
+    await fs.mkdir(path.join(__dirname, 'temp'), { recursive: true });
+    
+    // Write file
+    await fs.writeFile(filePath, csv, 'utf8');
+    
+    // Send file for download
+    res.download(filePath, filename, (err) => {
+      // Clean up temp file
+      fs.unlink(filePath).catch(() => {});
+    });
+  } catch (error: any) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Convert custom JSON to CSV
+app.post('/export/custom', (req, res) => {
+  try {
+    const { data, options = {} } = req.body;
+    
+    if (!Array.isArray(data)) {
+      return res.status(400).json({ error: 'Data must be an array' });
+    }
+    
+    // Limit to 10,000 records for safety
+    const safeData = data.slice(0, 10000);
+    
+    const csv = jsonToCsv(safeData, {
+      delimiter: options.delimiter || ',',
+      renameMap: options.renameMap || {}
+    });
+    
+    res.json({
+      success: true,
+      records: safeData.length,
+      csvSize: csv.length,
+      csv: options.returnCsv !== false ? csv : undefined
+    });
+  } catch (error: any) {
+    res.status(500).json({ 
+      success: false, 
+      error: error.message,
+      type: error.constructor.name
+    });
+  }
+});
+
+// Example with CSV injection protection
+app.get('/export/safe', (req, res) => {
+  const dangerousData = [
+    { id: 1, input: '=cmd|"/c calc.exe"!A0', note: 'Malicious formula' },
+    { id: 2, input: '@SUM(A1:A10)', note: 'Excel function' },
+    { id: 3, input: '+1-1', note: 'Another formula' },
+    { id: 4, input: '-2+2', note: 'Formula with minus' },
+    { id: 5, input: 'Normal text', note: 'Safe content' },
+    { id: 6, input: 'Text with, commas', note: 'With comma' },
+    { id: 7, input: 'Text with\nnewlines', note: 'With newline' },
+    { id: 8, input: 'Text with "quotes"', note: 'With quotes' }
+  ];
+  
+  const csv = jsonToCsv(dangerousData, { delimiter: ',' });
+  
+  res.set('Content-Type', 'text/plain');
+  res.send(`# CSV Injection Protection Demo\n\n${csv}`);
+});
+
+// Start server
+app.listen(PORT, () => {
+  console.log(`🚀 Express API running at http://localhost:${PORT}`);
+  console.log('📊 Try these endpoints:');
+  console.log(`   http://localhost:${PORT}/export/users`);
+  console.log(`   http://localhost:${PORT}/export/users/download`);
+  console.log(`   http://localhost:${PORT}/export/safe`);
+  console.log('\nExample POST to /export/custom:');
+  console.log(`curl -X POST http://localhost:${PORT}/export/custom \\
+  -H "Content-Type: application/json" \\
+  -d '{"data":[{"name":"John","age":30}],"options":{"delimiter":";"}}'`);
+});
+
+export default app;

package/examples/large-dataset-example.js

@@ -180,6 +180,3 @@ module.exports = {
   benchmarkConversion,
   saveCsvInChunks
 };
-
-
-

package/examples/large-dataset-example.ts

@@ -0,0 +1,204 @@
+// Large dataset example for jtcsv
+// Demonstrates handling of large datasets and memory efficiency
+
+import { jsonToCsv } from '../index-core.js';
+import fs from 'fs';
+import path from 'path';
+
+interface DatasetRecord {
+  id: number;
+  timestamp: string;
+  userId: string;
+  action: string;
+  amount: number;
+  success: boolean;
+  metadata: {
+    ip: string;
+    userAgent: string;
+    sessionId: string;
+  };
+  tags: string[];
+}
+
+interface MemoryUsage {
+  rss: number;
+  heapTotal: number;
+  heapUsed: number;
+  external: number;
+}
+
+// Generate large dataset
+export function generateLargeDataset(rows: number): DatasetRecord[] {
+  console.log(`Generating ${rows.toLocaleString()} records...`);
+  
+  const dataset: DatasetRecord[] = [];
+  const batchSize = 10000;
+  const batches = Math.ceil(rows / batchSize);
+  
+  for (let batch = 0; batch < batches; batch++) {
+    const start = batch * batchSize;
+    const end = Math.min(start + batchSize, rows);
+    
+    for (let i = start; i < end; i++) {
+      dataset.push({
+        id: i + 1,
+        timestamp: new Date(Date.now() - Math.random() * 31536000000).toISOString(),
+        userId: `user_${Math.floor(Math.random() * 10000)}`,
+        action: ['login', 'purchase', 'view', 'click', 'logout'][Math.floor(Math.random() * 5)],
+        amount: Math.random() * 1000,
+        success: Math.random() > 0.1,
+        metadata: {
+          ip: `${Math.floor(Math.random() * 255)}.${Math.floor(Math.random() * 255)}.${Math.floor(Math.random() * 255)}.${Math.floor(Math.random() * 255)}`,
+          userAgent: `Browser/${Math.floor(Math.random() * 100)}`,
+          sessionId: `session_${Math.random().toString(36).substr(2, 9)}`
+        },
+        tags: Array.from({ length: Math.floor(Math.random() * 5) + 1 }, 
+          () => ['tag' + Math.floor(Math.random() * 20), 'cat' + Math.floor(Math.random() * 10)]
+        ).flat()
+      });
+    }
+    
+    if ((batch + 1) % 10 === 0 || batch === batches - 1) {
+      console.log(`  Generated ${Math.min((batch + 1) * batchSize, rows).toLocaleString()} records`);
+    }
+  }
+  
+  return dataset;
+}
+
+// Memory usage helper
+export function getMemoryUsage(): MemoryUsage {
+  const used = process.memoryUsage();
+  return {
+    rss: Math.round(used.rss / 1024 / 1024),
+    heapTotal: Math.round(used.heapTotal / 1024 / 1024),
+    heapUsed: Math.round(used.heapUsed / 1024 / 1024),
+    external: Math.round(used.external / 1024 / 1024)
+  };
+}
+
+// Benchmark conversion
+export async function benchmarkConversion(dataset: DatasetRecord[], name: string): Promise<string | null> {
+  console.log(`\n${name}:`);
+  console.log(`  Records: ${dataset.length.toLocaleString()}`);
+  
+  const memBefore = getMemoryUsage();
+  console.log(`  Memory before: ${memBefore.heapUsed}MB heap`);
+  
+  const startTime = performance.now();
+  
+  try {
+    const csv = jsonToCsv(dataset, {
+      delimiter: ','
+    });
+    
+    const endTime = performance.now();
+    const memAfter = getMemoryUsage();
+    
+    console.log(`  Conversion time: ${(endTime - startTime).toFixed(2)}ms`);
+    console.log(`  Memory after: ${memAfter.heapUsed}MB heap`);
+    console.log(`  Memory delta: ${(memAfter.heapUsed - memBefore.heapUsed)}MB`);
+    console.log(`  CSV size: ${(csv.length / 1024 / 1024).toFixed(2)}MB`);
+    console.log(`  Rows/second: ${Math.round((dataset.length / (endTime - startTime)) * 1000).toLocaleString()}`);
+    
+    return csv;
+  } catch (error: unknown) {
+    console.error(`  Error: ${(error as Error).message}`);
+    return null;
+  }
+}
+
+// Save CSV in chunks (for very large files)
+export function saveCsvInChunks(csv: string, filename: string, chunkSize = 10 * 1024 * 1024): Promise<void> { // 10MB chunks
+  console.log(`\nSaving CSV in chunks to ${filename}...`);
+  
+  const totalChunks = Math.ceil(csv.length / chunkSize);
+  const writeStream = fs.createWriteStream(filename);
+  
+  return new Promise((resolve, reject) => {
+    writeStream.on('error', reject);
+    writeStream.on('finish', resolve);
+    
+    for (let i = 0; i < totalChunks; i++) {
+      const start = i * chunkSize;
+      const end = Math.min(start + chunkSize, csv.length);
+      const chunk = csv.substring(start, end);
+      
+      if (!writeStream.write(chunk)) {
+        // Wait for drain if buffer is full
+        writeStream.once('drain', () => {
+          if (i % 10 === 0 || i === totalChunks - 1) {
+            console.log(`  Written chunk ${i + 1}/${totalChunks} (${Math.round((i + 1) / totalChunks * 100)}%)`);
+          }
+        });
+      } else {
+        if (i % 10 === 0 || i === totalChunks - 1) {
+          console.log(`  Written chunk ${i + 1}/${totalChunks} (${Math.round((i + 1) / totalChunks * 100)}%)`);
+        }
+      }
+    }
+    
+    writeStream.end();
+  });
+}
+
+// Main function
+export async function main(): Promise<void> {
+  console.log('📊 jtcsv Large Dataset Demo');
+  console.log('=' .repeat(50));
+  
+  // Test with different dataset sizes
+  const testSizes = [1000, 10000, 50000];
+  
+  for (const size of testSizes) {
+    const dataset = generateLargeDataset(size);
+    const csv = await benchmarkConversion(dataset, `Dataset: ${size.toLocaleString()} records`);
+    
+    if (csv && size === 50000) {
+      // Save the largest dataset as example
+      const filename = `large-dataset-${size}-records.csv`;
+      await saveCsvInChunks(csv, filename);
+      
+      const stats = fs.statSync(filename);
+      console.log(`\n✅ Saved to ${filename}`);
+      console.log(`   File size: ${(stats.size / 1024 / 1024).toFixed(2)}MB`);
+      
+      // Show first few lines
+      const sample = csv.split('\n').slice(0, 3).join('\n');
+      console.log('\nSample output (first 3 lines):');
+      console.log(sample);
+      console.log('...');
+    }
+    
+    // Force garbage collection if available
+    if (global.gc) {
+      (global.gc as () => void)();
+    }
+  }
+  
+  console.log('\n🎯 Performance Summary:');
+  console.log('| Records | Approx. Time | Memory Usage | CSV Size |');
+  console.log('|---------|--------------|--------------|----------|');
+  console.log('| 1,000   | ~5ms         | ~2MB         | ~0.5MB   |');
+  console.log('| 10,000  | ~50ms        | ~10MB        | ~5MB     |');
+  console.log('| 50,000  | ~250ms       | ~40MB        | ~25MB    |');
+  console.log('| 100,000 | ~500ms*      | ~80MB*       | ~50MB*   |');
+  console.log('\n* Estimated values');
+  
+  console.log('\n💡 Tips for very large datasets:');
+  console.log('1. Use the maxRecords option optionally to limit processing');
+  console.log('2. Process data in batches if memory is limited');
+  console.log('3. Use saveAsCsv() for secure file writing');
+  console.log('4. Monitor memory usage with process.memoryUsage()');
+}
+
+// Run if called directly
+if (require.main === module) {
+  main().catch(console.error);
+}
+
+export default {
+  generateLargeDataset,
+  benchmarkConversion,
+  saveCsvInChunks
+};

package/examples/ndjson-processing.js

@@ -168,7 +168,7 @@ async function ndjsonToCsvConversion() {
   });
 
   // Collect CSV output
-  let csvOutput = '';
+  const csvOutput = '';
   transformStream.writable.getWriter();
 
   // Manual stream processing for demo