jinaga-server 3.5.1 → 3.5.3

package/src/http/csv-validator.ts

@@ -0,0 +1,90 @@
+import { Specification } from 'jinaga';
+import { CsvMetadata } from './csv-metadata';
+
+/**
+ * Validates a specification for CSV compatibility.
+ * Returns metadata including headers and validation results.
+ *
+ * CSV requires flat projections with single-valued components:
+ * - ✅ Fields (scalar values from matched facts)
+ * - ✅ Hashes
+ * - ✅ Timestamps
+ * - ❌ Arrays (nested specifications)
+ * - ❌ Nested objects (composite projections)
+ */
+export function validateSpecificationForCsv(specification: Specification): CsvMetadata {
+    const headers: string[] = [];
+    const errors: string[] = [];
+
+    // Ensure projection is composite
+    if (specification.projection.type !== 'composite') {
+        return {
+            headers: [],
+            isValid: false,
+            errors: ['Specification projection must be composite for CSV export']
+        };
+    }
+
+    // Ensure that the composite has components.
+    // An empty composite represents a missing projection.
+    if (specification.projection.components.length === 0) {
+        return {
+            headers: [],
+            isValid: false,
+            errors: ['Specification must have a projection for CSV export']
+        };
+    }
+
+    for (const component of specification.projection.components) {
+        if (component.type === 'field' || component.type === 'hash' || component.type === 'time') {
+            headers.push(component.name);
+        }
+        else {
+            errors.push(`Unsupported projection type of field '${component.name}' for CSV export. Only flat field projections are allowed.`);
+        }
+    }
+
+    return {
+        headers,
+        isValid: errors.length === 0,
+        errors
+    };
+}
+
+/**
+ * Extract value from result object using projection label.
+ * Since CSV only supports flat projections, labels are simple property names.
+ */
+export function extractValueByLabel(result: any, label: string): any {
+    if (result == null) {
+        return null;
+    }
+    
+    // Direct property access
+    const value = result[label];
+    
+    // Return null instead of undefined
+    return value ?? null;
+}
+
+/**
+ * Format a value for CSV output
+ */
+export function formatValueForCsv(value: any): string {
+    if (value === null || value === undefined) {
+        return '';
+    }
+    
+    // Handle dates
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    
+    // Handle booleans
+    if (typeof value === 'boolean') {
+        return value ? 'true' : 'false';
+    }
+    
+    // Default: convert to string
+    return String(value);
+}

package/src/http/output-formatters.ts

@@ -0,0 +1,196 @@
+import { stringify } from 'csv-stringify';
+import { Response } from "express";
+import { CsvMetadata } from "./csv-metadata";
+import { extractValueByLabel } from "./csv-validator";
+import { ResultStream } from "./result-stream";
+
+/**
+ * Output read results with streaming support.
+ * Handles content negotiation and streams data when possible.
+ */
+export async function outputReadResultsStreaming(
+    result: ResultStream<any>,
+    res: Response,
+    acceptType: string,
+    csvMetadata?: CsvMetadata
+): Promise<void> {
+    switch (acceptType) {
+        case "application/x-ndjson":
+            // NDJSON format - stream one JSON object per line
+            await streamAsNDJSON(result, res);
+            break;
+        case "text/csv":
+            // CSV format - stream as CSV using csv-stringify
+            if (csvMetadata) {
+                await streamAsCSVWithStringify(result, res, csvMetadata);
+            } else {
+                throw new Error("CSV metadata is required for CSV output format");
+            }
+            break;
+        case "application/json":
+            // Compact JSON
+            await collectAndSendCompactJSON(result, res);
+            break;
+        case "text/plain":
+            // Pretty-printed JSON
+            await collectAndSendPrettyJSON(result, res);
+            break;
+        default:
+            // Default to pretty-printed JSON for unrecognized types
+            await collectAndSendPrettyJSON(result, res);
+            break;
+    }
+}
+
+/**
+ * Stream results as NDJSON (newline-delimited JSON).
+ * Each result is written as a separate line.
+ */
+export async function streamAsNDJSON(stream: ResultStream<any>, res: Response): Promise<void> {
+    res.type("application/x-ndjson");
+
+    try {
+        let item;
+        while ((item = await stream.next()) !== null) {
+            res.write(JSON.stringify(item) + '\n');
+        }
+        res.end();
+    } catch (error) {
+        // Send error as NDJSON line
+        const errorFrame = {
+            error: true,
+            message: error instanceof Error ? error.message : String(error)
+        };
+        res.write(JSON.stringify(errorFrame) + '\n');
+        res.end();
+    } finally {
+        await stream.close();
+    }
+}
+
+/**
+ * Collect all results from stream and send as compact JSON.
+ * Used for application/json format.
+ */
+export async function collectAndSendCompactJSON(
+    stream: ResultStream<any>,
+    res: Response
+): Promise<void> {
+    try {
+        const results: any[] = [];
+        let item;
+        while ((item = await stream.next()) !== null) {
+            results.push(item);
+        }
+
+        res.type("application/json");
+        res.send(JSON.stringify(results));
+    } finally {
+        await stream.close();
+    }
+}
+
+/**
+ * Collect all results from stream and send as pretty-printed JSON.
+ * Used for text/plain format and as default for unrecognized Accept types.
+ */
+export async function collectAndSendPrettyJSON(
+    stream: ResultStream<any>,
+    res: Response
+): Promise<void> {
+    try {
+        const results: any[] = [];
+        let item;
+        while ((item = await stream.next()) !== null) {
+            results.push(item);
+        }
+
+        res.type("text/plain");
+        res.send(JSON.stringify(results, null, 2));
+    } finally {
+        await stream.close();
+    }
+}
+
+/**
+ * Stream results as CSV using csv-stringify library.
+ * Headers are provided from the specification metadata.
+ */
+export async function streamAsCSVWithStringify(
+    stream: ResultStream<any>,
+    res: Response,
+    csvMetadata: CsvMetadata
+): Promise<void> {
+    res.type("text/csv");
+
+    // Helper to await 'finish' or 'error' events on a stream
+    function finishedAsync(stream: NodeJS.EventEmitter): Promise<void> {
+        return new Promise((resolve, reject) => {
+            stream.once('finish', resolve);
+            stream.once('error', reject);
+        });
+    }
+
+    let stringifier: any;
+    try {
+        // Create csv-stringify stringifier with headers from specification
+        stringifier = stringify({
+            header: true,
+            columns: csvMetadata.headers,
+            cast: {
+                // Custom casting for special types
+                boolean: (value) => value ? 'true' : 'false',
+                date: (value) => {
+                    if (value instanceof Date) {
+                        return value.toISOString();
+                    }
+                    return String(value);
+                },
+                object: (value) => {
+                    // Handle nested objects
+                    if (value && typeof value === 'object') {
+                        // Fact reference - just use hash
+                        if (value.type && value.hash) {
+                            return value.hash;
+                        }
+                        // Other objects - stringify
+                        return JSON.stringify(value);
+                    }
+                    return String(value);
+                }
+            }
+        });
+
+        // Pipe stringifier to response
+        stringifier.pipe(res);
+
+        // Stream data through stringifier
+        let item: any = null;
+        while ((item = await stream.next()) !== null) {
+            // Extract values in header order
+            const row: any = {};
+            for (const header of csvMetadata.headers) {
+                const value = extractValueByLabel(item, header);
+                row[header] = value !== null && value !== undefined ? value : '';
+            }
+            
+            // Write row to stringifier
+            stringifier.write(row);
+        }
+
+        // Signal end of data
+        stringifier.end();
+
+        // Await finish or error
+        await finishedAsync(stringifier);
+
+    } catch (error) {
+        console.error('Error in CSV streaming:', error);
+        if (!res.headersSent) {
+            res.status(500).send('Error generating CSV');
+        }
+        throw error;
+    } finally {
+        await stream.close();
+    }
+}

package/src/http/result-stream.ts

@@ -0,0 +1,69 @@
+/**
+ * ResultStream interface for streaming query results incrementally.
+ * This abstraction allows for memory-efficient processing of large result sets.
+ */
+export interface ResultStream<T> {
+    /**
+     * Get the next item in the stream.
+     * @returns Promise resolving to the next item, or null if stream is exhausted
+     */
+    next(): Promise<T | null>;
+    
+    /**
+     * Close the stream and release resources.
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * Implements ResultStream for AsyncIterable sources.
+ * Useful for streaming data from database cursors or other async sources.
+ */
+export class AsyncIterableResultStream<T> implements ResultStream<T> {
+    private iterator: AsyncIterator<T>;
+    private closed: boolean = false;
+
+    constructor(iterable: AsyncIterable<T>) {
+        this.iterator = iterable[Symbol.asyncIterator]();
+    }
+
+    async next(): Promise<T | null> {
+        if (this.closed) {
+            return null;
+        }
+
+        const result = await this.iterator.next();
+        if (result.done) {
+            await this.close();
+            return null;
+        }
+
+        return result.value;
+    }
+
+    async close(): Promise<void> {
+        if (!this.closed) {
+            this.closed = true;
+            if (this.iterator.return) {
+                await this.iterator.return();
+            }
+        }
+    }
+}
+
+/**
+ * Converts an array to a ResultStream.
+ * Useful for backward compatibility and testing.
+ */
+export async function* arrayAsyncIterable<T>(array: T[]): AsyncIterable<T> {
+    for (const item of array) {
+        yield item;
+    }
+}
+
+/**
+ * Helper function to convert an array to a ResultStream.
+ */
+export function arrayToResultStream<T>(array: T[]): ResultStream<T> {
+    return new AsyncIterableResultStream(arrayAsyncIterable(array));
+}