jinaga-server 3.5.2 → 3.5.3

package/src/http/csv-validator.ts

@@ -0,0 +1,90 @@
+import { Specification } from 'jinaga';
+import { CsvMetadata } from './csv-metadata';
+
+/**
+ * Validates a specification for CSV compatibility.
+ * Returns metadata including headers and validation results.
+ *
+ * CSV requires flat projections with single-valued components:
+ * - ✅ Fields (scalar values from matched facts)
+ * - ✅ Hashes
+ * - ✅ Timestamps
+ * - ❌ Arrays (nested specifications)
+ * - ❌ Nested objects (composite projections)
+ */
+export function validateSpecificationForCsv(specification: Specification): CsvMetadata {
+    const headers: string[] = [];
+    const errors: string[] = [];
+
+    // Ensure projection is composite
+    if (specification.projection.type !== 'composite') {
+        return {
+            headers: [],
+            isValid: false,
+            errors: ['Specification projection must be composite for CSV export']
+        };
+    }
+
+    // Ensure that the composite has components.
+    // An empty composite represents a missing projection.
+    if (specification.projection.components.length === 0) {
+        return {
+            headers: [],
+            isValid: false,
+            errors: ['Specification must have a projection for CSV export']
+        };
+    }
+
+    for (const component of specification.projection.components) {
+        if (component.type === 'field' || component.type === 'hash' || component.type === 'time') {
+            headers.push(component.name);
+        }
+        else {
+            errors.push(`Unsupported projection type of field '${component.name}' for CSV export. Only flat field projections are allowed.`);
+        }
+    }
+
+    return {
+        headers,
+        isValid: errors.length === 0,
+        errors
+    };
+}
+
+/**
+ * Extract value from result object using projection label.
+ * Since CSV only supports flat projections, labels are simple property names.
+ */
+export function extractValueByLabel(result: any, label: string): any {
+    if (result == null) {
+        return null;
+    }
+    
+    // Direct property access
+    const value = result[label];
+    
+    // Return null instead of undefined
+    return value ?? null;
+}
+
+/**
+ * Format a value for CSV output
+ */
+export function formatValueForCsv(value: any): string {
+    if (value === null || value === undefined) {
+        return '';
+    }
+    
+    // Handle dates
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    
+    // Handle booleans
+    if (typeof value === 'boolean') {
+        return value ? 'true' : 'false';
+    }
+    
+    // Default: convert to string
+    return String(value);
+}

package/src/http/output-formatters.ts

@@ -0,0 +1,196 @@
+import { stringify } from 'csv-stringify';
+import { Response } from "express";
+import { CsvMetadata } from "./csv-metadata";
+import { extractValueByLabel } from "./csv-validator";
+import { ResultStream } from "./result-stream";
+
+/**
+ * Output read results with streaming support.
+ * Handles content negotiation and streams data when possible.
+ */
+export async function outputReadResultsStreaming(
+    result: ResultStream<any>,
+    res: Response,
+    acceptType: string,
+    csvMetadata?: CsvMetadata
+): Promise<void> {
+    switch (acceptType) {
+        case "application/x-ndjson":
+            // NDJSON format - stream one JSON object per line
+            await streamAsNDJSON(result, res);
+            break;
+        case "text/csv":
+            // CSV format - stream as CSV using csv-stringify
+            if (csvMetadata) {
+                await streamAsCSVWithStringify(result, res, csvMetadata);
+            } else {
+                throw new Error("CSV metadata is required for CSV output format");
+            }
+            break;
+        case "application/json":
+            // Compact JSON
+            await collectAndSendCompactJSON(result, res);
+            break;
+        case "text/plain":
+            // Pretty-printed JSON
+            await collectAndSendPrettyJSON(result, res);
+            break;
+        default:
+            // Default to pretty-printed JSON for unrecognized types
+            await collectAndSendPrettyJSON(result, res);
+            break;
+    }
+}
+
+/**
+ * Stream results as NDJSON (newline-delimited JSON).
+ * Each result is written as a separate line.
+ */
+export async function streamAsNDJSON(stream: ResultStream<any>, res: Response): Promise<void> {
+    res.type("application/x-ndjson");
+
+    try {
+        let item;
+        while ((item = await stream.next()) !== null) {
+            res.write(JSON.stringify(item) + '\n');
+        }
+        res.end();
+    } catch (error) {
+        // Send error as NDJSON line
+        const errorFrame = {
+            error: true,
+            message: error instanceof Error ? error.message : String(error)
+        };
+        res.write(JSON.stringify(errorFrame) + '\n');
+        res.end();
+    } finally {
+        await stream.close();
+    }
+}
+
+/**
+ * Collect all results from stream and send as compact JSON.
+ * Used for application/json format.
+ */
+export async function collectAndSendCompactJSON(
+    stream: ResultStream<any>,
+    res: Response
+): Promise<void> {
+    try {
+        const results: any[] = [];
+        let item;
+        while ((item = await stream.next()) !== null) {
+            results.push(item);
+        }
+
+        res.type("application/json");
+        res.send(JSON.stringify(results));
+    } finally {
+        await stream.close();
+    }
+}
+
+/**
+ * Collect all results from stream and send as pretty-printed JSON.
+ * Used for text/plain format and as default for unrecognized Accept types.
+ */
+export async function collectAndSendPrettyJSON(
+    stream: ResultStream<any>,
+    res: Response
+): Promise<void> {
+    try {
+        const results: any[] = [];
+        let item;
+        while ((item = await stream.next()) !== null) {
+            results.push(item);
+        }
+
+        res.type("text/plain");
+        res.send(JSON.stringify(results, null, 2));
+    } finally {
+        await stream.close();
+    }
+}
+
+/**
+ * Stream results as CSV using csv-stringify library.
+ * Headers are provided from the specification metadata.
+ */
+export async function streamAsCSVWithStringify(
+    stream: ResultStream<any>,
+    res: Response,
+    csvMetadata: CsvMetadata
+): Promise<void> {
+    res.type("text/csv");
+
+    // Helper to await 'finish' or 'error' events on a stream
+    function finishedAsync(stream: NodeJS.EventEmitter): Promise<void> {
+        return new Promise((resolve, reject) => {
+            stream.once('finish', resolve);
+            stream.once('error', reject);
+        });
+    }
+
+    let stringifier: any;
+    try {
+        // Create csv-stringify stringifier with headers from specification
+        stringifier = stringify({
+            header: true,
+            columns: csvMetadata.headers,
+            cast: {
+                // Custom casting for special types
+                boolean: (value) => value ? 'true' : 'false',
+                date: (value) => {
+                    if (value instanceof Date) {
+                        return value.toISOString();
+                    }
+                    return String(value);
+                },
+                object: (value) => {
+                    // Handle nested objects
+                    if (value && typeof value === 'object') {
+                        // Fact reference - just use hash
+                        if (value.type && value.hash) {
+                            return value.hash;
+                        }
+                        // Other objects - stringify
+                        return JSON.stringify(value);
+                    }
+                    return String(value);
+                }
+            }
+        });
+
+        // Pipe stringifier to response
+        stringifier.pipe(res);
+
+        // Stream data through stringifier
+        let item: any = null;
+        while ((item = await stream.next()) !== null) {
+            // Extract values in header order
+            const row: any = {};
+            for (const header of csvMetadata.headers) {
+                const value = extractValueByLabel(item, header);
+                row[header] = value !== null && value !== undefined ? value : '';
+            }
+            
+            // Write row to stringifier
+            stringifier.write(row);
+        }
+
+        // Signal end of data
+        stringifier.end();
+
+        // Await finish or error
+        await finishedAsync(stringifier);
+
+    } catch (error) {
+        console.error('Error in CSV streaming:', error);
+        if (!res.headersSent) {
+            res.status(500).send('Error generating CSV');
+        }
+        throw error;
+    } finally {
+        await stream.close();
+    }
+}

package/src/http/result-stream.ts

@@ -0,0 +1,69 @@
+/**
+ * ResultStream interface for streaming query results incrementally.
+ * This abstraction allows for memory-efficient processing of large result sets.
+ */
+export interface ResultStream<T> {
+    /**
+     * Get the next item in the stream.
+     * @returns Promise resolving to the next item, or null if stream is exhausted
+     */
+    next(): Promise<T | null>;
+    
+    /**
+     * Close the stream and release resources.
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * Implements ResultStream for AsyncIterable sources.
+ * Useful for streaming data from database cursors or other async sources.
+ */
+export class AsyncIterableResultStream<T> implements ResultStream<T> {
+    private iterator: AsyncIterator<T>;
+    private closed: boolean = false;
+
+    constructor(iterable: AsyncIterable<T>) {
+        this.iterator = iterable[Symbol.asyncIterator]();
+    }
+
+    async next(): Promise<T | null> {
+        if (this.closed) {
+            return null;
+        }
+
+        const result = await this.iterator.next();
+        if (result.done) {
+            await this.close();
+            return null;
+        }
+
+        return result.value;
+    }
+
+    async close(): Promise<void> {
+        if (!this.closed) {
+            this.closed = true;
+            if (this.iterator.return) {
+                await this.iterator.return();
+            }
+        }
+    }
+}
+
+/**
+ * Converts an array to a ResultStream.
+ * Useful for backward compatibility and testing.
+ */
+export async function* arrayAsyncIterable<T>(array: T[]): AsyncIterable<T> {
+    for (const item of array) {
+        yield item;
+    }
+}
+
+/**
+ * Helper function to convert an array to a ResultStream.
+ */
+export function arrayToResultStream<T>(array: T[]): ResultStream<T> {
+    return new AsyncIterableResultStream(arrayAsyncIterable(array));
+}

package/src/http/router.ts

@@ -32,7 +32,11 @@ import {
     UserIdentity,
     verifyEnvelopes
 } from "jinaga";
+import { CsvMetadata } from "./csv-metadata";
+import { validateSpecificationForCsv } from "./csv-validator";
 import { createLineReader } from "./line-reader";
+import { outputReadResultsStreaming } from "./output-formatters";
+import { arrayToResultStream, ResultStream } from "./result-stream";
 import { Stream } from "./stream";
 
 function getOrStream<U>(
@@ -213,26 +217,6 @@ function post<T, U>(
     };
 }
 
-function postString<U>(method: (user: RequestUser, message: string) => Promise<U>): Handler {
-    return (req, res, next) => {
-        const user = <RequestUser>(req as any).user;
-        const input = parseString(req.body);
-        if (!input || typeof (input) !== 'string') {
-            res.type("text");
-            res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
-        }
-        else {
-            method(user, input)
-                .then(response => {
-                    res.type("text");
-                    res.send(JSON.stringify(response, null, 2));
-                    next();
-                })
-                .catch(error => handleError(error, req, res, next));
-        }
-    };
-}
-
 function postCreate<T>(
     parse: (request: Request) => T,
     method: (user: RequestUser, message: T) => Promise<void>
@@ -268,6 +252,40 @@ function postStringCreate(method: (user: RequestUser, message: string) => Promis
     };
 }
 
+function postReadWithStreaming(
+    method: (user: RequestUser, message: string, acceptType: string) => Promise<{
+        resultStream: ResultStream<any>,
+        csvMetadata?: CsvMetadata
+    }>
+): Handler {
+    return (req, res, next) => {
+        const user = <RequestUser>(req as any).user;
+        const input = parseString(req.body);
+        if (!input || typeof (input) !== 'string') {
+            res.type("text");
+            res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
+        }
+        else {
+            // Check if Accept header explicitly prefers a specific format
+            const acceptHeader = req.get('Accept');
+            let acceptType: string = 'text/plain'; // Default for backward compatibility
+
+            if (acceptHeader && acceptHeader !== '*/*') {
+                // Only use req.accepts() when there's a specific preference
+                const preferredType = req.accepts(['text/csv', 'application/x-ndjson', 'application/json', 'text/plain']);
+                acceptType = preferredType ? String(preferredType) : 'text/plain';
+            }
+            
+            method(user, input, acceptType)
+                .then(({resultStream, csvMetadata}) => {
+                    outputReadResults(resultStream, res, acceptType, csvMetadata);
+                    next();
+                })
+                .catch(error => handleError(error, req, res, next));
+        }
+    };
+}
+
 function serializeUserIdentity(user: RequestUser | null): UserIdentity | null {
     if (!user) {
         return null;
@@ -324,6 +342,22 @@ function outputFeeds(result: FeedsResponse, res: Response, accepts: (type: strin
     res.send(JSON.stringify(result));
 }
 
+function outputReadResults(
+    result: ResultStream<any>, 
+    res: Response, 
+    acceptType: string,
+    csvMetadata?: CsvMetadata
+) {
+    // Use streaming output formatter
+    outputReadResultsStreaming(result, res, acceptType, csvMetadata)
+        .catch(error => {
+            console.error('Error in outputReadResults:', error);
+            if (!res.headersSent) {
+                res.status(500).send('Internal server error');
+            }
+        });
+}
+
 export interface RequestUser {
     provider: string;
     id: string;
@@ -352,7 +386,13 @@ export class HttpRouter {
             (user, graphSource) => this.save(user, graphSource)
         ));
 
-        router.post('/read', applyAllowOrigin, postString((user, input: string) => this.read(user, input)));
+        router.post('/read', applyAllowOrigin, postReadWithStreaming((user, input: string, acceptType: string) =>
+            this.readWithStreaming(
+                user,
+                input,
+                acceptType === 'text/csv' ? preProcessForCsv : preProcessForOther
+            )
+        ));
         router.post('/write', applyAllowOrigin, postStringCreate((user, input: string) => this.write(user, input)));
         router.post('/feeds', applyAllowOrigin, post(
             parseString,
@@ -423,26 +463,46 @@ export class HttpRouter {
         });
     }
 
-    private read(user: RequestUser | null, input: string): Promise<any[]> {
-        return Trace.dependency("read", "", async () => {
+    /**
+     * Read with streaming support.
+     * Uses readStream if available on the authorization object, otherwise falls back to read().
+     * 
+     * When CSV format is requested, validates that the specification contains only flat projections.
+     */
+    private async readWithStreaming(
+        user: RequestUser | null, 
+        input: string,
+        preProcess: (specification: Specification) => CsvMetadata | undefined
+    ): Promise<{
+        resultStream: ResultStream<any>,
+        csvMetadata?: CsvMetadata
+    }> {
+        return Trace.dependency("readWithStreaming", "", async () => {
             const knownFacts = await this.getKnownFacts(user);
             const parser = new SpecificationParser(input);
             parser.skipWhitespace();
             const declaration = parser.parseDeclaration(knownFacts);
             const specification = parser.parseSpecification();
             parser.expectEnd();
-            const start = this.selectStart(specification, declaration);
 
             var failures: string[] = this.factManager.testSpecificationForCompliance(specification);
             if (failures.length > 0) {
                 throw new Invalid(failures.join("\n"));
             }
 
+            const csvMetadata = preProcess(specification);
+
             const userIdentity = serializeUserIdentity(user);
+            const start = this.selectStart(specification, declaration);
             const results = await this.authorization.read(userIdentity, start, specification);
             const extracted = extractResults(results);
             Trace.counter("facts_read", extracted.count);
-            return extracted.result;
+            const resultStream = arrayToResultStream(extracted.result);
+
+            return {
+                resultStream,
+                csvMetadata
+            };
         });
     }
 
@@ -486,8 +546,8 @@ export class HttpRouter {
             }
             // Verify that the input type matches the start fact type
             for (let i = 0; i < start.length; i++) {
-                if (start[i].type !== specification.given[i].type) {
-                    throw new Invalid(`The type of start fact ${i} (${start[i].type}) does not match the type of input ${i} (${specification.given[i].type})`);
+                if (start[i].type !== specification.given[i].label.type) {
+                    throw new Invalid(`The type of start fact ${i} (${start[i].type}) does not match the type of input ${i} (${specification.given[i].label.type})`);
                 }
             }
             // Verify that the specification is compliant with purge conditions
@@ -496,9 +556,9 @@ export class HttpRouter {
                 throw new Invalid(failures.join("\n"));
             }
             
-            const namedStart = specification.given.reduce((map, label, index) => ({
+            const namedStart = specification.given.reduce((map, g, index) => ({
                 ...map,
-                [label.name]: start[index]
+                [g.label.name]: start[index]
             }), {} as ReferencesByName);
 
             // Verify that I can distribute all feeds to the user.
@@ -529,7 +589,7 @@ export class HttpRouter {
             const bookmark = query["b"] as string ?? "";
 
             const userIdentity = serializeUserIdentity(user);
-            const start = feedDefinition.feed.given.map(label => feedDefinition.namedStart[label.name]);
+            const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
             const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
             // Return distinct fact references from all the tuples.
             const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
@@ -565,7 +625,7 @@ export class HttpRouter {
         console.log(`[StreamFeed:${connectionId}] Initial bookmark: ${bookmark || 'empty'}`);
 
         const userIdentity = serializeUserIdentity(user);
-        const start = feedDefinition.feed.given.map(label => feedDefinition.namedStart[label.name]);
+        const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
         const givenHash = computeObjectHash(feedDefinition.namedStart);
 
         console.log(`[StreamFeed:${connectionId}] Feed setup - Given hash: ${givenHash.substring(0, 8)}..., Start facts: ${start.length}`);
@@ -812,9 +872,9 @@ export class HttpRouter {
     private selectStart(specification: Specification, declaration: Declaration): FactReference[] {
         // Select starting facts that match the inputs
         return specification.given.map(input => {
-            const declaredFact = declaration.find(d => d.name === input.name);
+            const declaredFact = declaration.find(d => d.name === input.label.name);
             if (!declaredFact) {
-                throw new Invalid(`No fact named ${input.name} was declared`);
+                throw new Invalid(`No fact named ${input.label.name} was declared`);
             }
             return declaredFact.declared.reference;
         });
@@ -893,6 +953,25 @@ export class HttpRouter {
     }
 }
 
+function preProcessForCsv(specification: Specification): CsvMetadata | undefined {
+    const csvMetadata = validateSpecificationForCsv(specification);
+
+    if (!csvMetadata.isValid) {
+        throw new Invalid(
+            `Specification is not compatible with CSV format:\n\n` +
+            csvMetadata.errors.join('\n\n') +
+            `\n\nHint: CSV requires flat projections with single-valued fields. ` +
+            `Avoid nested specifications.`
+        );
+    }
+    return csvMetadata;
+}
+
+function preProcessForOther(specification: Specification): CsvMetadata | undefined {
+    // For non-CSV formats, no special preprocessing is needed.
+    return undefined;
+}
+
 function parseString(input: any): string {
     if (typeof input !== 'string') {
         throw new Invalid("Expected a string. Check the content type of the request.");

package/src/postgres/postgres-store.ts

@@ -46,6 +46,10 @@ import { ResultSetFact, ResultSetRow, ResultSetTree, resultSqlFromSpecification,
 import { sqlFromFeed } from "./specification-sql";
 import { purgeDescendantsSql, purgeSqlFromSpecification } from "./purge-sql";
 
+interface PostgresFactRecord extends FactRecord {
+    timestamp: Date;  // Maps to date_learned column
+}
+
 interface FactTypeResult {
     rows: {
         fact_type_id: number;
@@ -91,6 +95,7 @@ interface AncestorResult {
         data: string;
         public_key: string;
         signature: string;
+        date_learned: Date;
     }[];
 }
 
@@ -353,15 +358,15 @@ export class PostgresStore implements Storage {
         const factParameters = flatten(references, (f) =>
             [f.hash, factTypes.get(f.type)]);
         const sql =
-            'SELECT f.fact_type_id, f.name, f.hash, f.data, s.public_key, s.signature ' +
-            'FROM (SELECT f.fact_id, f.fact_type_id, t.name, f.hash, f.data ' +
+            'SELECT f.fact_type_id, f.name, f.hash, f.data, f.date_learned, s.public_key, s.signature ' +
+            'FROM (SELECT f.fact_id, f.fact_type_id, t.name, f.hash, f.data, f.date_learned ' +
                 `FROM ${this.schema}.fact f ` +
                 `JOIN ${this.schema}.fact_type t ` +
                 '  ON f.fact_type_id = t.fact_type_id ' +
                 'JOIN (VALUES ' + factValues.join(', ') + ') AS v (hash, fact_type_id) ' +
                 '  ON v.fact_type_id = f.fact_type_id AND v.hash = f.hash ' +
                 'UNION ' +
-                'SELECT f2.fact_id, f2.fact_type_id, t.name, f2.hash, f2.data ' +
+                'SELECT f2.fact_id, f2.fact_type_id, t.name, f2.hash, f2.data, f2.date_learned ' +
                 `FROM ${this.schema}.fact f1 ` +
                 'JOIN (VALUES ' + factValues.join(', ') + ') AS v (hash, fact_type_id) ' +
                 '  ON v.fact_type_id = f1.fact_type_id AND v.hash = f1.hash ' +
@@ -387,11 +392,12 @@ export class PostgresStore implements Storage {
         this.factTypeMap = mergeFactTypes(this.factTypeMap, resultFactTypes);
         const records = result.rows.map((r) => {
             const { fields, predecessors }: { fields: {}, predecessors: PredecessorCollection } = r.data as any;
-            const fact: FactRecord = {
+            const fact: PostgresFactRecord = {
                 type: r.name,
                 hash: r.hash,
                 fields,
-                predecessors
+                predecessors,
+                timestamp: r.date_learned
             };
             const signature: FactSignature = {
                 publicKey: r.public_key,
@@ -523,7 +529,8 @@ async function executeQueryTree(sqlQueryTree: SqlQueryTree, connection: PoolClie
             const fact: ResultSetFact = {
                 hash: dataRow[`hash${label.index}`],
                 factId: dataRow[`id${label.index}`],
-                data: dataRow[`data${label.index}`]
+                data: dataRow[`data${label.index}`],
+                timestamp: dataRow[`timestamp${label.index}`],
             };
             return {
                 ...acc,

package/src/postgres/purge-sql.ts

@@ -47,9 +47,25 @@ export function purgeSqlFromSpecification(specification: Specification, factType
 
     let queryDescription = QueryDescription.unsatisfiable;
     let knownFacts: FactByLabel = {};
-    const given = specification.given;
-    const start: FactReference[] = specification.given.map(g => ({ type: g.type, hash: 'xxxxx' }));
+    const given = specification.given.map(g => g.label);
+    const start: FactReference[] = specification.given.map(g => ({ type: g.label.type, hash: 'xxxxx' }));
+    
+    // Process main matches
     ({ queryDescription, knownFacts } = queryDescriptionBuilder.addEdges(queryDescription, given, start, knownFacts, [], specification.matches));
+    
+    // Process conditions from given
+    for (const givenItem of specification.given) {
+        if (givenItem.conditions && givenItem.conditions.length > 0) {
+            for (const condition of givenItem.conditions) {
+                const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
+                const { queryDescription: queryDescriptionConditional } = queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, given, start, knownFacts, conditionalPath, condition.matches);
+                
+                if (queryDescriptionConditional.isSatisfiable()) {
+                    queryDescription = queryDescriptionConditional;
+                }
+            }
+        }
+    }
 
     if (!queryDescription.isSatisfiable()) {
         return null;