jinaga-server 3.5.2 → 3.5.4

package/src/postgres/purge-sql.ts

@@ -47,9 +47,25 @@ export function purgeSqlFromSpecification(specification: Specification, factType
 
     let queryDescription = QueryDescription.unsatisfiable;
     let knownFacts: FactByLabel = {};
-    const given = specification.given;
-    const start: FactReference[] = specification.given.map(g => ({ type: g.type, hash: 'xxxxx' }));
+    const given = specification.given.map(g => g.label);
+    const start: FactReference[] = specification.given.map(g => ({ type: g.label.type, hash: 'xxxxx' }));
+    
+    // Process main matches
     ({ queryDescription, knownFacts } = queryDescriptionBuilder.addEdges(queryDescription, given, start, knownFacts, [], specification.matches));
+    
+    // Process conditions from given
+    for (const givenItem of specification.given) {
+        if (givenItem.conditions && givenItem.conditions.length > 0) {
+            for (const condition of givenItem.conditions) {
+                const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
+                const { queryDescription: queryDescriptionConditional } = queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, given, start, knownFacts, conditionalPath, condition.matches);
+                
+                if (queryDescriptionConditional.isSatisfiable()) {
+                    queryDescription = queryDescriptionConditional;
+                }
+            }
+        }
+    }
 
     if (!queryDescription.isSatisfiable()) {
         return null;

package/src/postgres/specification-result-sql.ts

@@ -6,7 +6,6 @@ import {
     FieldProjection,
     HashProjection,
     hydrateFromTree,
-    Label,
     Match,
     PredecessorCollection,
     ProjectedResult,
@@ -14,7 +13,9 @@ import {
     ReferencesByName,
     SingularProjection,
     Specification,
+    SpecificationGiven,
     SpecificationProjection,
+    TimeProjection,
     validateGiven
 } from "jinaga";
 
@@ -24,7 +25,7 @@ import { EdgeDescription, ExistentialConditionDescription, FactByLabel, QueryDes
 function generateResultSqlQuery(queryDescription: QueryDescription, schema: string): SpecificationSqlQuery {
     const allLabels = [ ...queryDescription.inputs, ...queryDescription.outputs ];
     const columns = allLabels
-        .map(label => `f${label.factIndex}.hash as hash${label.factIndex}, f${label.factIndex}.fact_id as id${label.factIndex}, f${label.factIndex}.data as data${label.factIndex}`)
+        .map(label => `f${label.factIndex}.hash as hash${label.factIndex}, f${label.factIndex}.fact_id as id${label.factIndex}, f${label.factIndex}.data as data${label.factIndex}, f${label.factIndex}.date_learned as timestamp${label.factIndex}`)
         .join(", ");
     const firstEdge = queryDescription.edges[0];
     const predecessorFact = queryDescription.inputs.find(i => i.factIndex === firstEdge.predecessorFactIndex);
@@ -32,6 +33,8 @@ function generateResultSqlQuery(queryDescription: QueryDescription, schema: stri
     const firstFactIndex = predecessorFact ? predecessorFact.factIndex : successorFact!.factIndex;
     const writtenFactIndexes = new Set<number>().add(firstFactIndex);
     const joins: string[] = generateJoins(queryDescription.edges, writtenFactIndexes, schema);
+    // Add all input fact indexes to writtenFactIndexes since they're available in the FROM clause
+    queryDescription.inputs.forEach(input => writtenFactIndexes.add(input.factIndex));
     const inputWhereClauses = queryDescription.inputs
         .map(input => `f${input.factIndex}.fact_type_id = $${input.factTypeParameter} AND f${input.factIndex}.hash = $${input.factHashParameter}`)
         .join(" AND ");
@@ -197,6 +200,7 @@ export interface ResultSetFact {
     hash: string;
     factId: number;
     data: ResultSetData;
+    timestamp: Date;
 }
 
 export interface ResultSetRow {
@@ -411,6 +415,9 @@ export class ResultComposer {
         else if (this.resultProjection.type === "fact") {
             return this.factValue(this.resultProjection, row, factRecords);
         }
+        else if (this.resultProjection.type === "time") {
+            return this.timeValue(this.resultProjection, row);
+        }
         else {
             const _exhaustiveCheck: never = this.resultProjection;
             throw new Error(`Unknown projection type ${(this.resultProjection as any).type}`);
@@ -439,6 +446,9 @@ export class ResultComposer {
             // This should have already been taken care of
             return null;
         }
+        else if (projection.type === "time") {
+            return this.timeValue(projection, row);
+        }
         else {
             const _exhaustiveCheck: never = projection;
             throw new Error(`Unknown projection type ${(projection as any).type}`);
@@ -465,6 +475,11 @@ export class ResultComposer {
         return fact;
     }
 
+    private timeValue(projection: TimeProjection, row: ResultSetRow): any {
+        const label = this.getLabel(projection.label);
+        return row[label.index].timestamp;
+    }
+
     private getLabel(name: string) {
         const label = this.sqlQuery.labels.find(label => label.name === name);
         if (!label) {
@@ -487,15 +502,62 @@ class ResultDescriptionBuilder {
     buildDescription(start: FactReference[], specification: Specification): ResultDescription {
         validateGiven(start, specification);
 
-        return this.createResultDescription(QueryDescription.unsatisfiable, specification.given, start, specification.matches, specification.projection, {}, []);
+        const labels = specification.given.map(g => g.label);
+        let queryDescription = QueryDescription.unsatisfiable;
+        let knownFacts: FactByLabel = {};
+        
+        // First process main matches to build the result description
+        const resultDescription = this.createResultDescription(queryDescription, specification.given, start, specification.matches, specification.projection, knownFacts, []);
+        
+        // Then process conditions from given if we have a satisfiable query
+        if (resultDescription.queryDescription.isSatisfiable()) {
+            queryDescription = resultDescription.queryDescription;
+            knownFacts = resultDescription.queryDescription.outputs.reduce((acc, output) => ({
+                ...acc,
+                [output.label]: { factIndex: output.factIndex, type: output.type }
+            }), {} as FactByLabel);
+            
+            // Also include inputs in knownFacts
+            for (const input of resultDescription.queryDescription.inputs) {
+                if (!knownFacts[input.label]) {
+                    knownFacts[input.label] = { factIndex: input.factIndex, type: input.type };
+                }
+            }
+            
+            // Process conditions from each given
+            for (const givenItem of specification.given) {
+                if (givenItem.conditions && givenItem.conditions.length > 0) {
+                    for (const condition of givenItem.conditions) {
+                        const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
+                        const { queryDescription: queryDescriptionConditional } = this.queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, labels, start, knownFacts, conditionalPath, condition.matches);
+                        
+                        if (queryDescriptionConditional.isSatisfiable()) {
+                            queryDescription = queryDescriptionConditional;
+                        }
+                    }
+                }
+            }
+            
+            // Return updated result description with conditions applied
+            return {
+                ...resultDescription,
+                queryDescription
+            };
+        }
+        
+        return resultDescription;
     }
 
-    private createResultDescription(queryDescription: QueryDescription, given: Label[], start: FactReference[], matches: Match[], projection: Projection, knownFacts: FactByLabel, path: number[]): ResultDescription {
-        const givenTuple = given.reduce((acc, label, index) => ({
+    private createResultDescription(queryDescription: QueryDescription, given: SpecificationGiven[], start: FactReference[], matches: Match[], projection: Projection, knownFacts: FactByLabel, path: number[]): ResultDescription {
+        const givenTuple = given.reduce((acc, givenItem, index) => ({
             ...acc,
-            [label.name]: start[index]
+            [givenItem.label.name]: start[index]
         }), {} as ReferencesByName);
-        ({ queryDescription, knownFacts } = this.queryDescriptionBuilder.addEdges(queryDescription, given, start, knownFacts, path, matches));
+        
+        // Process main matches
+        const labels = given.map(g => g.label);
+        ({ queryDescription, knownFacts } = this.queryDescriptionBuilder.addEdges(queryDescription, labels, start, knownFacts, path, matches));
+        
         if (!queryDescription.isSatisfiable()) {
             // Abort the branch if the query is not satisfiable
             return {
@@ -513,7 +575,7 @@ class ResultDescriptionBuilder {
             const specificationProjections = projection.components
                 .filter(projection => projection.type === "specification") as ({ name: string } & SpecificationProjection)[];
             const singularProjections = projection.components
-                .filter(projection => projection.type === "field" || projection.type === "hash" || projection.type === "fact") as ({ name: string } & SingularProjection)[];
+                .filter(projection => projection.type === "field" || projection.type === "hash" || projection.type === "fact" || projection.type === "time") as ({ name: string } & SingularProjection)[];
             for (const child of specificationProjections) {
                 const childResultDescription = this.createResultDescription(queryDescription, given, start, child.matches, child.projection, knownFacts, []);
                 childResultDescriptions.push({

package/src/postgres/specification-sql.ts

@@ -40,9 +40,13 @@ function generateSqlQuery(queryDescription: QueryDescription, schema: string, bo
         .filter(c => c.exists === false))
         .map(notExistsWhereClause => ` AND NOT EXISTS (${generateNotExistsWhereClause(schema, notExistsWhereClause, writtenFactIndexes)})`)
         .join("");
+    const existsWhereClauses = (queryDescription.existentialConditions
+        .filter(c => c.exists === true))
+        .map(existsWhereClause => ` AND EXISTS (${generateNotExistsWhereClause(schema, existsWhereClause, writtenFactIndexes)})`)
+        .join("");
     const bookmarkParameter = queryDescription.parameters.length + 1;
     const limitParameter = bookmarkParameter + 1;
-    const sql = `SELECT ${hashes}, sort(array[${factIds}], 'desc') as bookmark FROM ${schema}.fact f${firstFactIndex}${joins.join("")} WHERE ${inputWhereClauses}${notExistsWhereClauses} AND sort(array[${factIds}], 'desc') > $${bookmarkParameter} ORDER BY bookmark ASC LIMIT $${limitParameter}`;
+    const sql = `SELECT ${hashes}, sort(array[${factIds}], 'desc') as bookmark FROM ${schema}.fact f${firstFactIndex}${joins.join("")} WHERE ${inputWhereClauses}${notExistsWhereClauses}${existsWhereClauses} AND sort(array[${factIds}], 'desc') > $${bookmarkParameter} ORDER BY bookmark ASC LIMIT $${limitParameter}`;
     const bookmarkValue: number[] = parseBookmark(bookmark);
     return {
         sql,
@@ -165,7 +169,7 @@ export function sqlFromSpecification(start: FactReference[], schema: string, boo
     validateGiven(start, specification);
 
     const labeledStart = start.map((s, index) => ({
-        name: specification.given[index].name,
+        name: specification.given[index].label.name,
         reference: s
     })).reduce((map, s) => {
         map.set(s.name, s.reference);
@@ -179,9 +183,9 @@ export function sqlFromSpecification(start: FactReference[], schema: string, boo
         bookmark: bookmarks[index]
     }));
     const queryDescriptionsAndBookmarks = feedAndBookmark.map(fb => {
-        const feedStart = fb.feed.given.map(s => labeledStart.get(s.name)!);
+        const feedStart = fb.feed.given.map(s => labeledStart.get(s.label.name)!);
         return {
-            queryDescription: buildQueryDescription(queryDescriptionBuilder, fb.feed, feedStart),
+            queryDescription: buildQueryDescription(queryDescriptionBuilder, fb.feed, feedStart, specification.given),
             bookmark: fb.bookmark
         };
     });
@@ -202,11 +206,35 @@ export function sqlFromFeed(feed: Specification, start: FactReference[], schema:
     return sql;
 }
 
-function buildQueryDescription(queryDescriptionBuilder: QueryDescriptionBuilder, specification: Specification, start: FactReference[]): QueryDescription {
-    const { queryDescription } = queryDescriptionBuilder.addEdges(
+function buildQueryDescription(queryDescriptionBuilder: QueryDescriptionBuilder, specification: Specification, start: FactReference[], originalGiven?: any[]): QueryDescription {
+    const labels = specification.given.map(g => g.label);
+    let knownFacts = {};
+    
+    // Process main matches
+    let { queryDescription, knownFacts: updatedKnownFacts } = queryDescriptionBuilder.addEdges(
         QueryDescription.unsatisfiable,
-        specification.given,
-        start, {}, [],
+        labels,
+        start,
+        knownFacts,
+        [],
         specification.matches);
+    knownFacts = updatedKnownFacts;
+    
+    // Process conditions from given (use originalGiven if provided, otherwise use specification.given)
+    const givenToProcess = originalGiven || specification.given;
+    for (const givenItem of givenToProcess) {
+        if (givenItem.conditions && givenItem.conditions.length > 0) {
+            for (const condition of givenItem.conditions) {
+                const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
+                const { queryDescription: queryDescriptionConditional, knownFacts: updatedKnownFacts } = queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, labels, start, knownFacts, conditionalPath, condition.matches);
+                
+                if (queryDescriptionConditional.isSatisfiable()) {
+                    queryDescription = queryDescriptionConditional;
+                    knownFacts = updatedKnownFacts;
+                }
+            }
+        }
+    }
+    
     return queryDescription;
 }

package/test/http/csv-validatorSpec.ts

@@ -0,0 +1,205 @@
+import { validateSpecificationForCsv, extractValueByLabel, formatValueForCsv } from '../../src/http/csv-validator';
+import { Specification, SpecificationParser, CompositeProjection } from 'jinaga';
+
+describe('CSV Validator', () => {
+    describe('validateSpecificationForCsv', () => {
+        // Helper function to create a specification from declarative syntax
+        function GivenSpecificationFromInput(input: string): Specification {
+            const parser = new SpecificationParser(input);
+            return parser.parseSpecification();
+        }
+
+        it('should accept flat field projections', () => {
+            const input = `(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => {
+                name = successor.name
+                count = successor.count
+            }`;
+            const specification = GivenSpecificationFromInput(input);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(true);
+            expect(metadata.headers).toEqual(['name', 'count']);
+            expect(metadata.errors).toEqual([]);
+        });
+
+        // Acceptance tests for future hash/timestamp support
+        it('should accept hash projections', () => {
+            const input = `(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => {
+                hash = #successor
+            }`;
+            const specification = GivenSpecificationFromInput(input);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.errors).toEqual([]);
+            expect(metadata.isValid).toBe(true);
+            expect(metadata.headers).toEqual(['hash']);
+        });
+
+        it('should accept timestamp projections', () => {
+            const input = `(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => {
+                timestamp = @successor
+            }`;
+            const specification = GivenSpecificationFromInput(input);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(true);
+            expect(metadata.headers).toEqual(['timestamp']);
+        });
+
+        it('should reject nested projections', () => {
+            const specification = GivenSpecificationFromInput(`(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => {
+                nested = {
+                    grandchild: Grandchild [
+                        grandchild->successor: Successor = successor
+                    ]
+                } => {
+                    name = grandchild.name
+                }
+            }`);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(false);
+            expect(metadata.errors.length).toBeGreaterThan(0);
+            expect(metadata.errors).toEqual([`Unsupported projection type of field 'nested' for CSV export. Only flat field projections are allowed.`]);
+        });
+
+        it('should reject fact projections', () => {
+            const specification = GivenSpecificationFromInput(`(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => {
+                successor = successor
+            }`);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(false);
+            expect(metadata.errors.length).toBeGreaterThan(0);
+            expect(metadata.errors).toEqual([`Unsupported projection type of field 'successor' for CSV export. Only flat field projections are allowed.`]);
+        });
+
+        it('should reject non-composite top-level projections', () => {
+            const specification = GivenSpecificationFromInput(`(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => successor.name`);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(false);
+            expect(metadata.errors).toContain('Specification projection must be composite for CSV export');
+        });
+
+        it('should reject specification without projection', () => {
+            const specification = GivenSpecificationFromInput(`(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            }`);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(false);
+            expect(metadata.errors).toEqual(['Specification must have a projection for CSV export']);
+        });
+
+        it('should handle mixed valid and invalid projections', () => {
+            const specification = GivenSpecificationFromInput(`(root: Root) {
+                successor: Successor [
+                    successor->root: Root = root
+                ]
+            } => {
+                name = successor.name
+                nested = {
+                    grandchild: Grandchild [
+                        grandchild->successor: Successor = successor
+                    ]
+                } => {
+                    name = grandchild.name
+                }
+                count = successor.count
+                successor = successor
+            }`);
+
+            const metadata = validateSpecificationForCsv(specification);
+            
+            expect(metadata.isValid).toBe(false);
+            expect(metadata.headers).toEqual(['name', 'count']); // Only valid field projections
+            expect(metadata.errors).toEqual([
+                `Unsupported projection type of field 'nested' for CSV export. Only flat field projections are allowed.`,
+                `Unsupported projection type of field 'successor' for CSV export. Only flat field projections are allowed.`
+            ]);
+        });
+    });
+
+    describe('extractValueByLabel', () => {
+        it('should extract direct property', () => {
+            const result = { name: 'Alice', age: 30 };
+            
+            expect(extractValueByLabel(result, 'name')).toBe('Alice');
+            expect(extractValueByLabel(result, 'age')).toBe(30);
+        });
+
+        it('should return null for missing properties', () => {
+            const result = { name: 'Alice' };
+            
+            expect(extractValueByLabel(result, 'age')).toBeNull();
+        });
+
+        it('should return null for null result', () => {
+            expect(extractValueByLabel(null, 'name')).toBeNull();
+        });
+
+        it('should return null for undefined result', () => {
+            expect(extractValueByLabel(undefined, 'name')).toBeNull();
+        });
+    });
+
+    describe('formatValueForCsv', () => {
+        it('should return empty string for null', () => {
+            expect(formatValueForCsv(null)).toBe('');
+        });
+
+        it('should return empty string for undefined', () => {
+            expect(formatValueForCsv(undefined)).toBe('');
+        });
+
+        it('should format Date as ISO string', () => {
+            const date = new Date('2024-01-15T10:30:00.000Z');
+            expect(formatValueForCsv(date)).toBe('2024-01-15T10:30:00.000Z');
+        });
+
+        it('should format boolean as string', () => {
+            expect(formatValueForCsv(true)).toBe('true');
+            expect(formatValueForCsv(false)).toBe('false');
+        });
+
+        it('should convert primitives to string', () => {
+            expect(formatValueForCsv('hello')).toBe('hello');
+            expect(formatValueForCsv(42)).toBe('42');
+            expect(formatValueForCsv(3.14)).toBe('3.14');
+        });
+    });
+});

package/test/http/result-streamSpec.ts

@@ -0,0 +1,151 @@
+import { AsyncIterableResultStream, arrayToResultStream, ResultStream } from '../../src/http/result-stream';
+
+describe('ResultStream', () => {
+    describe('AsyncIterableResultStream', () => {
+        it('should return items sequentially', async () => {
+            async function* generator() {
+                yield 1;
+                yield 2;
+                yield 3;
+            }
+
+            const stream = new AsyncIterableResultStream(generator());
+
+            expect(await stream.next()).toBe(1);
+            expect(await stream.next()).toBe(2);
+            expect(await stream.next()).toBe(3);
+            expect(await stream.next()).toBeNull();
+        });
+
+        it('should return null when exhausted', async () => {
+            async function* generator() {
+                yield 'item';
+            }
+
+            const stream = new AsyncIterableResultStream(generator());
+
+            expect(await stream.next()).toBe('item');
+            expect(await stream.next()).toBeNull();
+            expect(await stream.next()).toBeNull(); // Should still return null
+        });
+
+        it('should stop iteration after close()', async () => {
+            async function* generator() {
+                yield 1;
+                yield 2;
+                yield 3;
+            }
+
+            const stream = new AsyncIterableResultStream(generator());
+
+            expect(await stream.next()).toBe(1);
+            await stream.close();
+            expect(await stream.next()).toBeNull();
+            expect(await stream.next()).toBeNull();
+        });
+
+        it('should handle close() being called multiple times', async () => {
+            async function* generator() {
+                yield 1;
+            }
+
+            const stream = new AsyncIterableResultStream(generator());
+
+            await stream.close();
+            await stream.close(); // Should not throw
+            expect(await stream.next()).toBeNull();
+        });
+
+        it('should handle empty stream', async () => {
+            async function* generator() {
+                // Empty generator
+            }
+
+            const stream = new AsyncIterableResultStream(generator());
+
+            expect(await stream.next()).toBeNull();
+        });
+    });
+
+    describe('arrayToResultStream', () => {
+        it('should convert array to working stream', async () => {
+            const array = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+            const stream = arrayToResultStream(array);
+
+            const results: number[] = [];
+            let item;
+            while ((item = await stream.next()) !== null) {
+                results.push(item);
+            }
+
+            expect(results).toEqual(array);
+            expect(results.length).toBe(10);
+        });
+
+        it('should yield all items in order', async () => {
+            const array = ['first', 'second', 'third'];
+            const stream = arrayToResultStream(array);
+
+            expect(await stream.next()).toBe('first');
+            expect(await stream.next()).toBe('second');
+            expect(await stream.next()).toBe('third');
+            expect(await stream.next()).toBeNull();
+        });
+
+        it('should handle empty array', async () => {
+            const stream = arrayToResultStream([]);
+
+            expect(await stream.next()).toBeNull();
+        });
+
+        it('should handle array with objects', async () => {
+            const array = [
+                { id: 1, name: 'Alice' },
+                { id: 2, name: 'Bob' }
+            ];
+            const stream = arrayToResultStream(array);
+
+            const first = await stream.next();
+            expect(first).toEqual({ id: 1, name: 'Alice' });
+
+            const second = await stream.next();
+            expect(second).toEqual({ id: 2, name: 'Bob' });
+
+            expect(await stream.next()).toBeNull();
+        });
+
+        it('should allow early termination with close()', async () => {
+            const array = [1, 2, 3, 4, 5];
+            const stream = arrayToResultStream(array);
+
+            expect(await stream.next()).toBe(1);
+            expect(await stream.next()).toBe(2);
+            
+            await stream.close();
+            
+            expect(await stream.next()).toBeNull();
+        });
+    });
+
+    describe('Stream cleanup', () => {
+        it('should call iterator.return() on close', async () => {
+            let returnCalled = false;
+
+            async function* generator() {
+                try {
+                    yield 1;
+                    yield 2;
+                } finally {
+                    returnCalled = true;
+                }
+            }
+
+            const stream = new AsyncIterableResultStream(generator());
+
+            await stream.next(); // Get first item
+            await stream.close();
+
+            expect(returnCalled).toBe(true);
+        });
+    });
+});