jinaga-server 3.5.2 → 3.5.3

package/integration-test/jinaga-test/read-streaming.test.js

@@ -0,0 +1,199 @@
+const request = require('supertest');
+const { JinagaServer } = require('./jinaga-server');
+const express = require('express');
+
+const host = "db";
+const connectionString = `postgresql://dev:devpw@${host}:5432/integrationtest`;
+
+// Test model classes
+class Root {
+  static Type = "StreamTest.Root";
+  type = Root.Type;
+
+  constructor(identifier) {
+    this.identifier = identifier;
+  }
+}
+
+class Successor {
+  static Type = "StreamTest.Successor";
+  type = Successor.Type;
+
+  constructor(identifier, predecessor) {
+    this.identifier = identifier;
+    this.pred = predecessor;
+  }
+}
+
+// Helper to create test app
+async function createTestApp() {
+  const { handler, j, close } = JinagaServer.create({
+    pgKeystore: connectionString,
+    pgStore: connectionString
+  });
+
+  const app = express();
+  app.use(express.json());
+  app.use(express.text());
+  app.use(handler);
+
+  return { app, j, close };
+}
+
+// Helper to parse NDJSON
+function parseNDJSON(text) {
+  return text
+    .trim()
+    .split('\n')
+    .filter(line => line.trim() !== '')
+    .map(line => JSON.parse(line));
+}
+
+describe('Read Endpoint - Phase 2: Streaming Infrastructure', () => {
+  let app;
+  let j;
+  let close;
+
+  beforeEach(async () => {
+    ({ app, j, close } = await createTestApp());
+    await j.local();
+  });
+
+  afterEach(async () => {
+    if (close) {
+      await close();
+    }
+  });
+
+  describe('2.3: NDJSON Streaming Output', () => {
+    it('should stream NDJSON without loading all into memory', async () => {
+      // Create 5000 successors
+      const root = await j.fact(new Root('stream-test-' + Date.now()));
+      const rootHash = j.hash(root);
+      
+      // Create in batches
+      for (let i = 0; i < 5000; i++) {
+        await j.fact(new Successor(`stream-successor-${i}`, root));
+      }
+
+      const specification = `(root: StreamTest.Root) {
+  s: StreamTest.Successor [
+    s->pred: StreamTest.Root = root
+  ]
+} => {
+  s = s
+}`;
+
+      const startTime = Date.now();
+      const response = await request(app)
+        .post('/read')
+        .set('Content-Type', 'text/plain')
+        .set('Accept', 'application/x-ndjson')
+        .send(`let root: StreamTest.Root = #${rootHash}\n${specification}`);
+
+      const duration = Date.now() - startTime;
+
+      if (response.status !== 200) {
+        expect(response.text).toBe('');
+        expect(response.status).toBe(200);
+      }
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('application/x-ndjson');
+      
+      // Parse NDJSON response
+      const results = parseNDJSON(response.text);
+      expect(results.length).toBe(5000);
+      
+      // Should start streaming within 1 second
+      expect(duration).toBeLessThan(10000); // 10 seconds for large set
+      
+      // Verify structure
+      expect(results[0]).toHaveProperty('s');
+      expect(results[0].s).toHaveProperty('type', 'StreamTest.Successor');
+    }, 120000); // 120 second timeout for large dataset
+  });
+
+  describe('2.4: Stream Error Handling', () => {
+    it('should handle normal completion without errors', async () => {
+      const root = await j.fact(new Root('error-test-' + Date.now()));
+      const rootHash = j.hash(root);
+      await j.fact(new Successor(`error-successor-1`, root));
+      await j.fact(new Successor(`error-successor-2`, root));
+      await j.fact(new Successor(`error-successor-3`, root));
+
+      const specification = `(root: StreamTest.Root) {
+  s: StreamTest.Successor [
+    s->pred: StreamTest.Root = root
+  ]
+} => {
+  s = s
+}`;
+
+      const response = await request(app)
+        .post('/read')
+        .set('Content-Type', 'text/plain')
+        .set('Accept', 'application/x-ndjson')
+        .send(`let root: StreamTest.Root = #${rootHash}\n${specification}`);
+
+      if (response.status !== 200) {
+        expect(response.text).toBe('');
+        expect(response.status).toBe(200);
+      }
+      expect(response.status).toBe(200);
+      
+      const results = parseNDJSON(response.text);
+      expect(results.length).toBe(3);
+      
+      // None of the results should be error frames
+      results.forEach(item => {
+        expect(item.error).toBeUndefined();
+      });
+    });
+  });
+
+  describe('2.5: Backward Compatibility with Arrays', () => {
+    it('should accept and format array results', async () => {
+      const root = await j.fact(new Root('compat-test-' + Date.now()));
+      const rootHash = j.hash(root);
+      await j.fact(new Successor(`compat-successor-1`, root));
+      await j.fact(new Successor(`compat-successor-2`, root));
+
+      const specification = `(root: StreamTest.Root) {
+  s: StreamTest.Successor [
+    s->pred: StreamTest.Root = root
+  ]
+} => {
+  s = s
+}`;
+
+      // Test all formats work with array-based results
+      const jsonResponse = await request(app)
+        .post('/read')
+        .set('Content-Type', 'text/plain')
+        .set('Accept', 'application/json')
+        .send(`let root: StreamTest.Root = #${rootHash}\n${specification}`);
+
+      if (jsonResponse.status !== 200) {
+        expect(jsonResponse.text).toBe('');
+        expect(jsonResponse.status).toBe(200);
+      }
+      expect(jsonResponse.status).toBe(200);
+      const jsonResults = JSON.parse(jsonResponse.text);
+      expect(jsonResults.length).toBe(2);
+
+      const ndjsonResponse = await request(app)
+        .post('/read')
+        .set('Content-Type', 'text/plain')
+        .set('Accept', 'application/x-ndjson')
+        .send(`let root: StreamTest.Root = #${rootHash}\n${specification}`);
+
+      if (ndjsonResponse.status !== 200) {
+        expect(ndjsonResponse.text).toBe('');
+        expect(ndjsonResponse.status).toBe(200);
+      }
+      expect(ndjsonResponse.status).toBe(200);
+      const ndjsonResults = parseNDJSON(ndjsonResponse.text);
+      expect(ndjsonResults.length).toBe(2);
+    });
+  });
+});

package/integration-test/jinaga-test/specification-conditions.test.js

@@ -0,0 +1,147 @@
+const { JinagaServer } = require("./jinaga-server");
+const { parseSpecification } = require("./test-helpers");
+
+const host = "db";
+// const host = "localhost";
+const connectionString = `postgresql://dev:devpw@${host}:5432/integrationtest`;
+
+class Department {
+  constructor(identifier) {
+    this.type = Department.Type;
+    this.identifier = identifier;
+  }
+}
+Department.Type = "IntegrationTest.Conditions.Department";
+
+class DepartmentActive {
+  constructor(department) {
+    this.type = DepartmentActive.Type;
+    this.department = department;
+  }
+}
+DepartmentActive.Type = "IntegrationTest.Conditions.DepartmentActive";
+
+class Employee {
+  constructor(department, employeeId) {
+    this.type = Employee.Type;
+    this.department = department;
+    this.employeeId = employeeId;
+  }
+}
+Employee.Type = "IntegrationTest.Conditions.Employee";
+
+describe("Specifications with conditions", () => {
+  let j;
+  let close;
+
+  beforeEach(() => {
+      ({ j, close } = JinagaServer.create({
+          pgKeystore: connectionString,
+          pgStore:    connectionString
+      }));
+  });
+
+  afterEach(async () => {
+      await close();
+  });
+
+  it("should return employees when department is active (has DepartmentActive fact)", async () => {
+    // Create an active department (has DepartmentActive marker fact)
+    const department = await j.fact(new Department("HR-001"));
+    await j.fact(new DepartmentActive(department));
+    
+    // Add employees to the department
+    await j.fact(new Employee(department, "EMP-001"));
+    await j.fact(new Employee(department, "EMP-002"));
+
+    // Query with positive existential condition: only return employees for departments that HAVE an active marker
+    const specification = parseSpecification(`
+      (department: IntegrationTest.Conditions.Department [
+        E {
+          active: IntegrationTest.Conditions.DepartmentActive [
+            active->department: IntegrationTest.Conditions.Department = department
+          ]
+        }
+      ]) {
+        employee: IntegrationTest.Conditions.Employee [
+          employee->department: IntegrationTest.Conditions.Department = department
+        ]
+      } => employee
+    `);
+
+    const results = await j.query(specification, department);
+
+    // Should return both employees since department has active marker
+    expect(results.length).toBe(2);
+    expect(results).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ employeeId: "EMP-001" }),
+        expect.objectContaining({ employeeId: "EMP-002" })
+      ])
+    );
+  });
+
+  it("should NOT return employees for inactive department (with positive existential condition)", async () => {
+    // Create an inactive department (no DepartmentActive marker fact)
+    const department = await j.fact(new Department("HR-002"));
+    
+    // Add employees to the department
+    await j.fact(new Employee(department, "EMP-003"));
+    await j.fact(new Employee(department, "EMP-004"));
+
+    // Query with positive existential condition: only return employees for departments that HAVE an active marker
+    const specification = parseSpecification(`
+      (department: IntegrationTest.Conditions.Department [
+        E {
+          active: IntegrationTest.Conditions.DepartmentActive [
+            active->department: IntegrationTest.Conditions.Department = department
+          ]
+        }
+      ]) {
+        employee: IntegrationTest.Conditions.Employee [
+          employee->department: IntegrationTest.Conditions.Department = department
+        ]
+      } => employee
+    `);
+
+    const results = await j.query(specification, department);
+
+    // Should return NO employees since department has no active marker
+    expect(results).toEqual([]);
+  });
+
+  it("should return employees for inactive department using negative existential condition", async () => {
+    // Create two departments: one active, one inactive
+    const activeDept = await j.fact(new Department("HR-ACTIVE"));
+    await j.fact(new DepartmentActive(activeDept));
+    await j.fact(new Employee(activeDept, "EMP-ACTIVE-1"));
+    
+    const inactiveDept = await j.fact(new Department("HR-INACTIVE"));
+    // No DepartmentActive fact for this one
+    await j.fact(new Employee(inactiveDept, "EMP-INACTIVE-1"));
+
+    // Query with negative existential condition: only return employees for departments that DON'T HAVE an active marker
+    const specification = parseSpecification(`
+      (department: IntegrationTest.Conditions.Department [
+        !E {
+          active: IntegrationTest.Conditions.DepartmentActive [
+            active->department: IntegrationTest.Conditions.Department = department
+          ]
+        }
+      ]) {
+        employee: IntegrationTest.Conditions.Employee [
+          employee->department: IntegrationTest.Conditions.Department = department
+        ]
+      } => employee
+    `);
+
+    // Query active department - should return NO employees (has active marker, condition excludes it)
+    const activeResults = await j.query(specification, activeDept);
+    expect(activeResults).toEqual([]);
+
+    // Query inactive department - should return employees (no active marker, condition includes it)
+    const inactiveResults = await j.query(specification, inactiveDept);
+    expect(inactiveResults.length).toBe(1);
+    expect(inactiveResults[0]).toEqual(expect.objectContaining({ employeeId: "EMP-INACTIVE-1" }));
+  });
+});

package/integration-test/jinaga-test/test-helpers.js

@@ -0,0 +1,12 @@
+const { SpecificationParser, SpecificationOf } = require("jinaga");
+
+function parseSpecification(input) {
+  const parser = new SpecificationParser(input);
+  parser.skipWhitespace();
+  const spec = parser.parseSpecification();
+  return new SpecificationOf(spec);
+}
+
+module.exports = {
+  parseSpecification
+};

package/integration-test/jinaga-test/time-projection.test.js

@@ -0,0 +1,235 @@
+const { parseSpecification } = require("./test-helpers");
+const { JinagaServer } = require("./jinaga-server");
+
+const host = "db";
+// const host = "localhost";
+const connectionString = `postgresql://dev:devpw@${host}:5432/integrationtest`;
+
+class Tenant {
+    constructor(identifier) {
+        this.type = Tenant.Type;
+        this.identifier = identifier;
+    }
+}
+Tenant.Type = "IntegrationTest.TimeProjection.Tenant";
+
+class Event {
+    constructor(tenant, description) {
+        this.type = Event.Type;
+        this.tenant = tenant;
+        this.description = description;
+    }
+}
+Event.Type = "IntegrationTest.TimeProjection.Event";
+
+class EventCancelled {
+    constructor(event, reason) {
+        this.type = EventCancelled.Type;
+        this.event = event;
+        this.reason = reason;
+    }
+}
+EventCancelled.Type = "IntegrationTest.TimeProjection.EventCancelled";
+
+describe("Time projection", () => {
+    let j;
+    let close;
+    
+    beforeEach(() => {
+        ({ j, close } = JinagaServer.create({
+            pgKeystore: connectionString,
+            pgStore:    connectionString
+        }));
+    });
+    
+    afterEach(async () => {
+        await close();
+    });
+    
+    it("should retrieve timestamp for a fact", async () => {
+        const beforeTime = new Date();
+        const tenant = await j.fact(new Tenant("tenant-1"));
+        const event = await j.fact(new Event(tenant, "Test event"));
+        const afterTime = new Date();
+        
+        const specification = parseSpecification(`
+            (tenant: IntegrationTest.TimeProjection.Tenant) {
+                event: IntegrationTest.TimeProjection.Event [
+                    event->tenant: IntegrationTest.TimeProjection.Tenant = tenant
+                ]
+            } => @event
+        `);
+        
+        const results = await j.query(specification, tenant);
+        
+        expect(results).toHaveLength(1);
+        expect(results[0]).toBeInstanceOf(Date);
+        expect(results[0].getTime()).toBeGreaterThanOrEqual(beforeTime.getTime() - 1000);
+        expect(results[0].getTime()).toBeLessThanOrEqual(afterTime.getTime() + 1000);
+    });
+
+    it("should retrieve timestamps for multiple facts in order", async () => {
+        const tenant = await j.fact(new Tenant("tenant-2"));
+        const event1 = await j.fact(new Event(tenant, "First event"));
+        
+        // Add a small delay to ensure timestamps differ
+        await new Promise(resolve => setTimeout(resolve, 10));
+        
+        const event2 = await j.fact(new Event(tenant, "Second event"));
+        
+        await new Promise(resolve => setTimeout(resolve, 10));
+        
+        const event3 = await j.fact(new Event(tenant, "Third event"));
+        
+        const specification = parseSpecification(`
+            (tenant: IntegrationTest.TimeProjection.Tenant) {
+                event: IntegrationTest.TimeProjection.Event [
+                    event->tenant: IntegrationTest.TimeProjection.Tenant = tenant
+                ]
+            } => {
+                event = event
+                timestamp = @event
+            }
+        `);
+        
+        const results1 = await j.query(specification, tenant);
+        const results2 = await j.query(specification, tenant);
+        const results3 = await j.query(specification, tenant);
+        
+        expect(results1).toHaveLength(3);
+        expect(results2).toHaveLength(3);
+        expect(results3).toHaveLength(3);
+        
+        const time1 = results1[0].timestamp;
+        const time2 = results2[0].timestamp;
+        const time3 = results3[0].timestamp;
+        
+        expect(time1).toBeInstanceOf(Date);
+        expect(time2).toBeInstanceOf(Date);
+        expect(time3).toBeInstanceOf(Date);
+        
+        // Verify timestamps are ordered (allowing for small timing variations)
+        expect(time2.getTime()).toBeGreaterThanOrEqual(time1.getTime());
+        expect(time3.getTime()).toBeGreaterThanOrEqual(time2.getTime());
+    });
+
+    it("should project timestamp along with fact data", async () => {
+        const tenant = await j.fact(new Tenant("tenant-3"));
+        const event = await j.fact(new Event(tenant, "Complex test event"));
+        
+        const specification = parseSpecification(`
+            (tenant: IntegrationTest.TimeProjection.Tenant) {
+                event: IntegrationTest.TimeProjection.Event [
+                    event->tenant: IntegrationTest.TimeProjection.Tenant = tenant
+                ]
+            } => {
+                description = event.description
+                timestamp = @event
+            }
+        `);
+        
+        const results = await j.query(specification, tenant);
+        
+        expect(results).toHaveLength(1);
+        expect(results[0].description).toBe("Complex test event");
+        expect(results[0].timestamp).toBeInstanceOf(Date);
+        expect(results[0].timestamp.getTime()).toBeLessThanOrEqual(Date.now() + 1000);
+    });
+
+    it("should project timestamps in complex query with related facts", async () => {
+        const tenant = await j.fact(new Tenant("tenant-4"));
+        const event = await j.fact(new Event(tenant, "Event to be cancelled"));
+        
+        // Add delay to ensure different timestamps
+        await new Promise(resolve => setTimeout(resolve, 10));
+        
+        const cancellation = await j.fact(new EventCancelled(event, "No longer needed"));
+        
+        const specification = parseSpecification(`
+            (tenant: IntegrationTest.TimeProjection.Tenant) {
+                event: IntegrationTest.TimeProjection.Event [
+                    event->tenant: IntegrationTest.TimeProjection.Tenant = tenant
+                ]
+                cancellation: IntegrationTest.TimeProjection.EventCancelled [
+                    cancellation->event: IntegrationTest.TimeProjection.Event = event
+                ]
+            } => {
+                event = event
+                eventTimestamp = @event
+                cancellation = cancellation
+                cancellationTimestamp = @cancellation
+            }
+        `);
+        
+        const results = await j.query(specification, tenant);
+        
+        expect(results).toHaveLength(1);
+        expect(results[0].event).toEqual(event);
+        expect(results[0].eventTimestamp).toBeInstanceOf(Date);
+        expect(results[0].cancellation).toEqual(cancellation);
+        expect(results[0].cancellationTimestamp).toBeInstanceOf(Date);
+        
+        // Cancellation timestamp should be after or equal to event timestamp
+        expect(results[0].cancellationTimestamp.getTime())
+            .toBeGreaterThanOrEqual(results[0].eventTimestamp.getTime());
+    });
+
+    it("should project only timestamp without other data", async () => {
+        const tenant = await j.fact(new Tenant("tenant-6"));
+        const event = await j.fact(new Event(tenant, "Timestamp only"));
+        
+        const specification = parseSpecification(`
+            (tenant: IntegrationTest.TimeProjection.Tenant) {
+                event: IntegrationTest.TimeProjection.Event [
+                    event->tenant: IntegrationTest.TimeProjection.Tenant = tenant
+                ]
+            } => @event
+        `);
+        
+        const results = await j.query(specification, tenant);
+        
+        expect(results).toHaveLength(1);
+        expect(results[0]).toBeInstanceOf(Date);
+        // Should not include any other properties, just the Date
+        expect(typeof results[0].getTime).toBe('function');
+    });
+
+    it("should handle multiple time projections in same query", async () => {
+        const tenant = await j.fact(new Tenant("tenant-7"));
+        const event1 = await j.fact(new Event(tenant, "First"));
+        
+        await new Promise(resolve => setTimeout(resolve, 10));
+        
+        const event2 = await j.fact(new Event(tenant, "Second"));
+        
+        const specification = parseSpecification(`
+            (tenant: IntegrationTest.TimeProjection.Tenant) {
+                event: IntegrationTest.TimeProjection.Event [
+                    event->tenant: IntegrationTest.TimeProjection.Tenant = tenant
+                ]
+            } => {
+                description = event.description
+                timestamp = @event
+            }
+        `);
+        
+        const results1 = await j.query(specification, tenant);
+        const results2 = await j.query(specification, tenant);
+        
+        expect(results1).toHaveLength(2);
+        expect(results2).toHaveLength(2);
+        
+        // Find the specific events in the results
+        const result1First = results1.find(r => r.description === "First");
+        const result2Second = results2.find(r => r.description === "Second");
+        
+        expect(result1First).toBeDefined();
+        expect(result2Second).toBeDefined();
+        expect(result1First.timestamp).toBeInstanceOf(Date);
+        expect(result2Second.timestamp).toBeInstanceOf(Date);
+        
+        // Verify second event timestamp is not before first
+        expect(result2Second.timestamp.getTime())
+            .toBeGreaterThanOrEqual(result1First.timestamp.getTime());
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jinaga-server",
-  "version": "3.5.2",
+  "version": "3.5.3",
   "description": "Data management for web and mobile applications.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -47,8 +47,9 @@
     "typescript": "^4.7.2"
   },
   "dependencies": {
+    "csv-stringify": "^6.6.0",
     "express": "^4.21.2",
-    "jinaga": "^6.7.9",
+    "jinaga": "^6.7.14",
     "pg": "^8.13.1"
   }
 }

package/src/authorization/authorization-keystore.ts

@@ -55,9 +55,9 @@ export class AuthorizationKeystore implements Authorization {
             const userReference: FactReference | null = userIdentity
                 ? await this.keystore.getUserFact(userIdentity)
                 : null;
-            const namedStart = specification.given.reduce((map, label, index) => ({
+            const namedStart = specification.given.reduce((map, g, index) => ({
                 ...map,
-                [label.name]: start[index]
+                [g.label.name]: start[index]
             }), {} as ReferencesByName);
             // Break the specification into feeds and check distribution.
             const feeds = buildFeeds(specification);
@@ -74,9 +74,9 @@ export class AuthorizationKeystore implements Authorization {
             const userReference: FactReference | null = userIdentity
                 ? await this.keystore.getUserFact(userIdentity)
                 : null;
-            const namedStart = specification.given.reduce((map, label, index) => ({
+            const namedStart = specification.given.reduce((map, g, index) => ({
                 ...map,
-                [label.name]: start[index]
+                [g.label.name]: start[index]
             }), {} as ReferencesByName);
             const canDistribute = await this.distributionEngine.canDistributeToAll([specification], namedStart, userReference);
             if (canDistribute.type === "failure") {

package/src/http/csv-metadata.ts

@@ -0,0 +1,34 @@
+/**
+ * Metadata for CSV export derived from specification
+ */
+export interface CsvMetadata {
+    /** Column headers in order (from specification projection labels) */
+    headers: string[];
+    
+    /** Whether the projection is valid for CSV */
+    isValid: boolean;
+    
+    /** Validation errors if any */
+    errors: string[];
+}
+
+/**
+ * Projection component types
+ */
+export type ProjectionComponentType =
+    | 'field'        // Scalar field (string, number, boolean)
+    | 'hash'         // Fact hash
+    | 'type'         // Fact type
+    | 'array'        // Array/collection (invalid for CSV)
+    | 'nested'       // Nested object (invalid for CSV)
+    | 'unknown';
+
+/**
+ * Information about a single projection component
+ */
+export interface ProjectionComponent {
+    label: string;
+    type: ProjectionComponentType;
+    isValid: boolean;
+    reason?: string;
+}