jinaga-server 1.0.0 → 1.1.0

package/src/http/node-http.ts

@@ -63,7 +63,7 @@ export class NodeHttpConnection implements HttpConnection {
                     if (res.statusCode === 403) {
                         reject(response);
                     }
-                    else if (res.statusCode>= 400) {
+                    else if (!res.statusCode || res.statusCode>= 400) {
                         resolve({
                             result: "retry",
                             error: response
@@ -100,7 +100,10 @@ export class NodeHttpConnection implements HttpConnection {
     }
 }
 
-function isJsonResult(contentType: string) {
+function isJsonResult(contentType: string | undefined) {
+    if (!contentType) {
+        return false;
+    }
     const json = contentType.split(";").indexOf("application/json");
     return json >= 0;
 }

package/src/http/router.ts

@@ -1,8 +1,12 @@
 import { Handler, Router } from "express";
 import {
     Authorization,
+    buildFeeds,
+    computeObjectHash,
     Declaration,
     FactRecord,
+    FactReference,
+    Feed,
     Forbidden,
     fromDescriptiveString,
     LoadMessage,
@@ -11,20 +15,48 @@ import {
     QueryMessage,
     QueryResponse,
     SaveMessage,
-    SaveResponse,
+    Specification,
     SpecificationParser,
     Trace,
     UserIdentity,
 } from "jinaga";
+import { FeedResponse, FeedsResponse } from "jinaga/dist/http/messages";
 
-function get<U>(method: ((req: RequestUser) => Promise<U>)): Handler {
+import { FeedCache } from "./feed-cache";
+
+interface ParsedQs { [key: string]: undefined | string | string[] | ParsedQs | ParsedQs[] }
+
+function get<U>(method: ((req: RequestUser, params?: { [key: string]: string }, query?: ParsedQs) => Promise<U>)): Handler {
+    return (req, res, next) => {
+        const user = <RequestUser>req.user;
+        method(user, req.params, req.query)
+            .then(response => {
+                if (!response) {
+                    res.sendStatus(404);
+                    next();
+                }
+                else {
+                    res.setHeader('Content-Type', 'application/json');
+                    res.send(JSON.stringify(response));
+                    next();
+                }
+            })
+            .catch(error => {
+                Trace.error(error);
+                res.sendStatus(500);
+                next();
+            });
+    };
+}
+
+function getAuthenticate<U>(method: ((req: RequestUser, params?: { [key: string]: string }) => Promise<U>)): Handler {
     return (req, res, next) => {
         const user = <RequestUser>req.user;
         if (!user) {
             res.sendStatus(401);
         }
         else {
-            method(user)
+            method(user, req.params)
                 .then(response => {
                     res.setHeader('Content-Type', 'application/json');
                     res.send(JSON.stringify(response));
@@ -39,18 +71,24 @@ function get<U>(method: ((req: RequestUser) => Promise<U>)): Handler {
     };
 }
 
-function post<T, U>(method: (user: RequestUser, message: T) => Promise<U>): Handler {
+function post<T, U>(method: (user: RequestUser, message: T, params?: { [key: string]: string }) => Promise<U>): Handler {
     return (req, res, next) => {
         const user = <RequestUser>req.user;
         const message = <T>req.body;
         if (!message) {
             throw new Error('Ensure that you have called app.use(express.json()).');
         }
-        method(user, message)
+        method(user, message, req.params)
             .then(response => {
-                res.setHeader('Content-Type', 'application/json');
-                res.send(JSON.stringify(response));
-                next();
+                if (!response) {
+                    res.sendStatus(404);
+                    next();
+                }
+                else {
+                    res.setHeader('Content-Type', 'application/json');
+                    res.send(JSON.stringify(response));
+                    next();
+                }
             })
             .catch(error => {
                 if (error instanceof Forbidden) {
@@ -100,6 +138,65 @@ function postString<U>(method: (user: RequestUser, message: string) => Promise<U
     };
 }
 
+function postCreate<T>(method: (user: RequestUser, message: T) => Promise<void>): Handler {
+    return (req, res, next) => {
+        const user = <RequestUser>req.user;
+        const message = <T>req.body;
+        if (!message) {
+            throw new Error('Ensure that you have called app.use(express.json()).');
+        }
+        method(user, message)
+            .then(_ => {
+                res.sendStatus(201);
+                next();
+            })
+            .catch(error => {
+                if (error instanceof Forbidden) {
+                    Trace.warn(error.message);
+                    res.setHeader('Content-Type', 'text/plain');
+                    res.status(403).send(error.message);
+                }
+                else {
+                    Trace.error(error);
+                    res.setHeader('Content-Type', 'text/plain');
+                    res.sendStatus(500).send(error.message);
+                }
+                next();
+            });
+    };
+}
+
+function postStringCreate(method: (user: RequestUser, message: string) => Promise<void>): Handler {
+    return (req, res, next) => {
+        const user = <RequestUser>req.user;
+        const input = <string>req.body;
+        if (!input || typeof(input) !== 'string') {
+            res.setHeader('Content-Type', 'text/plain');
+            res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
+        }
+        else {
+            method(user, input)
+                .then(_ => {
+                    res.sendStatus(201);
+                    next();
+                })
+                .catch(error => {
+                    if (error instanceof Forbidden) {
+                        Trace.warn(error.message);
+                        res.setHeader('Content-Type', 'text/plain');
+                        res.status(403).send(error.message);
+                    }
+                    else {
+                        Trace.error(error);
+                        res.setHeader('Content-Type', 'text/plain');
+                        res.status(400).send(error.message);
+                    }
+                    next();
+                });
+        }
+    };
+}
+
 function serializeUserIdentity(user: RequestUser) : UserIdentity {
     if (!user) {
         return null;
@@ -119,16 +216,17 @@ export interface RequestUser {
 export class HttpRouter {
     handler: Handler;
 
-    constructor(private authorization: Authorization) {
+    constructor(private authorization: Authorization, private feedCache: FeedCache) {
         const router = Router();
-        router.get('/login', get(user => this.login(user)));
+        router.get('/login', getAuthenticate(user => this.login(user)));
         router.post('/query', post((user, queryMessage: QueryMessage) => this.query(user, queryMessage)));
         router.post('/load', post((user, loadMessage: LoadMessage) => this.load(user, loadMessage)));
-        router.post('/save', post((user, saveMessage: SaveMessage) => this.save(user, saveMessage)));
+        router.post('/save', postCreate((user, saveMessage: SaveMessage) => this.save(user, saveMessage)));
 
         router.post('/read', postString((user, input: string) => this.read(user, input)));
-        router.post('/write', post((user, input: string) => this.write(user, input)));
-        router.post('/feed', post((user, input: string) => this.feed(user, input)));
+        router.post('/write', postStringCreate((user, input: string) => this.write(user, input)));
+        router.post('/feeds', post((user, input: string) => this.feeds(user, input)));
+        router.get('/feeds/:hash', get((user, params, query) => this.feed(user, params, query)));
 
         this.handler = router;
     }
@@ -161,34 +259,25 @@ export class HttpRouter {
         };
     }
 
-    private async save(user: RequestUser, saveMessage: SaveMessage) : Promise<SaveResponse> {
+    private async save(user: RequestUser, saveMessage: SaveMessage) : Promise<void> {
         const userIdentity = serializeUserIdentity(user);
         await this.authorization.save(userIdentity, saveMessage.facts);
-        return {};
     }
 
     private async read(user: RequestUser, input: string): Promise<any[]> {
         const knownFacts = await this.getKnownFacts(user);
         const parser = new SpecificationParser(input);
         parser.skipWhitespace();
-        var declaration = parser.parseDeclaration(knownFacts);
-        var specification = parser.parseSpecification();
-
-        // Select starting facts that match the inputs
-        const start = specification.given.map(input => {
-            const declaredFact = declaration.find(d => d.name === input.name);
-            if (!declaredFact) {
-                throw new Error(`No fact named ${input.name} was declared`);
-            }
-            return declaredFact.declared.reference;
-        });
+        const declaration = parser.parseDeclaration(knownFacts);
+        const specification = parser.parseSpecification();
+        const start = this.selectStart(specification, declaration);
 
         const userIdentity = serializeUserIdentity(user);
         const results = await this.authorization.read(userIdentity, start, specification);
         return results;
     }
 
-    private async write(user: RequestUser, input: string): Promise<SaveResponse> {
+    private async write(user: RequestUser, input: string): Promise<void> {
         const knownFacts = await this.getKnownFacts(user);
         const parser = new SpecificationParser(input);
         parser.skipWhitespace();
@@ -204,11 +293,54 @@ export class HttpRouter {
 
         const userIdentity = serializeUserIdentity(user);
         await this.authorization.save(userIdentity, factRecords);
-        return {};
     }
 
-    private feed(user: RequestUser, input: string): Promise<string> {
-        throw new Error("Method not implemented.");
+    private async feeds(user: RequestUser, input: string): Promise<FeedsResponse> {
+        const knownFacts = await this.getKnownFacts(user);
+        const parser = new SpecificationParser(input);
+        parser.skipWhitespace();
+        const declaration = parser.parseDeclaration(knownFacts);
+        const specification = parser.parseSpecification();
+        const start = this.selectStart(specification, declaration);
+
+        const feedDefinitions = buildFeeds(start, specification).map(feed => ({
+            hash: urlSafeHash(feed),
+            feed
+        }));
+        // Store all feeds in the cache.
+        for (const feedDefinition of feedDefinitions) {
+            await this.feedCache.storeFeed(feedDefinition.hash, feedDefinition.feed);
+        }
+
+        return {
+            feeds: feedDefinitions.map(f => f.hash)
+        }
+    }
+
+    private async feed(user: RequestUser, params: { [key: string]: string }, query: ParsedQs): Promise<FeedResponse | null> {
+        const feedHash = params["hash"];
+        if (!feedHash) {
+            return null;
+        }
+
+        const feed = await this.feedCache.getFeed(feedHash);
+        if (!feed) {
+            return null;
+        }
+
+        const bookmark = query["b"] as string ?? "";
+
+        const userIdentity = serializeUserIdentity(user);
+        const results = await this.authorization.feed(userIdentity, feed, bookmark);
+        // Return distinct fact references from all the tuples.
+        const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
+            self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
+        );
+        const response: FeedResponse = {
+            references,
+            bookmark: results.bookmark
+        };
+        return response;
     }
 
     private async getKnownFacts(user: RequestUser): Promise<Declaration> {
@@ -234,4 +366,20 @@ export class HttpRouter {
             return [];
         }
     }
-}
+
+    private selectStart(specification: Specification, declaration: Declaration) : FactReference[] {
+        // Select starting facts that match the inputs
+        return specification.given.map(input => {
+            const declaredFact = declaration.find(d => d.name === input.name);
+            if (!declaredFact) {
+                throw new Error(`No fact named ${input.name} was declared`);
+            }
+            return declaredFact.declared.reference;
+        });
+    }
+}
+
+function urlSafeHash(feed: Feed): string {
+    const base64 = computeObjectHash(feed);
+    return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+}

package/src/jinaga-server.ts

@@ -4,11 +4,11 @@ import {
     AuthorizationNoOp,
     AuthorizationRules,
     Cache,
-    Feed,
-    FeedImpl,
     Fork,
     Jinaga,
     MemoryStore,
+    ObservableSource,
+    ObservableSourceImpl,
     PassThroughFork,
     Storage,
     SyncStatusNotifier,
@@ -23,6 +23,7 @@ import { AuthorizationKeystore } from "./authorization/authorization-keystore";
 import { NodeHttpConnection } from "./http/node-http";
 import { HttpRouter, RequestUser } from "./http/router";
 import { Keystore } from "./keystore";
+import { MemoryFeedCache } from "./memory/memory-feed-cache";
 import { PostgresKeystore } from "./postgres/postgres-keystore";
 import { PostgresStore } from "./postgres/postgres-store";
 
@@ -51,12 +52,13 @@ export class JinagaServer {
     static create(config: JinagaServerConfig): JinagaServerInstance {
         const syncStatusNotifier = new SyncStatusNotifier();
         const store = createStore(config);
-        const feed = new FeedImpl(store);
-        const fork = createFork(config, feed, syncStatusNotifier);
+        const source = new ObservableSourceImpl(store);
+        const fork = createFork(config, source, syncStatusNotifier);
         const keystore = new PostgresKeystore(config.pgKeystore);
         const authorizationRules = config.authorization ? config.authorization(new AuthorizationRules()) : null;
         const authorization = createAuthorization(authorizationRules, fork, keystore);
-        const router = new HttpRouter(authorization);
+        const feedCache = new MemoryFeedCache();
+        const router = new HttpRouter(authorization, feedCache);
         const authentication = new AuthenticationDevice(fork, keystore, localDeviceIdentity);
         const memory = new MemoryStore();
         const j: Jinaga = new Jinaga(authentication, memory, syncStatusNotifier);
@@ -69,7 +71,7 @@ export class JinagaServer {
             handler: router.handler,
             j,
             withSession: (req, callback) => {
-                return withSession(feed, keystore, authorizationRules, req, callback);
+                return withSession(source, keystore, authorizationRules, req, callback);
             },
             close
         }
@@ -87,7 +89,7 @@ function createStore(config: JinagaServerConfig): Storage {
     }
 }
 
-function createFork(config: JinagaServerConfig, feed: Feed, syncStatusNotifier: SyncStatusNotifier): Fork {
+function createFork(config: JinagaServerConfig, feed: ObservableSource, syncStatusNotifier: SyncStatusNotifier): Fork {
     if (config.httpEndpoint) {
         const httpConnection = new NodeHttpConnection(config.httpEndpoint);
         const httpTimeoutSeconds = config.httpTimeoutSeconds || 5;
@@ -102,7 +104,7 @@ function createFork(config: JinagaServerConfig, feed: Feed, syncStatusNotifier:
     }
 }
 
-function createAuthorization(authorizationRules: AuthorizationRules | null, feed: Feed, keystore: Keystore | null): Authorization {
+function createAuthorization(authorizationRules: AuthorizationRules | null, feed: ObservableSource, keystore: Keystore | null): Authorization {
     if (keystore) {
         const authorization = new AuthorizationKeystore(feed, keystore, authorizationRules);
         return authorization;
@@ -112,7 +114,7 @@ function createAuthorization(authorizationRules: AuthorizationRules | null, feed
     }
 }
 
-async function withSession(feed: Feed, keystore: Keystore, authorizationRules: AuthorizationRules | null, req: Request, callback: ((j: Jinaga) => Promise<void>)) {
+async function withSession(feed: ObservableSource, keystore: Keystore, authorizationRules: AuthorizationRules | null, req: Request, callback: ((j: Jinaga) => Promise<void>)) {
     const user = <RequestUser>req.user;
     const userIdentity: UserIdentity = {
         provider: user.provider,

package/src/memory/memory-feed-cache.ts

@@ -0,0 +1,15 @@
+import { Feed } from "jinaga";
+import { FeedCache } from "../http/feed-cache";
+
+export class MemoryFeedCache implements FeedCache {
+    private feeds: { [feedHash: string]: Feed } = {};
+
+    storeFeed(feedHash: string, feed: Feed): Promise<void> {
+        this.feeds[feedHash] = feed;
+        return Promise.resolve();
+    }
+
+    getFeed(feedHash: string): Promise<Feed | undefined> {
+        return Promise.resolve(this.feeds[feedHash]);
+    }
+}

package/src/postgres/postgres-store.ts

@@ -2,16 +2,17 @@ import {
     canonicalPredecessors,
     Direction,
     ExistentialCondition,
-    FactBookmark,
     FactEnvelope,
     FactPath,
     FactRecord,
     FactReference,
     factReferenceEquals,
-    FactStream,
     FactTuple,
+    Feed,
     getAllFactTypes,
+    getAllFactTypesFromFeed,
     getAllRoles,
+    getAllRolesFromFeed,
     Join,
     PredecessorCollection,
     PropertyCondition,
@@ -20,6 +21,7 @@ import {
     Step,
     Storage,
 } from "jinaga";
+import { FactFeed } from "jinaga/dist/storage";
 import { PoolClient } from "pg";
 
 import { distinct, flatten } from "../util/fn";
@@ -47,9 +49,8 @@ import {
     PublicKeyMap,
     RoleMap,
 } from "./maps";
-import { SpecificationLabel } from "./query-description";
 import { ResultSetTree, resultSqlFromSpecification, SqlQueryTree } from "./specification-result-sql";
-import { sqlFromSpecification } from "./specification-sql";
+import { sqlFromFeed } from "./specification-sql";
 import { sqlFromSteps } from "./sql";
 
 interface FactTypeResult {
@@ -98,11 +99,9 @@ interface AncestorResult {
     }[];
 }
 
-function loadFactReference(r: Row): FactReference {
-    return {
-        type: r.type,
-        hash: r.hash
-    };
+interface SpecificationLabel {
+    type: string;
+    index: number;
 }
 
 function loadFactTuple(labels: SpecificationLabel[], row: Row): FactTuple {
@@ -247,30 +246,25 @@ export class PostgresStore implements Storage {
         return composer.compose(resultSets);
     }
 
-    async streamsFromSpecification(start: FactReference[], bookmarks: FactBookmark[], limit: number, specification: Specification): Promise<FactStream[]> {
-        const factTypes = await this.loadFactTypesFromSpecification(specification);
-        const roleMap = await this.loadRolesFromSpecification(specification, factTypes);
+    async feed(feed: Feed, bookmark: string): Promise<FactFeed> {
+        const factTypes: FactTypeMap = await this.loadFactTypesFromFeed(feed);
+        const roleMap: RoleMap = await this.loadRolesFromFeed(feed, factTypes);
+        const sql = sqlFromFeed(feed, bookmark, 100, factTypes, roleMap);
+        if (!sql) {
+            return {
+                tuples: [],
+                bookmark
+            };
+        }
 
-        const sqlQueries = sqlFromSpecification(start, bookmarks, limit, specification, factTypes, roleMap);
-        const streams = await this.connectionFactory.with(async (connection) => {
-            const streams: FactStream[] = [];
-            for (const sqlQuery of sqlQueries) {
-                try {
-                    const { rows } = await connection.query(sqlQuery.sql, sqlQuery.parameters);
-                    const tuples = rows.map(row => loadFactTuple(sqlQuery.labels, row));
-                    streams.push({
-                        labels: sqlQuery.labels.map(l => l.name),
-                        tuples,
-                        bookmark: tuples.length > 0 ? tuples[tuples.length - 1].bookmark : sqlQuery.bookmark
-                    });
-                }
-                catch (error) {
-                    throw new Error(`Could not execute query "${sqlQuery.sql}", parameters ${sqlQuery.parameters}:\n${error}`);
-                }
-            }
-            return streams;
+        const { rows } = await this.connectionFactory.with(async (connection) => {
+            return await connection.query(sql.sql, sql.parameters);
         });
-        return streams;
+        const tuples = rows.map(row => loadFactTuple(sql.labels, row));
+        return {
+            tuples,
+            bookmark: tuples.length > 0 ? tuples[tuples.length - 1].bookmark : bookmark
+        };
     }
 
     private async loadFactTypesFromSteps(steps: Step[], startType: string): Promise<FactTypeMap> {
@@ -289,6 +283,21 @@ export class PostgresStore implements Storage {
         return factTypes;
     }
 
+    async loadFactTypesFromFeed(feed: Feed): Promise<FactTypeMap> {
+        const factTypes = this.factTypeMap;
+        const unknownFactTypes = getAllFactTypesFromFeed(feed)
+            .filter(factType => !factTypes.has(factType));
+        if (unknownFactTypes.length > 0) {
+            const loadedFactTypes = await this.connectionFactory.with(async (connection) => {
+                return await loadFactTypes(unknownFactTypes, connection);
+            });
+            const merged = mergeFactTypes(this.factTypeMap, loadedFactTypes);
+            this.factTypeMap = merged;
+            return merged;
+        }
+        return factTypes;
+    }
+
     async loadFactTypesFromSpecification(specification: Specification): Promise<FactTypeMap> {
         const factTypes = this.factTypeMap;
         const unknownFactTypes = getAllFactTypes(specification)
@@ -338,6 +347,25 @@ export class PostgresStore implements Storage {
         return roleMap;
     }
 
+    async loadRolesFromFeed(feed: Feed, factTypes: FactTypeMap): Promise<RoleMap> {
+        const roleMap = this.roleMap;
+        const unknownRoles = getAllRolesFromFeed(feed)
+            .map(r => ({
+                defining_fact_type_id: getFactTypeId(factTypes, r.definingFactType),
+                role: r.name
+            }))
+            .filter(r => !hasRole(roleMap, r.defining_fact_type_id, r.role));
+        if (unknownRoles.length > 0) {
+            const loadedRoles = await this.connectionFactory.with(async (connection) => {
+                return await loadRoles(unknownRoles, roleMap, connection);
+            });
+            const merged = mergeRoleMaps(this.roleMap, loadedRoles);
+            this.roleMap = merged;
+            return merged;
+        }
+        return roleMap;
+    }
+
     async whichExist(references: FactReference[]): Promise<FactReference[]> {
         if (references.length === 0) {
             return [];