jinaga-server 3.6.0 → 3.7.1

package/test/feeds/feedStreamSessionSpec.ts

@@ -0,0 +1,351 @@
+import {
+    buildFeeds,
+    computeObjectHash,
+    FactFeed,
+    FactManager,
+    FactReference,
+    FeedObject,
+    FeedResponse,
+    ProjectedResult,
+    ReferencesByName,
+    Specification,
+    SpecificationListener,
+    SpecificationParser
+} from "jinaga";
+
+import { FeedResult } from "../../src/authorization/authorization-keystore";
+import { FeedStreamSession } from "../../src/feeds/feed-stream-session";
+import { Stream } from "../../src/http/stream";
+
+// Build a real, invertible feed specification so the session can register
+// inverse and anchor listeners during start().
+function buildFeedDefinition(): { feedDefinition: FeedObject; givenHash: string } {
+    const anchorHash = "anchorhash000000000000000000000000000000000000";
+    const input =
+        `let p: Jinaga.User = #${anchorHash}\n` +
+        `(p: Jinaga.User) {\n` +
+        `    post: Test.Post [\n` +
+        `        post->author: Jinaga.User = p\n` +
+        `    ]\n` +
+        `} => post`;
+    const parser = new SpecificationParser(input);
+    parser.skipWhitespace();
+    const declaration = parser.parseDeclaration([]);
+    const specification = parser.parseSpecification();
+    const start = specification.given.map(g => {
+        const declared = declaration.find(d => d.name === g.label.name);
+        return declared!.declared.reference;
+    });
+    const namedStart = specification.given.reduce((map, g, index) => ({
+        ...map,
+        [g.label.name]: start[index]
+    }), {} as ReferencesByName);
+    const feed = buildFeeds(specification)[0];
+    const feedDefinition: FeedObject = { feed, namedStart };
+    const givenHash = computeObjectHash(namedStart);
+    return { feedDefinition, givenHash };
+}
+
+// A FactManager test double that records listeners and lets tests drive them.
+class FakeFactManager {
+    public listeners: { specification: Specification; onResult: (results: ProjectedResult[]) => Promise<void> }[] = [];
+    public removed: SpecificationListener[] = [];
+
+    addSpecificationListener(
+        specification: Specification,
+        onResult: (results: ProjectedResult[]) => Promise<void>
+    ): SpecificationListener {
+        const listener = { specification, onResult } as unknown as SpecificationListener;
+        this.listeners.push({ specification, onResult });
+        return listener;
+    }
+
+    removeSpecificationListener(listener: SpecificationListener): void {
+        this.removed.push(listener);
+    }
+}
+
+function ref(type: string, hash: string): FactReference {
+    return { type, hash };
+}
+
+function tuple(...facts: FactReference[]): { facts: FactReference[]; bookmark: string } {
+    return { facts, bookmark: "" };
+}
+
+function page(bookmark: string, tuples: { facts: FactReference[]; bookmark: string }[]): FeedResult {
+    const feed: FactFeed = { tuples, bookmark };
+    return { type: "success", feed };
+}
+
+function emptyPage(bookmark: string): FeedResult {
+    return page(bookmark, []);
+}
+
+function makeSession(
+    query: (bookmark: string) => Promise<FeedResult>,
+    options: {
+        initialBookmark?: string;
+        config?: ConstructorParameters<typeof FeedStreamSession>[8];
+        stream?: Stream<FeedResponse>;
+        factManager?: FakeFactManager;
+    } = {}
+): { session: FeedStreamSession; stream: Stream<FeedResponse>; received: FeedResponse[]; factManager: FakeFactManager } {
+    const { feedDefinition, givenHash } = buildFeedDefinition();
+    const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
+    const stream = options.stream ?? new Stream<FeedResponse>();
+    const received: FeedResponse[] = [];
+    stream.next(r => received.push(r));
+    const factManager = options.factManager ?? new FakeFactManager();
+    const session = new FeedStreamSession(
+        query,
+        factManager as unknown as FactManager,
+        feedDefinition,
+        start,
+        givenHash,
+        stream,
+        options.initialBookmark ?? "",
+        "test-conn",
+        options.config ?? {}
+    );
+    return { session, stream, received, factManager };
+}
+
+async function settle(): Promise<void> {
+    for (let i = 0; i < 10; i++) {
+        await new Promise<void>(resolve => setImmediate(resolve));
+    }
+}
+
+describe("FeedStreamSession", () => {
+    it("backfills to exhaustion then emits a SYNC frame", async () => {
+        const A = ref("Test.Post", "a");
+        const B = ref("Test.Post", "b");
+        const C = ref("Test.Post", "c");
+        const responses = [
+            page("bm1", [tuple(A), tuple(B)]),
+            page("bm2", [tuple(C)]),
+            emptyPage("bm2")
+        ];
+        let call = 0;
+        const { session, received } = makeSession(async () => responses[call++]);
+
+        session.start();
+        await settle();
+
+        expect(received).toEqual([
+            { references: [A, B], bookmark: "bm1" },
+            { references: [C], bookmark: "bm2" },
+            { references: [], bookmark: "bm2" }
+        ]);
+        // The last frame is the SYNC frame: empty references, current bookmark.
+        expect(received[received.length - 1].references).toEqual([]);
+    });
+
+    it("emits a SYNC frame immediately when there is no history", async () => {
+        const { session, received } = makeSession(async () => emptyPage(""));
+
+        session.start();
+        await settle();
+
+        expect(received).toEqual([{ references: [], bookmark: "" }]);
+    });
+
+    it("deduplicates facts across tuples within a page", async () => {
+        const A = ref("Test.Post", "a");
+        const B = ref("Test.Post", "b");
+        const C = ref("Test.Post", "c");
+        const responses = [
+            page("bm1", [tuple(A, B), tuple(B, C)]),
+            emptyPage("bm1")
+        ];
+        let call = 0;
+        const { session, received } = makeSession(async () => responses[call++]);
+
+        session.start();
+        await settle();
+
+        expect(received[0].references).toEqual([A, B, C]);
+    });
+
+    it("continues querying when facts arrive during a query (waitlist continuation)", async () => {
+        const A = ref("Test.Post", "a");
+        let call = 0;
+        let sessionRef: FeedStreamSession | null = null;
+        const query = jest.fn(async () => {
+            call++;
+            if (call === 1) {
+                // A fact arrives mid-query: enqueue it onto the waitlist.
+                sessionRef!.onInverseMatch([A]);
+                return emptyPage("");
+            }
+            if (call === 2) {
+                return page("bm1", [tuple(A)]);
+            }
+            return emptyPage("bm1");
+        });
+        const made = makeSession(query);
+        sessionRef = made.session;
+
+        made.session.start();
+        await settle();
+
+        // First empty + waitlist non-empty triggered a re-query that found A,
+        // then a final empty query produced the SYNC frame.
+        expect(made.received).toEqual([
+            { references: [A], bookmark: "bm1" },
+            { references: [], bookmark: "bm1" }
+        ]);
+        expect(query).toHaveBeenCalledTimes(3);
+    });
+
+    it("removes a fact from the waitlist only when its tuple is streamed", async () => {
+        const A = ref("Test.Post", "a");
+        const responses = [
+            page("bm1", [tuple(A)]),
+            emptyPage("bm1")
+        ];
+        let call = 0;
+        const { session, received } = makeSession(async () => responses[call++ ] ?? emptyPage("bm1"));
+
+        session.onInverseMatch([A]);
+        expect(session.waitlistSize).toBe(1);
+        await settle();
+
+        // Once A's tuple is streamed, it is removed and the cycle reaches SYNC.
+        expect(session.waitlistSize).toBe(0);
+        expect(received).toEqual([
+            { references: [A], bookmark: "bm1" },
+            { references: [], bookmark: "bm1" }
+        ]);
+    });
+
+    it("drops stale waitlist entries that never complete and emits SYNC", async () => {
+        const A = ref("Test.Post", "a");
+        const { session, received } = makeSession(async () => emptyPage(""));
+
+        session.onInverseMatch([A]);
+        expect(session.waitlistSize).toBe(1);
+        await settle();
+
+        // The query never returns A's tuple and no new trigger arrives, so the
+        // stale entry is dropped and a SYNC frame is emitted.
+        expect(session.waitlistSize).toBe(0);
+        expect(received).toEqual([{ references: [], bookmark: "" }]);
+    });
+
+    it("runs only one cycle at a time", async () => {
+        const A = ref("Test.Post", "a");
+        let concurrent = 0;
+        let maxConcurrent = 0;
+        let call = 0;
+        const query = jest.fn(async (_bookmark: string) => {
+            concurrent++;
+            maxConcurrent = Math.max(maxConcurrent, concurrent);
+            await new Promise<void>(resolve => setImmediate(resolve));
+            concurrent--;
+            call++;
+            return emptyPage("");
+        });
+        const { session } = makeSession(query);
+
+        // Fire several wake-ups in quick succession.
+        session.onInverseMatch([A]);
+        session.onInverseMatch([A]);
+        session.onInverseMatch([A]);
+        await settle();
+
+        expect(maxConcurrent).toBe(1);
+    });
+
+    it("caps the waitlist and closes the stream when exceeded", async () => {
+        const stream = new Stream<FeedResponse>();
+        const closeSpy = jest.spyOn(stream, "close");
+        // A query that never advances so the cycle cannot drain the waitlist.
+        const { session, factManager } = makeSession(
+            async () => emptyPage(""),
+            { stream, config: { waitlistCap: 2 } }
+        );
+
+        session.onInverseMatch([
+            ref("Test.Post", "a"),
+            ref("Test.Post", "b"),
+            ref("Test.Post", "c")
+        ]);
+
+        expect(closeSpy).toHaveBeenCalled();
+        // Listeners are torn down on graceful close.
+        expect(factManager.removed.length).toBe(factManager.listeners.length);
+    });
+
+    it("stops after the configured maximum number of pages", async () => {
+        let call = 0;
+        const query = jest.fn(async () => page(`bm${++call}`, [tuple(ref("Test.Post", `p${call}`))]));
+        const { session, received } = makeSession(query, { config: { maxInitialPages: 2, pageDelayEvery: 0 } });
+
+        session.start();
+        await settle();
+
+        // Exactly two pages streamed, then the cycle stops (no SYNC frame).
+        expect(query).toHaveBeenCalledTimes(2);
+        expect(received.length).toBe(2);
+        expect(received.every(r => r.references.length > 0)).toBe(true);
+    });
+
+    it("keeps the stream alive without SYNC when a query is denied", async () => {
+        const denied: FeedResult = { type: "denied", reason: "not authorized" };
+        const { session, received, stream } = makeSession(async () => denied);
+
+        session.start();
+        await settle();
+
+        expect(received).toEqual([]);
+        // Stream remains open so a later authorizing fact can wake the cycle.
+        expect((stream as any).closed).toBe(false);
+    });
+
+    it("only enqueues inverse results whose given subset matches", async () => {
+        const A = ref("Test.Post", "a");
+        const query = jest.fn(async () => emptyPage(""));
+        const { session, factManager } = makeSession(query);
+
+        session.start();
+        await settle();
+        query.mockClear();
+
+        const inverseListener = factManager.listeners[0];
+        expect(inverseListener).toBeDefined();
+
+        // A result whose tuple does not match this subscription's given.
+        const nonMatching: ProjectedResult = {
+            tuple: { p: ref("Jinaga.User", "someone-else"), post: A },
+            result: A
+        };
+        await inverseListener.onResult([nonMatching]);
+        await settle();
+
+        // The non-matching result is filtered out: no new cycle/query runs.
+        expect(query).not.toHaveBeenCalled();
+        expect(session.waitlistSize).toBe(0);
+    });
+
+    it("removes all listeners and stops cycling on dispose", async () => {
+        const A = ref("Test.Post", "a");
+        const query = jest.fn(async () => emptyPage(""));
+        const { session, factManager } = makeSession(query);
+
+        session.start();
+        await settle();
+        const registered = factManager.listeners.length;
+        expect(registered).toBeGreaterThan(0);
+
+        session.dispose();
+        expect(factManager.removed.length).toBe(registered);
+
+        query.mockClear();
+        // After dispose, further triggers are ignored.
+        session.onInverseMatch([A]);
+        await settle();
+        expect(query).not.toHaveBeenCalled();
+    });
+});

package/test/http/feedStreamSessionIntegrationSpec.ts

@@ -0,0 +1,202 @@
+import {
+    AuthorizationRules,
+    buildModel,
+    dehydrateFact,
+    FactManager,
+    FeedCache,
+    FeedResponse,
+    MemoryStore,
+    NetworkNoOp,
+    ObservableSource,
+    PassThroughFork,
+    User
+} from "jinaga";
+
+import { AuthorizationKeystore } from "../../src/authorization/authorization-keystore";
+import { HttpRouter, RequestUser } from "../../src/http/router";
+import { Stream } from "../../src/http/stream";
+import { MemoryKeystore } from "../../src/memory/memory-keystore";
+
+// End-to-end coverage of the FeedStreamSession wired through HttpRouter
+// against a real in-memory FactManager. Exercises backfill, mid-backfill
+// inserts, the SYNC frame, and post-SYNC real-time updates.
+class Post {
+    public static Type = "stream.Post" as const;
+    public type = Post.Type;
+    constructor(public author: User, public body: string) { }
+}
+
+const model = buildModel(b => b
+    .type(User)
+    .type(Post, m => m.predecessor("author", User))
+);
+
+const userIdentity = { provider: "mock", id: "subscriber" };
+
+interface Harness {
+    router: HttpRouter;
+    factManager: FactManager;
+    storage: MemoryStore;
+    requestUser: RequestUser;
+    author: User;
+    authorRef: { type: string; hash: string };
+}
+
+async function makeHarness(): Promise<Harness> {
+    const storage = new MemoryStore();
+    const keystore = new MemoryKeystore();
+    const userFact = await keystore.getOrCreateUserFact(userIdentity);
+    const author = new User(userFact.fields.publicKey);
+    const authorRef = dehydrateFact(author)[0];
+
+    const fork = new PassThroughFork(storage);
+    const observable = new ObservableSource(storage);
+    const network = new NetworkNoOp();
+    const factManager = new FactManager(fork, observable, storage, network, []);
+
+    const authorizationRules = new AuthorizationRules(model)
+        .any(User)
+        .any(Post);
+    const authorization = new AuthorizationKeystore(
+        factManager, storage, keystore, authorizationRules, null);
+    const feedCache = new FeedCache();
+
+    const router = new HttpRouter(factManager, authorization, feedCache, "*");
+
+    return {
+        router,
+        factManager,
+        storage,
+        requestUser: { provider: userIdentity.provider, id: userIdentity.id, profile: {} as any },
+        author,
+        authorRef
+    };
+}
+
+async function waitForMicrotasks(): Promise<void> {
+    for (let i = 0; i < 10; i++) {
+        await new Promise<void>(resolve => setImmediate(resolve));
+    }
+}
+
+function feedInput(authorHash: string): string {
+    return (
+        `let p: Jinaga.User = #${authorHash}\n` +
+        `(p: Jinaga.User) {\n` +
+        `    post: stream.Post [\n` +
+        `        post->author: Jinaga.User = p\n` +
+        `    ]\n` +
+        `} => post`
+    );
+}
+
+async function savePost(h: Harness, body: string): Promise<void> {
+    const post = new Post(h.author, body);
+    const envelopes = dehydrateFact(post).map(f => ({ fact: f, signatures: [] }));
+    await h.factManager.save(envelopes);
+}
+
+describe("HttpRouter streamFeed session integration", () => {
+    it("backfills all historical posts and ends with a SYNC frame", async () => {
+        const h = await makeHarness();
+        await h.factManager.save(dehydrateFact(h.author).map(f => ({ fact: f, signatures: [] })));
+        for (let i = 0; i < 5; i++) {
+            await savePost(h, `post-${i}`);
+        }
+
+        const feedsResponse = await (h.router as any).feeds(h.requestUser, feedInput(h.authorRef.hash));
+        const feedHash: string = feedsResponse.feeds[0];
+
+        const stream: Stream<FeedResponse> = await (h.router as any).streamFeed(
+            h.requestUser, { hash: feedHash }, {});
+        const received: FeedResponse[] = [];
+        stream.next(r => received.push(r));
+
+        await waitForMicrotasks();
+        stream.close();
+
+        const distinctPosts = new Set(received.flatMap(r => r.references)
+            .filter(r => r.type === Post.Type).map(r => r.hash));
+        expect(distinctPosts.size).toBe(5);
+        // The final frame is the SYNC signal: empty references.
+        expect(received[received.length - 1].references).toEqual([]);
+    });
+
+    it("does not lose posts inserted during backfill", async () => {
+        const h = await makeHarness();
+        await h.factManager.save(dehydrateFact(h.author).map(f => ({ fact: f, signatures: [] })));
+        await savePost(h, "historical");
+
+        const feedsResponse = await (h.router as any).feeds(h.requestUser, feedInput(h.authorRef.hash));
+        const feedHash: string = feedsResponse.feeds[0];
+
+        const stream: Stream<FeedResponse> = await (h.router as any).streamFeed(
+            h.requestUser, { hash: feedHash }, {});
+        const received: FeedResponse[] = [];
+        stream.next(r => received.push(r));
+
+        // Insert a post immediately, racing the backfill.
+        await savePost(h, "raced");
+        await waitForMicrotasks();
+        stream.close();
+
+        const distinctPosts = new Set(received.flatMap(r => r.references)
+            .filter(r => r.type === Post.Type).map(r => r.hash));
+        // Both the historical and the raced post are delivered.
+        expect(distinctPosts.size).toBe(2);
+        expect(received[received.length - 1].references).toEqual([]);
+    });
+
+    it("delivers post-SYNC updates in subsequent frames", async () => {
+        const h = await makeHarness();
+        await h.factManager.save(dehydrateFact(h.author).map(f => ({ fact: f, signatures: [] })));
+
+        const feedsResponse = await (h.router as any).feeds(h.requestUser, feedInput(h.authorRef.hash));
+        const feedHash: string = feedsResponse.feeds[0];
+
+        const stream: Stream<FeedResponse> = await (h.router as any).streamFeed(
+            h.requestUser, { hash: feedHash }, {});
+        const received: FeedResponse[] = [];
+        stream.next(r => received.push(r));
+
+        await waitForMicrotasks();
+        const framesBeforeUpdate = received.length;
+        // The first frame is the immediate SYNC (no history).
+        expect(received[framesBeforeUpdate - 1].references).toEqual([]);
+
+        // A new post after SYNC should arrive in a later frame.
+        await savePost(h, "after-sync");
+        await waitForMicrotasks();
+        stream.close();
+
+        const newFrames = received.slice(framesBeforeUpdate);
+        const distinctPosts = new Set(newFrames.flatMap(r => r.references)
+            .filter(r => r.type === Post.Type).map(r => r.hash));
+        expect(distinctPosts.size).toBe(1);
+    });
+
+    it("cleans up listeners on disconnect", async () => {
+        const h = await makeHarness();
+        await h.factManager.save(dehydrateFact(h.author).map(f => ({ fact: f, signatures: [] })));
+
+        const removeSpy = jest.spyOn(h.factManager, "removeSpecificationListener");
+
+        const feedsResponse = await (h.router as any).feeds(h.requestUser, feedInput(h.authorRef.hash));
+        const feedHash: string = feedsResponse.feeds[0];
+
+        const stream: Stream<FeedResponse> = await (h.router as any).streamFeed(
+            h.requestUser, { hash: feedHash }, {});
+        stream.next(() => { });
+        await waitForMicrotasks();
+
+        const removedBefore = removeSpy.mock.calls.length;
+        stream.close();
+
+        // Closing the stream disposes the session, removing every listener.
+        expect(removeSpy.mock.calls.length).toBeGreaterThan(removedBefore);
+
+        // A post saved after disconnect must not reach the closed stream.
+        await savePost(h, "after-close");
+        await waitForMicrotasks();
+    });
+});

package/test/http/graphSaveMultiHopAuthSpec.ts

@@ -0,0 +1,126 @@
+import {
+    AuthorizationRules,
+    buildModel,
+    dehydrateFact,
+    DistributionRules,
+    FactEnvelope,
+    FactManager,
+    FactRecord,
+    FeedCache,
+    GraphDeserializer,
+    GraphSerializer,
+    MemoryStore,
+    NetworkNoOp,
+    ObservableSource,
+    PassThroughFork,
+    User
+} from "jinaga";
+
+import { AuthorizationKeystore } from "../../src/authorization/authorization-keystore";
+import { HttpRouter, RequestUser } from "../../src/http/router";
+import { MemoryKeystore } from "../../src/memory/memory-keystore";
+
+class Filler {
+    static Type = "GraphSave.Filler" as const;
+    public type = Filler.Type;
+    constructor(public value: number) { }
+}
+
+class Workspace {
+    static Type = "GraphSave.Workspace" as const;
+    public type = Workspace.Type;
+    constructor(public owner: User, public identifier: string) { }
+}
+
+class Application {
+    static Type = "GraphSave.Application" as const;
+    public type = Application.Type;
+    constructor(public workspace: Workspace, public name: string) { }
+}
+
+const model = buildModel(b => b
+    .type(User)
+    .type(Filler)
+    .type(Workspace, w => w.predecessor("owner", User))
+    .type(Application, a => a.predecessor("workspace", Workspace))
+);
+
+function createReadLine(input: string) {
+    const lines = input.split("\n");
+    if (lines[lines.length - 1] === "") {
+        lines.pop();
+    }
+    return async () => {
+        const line = lines.shift();
+        return line !== undefined ? line : null;
+    };
+}
+
+// Regression test for issue #175: HttpRouter.save() used to pass its
+// authorization call directly as GraphDeserializer's per-flush callback, so
+// a graph body that split across flush boundaries (every 20 facts)
+// authorized each flush independently. A fact whose rule needs a multi-hop
+// predecessor chain landing in an earlier flush then failed with
+// "The fact <type>:<hash> is not defined." (surfaced to clients as a 500). This drives
+// the real application/x-jinaga-graph-v1 code path end to end.
+describe("POST /save with a multi-flush graph body", () => {
+    it("authorizes a fact whose multi-hop predecessors landed in a prior flush batch", async () => {
+        const store = new MemoryStore();
+        const keystore = new MemoryKeystore();
+        const ownerIdentity = { provider: "mock", id: "owner" };
+        const ownerFact = await keystore.getOrCreateUserFact(ownerIdentity);
+        const owner = new User(ownerFact.fields.publicKey);
+
+        const fork = new PassThroughFork(store);
+        const observable = new ObservableSource(store);
+        const network = new NetworkNoOp();
+        const factManager = new FactManager(fork, observable, store, network, []);
+
+        const authorizationRules = new AuthorizationRules(model)
+            .any(Filler.Type)
+            .any(User.Type)
+            .any(Workspace.Type)
+            .type(Application, app => app.workspace.owner);
+        const distributionRules = new DistributionRules([]);
+        const authorization = new AuthorizationKeystore(
+            factManager, store, keystore, authorizationRules, distributionRules);
+        const feedCache = new FeedCache();
+        const router = new HttpRouter(factManager, authorization, feedCache, "*");
+
+        const requestUser: RequestUser = {
+            provider: ownerIdentity.provider,
+            id: ownerIdentity.id,
+            profile: {} as any
+        };
+
+        // Pad the graph with enough leading facts that the real predecessor
+        // chain (owner -> workspace -> application) crosses the
+        // GraphDeserializer's default 20-fact flush threshold, landing
+        // Application in a later flush than its predecessors.
+        const fillerRecords: FactRecord[] = [];
+        for (let i = 0; i < 18; i++) {
+            fillerRecords.push(...dehydrateFact(new Filler(i)));
+        }
+        const chainRecords = dehydrateFact(new Application(new Workspace(owner, "workspace-1"), "application-1"));
+
+        const allRecords = [...fillerRecords, ...chainRecords];
+        expect(allRecords.length).toBeGreaterThan(20);
+        expect(allRecords[allRecords.length - 1].type).toBe(Application.Type);
+
+        const envelopes: FactEnvelope[] = allRecords.map(fact => ({ fact, signatures: [] }));
+
+        // Round-trip through the wire protocol so the test exercises the
+        // real GraphDeserializer flush boundary, not a hand-picked batch.
+        const lines: string[] = [];
+        new GraphSerializer(chunk => lines.push(chunk)).serialize(envelopes);
+        const wireText = lines.join("");
+        const readLine = createReadLine(wireText);
+        const graphSource = new GraphDeserializer(readLine);
+
+        await expect((router as any).save(requestUser, graphSource)).resolves.toBeUndefined();
+
+        const applicationRecord = chainRecords.find(f => f.type === Application.Type)!;
+        const saved = await store.whichExist([{ type: Application.Type, hash: applicationRecord.hash }]);
+        expect(saved.length).toBe(1);
+    });
+});