jinaga-server 3.5.5 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/authorization/authorization-keystore.d.ts +34 -2
  2. package/dist/authorization/authorization-keystore.js +59 -4
  3. package/dist/authorization/authorization-keystore.js.map +1 -1
  4. package/dist/feeds/feed-stream-session.d.ts +84 -0
  5. package/dist/feeds/feed-stream-session.js +331 -0
  6. package/dist/feeds/feed-stream-session.js.map +1 -0
  7. package/dist/http/router.d.ts +10 -3
  8. package/dist/http/router.js +186 -184
  9. package/dist/http/router.js.map +1 -1
  10. package/dist/index.d.ts +2 -1
  11. package/dist/index.js +5 -1
  12. package/dist/index.js.map +1 -1
  13. package/dist/jinaga-server.d.ts +7 -0
  14. package/dist/jinaga-server.js +19 -2
  15. package/dist/jinaga-server.js.map +1 -1
  16. package/dist/memory/memory-keystore.js +1 -1
  17. package/dist/memory/memory-keystore.js.map +1 -1
  18. package/dist/postgres/query-description.d.ts +1 -0
  19. package/dist/postgres/query-description.js +34 -3
  20. package/dist/postgres/query-description.js.map +1 -1
  21. package/dist/postgres/specification-sql.js +7 -1
  22. package/dist/postgres/specification-sql.js.map +1 -1
  23. package/docs/design/feeds-persistent-proxy-plan.md +11 -1
  24. package/integration-test/jinaga-test/distribution-multi-given.test.js +184 -0
  25. package/integration-test/jinaga-test/distribution-negating-feed.test.js +202 -0
  26. package/integration-test/jinaga-test/feed-existential-input.test.js +241 -0
  27. package/integration-test/jinaga-test/feeds-endpoint.test.js +140 -0
  28. package/integration-test/jinaga-test/package-lock.json +4 -4
  29. package/integration-test/jinaga-test/package.json +3 -2
  30. package/integration-test/jinaga-test/subscription-anchor-arrival.test.js +129 -0
  31. package/integration-test/jinaga-test/subscription-helpers.js +60 -0
  32. package/integration-test/jinaga-test/subscription-late-auth.test.js +245 -0
  33. package/integration-test/jinaga-test/subscription-no-rule.test.js +137 -0
  34. package/package.json +2 -2
  35. package/src/authorization/authorization-keystore.ts +106 -5
  36. package/src/feeds/feed-stream-session.ts +386 -0
  37. package/src/http/router.ts +212 -237
  38. package/src/index.ts +2 -1
  39. package/src/jinaga-server.ts +45 -5
  40. package/src/memory/memory-keystore.ts +1 -1
  41. package/src/postgres/query-description.ts +66 -11
  42. package/src/postgres/specification-sql.ts +7 -1
  43. package/test/authorization/intermediateProjectionFeedSpec.ts +189 -0
  44. package/test/authorization/subscriptionIntersectionSpec.ts +220 -0
  45. package/test/feeds/feedStreamSessionRegressionSpec.ts +227 -0
  46. package/test/feeds/feedStreamSessionSpec.ts +351 -0
  47. package/test/http/anchorListenerSpec.ts +117 -0
  48. package/test/http/feedStreamSessionIntegrationSpec.ts +202 -0
  49. package/test/http/lateAuthRecoverySpec.ts +435 -0
  50. package/test/http/streamFeedAnchorSpec.ts +171 -0
@@ -3,14 +3,14 @@ import {
3
3
  Authorization,
4
4
  buildFeeds,
5
5
  computeObjectHash,
6
- computeTupleSubsetHash,
7
6
  Declaration,
7
+ DistributionDenialCode,
8
8
  FactEnvelope,
9
9
  FactManager,
10
10
  FactRecord,
11
11
  FactReference,
12
12
  FeedCache,
13
- FeedObject,
13
+ FeedDecision,
14
14
  FeedResponse,
15
15
  FeedsResponse,
16
16
  Forbidden,
@@ -18,7 +18,6 @@ import {
18
18
  GraphSerializer,
19
19
  GraphSource,
20
20
  Invalid,
21
- invertSpecification,
22
21
  LoadMessage,
23
22
  LoadResponse,
24
23
  parseLoadMessage,
@@ -32,6 +31,8 @@ import {
32
31
  UserIdentity,
33
32
  verifyEnvelopes
34
33
  } from "jinaga";
34
+ import { DistributionIntersectionBranch, FeedResult, SubscriptionAuthorizer } from "../authorization/authorization-keystore";
35
+ import { FeedStreamSession, FeedStreamSessionConfig } from "../feeds/feed-stream-session";
35
36
  import { CsvMetadata } from "./csv-metadata";
36
37
  import { validateSpecificationForCsv } from "./csv-validator";
37
38
  import { createLineReader } from "./line-reader";
@@ -286,6 +287,10 @@ function postReadWithStreaming(
286
287
  };
287
288
  }
288
289
 
290
+ function subscriptionOwnerKey(userIdentity: UserIdentity | null): string | null {
291
+ return userIdentity ? `${userIdentity.provider}|${userIdentity.id}` : null;
292
+ }
293
+
289
294
  function serializeUserIdentity(user: RequestUser | null): UserIdentity | null {
290
295
  if (!user) {
291
296
  return null;
@@ -366,12 +371,22 @@ export interface RequestUser {
366
371
 
367
372
  export class HttpRouter {
368
373
  handler: Handler;
374
+ // Map from feed hash → owning user key, for feeds the server cleared
375
+ // via intersection. The cached spec self-filters by the lifted auth
376
+ // condition (the requesting user's fact is baked into start), so it
377
+ // is safe to skip the per-query distribution check — but ONLY for
378
+ // the same user who originally requested the subscription. Another
379
+ // authenticated user who somehow obtained the hash must go through
380
+ // the normal authorization.feed path, which will deny their request
381
+ // since the intersected spec is not authorizable in its own right.
382
+ private intersectedFeedOwners = new Map<string, string | null>();
369
383
 
370
384
  constructor(
371
385
  private factManager: FactManager,
372
- private authorization: Authorization,
386
+ private authorization: Authorization & SubscriptionAuthorizer,
373
387
  private feedCache: FeedCache,
374
- private allowedOrigin: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void)
388
+ private allowedOrigin: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void),
389
+ private feedStreamConfig: Partial<FeedStreamSessionConfig> = {}
375
390
  ) {
376
391
  const router = Router();
377
392
  const applyAllowOrigin = this.applyAllowOrigin.bind(this);
@@ -555,49 +570,187 @@ export class HttpRouter {
555
570
  if (failures.length > 0) {
556
571
  throw new Invalid(failures.join("\n"));
557
572
  }
558
-
573
+
559
574
  const namedStart = specification.given.reduce((map, g, index) => ({
560
575
  ...map,
561
576
  [g.label.name]: start[index]
562
577
  }), {} as ReferencesByName);
563
578
 
564
- // Verify that I can distribute all feeds to the user.
565
- const feeds = buildFeeds(specification);
579
+ // Resolve distribution. When the user is not authorized for the
580
+ // original spec but the distribution rules can rewrite it via
581
+ // intersection (jinaga.js#130), accept the subscription with
582
+ // empty results and let the inverse engine activate it later.
583
+ //
584
+ // Even when no rule applies at all, accept the subscription:
585
+ // the per-query distribution check inside streamFeed /
586
+ // feed returns "denied" and keeps the stream alive so the
587
+ // client can be notified the moment the authorizing fact
588
+ // arrives. Failing /feeds here would force the client to
589
+ // re-subscribe on a poll loop.
566
590
  const userIdentity = serializeUserIdentity(user);
567
- await this.authorization.verifyDistribution(userIdentity, feeds, namedStart);
591
+ const intersectResult = await this.authorization.verifyDistributionOrIntersect(userIdentity, specification, namedStart);
592
+ let branches: DistributionIntersectionBranch[];
593
+ let intersected: boolean;
594
+ if (intersectResult.type === "denied") {
595
+ // Structured, dashboard-observable signal that a feed was
596
+ // registered without an applicable distribution rule (issue
597
+ // #168 S3). The metric name is stable so a backend sees
598
+ // bounded cardinality; the denial code (a fixed 4-value enum)
599
+ // is the measurement dimension, so authoring errors
600
+ // (no-matching-rule / spec-more-restrictive-than-rule) are
601
+ // distinguishable from auth states (principal-excluded /
602
+ // not-authenticated) in CI and dashboards without grepping
603
+ // logs. The human-readable line is retained for correlation.
604
+ Trace.metric("distribution.unmatched", { [intersectResult.code]: 1 });
605
+ Trace.warn(`/feeds accepting spec without applicable distribution rule (${intersectResult.code}): ${intersectResult.reason}`);
606
+ branches = [{ start, specification }];
607
+ intersected = false;
608
+ } else {
609
+ branches = intersectResult.branches;
610
+ intersected = branches.length !== 1 || branches[0].specification !== specification;
611
+ }
568
612
 
569
- const feedHashes = this.feedCache.addFeeds(feeds, namedStart);
613
+ // Classify the spec's distribution decision once; it applies to
614
+ // every feed hash the spec produces (issue #168 S1). Registration
615
+ // and keep-alive are unchanged — reactive and denied feeds are
616
+ // still accepted and served exactly as before; this only reports
617
+ // the decision. `reactive` is authorized-via-intersection: denied
618
+ // now, self-heals when the authorizing fact arrives.
619
+ //
620
+ // S2: the decision is recomputed from verifyDistributionOrIntersect
621
+ // on every POST /feeds — it is NOT cached by feed hash — so a
622
+ // repeated request always returns the correct *per-user* decision.
623
+ // This matters because a non-intersected feed hash is shared across
624
+ // users (it excludes the requester's identity): the same hash is
625
+ // `authorized` for a permitted user and `denied` for another, and
626
+ // a `reactive` decision becomes `authorized` once the authorizing
627
+ // fact arrives. Caching the FeedDecision by hash would leak one
628
+ // user's decision to another. addFeeds stays idempotent on the
629
+ // hash; only the classification is per-request.
630
+ let decision: FeedDecision["decision"];
631
+ let decisionCode: DistributionDenialCode | undefined;
632
+ let decisionReason: string;
633
+ if (intersectResult.type === "denied") {
634
+ decision = "denied";
635
+ decisionCode = intersectResult.code;
636
+ decisionReason = intersectResult.reason;
637
+ } else if (intersected) {
638
+ decision = "reactive";
639
+ decisionCode = intersectResult.denial?.code;
640
+ decisionReason = intersectResult.denial?.reason
641
+ ?? "Authorized via intersection; awaiting the authorizing fact.";
642
+ } else {
643
+ decision = "authorized";
644
+ decisionCode = undefined;
645
+ decisionReason = "Distribution authorized.";
646
+ }
647
+
648
+ const ownerKey = subscriptionOwnerKey(userIdentity);
649
+ const feedHashes: string[] = [];
650
+ const decisions: FeedDecision[] = [];
651
+ for (const branch of branches) {
652
+ const branchNamedStart = branch.specification.given.reduce((map, g, index) => ({
653
+ ...map,
654
+ [g.label.name]: branch.start[index]
655
+ }), {} as ReferencesByName);
656
+ const branchFeeds = buildFeeds(branch.specification);
657
+ const branchHashes = this.feedCache.addFeeds(branchFeeds, branchNamedStart);
658
+ feedHashes.push(...branchHashes);
659
+ for (const hash of branchHashes) {
660
+ decisions.push({
661
+ feed: hash,
662
+ decision,
663
+ ...(decisionCode !== undefined ? { code: decisionCode } : {}),
664
+ reason: decisionReason
665
+ });
666
+ }
667
+ if (intersected) {
668
+ for (const hash of branchHashes) {
669
+ // Bind the hash to its owner so a different
670
+ // authenticated user who obtains it cannot reuse
671
+ // the cached spec — which carries the original
672
+ // owner's user fact in its start — to fetch
673
+ // facts authorized for that owner.
674
+ //
675
+ // The map allows only one owner per hash, which
676
+ // relies on the invariant that intersected specs
677
+ // bind the requesting user's fact into start —
678
+ // making the hash user-specific. If two distinct
679
+ // owners ever collide on the same hash, that
680
+ // invariant has broken in intersectForSubscribe;
681
+ // warn so we notice instead of silently
682
+ // overwriting.
683
+ const existing = this.intersectedFeedOwners.get(hash);
684
+ if (existing !== undefined && existing !== ownerKey) {
685
+ Trace.warn(`Intersected feed hash ${hash} re-bound from owner ${existing} to ${ownerKey}; intersection invariant may be broken`);
686
+ }
687
+ this.intersectedFeedOwners.set(hash, ownerKey);
688
+ }
689
+ }
690
+ }
570
691
 
571
692
  return {
572
- feeds: feedHashes
693
+ feeds: feedHashes,
694
+ decisions
573
695
  }
574
696
  });
575
697
  }
576
698
 
699
+ private async queryFeed(
700
+ feedHash: string,
701
+ userIdentity: UserIdentity | null,
702
+ specification: Specification,
703
+ start: FactReference[],
704
+ bookmark: string
705
+ ): Promise<FeedResult> {
706
+ const cachedOwner = this.intersectedFeedOwners.get(feedHash);
707
+ const requesterOwner = subscriptionOwnerKey(userIdentity);
708
+ if (cachedOwner !== undefined && cachedOwner === requesterOwner) {
709
+ const feed = await this.authorization.feedPreVerified(userIdentity, specification, start, bookmark);
710
+ return { type: "success", feed };
711
+ }
712
+ // For everyone else (different user, anonymous mismatch, or a
713
+ // non-intersected hash) go through the normal distribution-checked
714
+ // path. A "denied" result means the streaming / polling caller
715
+ // serves an empty page and keeps the subscription alive.
716
+ return await this.authorization.feedWithDistribution(userIdentity, specification, start, bookmark);
717
+ }
718
+
577
719
  private feed(user: RequestUser | null, params: { [key: string]: string }, query: qs.ParsedQs): Promise<FeedResponse | null> {
578
720
  return Trace.dependency("feed", params["hash"], async () => {
579
721
  const feedHash = params["hash"];
580
722
  if (!feedHash) {
723
+ // No hash in the route at all — a wrong URL, not a feed lookup.
581
724
  return null;
582
725
  }
583
726
 
584
727
  const feedDefinition = await this.feedCache.getFeed(feedHash);
585
728
  if (!feedDefinition) {
586
- return null;
729
+ // Known route, but the feed hash is unknown or expired. Signal
730
+ // this distinctly (issue #168 S4) so the client can re-register
731
+ // via POST /feeds instead of treating it as a routing error.
732
+ throw new FeedNotFound(feedHash);
587
733
  }
588
734
 
589
735
  const bookmark = query["b"] as string ?? "";
590
736
 
591
737
  const userIdentity = serializeUserIdentity(user);
592
738
  const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
593
- const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
739
+ const result = await this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, bookmark);
740
+ if (result.type === "denied") {
741
+ // The subscription has been accepted; treat poll-time
742
+ // distribution failures as an empty page so the client
743
+ // can keep polling and pick up results when an
744
+ // authorizing fact arrives.
745
+ return { references: [], bookmark } as FeedResponse;
746
+ }
594
747
  // Return distinct fact references from all the tuples.
595
- const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
748
+ const references = result.feed.tuples.flatMap(t => t.facts).filter((value, index, self) =>
596
749
  self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
597
750
  );
598
751
  const response: FeedResponse = {
599
752
  references,
600
- bookmark: results.bookmark
753
+ bookmark: result.feed.bookmark
601
754
  };
602
755
  return response;
603
756
  });
@@ -605,244 +758,51 @@ export class HttpRouter {
605
758
 
606
759
  private async streamFeed(user: RequestUser | null, params: { [key: string]: string }, query: qs.ParsedQs): Promise<Stream<FeedResponse> | null> {
607
760
  const connectionId = Math.random().toString(36).substring(2, 10);
608
- const startTime = Date.now();
609
-
610
- console.log(`[StreamFeed:${connectionId}] NEW CONNECTION - User: ${user?.id || 'anonymous'}, Hash: ${params["hash"]}`);
611
-
761
+
612
762
  const feedHash = params["hash"];
613
763
  if (!feedHash) {
614
- console.log(`[StreamFeed:${connectionId}] No feed hash provided`);
764
+ // No hash in the route at all — a wrong URL, not a feed lookup.
615
765
  return null;
616
766
  }
617
767
 
618
768
  const feedDefinition = await this.feedCache.getFeed(feedHash);
619
769
  if (!feedDefinition) {
620
- console.log(`[StreamFeed:${connectionId}] Feed definition not found for hash: ${feedHash}`);
621
- return null;
770
+ // Known route, but the feed hash is unknown or expired. Signal
771
+ // this distinctly (issue #168 S4) so the client can re-register
772
+ // via POST /feeds instead of treating it as a routing error.
773
+ throw new FeedNotFound(feedHash);
622
774
  }
623
775
 
624
- let bookmark = query["b"] as string ?? "";
625
- console.log(`[StreamFeed:${connectionId}] Initial bookmark: ${bookmark || 'empty'}`);
776
+ const bookmark = query["b"] as string ?? "";
626
777
 
627
778
  const userIdentity = serializeUserIdentity(user);
628
779
  const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
629
780
  const givenHash = computeObjectHash(feedDefinition.namedStart);
630
781
 
631
- console.log(`[StreamFeed:${connectionId}] Feed setup - Given hash: ${givenHash.substring(0, 8)}..., Start facts: ${start.length}`);
632
-
633
782
  const stream = new Stream<FeedResponse>();
634
-
635
- try {
636
- // Continuous initial query until exhausted
637
- console.log(`[StreamFeed:${connectionId}] Starting initial data streaming...`);
638
- const initialStart = Date.now();
639
- bookmark = await this.streamAllInitialResults(userIdentity, feedDefinition, start, bookmark, stream);
640
- const initialDuration = Date.now() - initialStart;
641
- console.log(`[StreamFeed:${connectionId}] Initial data streaming complete - Duration: ${initialDuration}ms, Final bookmark: ${bookmark || 'empty'}`);
642
-
643
- // Set up real-time listeners after initial data is complete
644
- console.log(`[StreamFeed:${connectionId}] Setting up real-time listeners...`);
645
- const inverses = invertSpecification(feedDefinition.feed);
646
- console.log(`[StreamFeed:${connectionId}] Created ${inverses.length} inverse specifications`);
647
-
648
- const listeners = inverses.map((inverse, index) => {
649
- console.log(`[StreamFeed:${connectionId}] Adding listener ${index + 1}/${inverses.length}`);
650
-
651
- return this.factManager.addSpecificationListener(
652
- inverse.inverseSpecification,
653
- async (results) => {
654
- const eventStart = Date.now();
655
- console.log(`[StreamFeed:${connectionId}] EVENT RECEIVED - Listener ${index + 1}, Results: ${results.length}`);
656
-
657
- try {
658
- // Filter out results that do not match the given.
659
- const matchingResults = results.filter(pr =>
660
- givenHash === computeTupleSubsetHash(pr.tuple, inverse.givenSubset));
661
-
662
- console.log(`[StreamFeed:${connectionId}] Filtered results - Matching: ${matchingResults.length}/${results.length}`);
663
-
664
- if (matchingResults.length != 0) {
665
- console.log(`[StreamFeed:${connectionId}] Processing matching results...`);
666
- const responseStart = Date.now();
667
- bookmark = await this.streamFeedResponse(userIdentity, feedDefinition, start, bookmark, stream, true);
668
- const responseDuration = Date.now() - responseStart;
669
- console.log(`[StreamFeed:${connectionId}] Feed response sent - Duration: ${responseDuration}ms, New bookmark: ${bookmark || 'empty'}`);
670
- } else {
671
- console.log(`[StreamFeed:${connectionId}] No matching results - skipping response`);
672
- }
673
- } catch (error) {
674
- console.error(`[StreamFeed:${connectionId}] ERROR processing event: ${error}`);
675
- }
676
-
677
- const eventDuration = Date.now() - eventStart;
678
- if (eventDuration > 100) {
679
- console.warn(`[StreamFeed:${connectionId}] SLOW event processing - Duration: ${eventDuration}ms`);
680
- }
681
- }
682
- );
683
- });
684
-
685
- console.log(`[StreamFeed:${connectionId}] All listeners registered - Count: ${listeners.length}`);
686
-
687
- stream.done(() => {
688
- const cleanupStart = Date.now();
689
- console.log(`[StreamFeed:${connectionId}] CLEANUP STARTED - Removing ${listeners.length} listeners`);
690
-
691
- for (let i = 0; i < listeners.length; i++) {
692
- try {
693
- this.factManager.removeSpecificationListener(listeners[i]);
694
- console.log(`[StreamFeed:${connectionId}] Removed listener ${i + 1}/${listeners.length}`);
695
- } catch (error) {
696
- console.error(`[StreamFeed:${connectionId}] ERROR removing listener ${i + 1}: ${error}`);
697
- }
698
- }
699
-
700
- const cleanupDuration = Date.now() - cleanupStart;
701
- const totalDuration = Date.now() - startTime;
702
- console.log(`[StreamFeed:${connectionId}] CLEANUP COMPLETE - Cleanup duration: ${cleanupDuration}ms, Total connection duration: ${totalDuration}ms`);
703
- });
704
-
705
- const setupDuration = Date.now() - startTime;
706
- console.log(`[StreamFeed:${connectionId}] Stream setup complete - Duration: ${setupDuration}ms`);
707
-
708
- return stream;
709
-
710
- } catch (error) {
711
- console.error(`[StreamFeed:${connectionId}] ERROR during setup: ${error}`);
712
- throw error;
713
- }
714
- }
715
783
 
716
- private async streamAllInitialResults(
717
- userIdentity: UserIdentity | null,
718
- feedDefinition: FeedObject,
719
- start: FactReference[],
720
- initialBookmark: string,
721
- stream: Stream<FeedResponse>
722
- ): Promise<string> {
723
- const streamId = Math.random().toString(36).substring(2, 8);
724
- console.log(`[InitialResults:${streamId}] Starting initial data fetch - Bookmark: ${initialBookmark || 'empty'}`);
725
-
726
- let bookmark = initialBookmark;
727
- let hasMoreResults = true;
728
- let pageCount = 0;
729
- let totalReferences = 0;
730
- const maxPages = 1000; // Safety limit to prevent infinite loops
731
- const startTime = Date.now();
732
-
733
- while (hasMoreResults && pageCount < maxPages) {
734
- const pageStart = Date.now();
735
- console.log(`[InitialResults:${streamId}] Fetching page ${pageCount + 1} - Bookmark: ${bookmark || 'empty'}`);
736
-
737
- const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
738
- const fetchDuration = Date.now() - pageStart;
739
-
740
- console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} fetched - Tuples: ${results.tuples.length}, Duration: ${fetchDuration}ms`);
741
-
742
- // Check if we got results
743
- if (results.tuples.length === 0) {
744
- console.log(`[InitialResults:${streamId}] No more results - ending pagination`);
745
- hasMoreResults = false;
746
- break;
747
- }
748
-
749
- // Process and send results
750
- const processStart = Date.now();
751
- const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
752
- self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
753
- );
754
- const processDuration = Date.now() - processStart;
755
-
756
- console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} processed - References: ${references.length}, Process duration: ${processDuration}ms`);
757
-
758
- if (references.length > 0) {
759
- const response: FeedResponse = {
760
- references,
761
- bookmark: results.bookmark
762
- };
763
-
764
- const feedStart = Date.now();
765
- stream.feed(response);
766
- const feedDuration = Date.now() - feedStart;
767
-
768
- totalReferences += references.length;
769
- console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} sent to stream - Feed duration: ${feedDuration}ms`);
770
- }
771
-
772
- // Update bookmark for next iteration
773
- const newBookmark = results.bookmark;
774
- if (newBookmark === bookmark) {
775
- console.log(`[InitialResults:${streamId}] Bookmark unchanged - ending pagination`);
776
- hasMoreResults = false;
777
- } else {
778
- bookmark = newBookmark;
779
- }
780
-
781
- // Check if we got fewer results than the page size (indicates end)
782
- if (results.tuples.length < 100) {
783
- console.log(`[InitialResults:${streamId}] Partial page received (${results.tuples.length} < 100) - ending pagination`);
784
- hasMoreResults = false;
785
- }
786
-
787
- pageCount++;
788
-
789
- // Add small delay to prevent overwhelming the database
790
- if (hasMoreResults && pageCount % 10 === 0) {
791
- console.log(`[InitialResults:${streamId}] Adding delay after ${pageCount} pages`);
792
- await new Promise(resolve => setTimeout(resolve, 10));
793
- }
794
- }
795
-
796
- const totalDuration = Date.now() - startTime;
797
- console.log(`[InitialResults:${streamId}] Initial data streaming complete - Pages: ${pageCount}, Total references: ${totalReferences}, Duration: ${totalDuration}ms, Final bookmark: ${bookmark || 'empty'}`);
798
-
799
- if (pageCount >= maxPages) {
800
- console.warn(`[InitialResults:${streamId}] Hit maximum page limit (${maxPages}) - possible infinite loop`);
801
- }
802
-
803
- return bookmark;
804
- }
805
-
806
- private async streamFeedResponse(userIdentity: UserIdentity | null, feedDefinition: FeedObject, start: FactReference[], bookmark: string, stream: Stream<FeedResponse>, skipIfEmpty = false): Promise<string> {
807
- const responseId = Math.random().toString(36).substring(2, 6);
808
- const startTime = Date.now();
809
-
810
- console.log(`[FeedResponse:${responseId}] Starting feed response - Bookmark: ${bookmark || 'empty'}, Skip if empty: ${skipIfEmpty}`);
811
-
812
- const feedStart = Date.now();
813
- const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
814
- const feedDuration = Date.now() - feedStart;
815
-
816
- console.log(`[FeedResponse:${responseId}] Authorization feed complete - Tuples: ${results.tuples.length}, Duration: ${feedDuration}ms`);
817
-
818
- // Return distinct fact references from all the tuples.
819
- const processStart = Date.now();
820
- const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
821
- self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
784
+ // The session serializes every fetch and stream operation through a
785
+ // single bookmark-driven query cycle. Inverse and anchor observers
786
+ // only enqueue references onto a waitlist and wake the cycle; they
787
+ // never fetch or stream directly. This removes the race between
788
+ // listener callbacks and bookmark mutation, guarantees tuple
789
+ // completeness, and ensures no update is lost between pages.
790
+ const session = new FeedStreamSession(
791
+ (b: string) => this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, b),
792
+ this.factManager,
793
+ feedDefinition,
794
+ start,
795
+ givenHash,
796
+ stream,
797
+ bookmark,
798
+ connectionId,
799
+ this.feedStreamConfig
822
800
  );
823
- const processDuration = Date.now() - processStart;
824
-
825
- console.log(`[FeedResponse:${responseId}] References processed - Count: ${references.length}, Duration: ${processDuration}ms`);
826
-
827
- if (!skipIfEmpty || references.length > 0) {
828
- const response: FeedResponse = {
829
- references,
830
- bookmark: results.bookmark
831
- };
832
-
833
- const streamStart = Date.now();
834
- stream.feed(response);
835
- const streamDuration = Date.now() - streamStart;
836
-
837
- console.log(`[FeedResponse:${responseId}] Response sent to stream - Duration: ${streamDuration}ms`);
838
- } else {
839
- console.log(`[FeedResponse:${responseId}] Skipping empty response`);
840
- }
841
-
842
- const totalDuration = Date.now() - startTime;
843
- console.log(`[FeedResponse:${responseId}] Feed response complete - New bookmark: ${results.bookmark || 'empty'}, Total duration: ${totalDuration}ms`);
844
-
845
- return results.bookmark;
801
+
802
+ stream.done(() => session.dispose());
803
+ session.start();
804
+
805
+ return stream;
846
806
  }
847
807
 
848
808
  private async getKnownFacts(user: RequestUser | null): Promise<Declaration> {
@@ -1005,9 +965,24 @@ function extractResults(obj: any): { result: any, count: number } {
1005
965
  }
1006
966
  }
1007
967
 
968
+ // A known/expired feed hash missed the feed cache. Distinct from a route
969
+ // miss (wrong URL) so clients can tell "unknown/expired feed hash" — which is
970
+ // recoverable by re-registering via POST /feeds — from "wrong URL" (issue
971
+ // #168 S4). handleError maps this to a 404 with a machine-readable body.
972
+ export class FeedNotFound extends Error {
973
+ constructor(public readonly feedHash: string) {
974
+ super(`Feed not found: ${feedHash}`);
975
+ this.name = "FeedNotFound";
976
+ }
977
+ }
978
+
1008
979
  function handleError(error: any, req: Request, res: Response, next: NextFunction) {
1009
980
  const requestPath = req.path;
1010
- if (error instanceof Forbidden) {
981
+ if (error instanceof FeedNotFound) {
982
+ Trace.warn(`Feed not found: ${error.feedHash} (Path: ${requestPath})`);
983
+ res.type("text");
984
+ res.status(404).send("feed_not_found");
985
+ } else if (error instanceof Forbidden) {
1011
986
  Trace.warn(`Forbidden: ${error.message} (Path: ${requestPath})`);
1012
987
  res.type("text");
1013
988
  res.status(403).send(error.message);
package/src/index.ts CHANGED
@@ -2,8 +2,9 @@ export { AuthenticationDevice } from "./authentication/authentication-device";
2
2
  export { AuthenticationSession } from "./authentication/authentication-session";
3
3
  export { AuthorizationKeystore } from "./authorization/authorization-keystore";
4
4
  export { DistributedFactCache } from "./authorization/distributed-fact-cache";
5
+ export { FeedStreamSession, FeedStreamSessionConfig, defaultFeedStreamSessionConfig } from "./feeds/feed-stream-session";
5
6
  export { createLineReader } from "./http/line-reader";
6
- export { HttpRouter, RequestUser } from "./http/router";
7
+ export { FeedNotFound, HttpRouter, RequestUser } from "./http/router";
7
8
  export { Stream } from "./http/stream";
8
9
  export { JinagaServer, JinagaServerConfig, JinagaServerInstance, tracePool } from "./jinaga-server";
9
10
  export { Keystore } from "./keystore";