jinaga-server 3.6.0 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authorization/authorization-keystore.d.ts +7 -1
- package/dist/authorization/authorization-keystore.js +9 -2
- package/dist/authorization/authorization-keystore.js.map +1 -1
- package/dist/feeds/feed-stream-session.d.ts +84 -0
- package/dist/feeds/feed-stream-session.js +331 -0
- package/dist/feeds/feed-stream-session.js.map +1 -0
- package/dist/http/router.d.ts +7 -3
- package/dist/http/router.js +96 -237
- package/dist/http/router.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/jinaga-server.d.ts +7 -0
- package/dist/jinaga-server.js +1 -1
- package/dist/jinaga-server.js.map +1 -1
- package/docs/design/feeds-persistent-proxy-plan.md +11 -1
- package/integration-test/jinaga-test/package-lock.json +4 -4
- package/integration-test/jinaga-test/package.json +1 -1
- package/package.json +2 -2
- package/src/authorization/authorization-keystore.ts +23 -4
- package/src/feeds/feed-stream-session.ts +386 -0
- package/src/http/router.ts +110 -288
- package/src/index.ts +2 -1
- package/src/jinaga-server.ts +9 -2
- package/test/authorization/intermediateProjectionFeedSpec.ts +189 -0
- package/test/authorization/subscriptionIntersectionSpec.ts +12 -0
- package/test/feeds/feedStreamSessionRegressionSpec.ts +227 -0
- package/test/feeds/feedStreamSessionSpec.ts +351 -0
- package/test/http/feedStreamSessionIntegrationSpec.ts +202 -0
- package/test/http/lateAuthRecoverySpec.ts +191 -1
- package/test/http/streamFeedAnchorSpec.ts +3 -1
package/src/http/router.ts
CHANGED
|
@@ -3,14 +3,14 @@ import {
|
|
|
3
3
|
Authorization,
|
|
4
4
|
buildFeeds,
|
|
5
5
|
computeObjectHash,
|
|
6
|
-
computeTupleSubsetHash,
|
|
7
6
|
Declaration,
|
|
7
|
+
DistributionDenialCode,
|
|
8
8
|
FactEnvelope,
|
|
9
9
|
FactManager,
|
|
10
10
|
FactRecord,
|
|
11
11
|
FactReference,
|
|
12
12
|
FeedCache,
|
|
13
|
-
|
|
13
|
+
FeedDecision,
|
|
14
14
|
FeedResponse,
|
|
15
15
|
FeedsResponse,
|
|
16
16
|
Forbidden,
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
GraphSerializer,
|
|
19
19
|
GraphSource,
|
|
20
20
|
Invalid,
|
|
21
|
-
invertSpecification,
|
|
22
21
|
LoadMessage,
|
|
23
22
|
LoadResponse,
|
|
24
23
|
parseLoadMessage,
|
|
@@ -33,6 +32,7 @@ import {
|
|
|
33
32
|
verifyEnvelopes
|
|
34
33
|
} from "jinaga";
|
|
35
34
|
import { DistributionIntersectionBranch, FeedResult, SubscriptionAuthorizer } from "../authorization/authorization-keystore";
|
|
35
|
+
import { FeedStreamSession, FeedStreamSessionConfig } from "../feeds/feed-stream-session";
|
|
36
36
|
import { CsvMetadata } from "./csv-metadata";
|
|
37
37
|
import { validateSpecificationForCsv } from "./csv-validator";
|
|
38
38
|
import { createLineReader } from "./line-reader";
|
|
@@ -385,7 +385,8 @@ export class HttpRouter {
|
|
|
385
385
|
private factManager: FactManager,
|
|
386
386
|
private authorization: Authorization & SubscriptionAuthorizer,
|
|
387
387
|
private feedCache: FeedCache,
|
|
388
|
-
private allowedOrigin: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void)
|
|
388
|
+
private allowedOrigin: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void),
|
|
389
|
+
private feedStreamConfig: Partial<FeedStreamSessionConfig> = {}
|
|
389
390
|
) {
|
|
390
391
|
const router = Router();
|
|
391
392
|
const applyAllowOrigin = this.applyAllowOrigin.bind(this);
|
|
@@ -591,7 +592,17 @@ export class HttpRouter {
|
|
|
591
592
|
let branches: DistributionIntersectionBranch[];
|
|
592
593
|
let intersected: boolean;
|
|
593
594
|
if (intersectResult.type === "denied") {
|
|
594
|
-
|
|
595
|
+
// Structured, dashboard-observable signal that a feed was
|
|
596
|
+
// registered without an applicable distribution rule (issue
|
|
597
|
+
// #168 S3). The metric name is stable so a backend sees
|
|
598
|
+
// bounded cardinality; the denial code (a fixed 4-value enum)
|
|
599
|
+
// is the measurement dimension, so authoring errors
|
|
600
|
+
// (no-matching-rule / spec-more-restrictive-than-rule) are
|
|
601
|
+
// distinguishable from auth states (principal-excluded /
|
|
602
|
+
// not-authenticated) in CI and dashboards without grepping
|
|
603
|
+
// logs. The human-readable line is retained for correlation.
|
|
604
|
+
Trace.metric("distribution.unmatched", { [intersectResult.code]: 1 });
|
|
605
|
+
Trace.warn(`/feeds accepting spec without applicable distribution rule (${intersectResult.code}): ${intersectResult.reason}`);
|
|
595
606
|
branches = [{ start, specification }];
|
|
596
607
|
intersected = false;
|
|
597
608
|
} else {
|
|
@@ -599,8 +610,44 @@ export class HttpRouter {
|
|
|
599
610
|
intersected = branches.length !== 1 || branches[0].specification !== specification;
|
|
600
611
|
}
|
|
601
612
|
|
|
613
|
+
// Classify the spec's distribution decision once; it applies to
|
|
614
|
+
// every feed hash the spec produces (issue #168 S1). Registration
|
|
615
|
+
// and keep-alive are unchanged — reactive and denied feeds are
|
|
616
|
+
// still accepted and served exactly as before; this only reports
|
|
617
|
+
// the decision. `reactive` is authorized-via-intersection: denied
|
|
618
|
+
// now, self-heals when the authorizing fact arrives.
|
|
619
|
+
//
|
|
620
|
+
// S2: the decision is recomputed from verifyDistributionOrIntersect
|
|
621
|
+
// on every POST /feeds — it is NOT cached by feed hash — so a
|
|
622
|
+
// repeated request always returns the correct *per-user* decision.
|
|
623
|
+
// This matters because a non-intersected feed hash is shared across
|
|
624
|
+
// users (it excludes the requester's identity): the same hash is
|
|
625
|
+
// `authorized` for a permitted user and `denied` for another, and
|
|
626
|
+
// a `reactive` decision becomes `authorized` once the authorizing
|
|
627
|
+
// fact arrives. Caching the FeedDecision by hash would leak one
|
|
628
|
+
// user's decision to another. addFeeds stays idempotent on the
|
|
629
|
+
// hash; only the classification is per-request.
|
|
630
|
+
let decision: FeedDecision["decision"];
|
|
631
|
+
let decisionCode: DistributionDenialCode | undefined;
|
|
632
|
+
let decisionReason: string;
|
|
633
|
+
if (intersectResult.type === "denied") {
|
|
634
|
+
decision = "denied";
|
|
635
|
+
decisionCode = intersectResult.code;
|
|
636
|
+
decisionReason = intersectResult.reason;
|
|
637
|
+
} else if (intersected) {
|
|
638
|
+
decision = "reactive";
|
|
639
|
+
decisionCode = intersectResult.denial?.code;
|
|
640
|
+
decisionReason = intersectResult.denial?.reason
|
|
641
|
+
?? "Authorized via intersection; awaiting the authorizing fact.";
|
|
642
|
+
} else {
|
|
643
|
+
decision = "authorized";
|
|
644
|
+
decisionCode = undefined;
|
|
645
|
+
decisionReason = "Distribution authorized.";
|
|
646
|
+
}
|
|
647
|
+
|
|
602
648
|
const ownerKey = subscriptionOwnerKey(userIdentity);
|
|
603
649
|
const feedHashes: string[] = [];
|
|
650
|
+
const decisions: FeedDecision[] = [];
|
|
604
651
|
for (const branch of branches) {
|
|
605
652
|
const branchNamedStart = branch.specification.given.reduce((map, g, index) => ({
|
|
606
653
|
...map,
|
|
@@ -609,6 +656,14 @@ export class HttpRouter {
|
|
|
609
656
|
const branchFeeds = buildFeeds(branch.specification);
|
|
610
657
|
const branchHashes = this.feedCache.addFeeds(branchFeeds, branchNamedStart);
|
|
611
658
|
feedHashes.push(...branchHashes);
|
|
659
|
+
for (const hash of branchHashes) {
|
|
660
|
+
decisions.push({
|
|
661
|
+
feed: hash,
|
|
662
|
+
decision,
|
|
663
|
+
...(decisionCode !== undefined ? { code: decisionCode } : {}),
|
|
664
|
+
reason: decisionReason
|
|
665
|
+
});
|
|
666
|
+
}
|
|
612
667
|
if (intersected) {
|
|
613
668
|
for (const hash of branchHashes) {
|
|
614
669
|
// Bind the hash to its owner so a different
|
|
@@ -635,7 +690,8 @@ export class HttpRouter {
|
|
|
635
690
|
}
|
|
636
691
|
|
|
637
692
|
return {
|
|
638
|
-
feeds: feedHashes
|
|
693
|
+
feeds: feedHashes,
|
|
694
|
+
decisions
|
|
639
695
|
}
|
|
640
696
|
});
|
|
641
697
|
}
|
|
@@ -664,12 +720,16 @@ export class HttpRouter {
|
|
|
664
720
|
return Trace.dependency("feed", params["hash"], async () => {
|
|
665
721
|
const feedHash = params["hash"];
|
|
666
722
|
if (!feedHash) {
|
|
723
|
+
// No hash in the route at all — a wrong URL, not a feed lookup.
|
|
667
724
|
return null;
|
|
668
725
|
}
|
|
669
726
|
|
|
670
727
|
const feedDefinition = await this.feedCache.getFeed(feedHash);
|
|
671
728
|
if (!feedDefinition) {
|
|
672
|
-
|
|
729
|
+
// Known route, but the feed hash is unknown or expired. Signal
|
|
730
|
+
// this distinctly (issue #168 S4) so the client can re-register
|
|
731
|
+
// via POST /feeds instead of treating it as a routing error.
|
|
732
|
+
throw new FeedNotFound(feedHash);
|
|
673
733
|
}
|
|
674
734
|
|
|
675
735
|
const bookmark = query["b"] as string ?? "";
|
|
@@ -698,304 +758,51 @@ export class HttpRouter {
|
|
|
698
758
|
|
|
699
759
|
private async streamFeed(user: RequestUser | null, params: { [key: string]: string }, query: qs.ParsedQs): Promise<Stream<FeedResponse> | null> {
|
|
700
760
|
const connectionId = Math.random().toString(36).substring(2, 10);
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
console.log(`[StreamFeed:${connectionId}] NEW CONNECTION - User: ${user?.id || 'anonymous'}, Hash: ${params["hash"]}`);
|
|
704
|
-
|
|
761
|
+
|
|
705
762
|
const feedHash = params["hash"];
|
|
706
763
|
if (!feedHash) {
|
|
707
|
-
|
|
764
|
+
// No hash in the route at all — a wrong URL, not a feed lookup.
|
|
708
765
|
return null;
|
|
709
766
|
}
|
|
710
767
|
|
|
711
768
|
const feedDefinition = await this.feedCache.getFeed(feedHash);
|
|
712
769
|
if (!feedDefinition) {
|
|
713
|
-
|
|
714
|
-
|
|
770
|
+
// Known route, but the feed hash is unknown or expired. Signal
|
|
771
|
+
// this distinctly (issue #168 S4) so the client can re-register
|
|
772
|
+
// via POST /feeds instead of treating it as a routing error.
|
|
773
|
+
throw new FeedNotFound(feedHash);
|
|
715
774
|
}
|
|
716
775
|
|
|
717
|
-
|
|
718
|
-
console.log(`[StreamFeed:${connectionId}] Initial bookmark: ${bookmark || 'empty'}`);
|
|
776
|
+
const bookmark = query["b"] as string ?? "";
|
|
719
777
|
|
|
720
778
|
const userIdentity = serializeUserIdentity(user);
|
|
721
779
|
const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
|
|
722
780
|
const givenHash = computeObjectHash(feedDefinition.namedStart);
|
|
723
781
|
|
|
724
|
-
console.log(`[StreamFeed:${connectionId}] Feed setup - Given hash: ${givenHash.substring(0, 8)}..., Start facts: ${start.length}`);
|
|
725
|
-
|
|
726
782
|
const stream = new Stream<FeedResponse>();
|
|
727
|
-
|
|
728
|
-
try {
|
|
729
|
-
// Continuous initial query until exhausted
|
|
730
|
-
console.log(`[StreamFeed:${connectionId}] Starting initial data streaming...`);
|
|
731
|
-
const initialStart = Date.now();
|
|
732
|
-
bookmark = await this.streamAllInitialResults(feedHash, userIdentity, feedDefinition, start, bookmark, stream);
|
|
733
|
-
const initialDuration = Date.now() - initialStart;
|
|
734
|
-
console.log(`[StreamFeed:${connectionId}] Initial data streaming complete - Duration: ${initialDuration}ms, Final bookmark: ${bookmark || 'empty'}`);
|
|
735
|
-
|
|
736
|
-
// Set up real-time listeners after initial data is complete
|
|
737
|
-
console.log(`[StreamFeed:${connectionId}] Setting up real-time listeners...`);
|
|
738
|
-
const inverses = invertSpecification(feedDefinition.feed);
|
|
739
|
-
console.log(`[StreamFeed:${connectionId}] Created ${inverses.length} inverse specifications`);
|
|
740
|
-
|
|
741
|
-
const listeners = inverses.map((inverse, index) => {
|
|
742
|
-
console.log(`[StreamFeed:${connectionId}] Adding listener ${index + 1}/${inverses.length}`);
|
|
743
|
-
|
|
744
|
-
return this.factManager.addSpecificationListener(
|
|
745
|
-
inverse.inverseSpecification,
|
|
746
|
-
async (results) => {
|
|
747
|
-
const eventStart = Date.now();
|
|
748
|
-
console.log(`[StreamFeed:${connectionId}] EVENT RECEIVED - Listener ${index + 1}, Results: ${results.length}`);
|
|
749
|
-
|
|
750
|
-
try {
|
|
751
|
-
// Filter out results that do not match the given.
|
|
752
|
-
const matchingResults = results.filter(pr =>
|
|
753
|
-
givenHash === computeTupleSubsetHash(pr.tuple, inverse.givenSubset));
|
|
754
|
-
|
|
755
|
-
console.log(`[StreamFeed:${connectionId}] Filtered results - Matching: ${matchingResults.length}/${results.length}`);
|
|
756
|
-
|
|
757
|
-
if (matchingResults.length != 0) {
|
|
758
|
-
console.log(`[StreamFeed:${connectionId}] Processing matching results...`);
|
|
759
|
-
const responseStart = Date.now();
|
|
760
|
-
bookmark = await this.streamFeedResponse(feedHash, userIdentity, feedDefinition, start, bookmark, stream, true);
|
|
761
|
-
const responseDuration = Date.now() - responseStart;
|
|
762
|
-
console.log(`[StreamFeed:${connectionId}] Feed response sent - Duration: ${responseDuration}ms, New bookmark: ${bookmark || 'empty'}`);
|
|
763
|
-
} else {
|
|
764
|
-
console.log(`[StreamFeed:${connectionId}] No matching results - skipping response`);
|
|
765
|
-
}
|
|
766
|
-
} catch (error) {
|
|
767
|
-
console.error(`[StreamFeed:${connectionId}] ERROR processing event: ${error}`);
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
const eventDuration = Date.now() - eventStart;
|
|
771
|
-
if (eventDuration > 100) {
|
|
772
|
-
console.warn(`[StreamFeed:${connectionId}] SLOW event processing - Duration: ${eventDuration}ms`);
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
);
|
|
776
|
-
});
|
|
777
|
-
|
|
778
|
-
console.log(`[StreamFeed:${connectionId}] All listeners registered - Count: ${listeners.length}`);
|
|
779
|
-
|
|
780
|
-
// Anchor listeners (jinaga.js#129): the inverse-listener path
|
|
781
|
-
// above only fires when a descendant of the given arrives. If
|
|
782
|
-
// the given fact itself isn't yet in the store when the client
|
|
783
|
-
// subscribes, neither the initial query nor any later inverse
|
|
784
|
-
// can deliver results until the given is recorded — and the
|
|
785
|
-
// given fact has no descendants whose save would re-trigger the
|
|
786
|
-
// inverse with the matching givenHash. Register a listener per
|
|
787
|
-
// unique anchor that re-runs streamFeedResponse the moment a
|
|
788
|
-
// saved fact matches the anchor's (type, hash).
|
|
789
|
-
const uniqueAnchors = start.filter((s, i, arr) =>
|
|
790
|
-
arr.findIndex(other => other.type === s.type && other.hash === s.hash) === i);
|
|
791
|
-
console.log(`[StreamFeed:${connectionId}] Registering ${uniqueAnchors.length} anchor listener(s)`);
|
|
792
|
-
const anchorListeners = uniqueAnchors.map(anchor => {
|
|
793
|
-
const anchorSpec: Specification = {
|
|
794
|
-
given: [{ label: { name: "x", type: anchor.type }, conditions: [] }],
|
|
795
|
-
matches: [],
|
|
796
|
-
projection: { type: "fact", label: "x" }
|
|
797
|
-
};
|
|
798
|
-
return this.factManager.addSpecificationListener(anchorSpec, async (results) => {
|
|
799
|
-
try {
|
|
800
|
-
const matched = results.some(pr => {
|
|
801
|
-
const ref = pr.tuple && (pr.tuple as any).x;
|
|
802
|
-
return ref && ref.type === anchor.type && ref.hash === anchor.hash;
|
|
803
|
-
});
|
|
804
|
-
if (matched) {
|
|
805
|
-
console.log(`[StreamFeed:${connectionId}] Anchor fact arrived - Type: ${anchor.type}, Hash: ${anchor.hash.substring(0, 8)}...`);
|
|
806
|
-
bookmark = await this.streamFeedResponse(feedHash, userIdentity, feedDefinition, start, bookmark, stream, true);
|
|
807
|
-
}
|
|
808
|
-
} catch (error) {
|
|
809
|
-
console.error(`[StreamFeed:${connectionId}] ERROR in anchor listener: ${error}`);
|
|
810
|
-
}
|
|
811
|
-
});
|
|
812
|
-
});
|
|
813
|
-
|
|
814
|
-
stream.done(() => {
|
|
815
|
-
const cleanupStart = Date.now();
|
|
816
|
-
console.log(`[StreamFeed:${connectionId}] CLEANUP STARTED - Removing ${listeners.length + anchorListeners.length} listener(s)`);
|
|
817
783
|
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
const totalDuration = Date.now() - startTime;
|
|
836
|
-
console.log(`[StreamFeed:${connectionId}] CLEANUP COMPLETE - Cleanup duration: ${cleanupDuration}ms, Total connection duration: ${totalDuration}ms`);
|
|
837
|
-
});
|
|
838
|
-
|
|
839
|
-
const setupDuration = Date.now() - startTime;
|
|
840
|
-
console.log(`[StreamFeed:${connectionId}] Stream setup complete - Duration: ${setupDuration}ms`);
|
|
841
|
-
|
|
842
|
-
return stream;
|
|
843
|
-
|
|
844
|
-
} catch (error) {
|
|
845
|
-
console.error(`[StreamFeed:${connectionId}] ERROR during setup: ${error}`);
|
|
846
|
-
throw error;
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
private async streamAllInitialResults(
|
|
851
|
-
feedHash: string,
|
|
852
|
-
userIdentity: UserIdentity | null,
|
|
853
|
-
feedDefinition: FeedObject,
|
|
854
|
-
start: FactReference[],
|
|
855
|
-
initialBookmark: string,
|
|
856
|
-
stream: Stream<FeedResponse>
|
|
857
|
-
): Promise<string> {
|
|
858
|
-
const streamId = Math.random().toString(36).substring(2, 8);
|
|
859
|
-
console.log(`[InitialResults:${streamId}] Starting initial data fetch - Bookmark: ${initialBookmark || 'empty'}`);
|
|
860
|
-
|
|
861
|
-
let bookmark = initialBookmark;
|
|
862
|
-
let hasMoreResults = true;
|
|
863
|
-
let pageCount = 0;
|
|
864
|
-
let totalReferences = 0;
|
|
865
|
-
const maxPages = 1000; // Safety limit to prevent infinite loops
|
|
866
|
-
const startTime = Date.now();
|
|
867
|
-
|
|
868
|
-
while (hasMoreResults && pageCount < maxPages) {
|
|
869
|
-
const pageStart = Date.now();
|
|
870
|
-
console.log(`[InitialResults:${streamId}] Fetching page ${pageCount + 1} - Bookmark: ${bookmark || 'empty'}`);
|
|
871
|
-
|
|
872
|
-
const queryResult = await this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, bookmark);
|
|
873
|
-
if (queryResult.type === "denied") {
|
|
874
|
-
// The subscription has already been accepted; tearing
|
|
875
|
-
// down the stream on a distribution failure would force
|
|
876
|
-
// the client to re-subscribe just to keep waiting.
|
|
877
|
-
// Treat as empty and let the listener path re-evaluate
|
|
878
|
-
// when an authorizing fact arrives.
|
|
879
|
-
console.log(`[InitialResults:${streamId}] Distribution failed (${queryResult.reason}) - serving empty and keeping stream live`);
|
|
880
|
-
return bookmark;
|
|
881
|
-
}
|
|
882
|
-
const results = queryResult.feed;
|
|
883
|
-
const fetchDuration = Date.now() - pageStart;
|
|
884
|
-
|
|
885
|
-
console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} fetched - Tuples: ${results.tuples.length}, Duration: ${fetchDuration}ms`);
|
|
886
|
-
|
|
887
|
-
// Check if we got results
|
|
888
|
-
if (results.tuples.length === 0) {
|
|
889
|
-
console.log(`[InitialResults:${streamId}] No more results - ending pagination`);
|
|
890
|
-
hasMoreResults = false;
|
|
891
|
-
break;
|
|
892
|
-
}
|
|
893
|
-
|
|
894
|
-
// Process and send results
|
|
895
|
-
const processStart = Date.now();
|
|
896
|
-
const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
|
|
897
|
-
self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
|
|
898
|
-
);
|
|
899
|
-
const processDuration = Date.now() - processStart;
|
|
900
|
-
|
|
901
|
-
console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} processed - References: ${references.length}, Process duration: ${processDuration}ms`);
|
|
902
|
-
|
|
903
|
-
if (references.length > 0) {
|
|
904
|
-
const response: FeedResponse = {
|
|
905
|
-
references,
|
|
906
|
-
bookmark: results.bookmark
|
|
907
|
-
};
|
|
908
|
-
|
|
909
|
-
const feedStart = Date.now();
|
|
910
|
-
stream.feed(response);
|
|
911
|
-
const feedDuration = Date.now() - feedStart;
|
|
912
|
-
|
|
913
|
-
totalReferences += references.length;
|
|
914
|
-
console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} sent to stream - Feed duration: ${feedDuration}ms`);
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
// Update bookmark for next iteration
|
|
918
|
-
const newBookmark = results.bookmark;
|
|
919
|
-
if (newBookmark === bookmark) {
|
|
920
|
-
console.log(`[InitialResults:${streamId}] Bookmark unchanged - ending pagination`);
|
|
921
|
-
hasMoreResults = false;
|
|
922
|
-
} else {
|
|
923
|
-
bookmark = newBookmark;
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
// Check if we got fewer results than the page size (indicates end)
|
|
927
|
-
if (results.tuples.length < 100) {
|
|
928
|
-
console.log(`[InitialResults:${streamId}] Partial page received (${results.tuples.length} < 100) - ending pagination`);
|
|
929
|
-
hasMoreResults = false;
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
pageCount++;
|
|
933
|
-
|
|
934
|
-
// Add small delay to prevent overwhelming the database
|
|
935
|
-
if (hasMoreResults && pageCount % 10 === 0) {
|
|
936
|
-
console.log(`[InitialResults:${streamId}] Adding delay after ${pageCount} pages`);
|
|
937
|
-
await new Promise(resolve => setTimeout(resolve, 10));
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
const totalDuration = Date.now() - startTime;
|
|
942
|
-
console.log(`[InitialResults:${streamId}] Initial data streaming complete - Pages: ${pageCount}, Total references: ${totalReferences}, Duration: ${totalDuration}ms, Final bookmark: ${bookmark || 'empty'}`);
|
|
943
|
-
|
|
944
|
-
if (pageCount >= maxPages) {
|
|
945
|
-
console.warn(`[InitialResults:${streamId}] Hit maximum page limit (${maxPages}) - possible infinite loop`);
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
return bookmark;
|
|
949
|
-
}
|
|
784
|
+
// The session serializes every fetch and stream operation through a
|
|
785
|
+
// single bookmark-driven query cycle. Inverse and anchor observers
|
|
786
|
+
// only enqueue references onto a waitlist and wake the cycle; they
|
|
787
|
+
// never fetch or stream directly. This removes the race between
|
|
788
|
+
// listener callbacks and bookmark mutation, guarantees tuple
|
|
789
|
+
// completeness, and ensures no update is lost between pages.
|
|
790
|
+
const session = new FeedStreamSession(
|
|
791
|
+
(b: string) => this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, b),
|
|
792
|
+
this.factManager,
|
|
793
|
+
feedDefinition,
|
|
794
|
+
start,
|
|
795
|
+
givenHash,
|
|
796
|
+
stream,
|
|
797
|
+
bookmark,
|
|
798
|
+
connectionId,
|
|
799
|
+
this.feedStreamConfig
|
|
800
|
+
);
|
|
950
801
|
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
const startTime = Date.now();
|
|
954
|
-
|
|
955
|
-
console.log(`[FeedResponse:${responseId}] Starting feed response - Bookmark: ${bookmark || 'empty'}, Skip if empty: ${skipIfEmpty}`);
|
|
956
|
-
|
|
957
|
-
const feedStart = Date.now();
|
|
958
|
-
const queryResult = await this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, bookmark);
|
|
959
|
-
if (queryResult.type === "denied") {
|
|
960
|
-
// The subscription stays live across distribution failures
|
|
961
|
-
// so the inverse / anchor listeners can deliver results
|
|
962
|
-
// once the authorizing fact arrives.
|
|
963
|
-
console.log(`[FeedResponse:${responseId}] Distribution failed (${queryResult.reason}) - keeping stream live`);
|
|
964
|
-
return bookmark;
|
|
965
|
-
}
|
|
966
|
-
const results = queryResult.feed;
|
|
967
|
-
const feedDuration = Date.now() - feedStart;
|
|
802
|
+
stream.done(() => session.dispose());
|
|
803
|
+
session.start();
|
|
968
804
|
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
// Return distinct fact references from all the tuples.
|
|
972
|
-
const processStart = Date.now();
|
|
973
|
-
const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
|
|
974
|
-
self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
|
|
975
|
-
);
|
|
976
|
-
const processDuration = Date.now() - processStart;
|
|
977
|
-
|
|
978
|
-
console.log(`[FeedResponse:${responseId}] References processed - Count: ${references.length}, Duration: ${processDuration}ms`);
|
|
979
|
-
|
|
980
|
-
if (!skipIfEmpty || references.length > 0) {
|
|
981
|
-
const response: FeedResponse = {
|
|
982
|
-
references,
|
|
983
|
-
bookmark: results.bookmark
|
|
984
|
-
};
|
|
985
|
-
|
|
986
|
-
const streamStart = Date.now();
|
|
987
|
-
stream.feed(response);
|
|
988
|
-
const streamDuration = Date.now() - streamStart;
|
|
989
|
-
|
|
990
|
-
console.log(`[FeedResponse:${responseId}] Response sent to stream - Duration: ${streamDuration}ms`);
|
|
991
|
-
} else {
|
|
992
|
-
console.log(`[FeedResponse:${responseId}] Skipping empty response`);
|
|
993
|
-
}
|
|
994
|
-
|
|
995
|
-
const totalDuration = Date.now() - startTime;
|
|
996
|
-
console.log(`[FeedResponse:${responseId}] Feed response complete - New bookmark: ${results.bookmark || 'empty'}, Total duration: ${totalDuration}ms`);
|
|
997
|
-
|
|
998
|
-
return results.bookmark;
|
|
805
|
+
return stream;
|
|
999
806
|
}
|
|
1000
807
|
|
|
1001
808
|
private async getKnownFacts(user: RequestUser | null): Promise<Declaration> {
|
|
@@ -1158,9 +965,24 @@ function extractResults(obj: any): { result: any, count: number } {
|
|
|
1158
965
|
}
|
|
1159
966
|
}
|
|
1160
967
|
|
|
968
|
+
// A known/expired feed hash missed the feed cache. Distinct from a route
|
|
969
|
+
// miss (wrong URL) so clients can tell "unknown/expired feed hash" — which is
|
|
970
|
+
// recoverable by re-registering via POST /feeds — from "wrong URL" (issue
|
|
971
|
+
// #168 S4). handleError maps this to a 404 with a machine-readable body.
|
|
972
|
+
export class FeedNotFound extends Error {
|
|
973
|
+
constructor(public readonly feedHash: string) {
|
|
974
|
+
super(`Feed not found: ${feedHash}`);
|
|
975
|
+
this.name = "FeedNotFound";
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
|
|
1161
979
|
function handleError(error: any, req: Request, res: Response, next: NextFunction) {
|
|
1162
980
|
const requestPath = req.path;
|
|
1163
|
-
if (error instanceof
|
|
981
|
+
if (error instanceof FeedNotFound) {
|
|
982
|
+
Trace.warn(`Feed not found: ${error.feedHash} (Path: ${requestPath})`);
|
|
983
|
+
res.type("text");
|
|
984
|
+
res.status(404).send("feed_not_found");
|
|
985
|
+
} else if (error instanceof Forbidden) {
|
|
1164
986
|
Trace.warn(`Forbidden: ${error.message} (Path: ${requestPath})`);
|
|
1165
987
|
res.type("text");
|
|
1166
988
|
res.status(403).send(error.message);
|
package/src/index.ts
CHANGED
|
@@ -2,8 +2,9 @@ export { AuthenticationDevice } from "./authentication/authentication-device";
|
|
|
2
2
|
export { AuthenticationSession } from "./authentication/authentication-session";
|
|
3
3
|
export { AuthorizationKeystore } from "./authorization/authorization-keystore";
|
|
4
4
|
export { DistributedFactCache } from "./authorization/distributed-fact-cache";
|
|
5
|
+
export { FeedStreamSession, FeedStreamSessionConfig, defaultFeedStreamSessionConfig } from "./feeds/feed-stream-session";
|
|
5
6
|
export { createLineReader } from "./http/line-reader";
|
|
6
|
-
export { HttpRouter, RequestUser } from "./http/router";
|
|
7
|
+
export { FeedNotFound, HttpRouter, RequestUser } from "./http/router";
|
|
7
8
|
export { Stream } from "./http/stream";
|
|
8
9
|
export { JinagaServer, JinagaServerConfig, JinagaServerInstance, tracePool } from "./jinaga-server";
|
|
9
10
|
export { Keystore } from "./keystore";
|
package/src/jinaga-server.ts
CHANGED
|
@@ -37,6 +37,7 @@ import { Pool } from "pg";
|
|
|
37
37
|
import { AuthenticationDevice } from "./authentication/authentication-device";
|
|
38
38
|
import { AuthenticationSession } from "./authentication/authentication-session";
|
|
39
39
|
import { AuthorizationKeystore, DistributionBranchesResult, FeedResult, SubscriptionAuthorizer } from "./authorization/authorization-keystore";
|
|
40
|
+
import { FeedStreamSessionConfig } from "./feeds/feed-stream-session";
|
|
40
41
|
import { HttpRouter, RequestUser } from "./http/router";
|
|
41
42
|
import { Keystore } from "./keystore";
|
|
42
43
|
import { PostgresKeystore } from "./postgres/postgres-keystore";
|
|
@@ -56,7 +57,13 @@ export type JinagaServerConfig = {
|
|
|
56
57
|
authorization?: (a: AuthorizationRules) => AuthorizationRules,
|
|
57
58
|
distribution?: (d: DistributionRules) => DistributionRules,
|
|
58
59
|
purgeConditions?: (p: PurgeConditions) => PurgeConditions,
|
|
59
|
-
origin?: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void)
|
|
60
|
+
origin?: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void),
|
|
61
|
+
/**
|
|
62
|
+
* Tuning for the per-connection feed streaming session
|
|
63
|
+
* (/feeds/:hash with Accept: application/x-jinaga-feed-stream).
|
|
64
|
+
* See FeedStreamSessionConfig for the available limits and defaults.
|
|
65
|
+
*/
|
|
66
|
+
feedStream?: Partial<FeedStreamSessionConfig>
|
|
60
67
|
};
|
|
61
68
|
|
|
62
69
|
export type JinagaServerInstance = {
|
|
@@ -91,7 +98,7 @@ export class JinagaServer {
|
|
|
91
98
|
const purgeConditions = createPurgeConditions(config);
|
|
92
99
|
const factManager = new FactManager(fork, source, store, network, purgeConditions);
|
|
93
100
|
const authorization = createAuthorization(authorizationRules, distributionRules, factManager, store, keystore);
|
|
94
|
-
const router = new HttpRouter(factManager, authorization, feedCache, config.origin || '*');
|
|
101
|
+
const router = new HttpRouter(factManager, authorization, feedCache, config.origin || '*', config.feedStream || {});
|
|
95
102
|
const j: Jinaga = new Jinaga(authentication, factManager, syncStatusNotifier);
|
|
96
103
|
|
|
97
104
|
async function close() {
|