jinaga-server 3.5.5 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authorization/authorization-keystore.d.ts +34 -2
- package/dist/authorization/authorization-keystore.js +59 -4
- package/dist/authorization/authorization-keystore.js.map +1 -1
- package/dist/feeds/feed-stream-session.d.ts +84 -0
- package/dist/feeds/feed-stream-session.js +331 -0
- package/dist/feeds/feed-stream-session.js.map +1 -0
- package/dist/http/router.d.ts +10 -3
- package/dist/http/router.js +186 -184
- package/dist/http/router.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/jinaga-server.d.ts +7 -0
- package/dist/jinaga-server.js +19 -2
- package/dist/jinaga-server.js.map +1 -1
- package/dist/memory/memory-keystore.js +1 -1
- package/dist/memory/memory-keystore.js.map +1 -1
- package/dist/postgres/query-description.d.ts +1 -0
- package/dist/postgres/query-description.js +34 -3
- package/dist/postgres/query-description.js.map +1 -1
- package/dist/postgres/specification-sql.js +7 -1
- package/dist/postgres/specification-sql.js.map +1 -1
- package/docs/design/feeds-persistent-proxy-plan.md +11 -1
- package/integration-test/jinaga-test/distribution-multi-given.test.js +184 -0
- package/integration-test/jinaga-test/distribution-negating-feed.test.js +202 -0
- package/integration-test/jinaga-test/feed-existential-input.test.js +241 -0
- package/integration-test/jinaga-test/feeds-endpoint.test.js +140 -0
- package/integration-test/jinaga-test/package-lock.json +4 -4
- package/integration-test/jinaga-test/package.json +3 -2
- package/integration-test/jinaga-test/subscription-anchor-arrival.test.js +129 -0
- package/integration-test/jinaga-test/subscription-helpers.js +60 -0
- package/integration-test/jinaga-test/subscription-late-auth.test.js +245 -0
- package/integration-test/jinaga-test/subscription-no-rule.test.js +137 -0
- package/package.json +2 -2
- package/src/authorization/authorization-keystore.ts +106 -5
- package/src/feeds/feed-stream-session.ts +386 -0
- package/src/http/router.ts +212 -237
- package/src/index.ts +2 -1
- package/src/jinaga-server.ts +45 -5
- package/src/memory/memory-keystore.ts +1 -1
- package/src/postgres/query-description.ts +66 -11
- package/src/postgres/specification-sql.ts +7 -1
- package/test/authorization/intermediateProjectionFeedSpec.ts +189 -0
- package/test/authorization/subscriptionIntersectionSpec.ts +220 -0
- package/test/feeds/feedStreamSessionRegressionSpec.ts +227 -0
- package/test/feeds/feedStreamSessionSpec.ts +351 -0
- package/test/http/anchorListenerSpec.ts +117 -0
- package/test/http/feedStreamSessionIntegrationSpec.ts +202 -0
- package/test/http/lateAuthRecoverySpec.ts +435 -0
- package/test/http/streamFeedAnchorSpec.ts +171 -0
package/src/http/router.ts
CHANGED
|
@@ -3,14 +3,14 @@ import {
|
|
|
3
3
|
Authorization,
|
|
4
4
|
buildFeeds,
|
|
5
5
|
computeObjectHash,
|
|
6
|
-
computeTupleSubsetHash,
|
|
7
6
|
Declaration,
|
|
7
|
+
DistributionDenialCode,
|
|
8
8
|
FactEnvelope,
|
|
9
9
|
FactManager,
|
|
10
10
|
FactRecord,
|
|
11
11
|
FactReference,
|
|
12
12
|
FeedCache,
|
|
13
|
-
|
|
13
|
+
FeedDecision,
|
|
14
14
|
FeedResponse,
|
|
15
15
|
FeedsResponse,
|
|
16
16
|
Forbidden,
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
GraphSerializer,
|
|
19
19
|
GraphSource,
|
|
20
20
|
Invalid,
|
|
21
|
-
invertSpecification,
|
|
22
21
|
LoadMessage,
|
|
23
22
|
LoadResponse,
|
|
24
23
|
parseLoadMessage,
|
|
@@ -32,6 +31,8 @@ import {
|
|
|
32
31
|
UserIdentity,
|
|
33
32
|
verifyEnvelopes
|
|
34
33
|
} from "jinaga";
|
|
34
|
+
import { DistributionIntersectionBranch, FeedResult, SubscriptionAuthorizer } from "../authorization/authorization-keystore";
|
|
35
|
+
import { FeedStreamSession, FeedStreamSessionConfig } from "../feeds/feed-stream-session";
|
|
35
36
|
import { CsvMetadata } from "./csv-metadata";
|
|
36
37
|
import { validateSpecificationForCsv } from "./csv-validator";
|
|
37
38
|
import { createLineReader } from "./line-reader";
|
|
@@ -286,6 +287,10 @@ function postReadWithStreaming(
|
|
|
286
287
|
};
|
|
287
288
|
}
|
|
288
289
|
|
|
290
|
+
function subscriptionOwnerKey(userIdentity: UserIdentity | null): string | null {
|
|
291
|
+
return userIdentity ? `${userIdentity.provider}|${userIdentity.id}` : null;
|
|
292
|
+
}
|
|
293
|
+
|
|
289
294
|
function serializeUserIdentity(user: RequestUser | null): UserIdentity | null {
|
|
290
295
|
if (!user) {
|
|
291
296
|
return null;
|
|
@@ -366,12 +371,22 @@ export interface RequestUser {
|
|
|
366
371
|
|
|
367
372
|
export class HttpRouter {
|
|
368
373
|
handler: Handler;
|
|
374
|
+
// Map from feed hash → owning user key, for feeds the server cleared
|
|
375
|
+
// via intersection. The cached spec self-filters by the lifted auth
|
|
376
|
+
// condition (the requesting user's fact is baked into start), so it
|
|
377
|
+
// is safe to skip the per-query distribution check — but ONLY for
|
|
378
|
+
// the same user who originally requested the subscription. Another
|
|
379
|
+
// authenticated user who somehow obtained the hash must go through
|
|
380
|
+
// the normal authorization.feed path, which will deny their request
|
|
381
|
+
// since the intersected spec is not authorizable in its own right.
|
|
382
|
+
private intersectedFeedOwners = new Map<string, string | null>();
|
|
369
383
|
|
|
370
384
|
constructor(
|
|
371
385
|
private factManager: FactManager,
|
|
372
|
-
private authorization: Authorization,
|
|
386
|
+
private authorization: Authorization & SubscriptionAuthorizer,
|
|
373
387
|
private feedCache: FeedCache,
|
|
374
|
-
private allowedOrigin: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void)
|
|
388
|
+
private allowedOrigin: string | string[] | ((origin: string, callback: (err: Error | null, allow?: boolean) => void) => void),
|
|
389
|
+
private feedStreamConfig: Partial<FeedStreamSessionConfig> = {}
|
|
375
390
|
) {
|
|
376
391
|
const router = Router();
|
|
377
392
|
const applyAllowOrigin = this.applyAllowOrigin.bind(this);
|
|
@@ -555,49 +570,187 @@ export class HttpRouter {
|
|
|
555
570
|
if (failures.length > 0) {
|
|
556
571
|
throw new Invalid(failures.join("\n"));
|
|
557
572
|
}
|
|
558
|
-
|
|
573
|
+
|
|
559
574
|
const namedStart = specification.given.reduce((map, g, index) => ({
|
|
560
575
|
...map,
|
|
561
576
|
[g.label.name]: start[index]
|
|
562
577
|
}), {} as ReferencesByName);
|
|
563
578
|
|
|
564
|
-
//
|
|
565
|
-
|
|
579
|
+
// Resolve distribution. When the user is not authorized for the
|
|
580
|
+
// original spec but the distribution rules can rewrite it via
|
|
581
|
+
// intersection (jinaga.js#130), accept the subscription with
|
|
582
|
+
// empty results and let the inverse engine activate it later.
|
|
583
|
+
//
|
|
584
|
+
// Even when no rule applies at all, accept the subscription:
|
|
585
|
+
// the per-query distribution check inside streamFeed /
|
|
586
|
+
// feed returns "denied" and keeps the stream alive so the
|
|
587
|
+
// client can be notified the moment the authorizing fact
|
|
588
|
+
// arrives. Failing /feeds here would force the client to
|
|
589
|
+
// re-subscribe on a poll loop.
|
|
566
590
|
const userIdentity = serializeUserIdentity(user);
|
|
567
|
-
await this.authorization.
|
|
591
|
+
const intersectResult = await this.authorization.verifyDistributionOrIntersect(userIdentity, specification, namedStart);
|
|
592
|
+
let branches: DistributionIntersectionBranch[];
|
|
593
|
+
let intersected: boolean;
|
|
594
|
+
if (intersectResult.type === "denied") {
|
|
595
|
+
// Structured, dashboard-observable signal that a feed was
|
|
596
|
+
// registered without an applicable distribution rule (issue
|
|
597
|
+
// #168 S3). The metric name is stable so a backend sees
|
|
598
|
+
// bounded cardinality; the denial code (a fixed 4-value enum)
|
|
599
|
+
// is the measurement dimension, so authoring errors
|
|
600
|
+
// (no-matching-rule / spec-more-restrictive-than-rule) are
|
|
601
|
+
// distinguishable from auth states (principal-excluded /
|
|
602
|
+
// not-authenticated) in CI and dashboards without grepping
|
|
603
|
+
// logs. The human-readable line is retained for correlation.
|
|
604
|
+
Trace.metric("distribution.unmatched", { [intersectResult.code]: 1 });
|
|
605
|
+
Trace.warn(`/feeds accepting spec without applicable distribution rule (${intersectResult.code}): ${intersectResult.reason}`);
|
|
606
|
+
branches = [{ start, specification }];
|
|
607
|
+
intersected = false;
|
|
608
|
+
} else {
|
|
609
|
+
branches = intersectResult.branches;
|
|
610
|
+
intersected = branches.length !== 1 || branches[0].specification !== specification;
|
|
611
|
+
}
|
|
568
612
|
|
|
569
|
-
|
|
613
|
+
// Classify the spec's distribution decision once; it applies to
|
|
614
|
+
// every feed hash the spec produces (issue #168 S1). Registration
|
|
615
|
+
// and keep-alive are unchanged — reactive and denied feeds are
|
|
616
|
+
// still accepted and served exactly as before; this only reports
|
|
617
|
+
// the decision. `reactive` is authorized-via-intersection: denied
|
|
618
|
+
// now, self-heals when the authorizing fact arrives.
|
|
619
|
+
//
|
|
620
|
+
// S2: the decision is recomputed from verifyDistributionOrIntersect
|
|
621
|
+
// on every POST /feeds — it is NOT cached by feed hash — so a
|
|
622
|
+
// repeated request always returns the correct *per-user* decision.
|
|
623
|
+
// This matters because a non-intersected feed hash is shared across
|
|
624
|
+
// users (it excludes the requester's identity): the same hash is
|
|
625
|
+
// `authorized` for a permitted user and `denied` for another, and
|
|
626
|
+
// a `reactive` decision becomes `authorized` once the authorizing
|
|
627
|
+
// fact arrives. Caching the FeedDecision by hash would leak one
|
|
628
|
+
// user's decision to another. addFeeds stays idempotent on the
|
|
629
|
+
// hash; only the classification is per-request.
|
|
630
|
+
let decision: FeedDecision["decision"];
|
|
631
|
+
let decisionCode: DistributionDenialCode | undefined;
|
|
632
|
+
let decisionReason: string;
|
|
633
|
+
if (intersectResult.type === "denied") {
|
|
634
|
+
decision = "denied";
|
|
635
|
+
decisionCode = intersectResult.code;
|
|
636
|
+
decisionReason = intersectResult.reason;
|
|
637
|
+
} else if (intersected) {
|
|
638
|
+
decision = "reactive";
|
|
639
|
+
decisionCode = intersectResult.denial?.code;
|
|
640
|
+
decisionReason = intersectResult.denial?.reason
|
|
641
|
+
?? "Authorized via intersection; awaiting the authorizing fact.";
|
|
642
|
+
} else {
|
|
643
|
+
decision = "authorized";
|
|
644
|
+
decisionCode = undefined;
|
|
645
|
+
decisionReason = "Distribution authorized.";
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
const ownerKey = subscriptionOwnerKey(userIdentity);
|
|
649
|
+
const feedHashes: string[] = [];
|
|
650
|
+
const decisions: FeedDecision[] = [];
|
|
651
|
+
for (const branch of branches) {
|
|
652
|
+
const branchNamedStart = branch.specification.given.reduce((map, g, index) => ({
|
|
653
|
+
...map,
|
|
654
|
+
[g.label.name]: branch.start[index]
|
|
655
|
+
}), {} as ReferencesByName);
|
|
656
|
+
const branchFeeds = buildFeeds(branch.specification);
|
|
657
|
+
const branchHashes = this.feedCache.addFeeds(branchFeeds, branchNamedStart);
|
|
658
|
+
feedHashes.push(...branchHashes);
|
|
659
|
+
for (const hash of branchHashes) {
|
|
660
|
+
decisions.push({
|
|
661
|
+
feed: hash,
|
|
662
|
+
decision,
|
|
663
|
+
...(decisionCode !== undefined ? { code: decisionCode } : {}),
|
|
664
|
+
reason: decisionReason
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
if (intersected) {
|
|
668
|
+
for (const hash of branchHashes) {
|
|
669
|
+
// Bind the hash to its owner so a different
|
|
670
|
+
// authenticated user who obtains it cannot reuse
|
|
671
|
+
// the cached spec — which carries the original
|
|
672
|
+
// owner's user fact in its start — to fetch
|
|
673
|
+
// facts authorized for that owner.
|
|
674
|
+
//
|
|
675
|
+
// The map allows only one owner per hash, which
|
|
676
|
+
// relies on the invariant that intersected specs
|
|
677
|
+
// bind the requesting user's fact into start —
|
|
678
|
+
// making the hash user-specific. If two distinct
|
|
679
|
+
// owners ever collide on the same hash, that
|
|
680
|
+
// invariant has broken in intersectForSubscribe;
|
|
681
|
+
// warn so we notice instead of silently
|
|
682
|
+
// overwriting.
|
|
683
|
+
const existing = this.intersectedFeedOwners.get(hash);
|
|
684
|
+
if (existing !== undefined && existing !== ownerKey) {
|
|
685
|
+
Trace.warn(`Intersected feed hash ${hash} re-bound from owner ${existing} to ${ownerKey}; intersection invariant may be broken`);
|
|
686
|
+
}
|
|
687
|
+
this.intersectedFeedOwners.set(hash, ownerKey);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
570
691
|
|
|
571
692
|
return {
|
|
572
|
-
feeds: feedHashes
|
|
693
|
+
feeds: feedHashes,
|
|
694
|
+
decisions
|
|
573
695
|
}
|
|
574
696
|
});
|
|
575
697
|
}
|
|
576
698
|
|
|
699
|
+
private async queryFeed(
|
|
700
|
+
feedHash: string,
|
|
701
|
+
userIdentity: UserIdentity | null,
|
|
702
|
+
specification: Specification,
|
|
703
|
+
start: FactReference[],
|
|
704
|
+
bookmark: string
|
|
705
|
+
): Promise<FeedResult> {
|
|
706
|
+
const cachedOwner = this.intersectedFeedOwners.get(feedHash);
|
|
707
|
+
const requesterOwner = subscriptionOwnerKey(userIdentity);
|
|
708
|
+
if (cachedOwner !== undefined && cachedOwner === requesterOwner) {
|
|
709
|
+
const feed = await this.authorization.feedPreVerified(userIdentity, specification, start, bookmark);
|
|
710
|
+
return { type: "success", feed };
|
|
711
|
+
}
|
|
712
|
+
// For everyone else (different user, anonymous mismatch, or a
|
|
713
|
+
// non-intersected hash) go through the normal distribution-checked
|
|
714
|
+
// path. A "denied" result means the streaming / polling caller
|
|
715
|
+
// serves an empty page and keeps the subscription alive.
|
|
716
|
+
return await this.authorization.feedWithDistribution(userIdentity, specification, start, bookmark);
|
|
717
|
+
}
|
|
718
|
+
|
|
577
719
|
private feed(user: RequestUser | null, params: { [key: string]: string }, query: qs.ParsedQs): Promise<FeedResponse | null> {
|
|
578
720
|
return Trace.dependency("feed", params["hash"], async () => {
|
|
579
721
|
const feedHash = params["hash"];
|
|
580
722
|
if (!feedHash) {
|
|
723
|
+
// No hash in the route at all — a wrong URL, not a feed lookup.
|
|
581
724
|
return null;
|
|
582
725
|
}
|
|
583
726
|
|
|
584
727
|
const feedDefinition = await this.feedCache.getFeed(feedHash);
|
|
585
728
|
if (!feedDefinition) {
|
|
586
|
-
|
|
729
|
+
// Known route, but the feed hash is unknown or expired. Signal
|
|
730
|
+
// this distinctly (issue #168 S4) so the client can re-register
|
|
731
|
+
// via POST /feeds instead of treating it as a routing error.
|
|
732
|
+
throw new FeedNotFound(feedHash);
|
|
587
733
|
}
|
|
588
734
|
|
|
589
735
|
const bookmark = query["b"] as string ?? "";
|
|
590
736
|
|
|
591
737
|
const userIdentity = serializeUserIdentity(user);
|
|
592
738
|
const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
|
|
593
|
-
const
|
|
739
|
+
const result = await this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, bookmark);
|
|
740
|
+
if (result.type === "denied") {
|
|
741
|
+
// The subscription has been accepted; treat poll-time
|
|
742
|
+
// distribution failures as an empty page so the client
|
|
743
|
+
// can keep polling and pick up results when an
|
|
744
|
+
// authorizing fact arrives.
|
|
745
|
+
return { references: [], bookmark } as FeedResponse;
|
|
746
|
+
}
|
|
594
747
|
// Return distinct fact references from all the tuples.
|
|
595
|
-
const references =
|
|
748
|
+
const references = result.feed.tuples.flatMap(t => t.facts).filter((value, index, self) =>
|
|
596
749
|
self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
|
|
597
750
|
);
|
|
598
751
|
const response: FeedResponse = {
|
|
599
752
|
references,
|
|
600
|
-
bookmark:
|
|
753
|
+
bookmark: result.feed.bookmark
|
|
601
754
|
};
|
|
602
755
|
return response;
|
|
603
756
|
});
|
|
@@ -605,244 +758,51 @@ export class HttpRouter {
|
|
|
605
758
|
|
|
606
759
|
private async streamFeed(user: RequestUser | null, params: { [key: string]: string }, query: qs.ParsedQs): Promise<Stream<FeedResponse> | null> {
|
|
607
760
|
const connectionId = Math.random().toString(36).substring(2, 10);
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
console.log(`[StreamFeed:${connectionId}] NEW CONNECTION - User: ${user?.id || 'anonymous'}, Hash: ${params["hash"]}`);
|
|
611
|
-
|
|
761
|
+
|
|
612
762
|
const feedHash = params["hash"];
|
|
613
763
|
if (!feedHash) {
|
|
614
|
-
|
|
764
|
+
// No hash in the route at all — a wrong URL, not a feed lookup.
|
|
615
765
|
return null;
|
|
616
766
|
}
|
|
617
767
|
|
|
618
768
|
const feedDefinition = await this.feedCache.getFeed(feedHash);
|
|
619
769
|
if (!feedDefinition) {
|
|
620
|
-
|
|
621
|
-
|
|
770
|
+
// Known route, but the feed hash is unknown or expired. Signal
|
|
771
|
+
// this distinctly (issue #168 S4) so the client can re-register
|
|
772
|
+
// via POST /feeds instead of treating it as a routing error.
|
|
773
|
+
throw new FeedNotFound(feedHash);
|
|
622
774
|
}
|
|
623
775
|
|
|
624
|
-
|
|
625
|
-
console.log(`[StreamFeed:${connectionId}] Initial bookmark: ${bookmark || 'empty'}`);
|
|
776
|
+
const bookmark = query["b"] as string ?? "";
|
|
626
777
|
|
|
627
778
|
const userIdentity = serializeUserIdentity(user);
|
|
628
779
|
const start = feedDefinition.feed.given.map(g => feedDefinition.namedStart[g.label.name]);
|
|
629
780
|
const givenHash = computeObjectHash(feedDefinition.namedStart);
|
|
630
781
|
|
|
631
|
-
console.log(`[StreamFeed:${connectionId}] Feed setup - Given hash: ${givenHash.substring(0, 8)}..., Start facts: ${start.length}`);
|
|
632
|
-
|
|
633
782
|
const stream = new Stream<FeedResponse>();
|
|
634
|
-
|
|
635
|
-
try {
|
|
636
|
-
// Continuous initial query until exhausted
|
|
637
|
-
console.log(`[StreamFeed:${connectionId}] Starting initial data streaming...`);
|
|
638
|
-
const initialStart = Date.now();
|
|
639
|
-
bookmark = await this.streamAllInitialResults(userIdentity, feedDefinition, start, bookmark, stream);
|
|
640
|
-
const initialDuration = Date.now() - initialStart;
|
|
641
|
-
console.log(`[StreamFeed:${connectionId}] Initial data streaming complete - Duration: ${initialDuration}ms, Final bookmark: ${bookmark || 'empty'}`);
|
|
642
|
-
|
|
643
|
-
// Set up real-time listeners after initial data is complete
|
|
644
|
-
console.log(`[StreamFeed:${connectionId}] Setting up real-time listeners...`);
|
|
645
|
-
const inverses = invertSpecification(feedDefinition.feed);
|
|
646
|
-
console.log(`[StreamFeed:${connectionId}] Created ${inverses.length} inverse specifications`);
|
|
647
|
-
|
|
648
|
-
const listeners = inverses.map((inverse, index) => {
|
|
649
|
-
console.log(`[StreamFeed:${connectionId}] Adding listener ${index + 1}/${inverses.length}`);
|
|
650
|
-
|
|
651
|
-
return this.factManager.addSpecificationListener(
|
|
652
|
-
inverse.inverseSpecification,
|
|
653
|
-
async (results) => {
|
|
654
|
-
const eventStart = Date.now();
|
|
655
|
-
console.log(`[StreamFeed:${connectionId}] EVENT RECEIVED - Listener ${index + 1}, Results: ${results.length}`);
|
|
656
|
-
|
|
657
|
-
try {
|
|
658
|
-
// Filter out results that do not match the given.
|
|
659
|
-
const matchingResults = results.filter(pr =>
|
|
660
|
-
givenHash === computeTupleSubsetHash(pr.tuple, inverse.givenSubset));
|
|
661
|
-
|
|
662
|
-
console.log(`[StreamFeed:${connectionId}] Filtered results - Matching: ${matchingResults.length}/${results.length}`);
|
|
663
|
-
|
|
664
|
-
if (matchingResults.length != 0) {
|
|
665
|
-
console.log(`[StreamFeed:${connectionId}] Processing matching results...`);
|
|
666
|
-
const responseStart = Date.now();
|
|
667
|
-
bookmark = await this.streamFeedResponse(userIdentity, feedDefinition, start, bookmark, stream, true);
|
|
668
|
-
const responseDuration = Date.now() - responseStart;
|
|
669
|
-
console.log(`[StreamFeed:${connectionId}] Feed response sent - Duration: ${responseDuration}ms, New bookmark: ${bookmark || 'empty'}`);
|
|
670
|
-
} else {
|
|
671
|
-
console.log(`[StreamFeed:${connectionId}] No matching results - skipping response`);
|
|
672
|
-
}
|
|
673
|
-
} catch (error) {
|
|
674
|
-
console.error(`[StreamFeed:${connectionId}] ERROR processing event: ${error}`);
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
const eventDuration = Date.now() - eventStart;
|
|
678
|
-
if (eventDuration > 100) {
|
|
679
|
-
console.warn(`[StreamFeed:${connectionId}] SLOW event processing - Duration: ${eventDuration}ms`);
|
|
680
|
-
}
|
|
681
|
-
}
|
|
682
|
-
);
|
|
683
|
-
});
|
|
684
|
-
|
|
685
|
-
console.log(`[StreamFeed:${connectionId}] All listeners registered - Count: ${listeners.length}`);
|
|
686
|
-
|
|
687
|
-
stream.done(() => {
|
|
688
|
-
const cleanupStart = Date.now();
|
|
689
|
-
console.log(`[StreamFeed:${connectionId}] CLEANUP STARTED - Removing ${listeners.length} listeners`);
|
|
690
|
-
|
|
691
|
-
for (let i = 0; i < listeners.length; i++) {
|
|
692
|
-
try {
|
|
693
|
-
this.factManager.removeSpecificationListener(listeners[i]);
|
|
694
|
-
console.log(`[StreamFeed:${connectionId}] Removed listener ${i + 1}/${listeners.length}`);
|
|
695
|
-
} catch (error) {
|
|
696
|
-
console.error(`[StreamFeed:${connectionId}] ERROR removing listener ${i + 1}: ${error}`);
|
|
697
|
-
}
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
const cleanupDuration = Date.now() - cleanupStart;
|
|
701
|
-
const totalDuration = Date.now() - startTime;
|
|
702
|
-
console.log(`[StreamFeed:${connectionId}] CLEANUP COMPLETE - Cleanup duration: ${cleanupDuration}ms, Total connection duration: ${totalDuration}ms`);
|
|
703
|
-
});
|
|
704
|
-
|
|
705
|
-
const setupDuration = Date.now() - startTime;
|
|
706
|
-
console.log(`[StreamFeed:${connectionId}] Stream setup complete - Duration: ${setupDuration}ms`);
|
|
707
|
-
|
|
708
|
-
return stream;
|
|
709
|
-
|
|
710
|
-
} catch (error) {
|
|
711
|
-
console.error(`[StreamFeed:${connectionId}] ERROR during setup: ${error}`);
|
|
712
|
-
throw error;
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
783
|
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
while (hasMoreResults && pageCount < maxPages) {
|
|
734
|
-
const pageStart = Date.now();
|
|
735
|
-
console.log(`[InitialResults:${streamId}] Fetching page ${pageCount + 1} - Bookmark: ${bookmark || 'empty'}`);
|
|
736
|
-
|
|
737
|
-
const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
|
|
738
|
-
const fetchDuration = Date.now() - pageStart;
|
|
739
|
-
|
|
740
|
-
console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} fetched - Tuples: ${results.tuples.length}, Duration: ${fetchDuration}ms`);
|
|
741
|
-
|
|
742
|
-
// Check if we got results
|
|
743
|
-
if (results.tuples.length === 0) {
|
|
744
|
-
console.log(`[InitialResults:${streamId}] No more results - ending pagination`);
|
|
745
|
-
hasMoreResults = false;
|
|
746
|
-
break;
|
|
747
|
-
}
|
|
748
|
-
|
|
749
|
-
// Process and send results
|
|
750
|
-
const processStart = Date.now();
|
|
751
|
-
const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
|
|
752
|
-
self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
|
|
753
|
-
);
|
|
754
|
-
const processDuration = Date.now() - processStart;
|
|
755
|
-
|
|
756
|
-
console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} processed - References: ${references.length}, Process duration: ${processDuration}ms`);
|
|
757
|
-
|
|
758
|
-
if (references.length > 0) {
|
|
759
|
-
const response: FeedResponse = {
|
|
760
|
-
references,
|
|
761
|
-
bookmark: results.bookmark
|
|
762
|
-
};
|
|
763
|
-
|
|
764
|
-
const feedStart = Date.now();
|
|
765
|
-
stream.feed(response);
|
|
766
|
-
const feedDuration = Date.now() - feedStart;
|
|
767
|
-
|
|
768
|
-
totalReferences += references.length;
|
|
769
|
-
console.log(`[InitialResults:${streamId}] Page ${pageCount + 1} sent to stream - Feed duration: ${feedDuration}ms`);
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
// Update bookmark for next iteration
|
|
773
|
-
const newBookmark = results.bookmark;
|
|
774
|
-
if (newBookmark === bookmark) {
|
|
775
|
-
console.log(`[InitialResults:${streamId}] Bookmark unchanged - ending pagination`);
|
|
776
|
-
hasMoreResults = false;
|
|
777
|
-
} else {
|
|
778
|
-
bookmark = newBookmark;
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
// Check if we got fewer results than the page size (indicates end)
|
|
782
|
-
if (results.tuples.length < 100) {
|
|
783
|
-
console.log(`[InitialResults:${streamId}] Partial page received (${results.tuples.length} < 100) - ending pagination`);
|
|
784
|
-
hasMoreResults = false;
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
pageCount++;
|
|
788
|
-
|
|
789
|
-
// Add small delay to prevent overwhelming the database
|
|
790
|
-
if (hasMoreResults && pageCount % 10 === 0) {
|
|
791
|
-
console.log(`[InitialResults:${streamId}] Adding delay after ${pageCount} pages`);
|
|
792
|
-
await new Promise(resolve => setTimeout(resolve, 10));
|
|
793
|
-
}
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
const totalDuration = Date.now() - startTime;
|
|
797
|
-
console.log(`[InitialResults:${streamId}] Initial data streaming complete - Pages: ${pageCount}, Total references: ${totalReferences}, Duration: ${totalDuration}ms, Final bookmark: ${bookmark || 'empty'}`);
|
|
798
|
-
|
|
799
|
-
if (pageCount >= maxPages) {
|
|
800
|
-
console.warn(`[InitialResults:${streamId}] Hit maximum page limit (${maxPages}) - possible infinite loop`);
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
return bookmark;
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
private async streamFeedResponse(userIdentity: UserIdentity | null, feedDefinition: FeedObject, start: FactReference[], bookmark: string, stream: Stream<FeedResponse>, skipIfEmpty = false): Promise<string> {
|
|
807
|
-
const responseId = Math.random().toString(36).substring(2, 6);
|
|
808
|
-
const startTime = Date.now();
|
|
809
|
-
|
|
810
|
-
console.log(`[FeedResponse:${responseId}] Starting feed response - Bookmark: ${bookmark || 'empty'}, Skip if empty: ${skipIfEmpty}`);
|
|
811
|
-
|
|
812
|
-
const feedStart = Date.now();
|
|
813
|
-
const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
|
|
814
|
-
const feedDuration = Date.now() - feedStart;
|
|
815
|
-
|
|
816
|
-
console.log(`[FeedResponse:${responseId}] Authorization feed complete - Tuples: ${results.tuples.length}, Duration: ${feedDuration}ms`);
|
|
817
|
-
|
|
818
|
-
// Return distinct fact references from all the tuples.
|
|
819
|
-
const processStart = Date.now();
|
|
820
|
-
const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) =>
|
|
821
|
-
self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
|
|
784
|
+
// The session serializes every fetch and stream operation through a
|
|
785
|
+
// single bookmark-driven query cycle. Inverse and anchor observers
|
|
786
|
+
// only enqueue references onto a waitlist and wake the cycle; they
|
|
787
|
+
// never fetch or stream directly. This removes the race between
|
|
788
|
+
// listener callbacks and bookmark mutation, guarantees tuple
|
|
789
|
+
// completeness, and ensures no update is lost between pages.
|
|
790
|
+
const session = new FeedStreamSession(
|
|
791
|
+
(b: string) => this.queryFeed(feedHash, userIdentity, feedDefinition.feed, start, b),
|
|
792
|
+
this.factManager,
|
|
793
|
+
feedDefinition,
|
|
794
|
+
start,
|
|
795
|
+
givenHash,
|
|
796
|
+
stream,
|
|
797
|
+
bookmark,
|
|
798
|
+
connectionId,
|
|
799
|
+
this.feedStreamConfig
|
|
822
800
|
);
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
const response: FeedResponse = {
|
|
829
|
-
references,
|
|
830
|
-
bookmark: results.bookmark
|
|
831
|
-
};
|
|
832
|
-
|
|
833
|
-
const streamStart = Date.now();
|
|
834
|
-
stream.feed(response);
|
|
835
|
-
const streamDuration = Date.now() - streamStart;
|
|
836
|
-
|
|
837
|
-
console.log(`[FeedResponse:${responseId}] Response sent to stream - Duration: ${streamDuration}ms`);
|
|
838
|
-
} else {
|
|
839
|
-
console.log(`[FeedResponse:${responseId}] Skipping empty response`);
|
|
840
|
-
}
|
|
841
|
-
|
|
842
|
-
const totalDuration = Date.now() - startTime;
|
|
843
|
-
console.log(`[FeedResponse:${responseId}] Feed response complete - New bookmark: ${results.bookmark || 'empty'}, Total duration: ${totalDuration}ms`);
|
|
844
|
-
|
|
845
|
-
return results.bookmark;
|
|
801
|
+
|
|
802
|
+
stream.done(() => session.dispose());
|
|
803
|
+
session.start();
|
|
804
|
+
|
|
805
|
+
return stream;
|
|
846
806
|
}
|
|
847
807
|
|
|
848
808
|
private async getKnownFacts(user: RequestUser | null): Promise<Declaration> {
|
|
@@ -1005,9 +965,24 @@ function extractResults(obj: any): { result: any, count: number } {
|
|
|
1005
965
|
}
|
|
1006
966
|
}
|
|
1007
967
|
|
|
968
|
+
// A known/expired feed hash missed the feed cache. Distinct from a route
|
|
969
|
+
// miss (wrong URL) so clients can tell "unknown/expired feed hash" — which is
|
|
970
|
+
// recoverable by re-registering via POST /feeds — from "wrong URL" (issue
|
|
971
|
+
// #168 S4). handleError maps this to a 404 with a machine-readable body.
|
|
972
|
+
export class FeedNotFound extends Error {
|
|
973
|
+
constructor(public readonly feedHash: string) {
|
|
974
|
+
super(`Feed not found: ${feedHash}`);
|
|
975
|
+
this.name = "FeedNotFound";
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
|
|
1008
979
|
function handleError(error: any, req: Request, res: Response, next: NextFunction) {
|
|
1009
980
|
const requestPath = req.path;
|
|
1010
|
-
if (error instanceof
|
|
981
|
+
if (error instanceof FeedNotFound) {
|
|
982
|
+
Trace.warn(`Feed not found: ${error.feedHash} (Path: ${requestPath})`);
|
|
983
|
+
res.type("text");
|
|
984
|
+
res.status(404).send("feed_not_found");
|
|
985
|
+
} else if (error instanceof Forbidden) {
|
|
1011
986
|
Trace.warn(`Forbidden: ${error.message} (Path: ${requestPath})`);
|
|
1012
987
|
res.type("text");
|
|
1013
988
|
res.status(403).send(error.message);
|
package/src/index.ts
CHANGED
|
@@ -2,8 +2,9 @@ export { AuthenticationDevice } from "./authentication/authentication-device";
|
|
|
2
2
|
export { AuthenticationSession } from "./authentication/authentication-session";
|
|
3
3
|
export { AuthorizationKeystore } from "./authorization/authorization-keystore";
|
|
4
4
|
export { DistributedFactCache } from "./authorization/distributed-fact-cache";
|
|
5
|
+
export { FeedStreamSession, FeedStreamSessionConfig, defaultFeedStreamSessionConfig } from "./feeds/feed-stream-session";
|
|
5
6
|
export { createLineReader } from "./http/line-reader";
|
|
6
|
-
export { HttpRouter, RequestUser } from "./http/router";
|
|
7
|
+
export { FeedNotFound, HttpRouter, RequestUser } from "./http/router";
|
|
7
8
|
export { Stream } from "./http/stream";
|
|
8
9
|
export { JinagaServer, JinagaServerConfig, JinagaServerInstance, tracePool } from "./jinaga-server";
|
|
9
10
|
export { Keystore } from "./keystore";
|