jinaga-server 3.5.1 → 3.5.3

package/test-concurrent-streams.js

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+
+/**
+ * Test script to simulate multiple concurrent clients connecting to streaming feeds
+ * This will help validate the diagnostic logging and reproduce the push notification issues
+ */
+
+const http = require('http');
+const { performance } = require('perf_hooks');
+
+// Configuration
+const SERVER_HOST = 'localhost';
+const SERVER_PORT = 8080;
+const FEED_HASH = 'your-feed-hash-here'; // Replace with actual feed hash
+const NUM_CLIENTS = 5;
+const CONNECTION_DURATION = 30000; // 30 seconds
+const STAGGER_DELAY = 1000; // 1 second between client connections
+
+class StreamingClient {
+    constructor(clientId, feedHash) {
+        this.clientId = clientId;
+        this.feedHash = feedHash;
+        this.messageCount = 0;
+        this.connected = false;
+        this.startTime = null;
+        this.lastMessageTime = null;
+    }
+
+    connect() {
+        return new Promise((resolve, reject) => {
+            console.log(`[Client-${this.clientId}] Connecting to feed: ${this.feedHash}`);
+            this.startTime = performance.now();
+
+            const options = {
+                hostname: SERVER_HOST,
+                port: SERVER_PORT,
+                path: `/feeds/${this.feedHash}`,
+                method: 'GET',
+                headers: {
+                    'Accept': 'application/x-jinaga-feed-stream',
+                    'Connection': 'keep-alive',
+                    'Cache-Control': 'no-cache'
+                }
+            };
+
+            const req = http.request(options, (res) => {
+                console.log(`[Client-${this.clientId}] Connected - Status: ${res.statusCode}`);
+                
+                if (res.statusCode !== 200) {
+                    console.error(`[Client-${this.clientId}] Connection failed with status: ${res.statusCode}`);
+                    reject(new Error(`HTTP ${res.statusCode}`));
+                    return;
+                }
+
+                this.connected = true;
+                resolve();
+
+                res.on('data', (chunk) => {
+                    const now = performance.now();
+                    this.messageCount++;
+                    this.lastMessageTime = now;
+                    
+                    const timeSinceStart = Math.round(now - this.startTime);
+                    const timeSinceLastMessage = this.lastMessageTime ? Math.round(now - this.lastMessageTime) : 0;
+                    
+                    console.log(`[Client-${this.clientId}] Message #${this.messageCount} received - Time: ${timeSinceStart}ms, Gap: ${timeSinceLastMessage}ms, Size: ${chunk.length} bytes`);
+                    
+                    try {
+                        const data = chunk.toString().trim();
+                        if (data) {
+                            const parsed = JSON.parse(data);
+                            console.log(`[Client-${this.clientId}] Parsed message - References: ${parsed.references?.length || 0}, Bookmark: ${parsed.bookmark?.substring(0, 8) || 'none'}...`);
+                        }
+                    } catch (error) {
+                        console.warn(`[Client-${this.clientId}] Failed to parse message: ${error.message}`);
+                    }
+                });
+
+                res.on('end', () => {
+                    const duration = Math.round(performance.now() - this.startTime);
+                    console.log(`[Client-${this.clientId}] Stream ended - Duration: ${duration}ms, Messages: ${this.messageCount}`);
+                    this.connected = false;
+                });
+
+                res.on('error', (error) => {
+                    console.error(`[Client-${this.clientId}] Stream error: ${error.message}`);
+                    this.connected = false;
+                });
+            });
+
+            req.on('error', (error) => {
+                console.error(`[Client-${this.clientId}] Request error: ${error.message}`);
+                reject(error);
+            });
+
+            req.setTimeout(CONNECTION_DURATION + 5000, () => {
+                console.log(`[Client-${this.clientId}] Request timeout`);
+                req.destroy();
+            });
+
+            req.end();
+
+            // Auto-disconnect after specified duration
+            setTimeout(() => {
+                if (this.connected) {
+                    console.log(`[Client-${this.clientId}] Auto-disconnecting after ${CONNECTION_DURATION}ms`);
+                    req.destroy();
+                }
+            }, CONNECTION_DURATION);
+        });
+    }
+
+    getStats() {
+        const duration = this.startTime ? Math.round(performance.now() - this.startTime) : 0;
+        return {
+            clientId: this.clientId,
+            messageCount: this.messageCount,
+            connected: this.connected,
+            duration: duration
+        };
+    }
+}
+
+async function runConcurrentTest() {
+    console.log(`Starting concurrent streaming test with ${NUM_CLIENTS} clients`);
+    console.log(`Server: ${SERVER_HOST}:${SERVER_PORT}`);
+    console.log(`Feed Hash: ${FEED_HASH}`);
+    console.log(`Connection Duration: ${CONNECTION_DURATION}ms`);
+    console.log(`Stagger Delay: ${STAGGER_DELAY}ms`);
+    console.log('---');
+
+    const clients = [];
+    const connectionPromises = [];
+
+    // Create and connect clients with staggered timing
+    for (let i = 1; i <= NUM_CLIENTS; i++) {
+        const client = new StreamingClient(i, FEED_HASH);
+        clients.push(client);
+
+        // Stagger the connections
+        setTimeout(async () => {
+            try {
+                await client.connect();
+            } catch (error) {
+                console.error(`[Client-${i}] Connection failed: ${error.message}`);
+            }
+        }, (i - 1) * STAGGER_DELAY);
+    }
+
+    // Monitor client statistics
+    const statsInterval = setInterval(() => {
+        console.log('\n=== CLIENT STATISTICS ===');
+        clients.forEach(client => {
+            const stats = client.getStats();
+            console.log(`Client-${stats.clientId}: Messages=${stats.messageCount}, Connected=${stats.connected}, Duration=${stats.duration}ms`);
+        });
+        console.log('========================\n');
+    }, 10000); // Every 10 seconds
+
+    // Wait for test completion
+    setTimeout(() => {
+        clearInterval(statsInterval);
+        
+        console.log('\n=== FINAL STATISTICS ===');
+        let totalMessages = 0;
+        let connectedClients = 0;
+        
+        clients.forEach(client => {
+            const stats = client.getStats();
+            totalMessages += stats.messageCount;
+            if (stats.connected) connectedClients++;
+            console.log(`Client-${stats.clientId}: Messages=${stats.messageCount}, Connected=${stats.connected}, Duration=${stats.duration}ms`);
+        });
+        
+        console.log(`\nSummary:`);
+        console.log(`- Total Messages Received: ${totalMessages}`);
+        console.log(`- Still Connected: ${connectedClients}/${NUM_CLIENTS}`);
+        console.log(`- Average Messages per Client: ${(totalMessages / NUM_CLIENTS).toFixed(1)}`);
+        
+        // Check for potential issues
+        const messageCounts = clients.map(c => c.getStats().messageCount);
+        const minMessages = Math.min(...messageCounts);
+        const maxMessages = Math.max(...messageCounts);
+        const variance = maxMessages - minMessages;
+        
+        if (variance > 5) {
+            console.log(`\n⚠️  POTENTIAL ISSUE DETECTED:`);
+            console.log(`   Message count variance: ${variance} (min: ${minMessages}, max: ${maxMessages})`);
+            console.log(`   This suggests some clients may not be receiving all notifications`);
+        } else {
+            console.log(`\n✅ Message distribution appears consistent`);
+        }
+        
+        console.log('\nTest completed. Check server logs for detailed diagnostic information.');
+        process.exit(0);
+    }, CONNECTION_DURATION + (NUM_CLIENTS * STAGGER_DELAY) + 5000);
+}
+
+// Handle command line arguments
+if (process.argv.length > 2) {
+    const feedHash = process.argv[2];
+    if (feedHash && feedHash !== 'your-feed-hash-here') {
+        FEED_HASH = feedHash;
+        console.log(`Using feed hash from command line: ${FEED_HASH}`);
+    }
+}
+
+if (FEED_HASH === 'your-feed-hash-here') {
+    console.error('Please provide a valid feed hash as a command line argument:');
+    console.error('node test-concurrent-streams.js <feed-hash>');
+    console.error('\nOr edit the FEED_HASH constant in this script.');
+    process.exit(1);
+}
+
+// Start the test
+runConcurrentTest().catch(error => {
+    console.error('Test failed:', error);
+    process.exit(1);
+});

package/documentation/streaming-feeds-full-results-solution.md

@@ -1,230 +0,0 @@
-# Solution: Full Results Delivery for Streamed Feeds Endpoint
-
-## Problem Analysis
-
-The current streaming feeds endpoint has a critical limitation: it only delivers the first 100 results from the initial query and then switches to push-only mode. This means clients never receive complete historical data for feeds with more than 100 matching results.
-
-## Proposed Solution: Continuous Initial Query with Pagination
-
-### Core Design Principle
-
-Modify the streaming implementation to **continue fetching and delivering all historical results** before transitioning to real-time push mode, while maintaining the streaming connection throughout the process.
-
-### Implementation Strategy
-
-#### 1. Enhanced `streamFeed` Method
-
-```typescript
-private async streamFeed(user: RequestUser | null, params: { [key: string]: string }, query: qs.ParsedQs): Promise<Stream<FeedResponse> | null> {
-    const feedHash = params["hash"];
-    if (!feedHash) {
-        return null;
-    }
-
-    const feedDefinition = await this.feedCache.getFeed(feedHash);
-    if (!feedDefinition) {
-        return null;
-    }
-
-    let bookmark = query["b"] as string ?? "";
-    const userIdentity = serializeUserIdentity(user);
-    const start = feedDefinition.feed.given.map(label => feedDefinition.namedStart[label.name]);
-    const givenHash = computeObjectHash(feedDefinition.namedStart);
-
-    const stream = new Stream<FeedResponse>();
-    
-    // NEW: Continuous initial query until exhausted
-    bookmark = await this.streamAllInitialResults(userIdentity, feedDefinition, start, bookmark, stream);
-    
-    // Set up real-time listeners after initial data is complete
-    const inverses = invertSpecification(feedDefinition.feed);
-    const listeners = inverses.map(inverse => this.factManager.addSpecificationListener(
-        inverse.inverseSpecification,
-        async (results) => {
-            const matchingResults = results.filter(pr =>
-                givenHash === computeTupleSubsetHash(pr.tuple, inverse.givenSubset));
-            if (matchingResults.length != 0) {
-                bookmark = await this.streamFeedResponse(userIdentity, feedDefinition, start, bookmark, stream, true);
-            }
-        }
-    ));
-    
-    stream.done(() => {
-        for (const listener of listeners) {
-            this.factManager.removeSpecificationListener(listener);
-        }
-    });
-    
-    return stream;
-}
-```
-
-#### 2. New `streamAllInitialResults` Method
-
-```typescript
-private async streamAllInitialResults(
-    userIdentity: UserIdentity | null, 
-    feedDefinition: FeedObject, 
-    start: FactReference[], 
-    initialBookmark: string, 
-    stream: Stream<FeedResponse>
-): Promise<string> {
-    let bookmark = initialBookmark;
-    let hasMoreResults = true;
-    let pageCount = 0;
-    const maxPages = 1000; // Safety limit to prevent infinite loops
-    
-    while (hasMoreResults && pageCount < maxPages) {
-        const results = await this.authorization.feed(userIdentity, feedDefinition.feed, start, bookmark);
-        
-        // Check if we got results
-        if (results.tuples.length === 0) {
-            hasMoreResults = false;
-            break;
-        }
-        
-        // Process and send results
-        const references = results.tuples.flatMap(t => t.facts).filter((value, index, self) => 
-            self.findIndex(f => f.hash === value.hash && f.type === value.type) === index
-        );
-        
-        if (references.length > 0) {
-            const response: FeedResponse = {
-                references,
-                bookmark: results.bookmark
-            };
-            stream.feed(response);
-        }
-        
-        // Update bookmark for next iteration
-        const newBookmark = results.bookmark;
-        if (newBookmark === bookmark) {
-            // No progress made, avoid infinite loop
-            hasMoreResults = false;
-        } else {
-            bookmark = newBookmark;
-        }
-        
-        // Check if we got fewer results than the page size (indicates end)
-        if (results.tuples.length < 100) {
-            hasMoreResults = false;
-        }
-        
-        pageCount++;
-        
-        // Add small delay to prevent overwhelming the database
-        if (hasMoreResults && pageCount % 10 === 0) {
-            await new Promise(resolve => setTimeout(resolve, 10));
-        }
-    }
-    
-    return bookmark;
-}
-```
-
-### Key Design Features
-
-#### 1. **Continuous Pagination**
-- Loops through all available pages until no more results
-- Uses bookmark progression to detect completion
-- Includes safety limits to prevent infinite loops
-
-#### 2. **Stream Integrity**
-- Maintains single persistent connection throughout
-- Delivers results incrementally as pages are fetched
-- No interruption to client connection
-
-#### 3. **Performance Safeguards**
-- Maximum page limit (1000 pages = 100,000 results max)
-- Periodic delays to prevent database overload
-- Early termination on duplicate bookmarks
-
-#### 4. **Backward Compatibility**
-- Preserves existing API contract
-- Maintains same response format
-- No changes required to client implementations
-
-### Alternative Approaches Considered
-
-#### Option A: Async Background Fetching
-**Rejected**: Would complicate error handling and resource cleanup
-
-#### Option B: Configurable Page Limits
-**Rejected**: Adds complexity without solving the core issue
-
-#### Option C: Separate Historical vs Real-time Endpoints
-**Rejected**: Breaks the unified streaming model
-
-### Implementation Considerations
-
-#### 1. **Database Impact**
-- Multiple sequential queries instead of single query
-- Potential increased load on connection pool
-- Mitigation: Rate limiting and connection reuse
-
-#### 2. **Memory Management**
-- Stream queue may grow during rapid pagination
-- Mitigation: Client processing should keep pace with delivery
-- Consider: Optional memory pressure monitoring
-
-#### 3. **Timeout Handling**
-- 5-minute stream timeout may be insufficient for large datasets
-- Consider: Dynamic timeout based on data volume
-- Alternative: Configurable timeout per feed
-
-#### 4. **Error Recovery**
-- Database errors during pagination should not break stream
-- Implement: Retry logic for individual page failures
-- Fallback: Continue from last successful bookmark
-
-### Testing Strategy
-
-#### 1. **Unit Tests**
-- Test pagination logic with various result set sizes
-- Verify bookmark progression and termination conditions
-- Test error scenarios and recovery
-
-#### 2. **Integration Tests**
-- Large dataset streaming (1000+ results)
-- Concurrent stream handling
-- Database connection pool behavior under load
-
-#### 3. **Performance Tests**
-- Memory usage during large result streaming
-- Database query performance impact
-- Client processing capability validation
-
-### Deployment Considerations
-
-#### 1. **Gradual Rollout**
-- Feature flag to enable/disable full pagination
-- Monitor database performance impact
-- Rollback capability if issues arise
-
-#### 2. **Monitoring**
-- Track average pages per stream
-- Monitor database connection pool utilization
-- Alert on streams exceeding safety limits
-
-#### 3. **Configuration**
-- Make page limit configurable
-- Allow timeout adjustment per deployment
-- Enable/disable rate limiting
-
-## Expected Outcomes
-
-### Benefits
-1. **Complete Data Delivery**: Clients receive all historical results
-2. **Unified Experience**: Single endpoint for both historical and real-time data
-3. **Streaming Efficiency**: Maintains connection throughout data delivery
-4. **Backward Compatibility**: No breaking changes to existing clients
-
-### Trade-offs
-1. **Increased Database Load**: More queries per stream initialization
-2. **Longer Initial Response Time**: Complete data delivery takes longer
-3. **Memory Usage**: Larger stream queues during pagination
-4. **Complexity**: More sophisticated error handling required
-
-## Conclusion
-
-This solution addresses the core limitation while maintaining the streaming architecture's benefits. The continuous pagination approach ensures complete data delivery without breaking the real-time streaming model, providing a robust foundation for applications requiring both historical completeness and live updates.