@risingwave/wavelet-server 0.1.4 → 0.2.4

package/src/webhook.ts

@@ -0,0 +1,67 @@
+import { createHmac } from 'node:crypto'
+import type { QueryDef, SqlFragment } from '@risingwave/wavelet'
+import type { ViewDiff } from './cursor-manager.js'
+
+const TIMEOUT_MS = 10000
+
+export class WebhookFanout {
+  private webhooks: Map<string, string> = new Map() // queryName -> url
+
+  constructor(
+    queries: Record<string, QueryDef | SqlFragment>,
+    private signingSecret?: string
+  ) {
+    for (const [name, def] of Object.entries(queries)) {
+      if ('_tag' in def && def._tag === 'sql') continue
+      const qd = def as QueryDef
+      if (qd.webhook) {
+        this.webhooks.set(name, qd.webhook)
+      }
+    }
+  }
+
+  async broadcast(queryName: string, diff: ViewDiff): Promise<void> {
+    const url = this.webhooks.get(queryName)
+    if (!url) return
+
+    const body = JSON.stringify({
+      query: queryName,
+      cursor: diff.cursor,
+      inserted: diff.inserted,
+      updated: diff.updated,
+      deleted: diff.deleted,
+    })
+
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'User-Agent': 'Wavelet-Webhook/1.0',
+    }
+
+    if (this.signingSecret) {
+      const signature = createHmac('sha256', this.signingSecret)
+        .update(body)
+        .digest('hex')
+      headers['X-Wavelet-Signature'] = `sha256=${signature}`
+    }
+
+    try {
+      const controller = new AbortController()
+      const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS)
+
+      await fetch(url, {
+        method: 'POST',
+        headers,
+        body,
+        signal: controller.signal,
+      })
+
+      clearTimeout(timeout)
+    } catch (err: any) {
+      if (err.name === 'AbortError') {
+        console.error(`[webhook] Timeout sending to ${url} for query ${queryName}`)
+      } else {
+        console.error(`[webhook] Failed to send to ${url} for query ${queryName}:`, err.message)
+      }
+    }
+  }
+}

package/src/ws-fanout.ts

@@ -1,23 +1,25 @@
 import { WebSocketServer, WebSocket } from 'ws'
 import type { IncomingMessage, Server } from 'node:http'
-import type { ViewDef, SqlFragment } from '@risingwave/wavelet'
+import type { QueryDef, SqlFragment } from '@risingwave/wavelet'
 import type { CursorManager, ViewDiff } from './cursor-manager.js'
 import type { JwtVerifier, JwtClaims } from './jwt.js'
 
 interface Subscriber {
   ws: WebSocket
-  viewName: string
+  queryName: string
   claims: JwtClaims | null
+  ready: boolean
+  pendingDiffs: ViewDiff[]
 }
 
 export class WebSocketFanout {
   private wss: WebSocketServer | null = null
-  private subscribers: Map<string, Set<Subscriber>> = new Map() // viewName -> subscribers
+  private subscribers: Map<string, Set<Subscriber>> = new Map() // queryName -> subscribers
 
   constructor(
     private cursorManager: CursorManager,
     private jwt: JwtVerifier,
-    private views: Record<string, ViewDef | SqlFragment>
+    private queries: Record<string, QueryDef | SqlFragment>
   ) {}
 
   attach(server: Server, pathPrefix?: string): void {
@@ -54,17 +56,17 @@ export class WebSocketFanout {
 
     if (!match) {
       throw new Error(
-        `Invalid path: ${url.pathname}. Use /subscribe/{viewName}. ` +
-        `Available views: ${Object.keys(this.views).join(', ')}`
+        `Invalid path: ${url.pathname}. Use /subscribe/{queryName}. ` +
+        `Available queries: ${Object.keys(this.queries).join(', ')}`
       )
     }
 
-    const viewName = match[1]
+    const queryName = match[1]
 
-    if (!this.views[viewName]) {
-      const available = Object.keys(this.views)
+    if (!this.queries[queryName]) {
+      const available = Object.keys(this.queries)
       throw new Error(
-        `View '${viewName}' not found. Available views: ${available.join(', ')}`
+        `Query '${queryName}' not found. Available queries: ${available.join(', ')}`
       )
     }
 
@@ -80,57 +82,119 @@ export class WebSocketFanout {
       claims = await this.jwt.verify(token)
     }
 
-    const subscriber: Subscriber = { ws, viewName, claims }
+    const subscriber: Subscriber = {
+      ws,
+      queryName,
+      claims,
+      ready: false,
+      pendingDiffs: [],
+    }
 
-    if (!this.subscribers.has(viewName)) {
-      this.subscribers.set(viewName, new Set())
+    if (!this.subscribers.has(queryName)) {
+      this.subscribers.set(queryName, new Set())
     }
-    this.subscribers.get(viewName)!.add(subscriber)
+    this.subscribers.get(queryName)!.add(subscriber)
 
     ws.on('close', () => {
-      this.subscribers.get(viewName)?.delete(subscriber)
+      this.subscribers.get(queryName)?.delete(subscriber)
     })
 
-    ws.send(JSON.stringify({ type: 'connected', view: viewName }))
+    // Heartbeat: detect dead connections behind proxies/load balancers
+    const pingInterval = setInterval(() => {
+      if (ws.readyState === WebSocket.OPEN) {
+        ws.ping()
+      }
+    }, 30000)
+    ws.on('close', () => clearInterval(pingInterval))
+    ws.on('pong', () => { /* connection alive */ })
+
+    ws.send(JSON.stringify({ type: 'connected', query: queryName }))
+
+    const bootstrap = await this.cursorManager.bootstrap(queryName)
+    const snapshotRows = this.filterSnapshotRows(queryName, bootstrap.snapshotRows, claims)
+    ws.send(JSON.stringify({
+      type: 'snapshot',
+      query: queryName,
+      rows: snapshotRows,
+    }))
+
+    for (const diff of bootstrap.diffs) {
+      const filteredDiff = this.filterDiffForSubscriber(queryName, diff, claims)
+      if (this.isEmptyDiff(filteredDiff)) continue
+      if (ws.readyState !== WebSocket.OPEN) break
+      ws.send(this.serializeDiffMessage(queryName, filteredDiff))
+    }
+
+    const handoffCursor = bootstrap.lastCursor
+    subscriber.ready = true
+    for (const diff of subscriber.pendingDiffs) {
+      if (ws.readyState !== WebSocket.OPEN) break
+      if (handoffCursor && this.compareCursor(diff.cursor, handoffCursor) <= 0) {
+        continue
+      }
+      ws.send(this.serializeDiffMessage(queryName, diff))
+    }
+    subscriber.pendingDiffs = []
   }
 
-  broadcast(viewName: string, diff: ViewDiff): void {
-    const subs = this.subscribers.get(viewName)
+  broadcast(queryName: string, diff: ViewDiff): void {
+    const subs = this.subscribers.get(queryName)
     if (!subs || subs.size === 0) return
 
-    const viewDef = this.views[viewName]
-    const filterBy = this.getFilterBy(viewDef)
-
     for (const sub of subs) {
       if (sub.ws.readyState !== WebSocket.OPEN) continue
 
-      let filteredDiff = diff
-      if (filterBy && sub.claims) {
-        filteredDiff = this.filterDiff(diff, filterBy, sub.claims)
-      }
+      const filteredDiff = this.filterDiffForSubscriber(queryName, diff, sub.claims)
+      if (this.isEmptyDiff(filteredDiff)) continue
 
-      if (
-        filteredDiff.inserted.length === 0 &&
-        filteredDiff.updated.length === 0 &&
-        filteredDiff.deleted.length === 0
-      ) {
+      if (!sub.ready) {
+        sub.pendingDiffs.push(filteredDiff)
         continue
       }
 
-      sub.ws.send(JSON.stringify({
-        type: 'diff',
-        view: viewName,
-        cursor: filteredDiff.cursor,
-        inserted: filteredDiff.inserted,
-        updated: filteredDiff.updated,
-        deleted: filteredDiff.deleted,
-      }))
+      sub.ws.send(this.serializeDiffMessage(queryName, filteredDiff))
     }
   }
 
+  private filterSnapshotRows(
+    queryName: string,
+    rows: Record<string, unknown>[],
+    claims: JwtClaims | null
+  ): Record<string, unknown>[] {
+    const queryDef = this.queries[queryName]
+    const filterBy = this.getFilterBy(queryDef)
+
+    if (filterBy && claims) {
+      const claimValue = claims[filterBy]
+      if (claimValue === undefined) return []
+
+      return rows.filter((row) => String(row[filterBy]) === String(claimValue))
+    }
+
+    return rows
+  }
+
+  private filterDiffForSubscriber(
+    queryName: string,
+    diff: ViewDiff,
+    claims: JwtClaims | null
+  ): ViewDiff {
+    const queryDef = this.queries[queryName]
+    const filterBy = this.getFilterBy(queryDef)
+
+    if (filterBy && claims) {
+      return this.filterDiff(diff, filterBy, claims)
+    }
+
+    return diff
+  }
+
   private filterDiff(diff: ViewDiff, filterBy: string, claims: JwtClaims): ViewDiff {
     const claimValue = claims[filterBy]
-    if (claimValue === undefined) return diff
+    if (claimValue === undefined) {
+      // No matching claim -- return empty diff, not all data
+      return { cursor: diff.cursor, inserted: [], updated: [], deleted: [] }
+    }
 
     const filterFn = (row: Record<string, unknown>) =>
       String(row[filterBy]) === String(claimValue)
@@ -143,9 +207,31 @@ export class WebSocketFanout {
     }
   }
 
-  private getFilterBy(viewDef: ViewDef | SqlFragment): string | undefined {
-    if ('_tag' in viewDef && viewDef._tag === 'sql') return undefined
-    return (viewDef as ViewDef).filterBy
+  private getFilterBy(queryDef: QueryDef | SqlFragment): string | undefined {
+    if ('_tag' in queryDef && queryDef._tag === 'sql') return undefined
+    return (queryDef as QueryDef).filterBy
+  }
+
+  private isEmptyDiff(diff: ViewDiff): boolean {
+    return diff.inserted.length === 0 && diff.updated.length === 0 && diff.deleted.length === 0
+  }
+
+  private serializeDiffMessage(queryName: string, diff: ViewDiff): string {
+    return JSON.stringify({
+      type: 'diff',
+      query: queryName,
+      cursor: diff.cursor,
+      inserted: diff.inserted,
+      updated: diff.updated,
+      deleted: diff.deleted,
+    })
+  }
+
+  private compareCursor(left: string, right: string): number {
+    const leftValue = BigInt(left)
+    const rightValue = BigInt(right)
+    if (leftValue === rightValue) return 0
+    return leftValue < rightValue ? -1 : 1
   }
 
   closeAll(): void {