@risingwave/wavelet-server 0.1.0

package/src/http-api.ts

@@ -0,0 +1,199 @@
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import pg from 'pg'
+import type { StreamDef, ViewDef, SqlFragment } from '@risingwave/wavelet'
+
+const { Client } = pg
+
+export class HttpApi {
+  private client: InstanceType<typeof Client> | null = null
+
+  constructor(
+    private connectionString: string,
+    private streams: Record<string, StreamDef>,
+    private views: Record<string, ViewDef | SqlFragment>
+  ) {}
+
+  async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    // CORS
+    res.setHeader('Access-Control-Allow-Origin', '*')
+    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization')
+
+    if (req.method === 'OPTIONS') {
+      res.writeHead(204)
+      res.end()
+      return
+    }
+
+    const url = new URL(req.url ?? '/', `http://${req.headers.host}`)
+
+    try {
+      // POST /v1/streams/{name}
+      const streamMatch = url.pathname.match(/^\/v1\/streams\/([^/]+)$/)
+      if (streamMatch && req.method === 'POST') {
+        await this.handleWrite(streamMatch[1], req, res)
+        return
+      }
+
+      // POST /v1/streams/{name}/batch
+      const batchMatch = url.pathname.match(/^\/v1\/streams\/([^/]+)\/batch$/)
+      if (batchMatch && req.method === 'POST') {
+        await this.handleBatchWrite(batchMatch[1], req, res)
+        return
+      }
+
+      // GET /v1/views/{name}
+      const viewMatch = url.pathname.match(/^\/v1\/views\/([^/]+)$/)
+      if (viewMatch && req.method === 'GET') {
+        await this.handleRead(viewMatch[1], url, res)
+        return
+      }
+
+      // GET /v1/health
+      if (url.pathname === '/v1/health') {
+        this.json(res, 200, { status: 'ok' })
+        return
+      }
+
+      // GET /v1/views - list available views
+      if (url.pathname === '/v1/views' && req.method === 'GET') {
+        this.json(res, 200, { views: Object.keys(this.views) })
+        return
+      }
+
+      // GET /v1/streams - list available streams
+      if (url.pathname === '/v1/streams' && req.method === 'GET') {
+        this.json(res, 200, { streams: Object.keys(this.streams) })
+        return
+      }
+
+      this.json(res, 404, {
+        error: 'Not found',
+        message: `${req.method} ${url.pathname} does not match any route.`,
+        routes: [
+          'GET  /v1/health',
+          'GET  /v1/views',
+          'GET  /v1/views/{name}',
+          'GET  /v1/streams',
+          'POST /v1/streams/{name}',
+          'POST /v1/streams/{name}/batch',
+        ],
+      })
+    } catch (err: any) {
+      console.error('HTTP error:', err)
+      this.json(res, 500, { error: err.message })
+    }
+  }
+
+  private async ensureClient(): Promise<InstanceType<typeof Client>> {
+    if (!this.client) {
+      this.client = new Client({ connectionString: this.connectionString })
+      await this.client.connect()
+    }
+    return this.client
+  }
+
+  private async handleWrite(streamName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
+    const stream = this.streams[streamName]
+    if (!stream) {
+      const available = Object.keys(this.streams)
+      this.json(res, 404, {
+        error: `Stream '${streamName}' not found.`,
+        available_streams: available,
+      })
+      return
+    }
+
+    const body = await this.readBody(req)
+    const data = JSON.parse(body)
+
+    const client = await this.ensureClient()
+    const columns = Object.keys(stream.columns)
+    const values = columns.map((col) => data[col])
+    const placeholders = columns.map((_, i) => `$${i + 1}`)
+
+    await client.query(
+      `INSERT INTO ${streamName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`,
+      values
+    )
+
+    this.json(res, 200, { ok: true })
+  }
+
+  private async handleBatchWrite(streamName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
+    const stream = this.streams[streamName]
+    if (!stream) {
+      const available = Object.keys(this.streams)
+      this.json(res, 404, {
+        error: `Stream '${streamName}' not found.`,
+        available_streams: available,
+      })
+      return
+    }
+
+    const body = await this.readBody(req)
+    const events: any[] = JSON.parse(body)
+
+    if (!Array.isArray(events)) {
+      this.json(res, 400, { error: 'Batch endpoint expects a JSON array.' })
+      return
+    }
+
+    const client = await this.ensureClient()
+    const columns = Object.keys(stream.columns)
+
+    for (const data of events) {
+      const values = columns.map((col) => data[col])
+      const placeholders = columns.map((_, i) => `$${i + 1}`)
+      await client.query(
+        `INSERT INTO ${streamName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`,
+        values
+      )
+    }
+
+    this.json(res, 200, { ok: true, count: events.length })
+  }
+
+  private async handleRead(viewName: string, url: URL, res: ServerResponse): Promise<void> {
+    if (!this.views[viewName]) {
+      const available = Object.keys(this.views)
+      this.json(res, 404, {
+        error: `View '${viewName}' not found.`,
+        available_views: available,
+      })
+      return
+    }
+
+    const client = await this.ensureClient()
+
+    // Build WHERE clause from query params
+    const params: string[] = []
+    const values: unknown[] = []
+    for (const [key, value] of url.searchParams.entries()) {
+      params.push(`${key} = $${params.length + 1}`)
+      values.push(value)
+    }
+
+    let sql = `SELECT * FROM ${viewName}`
+    if (params.length > 0) {
+      sql += ` WHERE ${params.join(' AND ')}`
+    }
+
+    const result = await client.query(sql, values)
+    this.json(res, 200, { view: viewName, rows: result.rows })
+  }
+
+  private json(res: ServerResponse, status: number, data: unknown): void {
+    res.writeHead(status, { 'Content-Type': 'application/json' })
+    res.end(JSON.stringify(data))
+  }
+
+  private readBody(req: IncomingMessage): Promise<string> {
+    return new Promise((resolve, reject) => {
+      let body = ''
+      req.on('data', (chunk) => { body += chunk })
+      req.on('end', () => resolve(body))
+      req.on('error', reject)
+    })
+  }
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+export { WaveletServer } from './server.js'
+export { loadConfig } from './config-loader.js'
+export { DdlManager } from './ddl-manager.js'
+export type { DdlAction } from './ddl-manager.js'
+
+async function main() {
+  const { loadConfig } = await import('./config-loader.js')
+  const { WaveletServer } = await import('./server.js')
+
+  const configPath = process.argv[2] || './wavelet.config.ts'
+  const config = await loadConfig(configPath)
+
+  const server = new WaveletServer(config)
+  await server.start()
+
+  const shutdown = async () => {
+    console.log('\nShutting down...')
+    await server.stop()
+    process.exit(0)
+  }
+  process.on('SIGINT', shutdown)
+  process.on('SIGTERM', shutdown)
+}
+
+// Only run main() when executed directly, not when imported as a library
+if (require.main === module) {
+  main().catch((err) => {
+    console.error('Fatal:', err)
+    process.exit(1)
+  })
+}

package/src/jwt.ts

@@ -0,0 +1,56 @@
+import * as jose from 'jose'
+
+export type JwtClaims = Record<string, unknown>
+
+export class JwtVerifier {
+  private secret: Uint8Array | null = null
+  private jwksUrl: string | null = null
+  private issuer: string | undefined
+  private audience: string | undefined
+
+  constructor(config?: { secret?: string; jwksUrl?: string; issuer?: string; audience?: string }) {
+    if (config?.secret) {
+      this.secret = new TextEncoder().encode(config.secret)
+    }
+    if (config?.jwksUrl) {
+      this.jwksUrl = config.jwksUrl
+    }
+    this.issuer = config?.issuer
+    this.audience = config?.audience
+  }
+
+  isConfigured(): boolean {
+    return this.secret !== null || this.jwksUrl !== null
+  }
+
+  async verify(token: string): Promise<JwtClaims> {
+    try {
+      if (this.secret) {
+        const { payload } = await jose.jwtVerify(token, this.secret, {
+          issuer: this.issuer,
+          audience: this.audience,
+        })
+        return payload as JwtClaims
+      }
+
+      if (this.jwksUrl) {
+        const jwks = jose.createRemoteJWKSet(new URL(this.jwksUrl))
+        const { payload } = await jose.jwtVerify(token, jwks, {
+          issuer: this.issuer,
+          audience: this.audience,
+        })
+        return payload as JwtClaims
+      }
+
+      throw new Error('JWT verification not configured')
+    } catch (err: any) {
+      if (err.code === 'ERR_JWT_EXPIRED') {
+        throw new Error('Token expired. Please refresh your authentication token.')
+      }
+      if (err.code === 'ERR_JWS_SIGNATURE_VERIFICATION_FAILED') {
+        throw new Error('Invalid token signature. Check your JWT secret configuration.')
+      }
+      throw new Error(`Authentication failed: ${err.message}`)
+    }
+  }
+}

package/src/server.ts

@@ -0,0 +1,82 @@
+import { createServer, type IncomingMessage, type ServerResponse, type Server } from 'node:http'
+import type { WaveletConfig } from '@risingwave/wavelet'
+import { CursorManager } from './cursor-manager.js'
+import { WebSocketFanout } from './ws-fanout.js'
+import { HttpApi } from './http-api.js'
+import { JwtVerifier } from './jwt.js'
+
+export class WaveletServer {
+  private httpServer: ReturnType<typeof createServer> | null = null
+  private cursorManager: CursorManager
+  private fanout: WebSocketFanout
+  private httpApi: HttpApi
+  private jwt: JwtVerifier
+
+  constructor(private config: WaveletConfig) {
+    this.jwt = new JwtVerifier(config.jwt)
+    this.cursorManager = new CursorManager(config.database, config.views ?? {})
+    this.fanout = new WebSocketFanout(this.cursorManager, this.jwt, config.views ?? {})
+    this.httpApi = new HttpApi(config.database, config.streams ?? {}, config.views ?? {})
+  }
+
+  async start(): Promise<void> {
+    const port = this.config.server?.port ?? 8080
+    const host = this.config.server?.host ?? '0.0.0.0'
+
+    this.httpServer = createServer((req, res) => this.httpApi.handle(req, res))
+    this.fanout.attach(this.httpServer)
+
+    await this.cursorManager.initialize()
+    this.cursorManager.startPolling((viewName, diffs) => {
+      this.fanout.broadcast(viewName, diffs)
+    })
+
+    await new Promise<void>((resolve) => {
+      this.httpServer!.listen(port, host, () => {
+        console.log(`Wavelet server listening on ${host}:${port}`)
+        console.log(`WebSocket: ws://${host}:${port}/subscribe/{view}`)
+        console.log(`HTTP API:  http://${host}:${port}/v1/`)
+        resolve()
+      })
+    })
+  }
+
+  /**
+   * Attach to an existing HTTP server instead of creating one.
+   * Used by Wavelet Cloud to run multiple instances on a shared port.
+   */
+  async attachTo(server: Server, opts?: { pathPrefix?: string }): Promise<{
+    handleHttp: (req: IncomingMessage, res: ServerResponse) => void
+  }> {
+    const prefix = opts?.pathPrefix ?? ''
+
+    this.fanout.attach(server, prefix)
+
+    await this.cursorManager.initialize()
+    this.cursorManager.startPolling((viewName, diffs) => {
+      this.fanout.broadcast(viewName, diffs)
+    })
+
+    const handleHttp = (req: IncomingMessage, res: ServerResponse) => {
+      if (prefix && req.url?.startsWith(prefix)) {
+        req.url = req.url.slice(prefix.length) || '/'
+      }
+      this.httpApi.handle(req, res)
+    }
+
+    return { handleHttp }
+  }
+
+  async stop(): Promise<void> {
+    this.cursorManager.stopPolling()
+    this.fanout.closeAll()
+    await this.cursorManager.close()
+    await new Promise<void>((resolve) => {
+      if (this.httpServer) {
+        this.httpServer.close(() => resolve())
+      } else {
+        resolve()
+      }
+    })
+  }
+}

package/src/ws-fanout.ts

@@ -0,0 +1,159 @@
+import { WebSocketServer, WebSocket } from 'ws'
+import type { IncomingMessage, Server } from 'node:http'
+import type { ViewDef, SqlFragment } from '@risingwave/wavelet'
+import type { CursorManager, ViewDiff } from './cursor-manager.js'
+import type { JwtVerifier, JwtClaims } from './jwt.js'
+
+interface Subscriber {
+  ws: WebSocket
+  viewName: string
+  claims: JwtClaims | null
+}
+
+export class WebSocketFanout {
+  private wss: WebSocketServer | null = null
+  private subscribers: Map<string, Set<Subscriber>> = new Map() // viewName -> subscribers
+
+  constructor(
+    private cursorManager: CursorManager,
+    private jwt: JwtVerifier,
+    private views: Record<string, ViewDef | SqlFragment>
+  ) {}
+
+  attach(server: Server, pathPrefix?: string): void {
+    this.wss = new WebSocketServer({ noServer: true })
+
+    server.on('upgrade', (req, socket, head) => {
+      const pathname = req.url?.split('?')[0] ?? ''
+      const subscribePrefix = (pathPrefix ?? '') + '/subscribe/'
+
+      if (pathname.startsWith(subscribePrefix)) {
+        this.wss!.handleUpgrade(req, socket, head, (ws) => {
+          this.wss!.emit('connection', ws, req)
+        })
+      }
+    })
+
+    this.wss.on('connection', async (ws: WebSocket, req: IncomingMessage) => {
+      // Strip path prefix before handling
+      if (pathPrefix && req.url?.startsWith(pathPrefix)) {
+        req.url = req.url.slice(pathPrefix.length)
+      }
+      try {
+        await this.handleConnection(ws, req)
+      } catch (err: any) {
+        ws.send(JSON.stringify({ error: err.message }))
+        ws.close(4000, err.message)
+      }
+    })
+  }
+
+  private async handleConnection(ws: WebSocket, req: IncomingMessage): Promise<void> {
+    const url = new URL(req.url ?? '', `http://${req.headers.host ?? 'localhost'}`)
+    const match = url.pathname?.match(/^\/subscribe\/(.+)$/)
+
+    if (!match) {
+      throw new Error(
+        `Invalid path: ${url.pathname}. Use /subscribe/{viewName}. ` +
+        `Available views: ${Object.keys(this.views).join(', ')}`
+      )
+    }
+
+    const viewName = match[1]
+
+    if (!this.views[viewName]) {
+      const available = Object.keys(this.views)
+      throw new Error(
+        `View '${viewName}' not found. Available views: ${available.join(', ')}`
+      )
+    }
+
+    // Verify JWT if configured
+    let claims: JwtClaims | null = null
+    const token = url.searchParams.get('token')
+      ?? req.headers.authorization?.replace('Bearer ', '')
+
+    if (this.jwt.isConfigured()) {
+      if (!token) {
+        throw new Error('Authentication required. Pass token as query param or Authorization header.')
+      }
+      claims = await this.jwt.verify(token)
+    }
+
+    const subscriber: Subscriber = { ws, viewName, claims }
+
+    if (!this.subscribers.has(viewName)) {
+      this.subscribers.set(viewName, new Set())
+    }
+    this.subscribers.get(viewName)!.add(subscriber)
+
+    ws.on('close', () => {
+      this.subscribers.get(viewName)?.delete(subscriber)
+    })
+
+    ws.send(JSON.stringify({ type: 'connected', view: viewName }))
+  }
+
+  broadcast(viewName: string, diff: ViewDiff): void {
+    const subs = this.subscribers.get(viewName)
+    if (!subs || subs.size === 0) return
+
+    const viewDef = this.views[viewName]
+    const filterBy = this.getFilterBy(viewDef)
+
+    for (const sub of subs) {
+      if (sub.ws.readyState !== WebSocket.OPEN) continue
+
+      let filteredDiff = diff
+      if (filterBy && sub.claims) {
+        filteredDiff = this.filterDiff(diff, filterBy, sub.claims)
+      }
+
+      if (
+        filteredDiff.inserted.length === 0 &&
+        filteredDiff.updated.length === 0 &&
+        filteredDiff.deleted.length === 0
+      ) {
+        continue
+      }
+
+      sub.ws.send(JSON.stringify({
+        type: 'diff',
+        view: viewName,
+        cursor: filteredDiff.cursor,
+        inserted: filteredDiff.inserted,
+        updated: filteredDiff.updated,
+        deleted: filteredDiff.deleted,
+      }))
+    }
+  }
+
+  private filterDiff(diff: ViewDiff, filterBy: string, claims: JwtClaims): ViewDiff {
+    const claimValue = claims[filterBy]
+    if (claimValue === undefined) return diff
+
+    const filterFn = (row: Record<string, unknown>) =>
+      String(row[filterBy]) === String(claimValue)
+
+    return {
+      cursor: diff.cursor,
+      inserted: diff.inserted.filter(filterFn),
+      updated: diff.updated.filter(filterFn),
+      deleted: diff.deleted.filter(filterFn),
+    }
+  }
+
+  private getFilterBy(viewDef: ViewDef | SqlFragment): string | undefined {
+    if ('_tag' in viewDef && viewDef._tag === 'sql') return undefined
+    return (viewDef as ViewDef).filterBy
+  }
+
+  closeAll(): void {
+    for (const [, subs] of this.subscribers) {
+      for (const sub of subs) {
+        sub.ws.close(1001, 'Server shutting down')
+      }
+    }
+    this.wss?.close()
+  }
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src"]
+}