@risingwave/wavelet-server 0.1.0

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@risingwave/wavelet-server",
+  "version": "0.1.0",
+  "description": "Wavelet server - WebSocket fanout layer for RisingWave",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx watch src/index.ts",
+    "start": "node dist/index.js"
+  },
+  "dependencies": {
+    "pg": "^8.13.0",
+    "ws": "^8.18.0",
+    "jose": "^6.0.0",
+    "@risingwave/wavelet": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/pg": "^8.11.0",
+    "@types/ws": "^8.5.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.7.0"
+  },
+  "license": "Apache-2.0"
+}

package/src/__tests__/cursor-parsing.test.ts

@@ -0,0 +1,68 @@
+import { describe, it, expect } from 'vitest'
+import { CursorManager } from '../cursor-manager.js'
+
+// Access private parseDiffs via cast
+function parseDiffs(rows: any[]) {
+  const cm = new CursorManager('postgres://dummy', {})
+  return (cm as any).parseDiffs(rows)
+}
+
+describe('parseDiffs', () => {
+  it('parses Insert ops (string form)', () => {
+    const rows = [
+      { op: 'Insert', rw_timestamp: '100', player_id: 'alice', score: 42 },
+    ]
+    const diff = parseDiffs(rows)
+    expect(diff.inserted).toHaveLength(1)
+    expect(diff.inserted[0]).toEqual({ player_id: 'alice', score: 42 })
+    expect(diff.deleted).toHaveLength(0)
+    expect(diff.updated).toHaveLength(0)
+  })
+
+  it('parses Insert ops (numeric form)', () => {
+    const rows = [
+      { op: '1', rw_timestamp: '100', name: 'bob' },
+    ]
+    const diff = parseDiffs(rows)
+    expect(diff.inserted).toHaveLength(1)
+    expect(diff.inserted[0]).toEqual({ name: 'bob' })
+  })
+
+  it('parses Delete ops', () => {
+    const rows = [
+      { op: 'Delete', rw_timestamp: '200', player_id: 'alice' },
+    ]
+    const diff = parseDiffs(rows)
+    expect(diff.deleted).toHaveLength(1)
+    expect(diff.deleted[0]).toEqual({ player_id: 'alice' })
+  })
+
+  it('parses Update ops (UpdateDelete + UpdateInsert)', () => {
+    const rows = [
+      { op: 'UpdateDelete', rw_timestamp: '300', player_id: 'alice', score: 10 },
+      { op: 'UpdateInsert', rw_timestamp: '300', player_id: 'alice', score: 20 },
+    ]
+    const diff = parseDiffs(rows)
+    expect(diff.deleted).toHaveLength(1)
+    expect(diff.deleted[0]).toEqual({ player_id: 'alice', score: 10 })
+    expect(diff.updated).toHaveLength(1)
+    expect(diff.updated[0]).toEqual({ player_id: 'alice', score: 20 })
+  })
+
+  it('returns empty diff for no rows', () => {
+    const diff = parseDiffs([])
+    expect(diff.inserted).toHaveLength(0)
+    expect(diff.updated).toHaveLength(0)
+    expect(diff.deleted).toHaveLength(0)
+    expect(diff.cursor).toBe('')
+  })
+
+  it('extracts cursor from last rw_timestamp', () => {
+    const rows = [
+      { op: 'Insert', rw_timestamp: '100', x: 1 },
+      { op: 'Insert', rw_timestamp: '200', x: 2 },
+    ]
+    const diff = parseDiffs(rows)
+    expect(diff.cursor).toBe('200')
+  })
+})

package/src/__tests__/http-api.test.ts

@@ -0,0 +1,130 @@
+import { describe, it, expect } from 'vitest'
+import http from 'node:http'
+import { HttpApi } from '../http-api.js'
+
+function createTestServer(api: HttpApi): Promise<http.Server> {
+  return new Promise((resolve) => {
+    const server = http.createServer((req, res) => api.handle(req, res))
+    server.listen(0, () => resolve(server))
+  })
+}
+
+function request(server: http.Server, method: string, path: string, body?: unknown): Promise<{ status: number; data: any }> {
+  const addr = server.address() as any
+  return new Promise((resolve, reject) => {
+    const req = http.request({
+      hostname: 'localhost',
+      port: addr.port,
+      method,
+      path,
+      headers: body ? { 'Content-Type': 'application/json' } : {},
+    }, (res) => {
+      let data = ''
+      res.on('data', (chunk) => { data += chunk })
+      res.on('end', () => {
+        try {
+          resolve({ status: res.statusCode!, data: JSON.parse(data) })
+        } catch {
+          resolve({ status: res.statusCode!, data })
+        }
+      })
+    })
+    req.on('error', reject)
+    if (body) req.write(JSON.stringify(body))
+    req.end()
+  })
+}
+
+describe('HttpApi', () => {
+  const streams = {
+    events: { columns: { user_id: 'string' as const, value: 'int' as const } },
+  }
+  const views = {
+    leaderboard: { _tag: 'sql' as const, text: 'SELECT 1' },
+  }
+
+  it('returns health check', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'GET', '/v1/health')
+      expect(res.status).toBe(200)
+      expect(res.data).toEqual({ status: 'ok' })
+    } finally {
+      server.close()
+    }
+  })
+
+  it('lists available views', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'GET', '/v1/views')
+      expect(res.status).toBe(200)
+      expect(res.data).toEqual({ views: ['leaderboard'] })
+    } finally {
+      server.close()
+    }
+  })
+
+  it('lists available streams', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'GET', '/v1/streams')
+      expect(res.status).toBe(200)
+      expect(res.data).toEqual({ streams: ['events'] })
+    } finally {
+      server.close()
+    }
+  })
+
+  it('returns 404 for unknown routes with helpful message', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'GET', '/v1/nonexistent')
+      expect(res.status).toBe(404)
+      expect(res.data.error).toBe('Not found')
+      expect(res.data.routes).toBeDefined()
+      expect(res.data.routes.length).toBeGreaterThan(0)
+    } finally {
+      server.close()
+    }
+  })
+
+  it('handles CORS preflight', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'OPTIONS', '/v1/views')
+      expect(res.status).toBe(204)
+    } finally {
+      server.close()
+    }
+  })
+
+  it('returns 404 for unknown stream', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'POST', '/v1/streams/nonexistent', { x: 1 })
+      expect(res.status).toBe(404)
+      expect(res.data.available_streams).toEqual(['events'])
+    } finally {
+      server.close()
+    }
+  })
+
+  it('returns 404 for unknown view', async () => {
+    const api = new HttpApi('postgres://dummy', streams, views)
+    const server = await createTestServer(api)
+    try {
+      const res = await request(server, 'GET', '/v1/views/nonexistent')
+      expect(res.status).toBe(404)
+      expect(res.data.available_views).toEqual(['leaderboard'])
+    } finally {
+      server.close()
+    }
+  })
+})

package/src/__tests__/integration.test.ts

@@ -0,0 +1,217 @@
+/**
+ * Integration tests that require a running RisingWave instance.
+ * Skipped in CI unless WAVELET_TEST_DATABASE_URL is set.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import pg from 'pg'
+import http from 'node:http'
+import { WebSocket } from 'ws'
+import { DdlManager } from '../ddl-manager.js'
+import { WaveletServer } from '../server.js'
+import { sql } from '@risingwave/wavelet'
+import type { WaveletConfig } from '@risingwave/wavelet'
+
+const { Client } = pg
+
+const DATABASE_URL = process.env.WAVELET_TEST_DATABASE_URL ?? 'postgres://root@localhost:4566/dev'
+
+// Check if RisingWave is reachable
+async function isRisingWaveAvailable(): Promise<boolean> {
+  const client = new Client({ connectionString: DATABASE_URL, connectionTimeoutMillis: 2000 })
+  try {
+    await client.connect()
+    await client.query('SELECT 1')
+    await client.end()
+    return true
+  } catch {
+    return false
+  }
+}
+
+// Use unique names to avoid collisions with other test runs
+const prefix = `test_${Date.now()}`
+const STREAM_NAME = `${prefix}_events`
+const VIEW_NAME = `${prefix}_totals`
+const SUB_NAME = `wavelet_sub_${VIEW_NAME}`
+
+const testConfig: WaveletConfig = {
+  database: DATABASE_URL,
+  streams: {
+    [STREAM_NAME]: {
+      columns: {
+        user_id: 'string',
+        value: 'int',
+      },
+    },
+  },
+  views: {
+    [VIEW_NAME]: sql`
+      SELECT user_id, SUM(value) AS total_value, COUNT(*) AS event_count
+      FROM ${STREAM_NAME}
+      GROUP BY user_id
+    `,
+  },
+  server: {
+    port: 0, // random port
+  },
+}
+
+describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: DDL Manager', () => {
+  let ddl: DdlManager
+
+  beforeAll(async () => {
+    ddl = new DdlManager(DATABASE_URL)
+    await ddl.connect()
+  })
+
+  afterAll(async () => {
+    // Cleanup: drop test objects
+    const client = new Client({ connectionString: DATABASE_URL })
+    await client.connect()
+    try { await client.query(`DROP SUBSCRIPTION IF EXISTS ${SUB_NAME}`) } catch {}
+    try { await client.query(`DROP MATERIALIZED VIEW IF EXISTS ${VIEW_NAME}`) } catch {}
+    try { await client.query(`DROP TABLE IF EXISTS ${STREAM_NAME}`) } catch {}
+    await client.end()
+    await ddl.close()
+  })
+
+  it('creates tables, views, and subscriptions', async () => {
+    const actions = await ddl.sync(testConfig)
+
+    const creates = actions.filter(a => a.type === 'create')
+    expect(creates.length).toBeGreaterThanOrEqual(3) // stream + view + subscription
+
+    const streamAction = creates.find(a => a.resource === 'stream' && a.name === STREAM_NAME)
+    expect(streamAction).toBeDefined()
+
+    const viewAction = creates.find(a => a.resource === 'view' && a.name === VIEW_NAME)
+    expect(viewAction).toBeDefined()
+
+    const subAction = creates.find(a => a.resource === 'subscription' && a.name === SUB_NAME)
+    expect(subAction).toBeDefined()
+  })
+
+  it('is idempotent - second sync reports unchanged', async () => {
+    const actions = await ddl.sync(testConfig)
+
+    const unchanged = actions.filter(a => a.type === 'unchanged')
+    expect(unchanged.length).toBeGreaterThanOrEqual(3)
+
+    const creates = actions.filter(a => a.type === 'create')
+    expect(creates.length).toBe(0)
+  })
+
+  it('can write and read data', async () => {
+    const client = new Client({ connectionString: DATABASE_URL })
+    await client.connect()
+
+    await client.query(`INSERT INTO ${STREAM_NAME} (user_id, value) VALUES ('alice', 10)`)
+    await client.query(`INSERT INTO ${STREAM_NAME} (user_id, value) VALUES ('alice', 20)`)
+    await client.query(`INSERT INTO ${STREAM_NAME} (user_id, value) VALUES ('bob', 5)`)
+
+    // Wait for MV to update
+    await new Promise(r => setTimeout(r, 2000))
+
+    const result = await client.query(`SELECT * FROM ${VIEW_NAME} ORDER BY total_value DESC`)
+    expect(result.rows.length).toBeGreaterThanOrEqual(2)
+
+    const alice = result.rows.find((r: any) => r.user_id === 'alice')
+    expect(alice).toBeDefined()
+    expect(Number(alice.total_value)).toBe(30)
+    expect(Number(alice.event_count)).toBe(2)
+
+    await client.end()
+  })
+})
+
+describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Server', () => {
+  let server: WaveletServer
+  let port: number
+
+  beforeAll(async () => {
+    // Ensure DDL is synced
+    const ddl = new DdlManager(DATABASE_URL)
+    await ddl.connect()
+    await ddl.sync(testConfig)
+    await ddl.close()
+
+    // Start server on random port
+    server = new WaveletServer({ ...testConfig, server: { port: 0 } })
+    await server.start()
+
+    // Get the actual port
+    const addr = (server as any).httpServer?.address()
+    port = addr?.port
+  }, 15000)
+
+  afterAll(async () => {
+    await server.stop()
+
+    // Cleanup
+    const client = new Client({ connectionString: DATABASE_URL })
+    await client.connect()
+    try { await client.query(`DROP SUBSCRIPTION IF EXISTS ${SUB_NAME}`) } catch {}
+    try { await client.query(`DROP MATERIALIZED VIEW IF EXISTS ${VIEW_NAME}`) } catch {}
+    try { await client.query(`DROP TABLE IF EXISTS ${STREAM_NAME}`) } catch {}
+    await client.end()
+  })
+
+  it('serves health check', async () => {
+    const res = await fetch(`http://localhost:${port}/v1/health`)
+    const data = await res.json()
+    expect(data.status).toBe('ok')
+  })
+
+  it('lists views', async () => {
+    const res = await fetch(`http://localhost:${port}/v1/views`)
+    const data = await res.json()
+    expect(data.views).toContain(VIEW_NAME)
+  })
+
+  it('writes events via HTTP and reads view', async () => {
+    // Write events
+    await fetch(`http://localhost:${port}/v1/streams/${STREAM_NAME}`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ user_id: 'integration_test', value: 42 }),
+    })
+
+    // Wait for MV update
+    await new Promise(r => setTimeout(r, 2000))
+
+    // Read view
+    const res = await fetch(`http://localhost:${port}/v1/views/${VIEW_NAME}`)
+    const data = await res.json()
+    expect(data.rows.length).toBeGreaterThan(0)
+  })
+
+  it('pushes diffs via WebSocket', async () => {
+    const diff = await new Promise<any>((resolve, reject) => {
+      const ws = new WebSocket(`ws://localhost:${port}/subscribe/${VIEW_NAME}`)
+
+      ws.on('message', (data: Buffer) => {
+        const msg = JSON.parse(data.toString())
+        if (msg.type === 'connected') {
+          // Write event to trigger diff
+          fetch(`http://localhost:${port}/v1/streams/${STREAM_NAME}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ user_id: 'ws_test_user', value: 7 }),
+          })
+        }
+        if (msg.type === 'diff') {
+          ws.close()
+          resolve(msg)
+        }
+      })
+
+      ws.on('error', reject)
+      setTimeout(() => { ws.close(); reject(new Error('WebSocket timeout')) }, 15000)
+    })
+
+    expect(diff.type).toBe('diff')
+    expect(diff.cursor).toBeDefined()
+    // Should have at least one insert or update
+    expect(diff.inserted.length + diff.updated.length).toBeGreaterThan(0)
+  }, 20000)
+})

package/src/__tests__/jwt.test.ts

@@ -0,0 +1,62 @@
+import { describe, it, expect } from 'vitest'
+import { JwtVerifier } from '../jwt.js'
+import * as jose from 'jose'
+
+describe('JwtVerifier', () => {
+  it('isConfigured returns false with no config', () => {
+    const v = new JwtVerifier()
+    expect(v.isConfigured()).toBe(false)
+  })
+
+  it('isConfigured returns true with secret', () => {
+    const v = new JwtVerifier({ secret: 'test-secret' })
+    expect(v.isConfigured()).toBe(true)
+  })
+
+  it('isConfigured returns true with jwksUrl', () => {
+    const v = new JwtVerifier({ jwksUrl: 'https://example.com/.well-known/jwks.json' })
+    expect(v.isConfigured()).toBe(true)
+  })
+
+  it('verifies a valid HS256 JWT', async () => {
+    const secret = 'my-test-secret-that-is-long-enough'
+    const v = new JwtVerifier({ secret })
+
+    const encodedSecret = new TextEncoder().encode(secret)
+    const token = await new jose.SignJWT({ tenant_id: 't1', role: 'user' })
+      .setProtectedHeader({ alg: 'HS256' })
+      .setIssuedAt()
+      .setExpirationTime('1h')
+      .sign(encodedSecret)
+
+    const claims = await v.verify(token)
+    expect(claims.tenant_id).toBe('t1')
+    expect(claims.role).toBe('user')
+  })
+
+  it('rejects expired tokens', async () => {
+    const secret = 'my-test-secret-that-is-long-enough'
+    const v = new JwtVerifier({ secret })
+
+    const encodedSecret = new TextEncoder().encode(secret)
+    const token = await new jose.SignJWT({ tenant_id: 't1' })
+      .setProtectedHeader({ alg: 'HS256' })
+      .setIssuedAt(Math.floor(Date.now() / 1000) - 7200)
+      .setExpirationTime(Math.floor(Date.now() / 1000) - 3600)
+      .sign(encodedSecret)
+
+    await expect(v.verify(token)).rejects.toThrow('expired')
+  })
+
+  it('rejects tokens with wrong secret', async () => {
+    const v = new JwtVerifier({ secret: 'correct-secret-that-is-long-enough' })
+
+    const wrongSecret = new TextEncoder().encode('wrong-secret-that-is-long-enough')
+    const token = await new jose.SignJWT({ tenant_id: 't1' })
+      .setProtectedHeader({ alg: 'HS256' })
+      .setExpirationTime('1h')
+      .sign(wrongSecret)
+
+    await expect(v.verify(token)).rejects.toThrow()
+  })
+})

package/src/config-loader.ts

@@ -0,0 +1,20 @@
+import { pathToFileURL } from 'node:url'
+import { resolve } from 'node:path'
+import type { WaveletConfig } from '@risingwave/wavelet'
+
+export async function loadConfig(configPath: string): Promise<WaveletConfig> {
+  const abs = resolve(configPath)
+  const mod = await import(pathToFileURL(abs).href)
+  const config = mod.default ?? mod
+
+  if (!config.database) {
+    throw new Error(
+      `wavelet.config.ts must export a config with a 'database' field.\n` +
+      `Example:\n` +
+      `  import { defineConfig } from '@risingwave/wavelet'\n` +
+      `  export default defineConfig({ database: 'postgres://...' })`
+    )
+  }
+
+  return config
+}