@risingwave/wavelet-server 0.1.3 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@risingwave/wavelet-server",
3
- "version": "0.1.3",
3
+ "version": "0.2.1",
4
4
  "description": "Wavelet server - WebSocket fanout layer for RisingWave",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -13,7 +13,7 @@
13
13
  "pg": "^8.13.0",
14
14
  "ws": "^8.18.0",
15
15
  "jose": "^6.0.0",
16
- "@risingwave/wavelet": "0.1.3"
16
+ "@risingwave/wavelet": "0.2.1"
17
17
  },
18
18
  "devDependencies": {
19
19
  "@types/pg": "^8.11.0",
@@ -36,15 +36,15 @@ function request(server: http.Server, method: string, path: string, body?: unkno
36
36
  }
37
37
 
38
38
  describe('HttpApi', () => {
39
- const streams = {
40
- events: { columns: { user_id: 'string' as const, value: 'int' as const } },
39
+ const events = {
40
+ game_events: { columns: { user_id: 'string' as const, value: 'int' as const } },
41
41
  }
42
- const views = {
42
+ const queries = {
43
43
  leaderboard: { _tag: 'sql' as const, text: 'SELECT 1' },
44
44
  }
45
45
 
46
46
  it('returns health check', async () => {
47
- const api = new HttpApi('postgres://dummy', streams, views)
47
+ const api = new HttpApi('postgres://dummy', events, queries)
48
48
  const server = await createTestServer(api)
49
49
  try {
50
50
  const res = await request(server, 'GET', '/v1/health')
@@ -55,32 +55,32 @@ describe('HttpApi', () => {
55
55
  }
56
56
  })
57
57
 
58
- it('lists available views', async () => {
59
- const api = new HttpApi('postgres://dummy', streams, views)
58
+ it('lists available queries', async () => {
59
+ const api = new HttpApi('postgres://dummy', events, queries)
60
60
  const server = await createTestServer(api)
61
61
  try {
62
- const res = await request(server, 'GET', '/v1/views')
62
+ const res = await request(server, 'GET', '/v1/queries')
63
63
  expect(res.status).toBe(200)
64
- expect(res.data).toEqual({ views: ['leaderboard'] })
64
+ expect(res.data).toEqual({ queries: ['leaderboard'] })
65
65
  } finally {
66
66
  server.close()
67
67
  }
68
68
  })
69
69
 
70
- it('lists available streams', async () => {
71
- const api = new HttpApi('postgres://dummy', streams, views)
70
+ it('lists available events', async () => {
71
+ const api = new HttpApi('postgres://dummy', events, queries)
72
72
  const server = await createTestServer(api)
73
73
  try {
74
- const res = await request(server, 'GET', '/v1/streams')
74
+ const res = await request(server, 'GET', '/v1/events')
75
75
  expect(res.status).toBe(200)
76
- expect(res.data).toEqual({ streams: ['events'] })
76
+ expect(res.data).toEqual({ events: ['game_events'] })
77
77
  } finally {
78
78
  server.close()
79
79
  }
80
80
  })
81
81
 
82
82
  it('returns 404 for unknown routes with helpful message', async () => {
83
- const api = new HttpApi('postgres://dummy', streams, views)
83
+ const api = new HttpApi('postgres://dummy', events, queries)
84
84
  const server = await createTestServer(api)
85
85
  try {
86
86
  const res = await request(server, 'GET', '/v1/nonexistent')
@@ -94,35 +94,35 @@ describe('HttpApi', () => {
94
94
  })
95
95
 
96
96
  it('handles CORS preflight', async () => {
97
- const api = new HttpApi('postgres://dummy', streams, views)
97
+ const api = new HttpApi('postgres://dummy', events, queries)
98
98
  const server = await createTestServer(api)
99
99
  try {
100
- const res = await request(server, 'OPTIONS', '/v1/views')
100
+ const res = await request(server, 'OPTIONS', '/v1/queries')
101
101
  expect(res.status).toBe(204)
102
102
  } finally {
103
103
  server.close()
104
104
  }
105
105
  })
106
106
 
107
- it('returns 404 for unknown stream', async () => {
108
- const api = new HttpApi('postgres://dummy', streams, views)
107
+ it('returns 404 for unknown event', async () => {
108
+ const api = new HttpApi('postgres://dummy', events, queries)
109
109
  const server = await createTestServer(api)
110
110
  try {
111
- const res = await request(server, 'POST', '/v1/streams/nonexistent', { x: 1 })
111
+ const res = await request(server, 'POST', '/v1/events/nonexistent', { x: 1 })
112
112
  expect(res.status).toBe(404)
113
- expect(res.data.available_streams).toEqual(['events'])
113
+ expect(res.data.available_events).toEqual(['game_events'])
114
114
  } finally {
115
115
  server.close()
116
116
  }
117
117
  })
118
118
 
119
- it('returns 404 for unknown view', async () => {
120
- const api = new HttpApi('postgres://dummy', streams, views)
119
+ it('returns 404 for unknown query', async () => {
120
+ const api = new HttpApi('postgres://dummy', events, queries)
121
121
  const server = await createTestServer(api)
122
122
  try {
123
- const res = await request(server, 'GET', '/v1/views/nonexistent')
123
+ const res = await request(server, 'GET', '/v1/queries/nonexistent')
124
124
  expect(res.status).toBe(404)
125
- expect(res.data.available_views).toEqual(['leaderboard'])
125
+ expect(res.data.available_queries).toEqual(['leaderboard'])
126
126
  } finally {
127
127
  server.close()
128
128
  }
@@ -36,7 +36,7 @@ const SUB_NAME = `wavelet_sub_${VIEW_NAME}`
36
36
 
37
37
  const testConfig: WaveletConfig = {
38
38
  database: DATABASE_URL,
39
- streams: {
39
+ events: {
40
40
  [STREAM_NAME]: {
41
41
  columns: {
42
42
  user_id: 'string',
@@ -44,7 +44,7 @@ const testConfig: WaveletConfig = {
44
44
  },
45
45
  },
46
46
  },
47
- views: {
47
+ queries: {
48
48
  [VIEW_NAME]: sql`
49
49
  SELECT user_id, SUM(value) AS total_value, COUNT(*) AS event_count
50
50
  FROM ${STREAM_NAME}
@@ -75,17 +75,17 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: DDL Manage
75
75
  await ddl.close()
76
76
  })
77
77
 
78
- it('creates tables, views, and subscriptions', async () => {
78
+ it('creates tables, materialized views, and subscriptions', async () => {
79
79
  const actions = await ddl.sync(testConfig)
80
80
 
81
81
  const creates = actions.filter(a => a.type === 'create')
82
- expect(creates.length).toBeGreaterThanOrEqual(3) // stream + view + subscription
82
+ expect(creates.length).toBeGreaterThanOrEqual(3) // event + query + subscription
83
83
 
84
- const streamAction = creates.find(a => a.resource === 'stream' && a.name === STREAM_NAME)
85
- expect(streamAction).toBeDefined()
84
+ const eventAction = creates.find(a => a.resource === 'event' && a.name === STREAM_NAME)
85
+ expect(eventAction).toBeDefined()
86
86
 
87
- const viewAction = creates.find(a => a.resource === 'view' && a.name === VIEW_NAME)
88
- expect(viewAction).toBeDefined()
87
+ const queryAction = creates.find(a => a.resource === 'query' && a.name === VIEW_NAME)
88
+ expect(queryAction).toBeDefined()
89
89
 
90
90
  const subAction = creates.find(a => a.resource === 'subscription' && a.name === SUB_NAME)
91
91
  expect(subAction).toBeDefined()
@@ -162,15 +162,15 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
162
162
  expect(data.status).toBe('ok')
163
163
  })
164
164
 
165
- it('lists views', async () => {
166
- const res = await fetch(`http://localhost:${port}/v1/views`)
165
+ it('lists queries', async () => {
166
+ const res = await fetch(`http://localhost:${port}/v1/queries`)
167
167
  const data = await res.json()
168
- expect(data.views).toContain(VIEW_NAME)
168
+ expect(data.queries).toContain(VIEW_NAME)
169
169
  })
170
170
 
171
- it('writes events via HTTP and reads view', async () => {
171
+ it('writes events via HTTP and reads query', async () => {
172
172
  // Write events
173
- await fetch(`http://localhost:${port}/v1/streams/${STREAM_NAME}`, {
173
+ await fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
174
174
  method: 'POST',
175
175
  headers: { 'Content-Type': 'application/json' },
176
176
  body: JSON.stringify({ user_id: 'integration_test', value: 42 }),
@@ -179,8 +179,8 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
179
179
  // Wait for MV update
180
180
  await new Promise(r => setTimeout(r, 2000))
181
181
 
182
- // Read view
183
- const res = await fetch(`http://localhost:${port}/v1/views/${VIEW_NAME}`)
182
+ // Read query
183
+ const res = await fetch(`http://localhost:${port}/v1/queries/${VIEW_NAME}`)
184
184
  const data = await res.json()
185
185
  expect(data.rows.length).toBeGreaterThan(0)
186
186
  })
@@ -193,7 +193,7 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
193
193
  const msg = JSON.parse(data.toString())
194
194
  if (msg.type === 'connected') {
195
195
  // Write event to trigger diff
196
- fetch(`http://localhost:${port}/v1/streams/${STREAM_NAME}`, {
196
+ fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
197
197
  method: 'POST',
198
198
  headers: { 'Content-Type': 'application/json' },
199
199
  body: JSON.stringify({ user_id: 'ws_test_user', value: 7 }),
@@ -1,5 +1,5 @@
1
1
  import pg from 'pg'
2
- import type { SqlFragment, ViewDef } from '@risingwave/wavelet'
2
+ import type { SqlFragment, QueryDef } from '@risingwave/wavelet'
3
3
 
4
4
  const { Client } = pg
5
5
 
@@ -16,12 +16,12 @@ export interface ViewDiff {
16
16
  deleted: Record<string, unknown>[]
17
17
  }
18
18
 
19
- type DiffCallback = (viewName: string, diff: ViewDiff) => void
19
+ type DiffCallback = (queryName: string, diff: ViewDiff) => void
20
20
 
21
21
  /**
22
22
  * Manages persistent subscription cursors against RisingWave.
23
23
  *
24
- * Each view gets its own dedicated pg connection and a persistent cursor.
24
+ * Each query gets its own dedicated pg connection and a persistent cursor.
25
25
  * Uses blocking FETCH (WITH timeout) so there is no polling interval -
26
26
  * diffs are dispatched as soon as RisingWave produces them.
27
27
  */
@@ -29,15 +29,15 @@ export class CursorManager {
29
29
  // Shared connection for DDL (CREATE SUBSCRIPTION) and ad-hoc queries
30
30
  private client: InstanceType<typeof Client> | null = null
31
31
 
32
- // Per-view dedicated connections for blocking FETCH
33
- private viewConnections: Map<string, InstanceType<typeof Client>> = new Map()
32
+ // Per-query dedicated connections for blocking FETCH
33
+ private queryConnections: Map<string, InstanceType<typeof Client>> = new Map()
34
34
  private cursorNames: Map<string, string> = new Map()
35
35
  private subscriptions: Map<string, string> = new Map()
36
36
  private running = false
37
37
 
38
38
  constructor(
39
39
  private connectionString: string,
40
- private views: Record<string, ViewDef | SqlFragment>
40
+ private queries: Record<string, QueryDef | SqlFragment>
41
41
  ) {}
42
42
 
43
43
  async initialize(): Promise<void> {
@@ -45,13 +45,13 @@ export class CursorManager {
45
45
  await this.client.connect()
46
46
  console.log('Connected to RisingWave')
47
47
 
48
- for (const [viewName] of Object.entries(this.views)) {
49
- const subName = `wavelet_sub_${viewName}`
48
+ for (const [queryName] of Object.entries(this.queries)) {
49
+ const subName = `wavelet_sub_${queryName}`
50
50
 
51
51
  // Create subscription if not exists (idempotent)
52
52
  try {
53
53
  await this.client.query(
54
- `CREATE SUBSCRIPTION ${subName} FROM ${viewName} WITH (retention = '24h')`
54
+ `CREATE SUBSCRIPTION ${subName} FROM ${queryName} WITH (retention = '24h')`
55
55
  )
56
56
  console.log(`Created subscription: ${subName}`)
57
57
  } catch (err: any) {
@@ -62,31 +62,31 @@ export class CursorManager {
62
62
  }
63
63
  }
64
64
 
65
- this.subscriptions.set(viewName, subName)
65
+ this.subscriptions.set(queryName, subName)
66
66
 
67
- // Create dedicated connection and persistent cursor for this view
67
+ // Create dedicated connection and persistent cursor for this query
68
68
  const conn = new Client({ connectionString: this.connectionString })
69
69
  await conn.connect()
70
70
 
71
- const cursorName = `wavelet_cur_${viewName}`
71
+ const cursorName = `wavelet_cur_${queryName}`
72
72
  await conn.query(`DECLARE ${cursorName} SUBSCRIPTION CURSOR FOR ${subName}`)
73
73
  console.log(`Opened persistent cursor: ${cursorName}`)
74
74
 
75
- this.viewConnections.set(viewName, conn)
76
- this.cursorNames.set(viewName, cursorName)
75
+ this.queryConnections.set(queryName, conn)
76
+ this.cursorNames.set(queryName, cursorName)
77
77
  }
78
78
  }
79
79
 
80
80
  /**
81
- * Start listening for diffs on all views.
82
- * Each view runs its own async loop with blocking FETCH.
81
+ * Start listening for diffs on all queries.
82
+ * Each query runs its own async loop with blocking FETCH.
83
83
  * No polling interval - FETCH blocks until data arrives or timeout.
84
84
  */
85
85
  startPolling(callback: DiffCallback): void {
86
86
  this.running = true
87
87
 
88
- for (const [viewName] of this.subscriptions.entries()) {
89
- this.listenLoop(viewName, callback)
88
+ for (const [queryName] of this.subscriptions.entries()) {
89
+ this.listenLoop(queryName, callback)
90
90
  }
91
91
  }
92
92
 
@@ -94,9 +94,9 @@ export class CursorManager {
94
94
  this.running = false
95
95
  }
96
96
 
97
- private async listenLoop(viewName: string, callback: DiffCallback): Promise<void> {
98
- const conn = this.viewConnections.get(viewName)
99
- const cursorName = this.cursorNames.get(viewName)
97
+ private async listenLoop(queryName: string, callback: DiffCallback): Promise<void> {
98
+ const conn = this.queryConnections.get(queryName)
99
+ const cursorName = this.cursorNames.get(queryName)
100
100
  if (!conn || !cursorName) return
101
101
 
102
102
  while (this.running) {
@@ -108,12 +108,12 @@ export class CursorManager {
108
108
 
109
109
  if (result.rows.length === 0) continue
110
110
 
111
- // Got at least one row. Drain any remaining rows without blocking.
111
+ // Got at least one row. Drain any remaining rows with timeout.
112
112
  const allRows = [...result.rows]
113
113
  let more = true
114
114
  while (more) {
115
115
  const batch = await conn.query(
116
- `FETCH 100 FROM ${cursorName}`
116
+ `FETCH 100 FROM ${cursorName} WITH (timeout = '1s')`
117
117
  )
118
118
  if (batch.rows.length > 0) {
119
119
  allRows.push(...batch.rows)
@@ -125,11 +125,11 @@ export class CursorManager {
125
125
  const diff = this.parseDiffs(allRows)
126
126
 
127
127
  if (diff.inserted.length > 0 || diff.updated.length > 0 || diff.deleted.length > 0) {
128
- callback(viewName, diff)
128
+ callback(queryName, diff)
129
129
  }
130
130
  } catch (err: any) {
131
131
  if (!this.running) return
132
- console.error(`[cursor-manager] Error fetching ${viewName}:`, err.message)
132
+ console.error(`[cursor-manager] Error fetching ${queryName}:`, err.message)
133
133
  // Back off on error, then retry
134
134
  await new Promise(r => setTimeout(r, 1000))
135
135
  }
@@ -187,9 +187,9 @@ export class CursorManager {
187
187
  async close(): Promise<void> {
188
188
  this.running = false
189
189
 
190
- // Close per-view connections
191
- for (const [viewName, conn] of this.viewConnections) {
192
- const cursorName = this.cursorNames.get(viewName)
190
+ // Close per-query connections
191
+ for (const [queryName, conn] of this.queryConnections) {
192
+ const cursorName = this.cursorNames.get(queryName)
193
193
  try {
194
194
  if (cursorName) await conn.query(`CLOSE ${cursorName}`)
195
195
  } catch {}
@@ -197,7 +197,7 @@ export class CursorManager {
197
197
  await conn.end()
198
198
  } catch {}
199
199
  }
200
- this.viewConnections.clear()
200
+ this.queryConnections.clear()
201
201
  this.cursorNames.clear()
202
202
 
203
203
  // Close shared connection
@@ -1,11 +1,11 @@
1
1
  import pg from 'pg'
2
- import type { WaveletConfig, StreamDef, ViewDef, SqlFragment } from '@risingwave/wavelet'
2
+ import type { WaveletConfig, EventDef, QueryDef, SqlFragment, PostgresCdcSource } from '@risingwave/wavelet'
3
3
 
4
4
  const { Client } = pg
5
5
 
6
6
  export interface DdlAction {
7
7
  type: 'create' | 'update' | 'delete' | 'unchanged'
8
- resource: 'stream' | 'view' | 'subscription'
8
+ resource: 'event' | 'source' | 'query' | 'subscription'
9
9
  name: string
10
10
  detail?: string
11
11
  }
@@ -26,13 +26,13 @@ function normalizeSql(sql: string): string {
26
26
  return stripped.replace(/\s+/g, ' ').trim().toLowerCase()
27
27
  }
28
28
 
29
- function getViewQuery(viewDef: ViewDef | SqlFragment): string {
30
- if ('_tag' in viewDef && viewDef._tag === 'sql') return viewDef.text
31
- return (viewDef as ViewDef).query.text
29
+ function getQuerySql(queryDef: QueryDef | SqlFragment): string {
30
+ if ('_tag' in queryDef && queryDef._tag === 'sql') return queryDef.text
31
+ return (queryDef as QueryDef).query.text
32
32
  }
33
33
 
34
- function buildCreateTableSql(name: string, streamDef: StreamDef): string {
35
- const cols = Object.entries(streamDef.columns)
34
+ function buildCreateTableSql(name: string, eventDef: EventDef): string {
35
+ const cols = Object.entries(eventDef.columns)
36
36
  .map(([colName, colType]) => {
37
37
  const sqlType = COLUMN_TYPE_MAP[colType]
38
38
  if (!sqlType) throw new Error(`Unknown column type "${colType}" for column "${colName}"`)
@@ -60,7 +60,7 @@ export class DdlManager {
60
60
  }
61
61
 
62
62
  /**
63
- * Sync all streams, views, and subscriptions to match the config.
63
+ * Sync all events, queries, and subscriptions to match the config.
64
64
  * Returns a list of actions taken.
65
65
  * Idempotent - safe to call multiple times.
66
66
  */
@@ -71,61 +71,78 @@ export class DdlManager {
71
71
 
72
72
  // 1. Fetch existing state from RisingWave
73
73
  const existingTables = await this.getExistingTables()
74
- const existingViews = await this.getExistingViews()
74
+ const existingMVs = await this.getExistingMaterializedViews()
75
75
  const existingSubscriptions = await this.getExistingSubscriptions()
76
76
 
77
- const desiredStreams = config.streams ?? {}
78
- const desiredViews = config.views ?? {}
77
+ const desiredEvents = config.events ?? config.streams ?? {}
78
+ const desiredSources = config.sources ?? {}
79
+ const desiredQueries = config.queries ?? config.views ?? {}
79
80
 
80
- // 2. Determine which streams (tables) to create or remove
81
- const desiredStreamNames = new Set(Object.keys(desiredStreams))
82
- const desiredViewNames = new Set(Object.keys(desiredViews))
81
+ // 2. Determine which events (tables) to create or remove
82
+ const desiredEventNames = new Set(Object.keys(desiredEvents))
83
+ const desiredQueryNames = new Set(Object.keys(desiredQueries))
83
84
 
84
- // 3. Sync streams - create missing tables
85
- for (const [streamName, streamDef] of Object.entries(desiredStreams)) {
86
- if (existingTables.has(streamName)) {
87
- actions.push({ type: 'unchanged', resource: 'stream', name: streamName })
85
+ // 3. Sync events - create missing tables
86
+ for (const [eventName, eventDef] of Object.entries(desiredEvents)) {
87
+ if (existingTables.has(eventName)) {
88
+ actions.push({ type: 'unchanged', resource: 'event', name: eventName })
88
89
  } else {
89
- await this.createTable(streamName, streamDef)
90
- actions.push({ type: 'create', resource: 'stream', name: streamName })
90
+ await this.createTable(eventName, eventDef)
91
+ actions.push({ type: 'create', resource: 'event', name: eventName })
91
92
  }
92
93
  }
93
94
 
94
- // 4. Sync views - create, update, or leave unchanged
95
- for (const [viewName, viewDef] of Object.entries(desiredViews)) {
96
- const subName = `wavelet_sub_${viewName}`
97
- const desiredSql = getViewQuery(viewDef)
98
- const existingSql = existingViews.get(viewName)
95
+ // 3b. Sync CDC sources - create Postgres CDC tables
96
+ const existingSources = await this.getExistingSources()
97
+ for (const [sourceName, sourceDef] of Object.entries(desiredSources)) {
98
+ if (sourceDef.type === 'postgres') {
99
+ for (const tableName of sourceDef.tables) {
100
+ const cdcTableName = `${sourceName}_${tableName}`
101
+ if (existingSources.has(cdcTableName) || existingTables.has(cdcTableName)) {
102
+ actions.push({ type: 'unchanged', resource: 'source', name: cdcTableName })
103
+ } else {
104
+ await this.createCdcSource(sourceName, tableName, sourceDef)
105
+ actions.push({ type: 'create', resource: 'source', name: cdcTableName })
106
+ }
107
+ }
108
+ }
109
+ }
110
+
111
+ // 4. Sync queries - create, update, or leave unchanged
112
+ for (const [queryName, queryDef] of Object.entries(desiredQueries)) {
113
+ const subName = `wavelet_sub_${queryName}`
114
+ const desiredSql = getQuerySql(queryDef)
115
+ const existingSql = existingMVs.get(queryName)
99
116
 
100
117
  if (existingSql === undefined) {
101
- // View does not exist - create MV and subscription
102
- await this.createMaterializedView(viewName, desiredSql)
103
- actions.push({ type: 'create', resource: 'view', name: viewName })
118
+ // MV does not exist - create MV and subscription
119
+ await this.createMaterializedView(queryName, desiredSql)
120
+ actions.push({ type: 'create', resource: 'query', name: queryName })
104
121
 
105
- await this.createSubscription(subName, viewName)
122
+ await this.createSubscription(subName, queryName)
106
123
  actions.push({ type: 'create', resource: 'subscription', name: subName })
107
124
  } else if (normalizeSql(existingSql) !== normalizeSql(desiredSql)) {
108
- // View SQL changed - drop subscription, drop MV, recreate
125
+ // Query SQL changed - drop subscription, drop MV, recreate
109
126
  if (existingSubscriptions.has(subName)) {
110
127
  await this.dropSubscription(subName)
111
- actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'dropped for view update' })
128
+ actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'dropped for query update' })
112
129
  }
113
130
 
114
- await this.dropMaterializedView(viewName)
115
- actions.push({ type: 'delete', resource: 'view', name: viewName, detail: 'dropped for update' })
131
+ await this.dropMaterializedView(queryName)
132
+ actions.push({ type: 'delete', resource: 'query', name: queryName, detail: 'dropped for update' })
116
133
 
117
- await this.createMaterializedView(viewName, desiredSql)
118
- actions.push({ type: 'create', resource: 'view', name: viewName, detail: 'recreated with updated SQL' })
134
+ await this.createMaterializedView(queryName, desiredSql)
135
+ actions.push({ type: 'create', resource: 'query', name: queryName, detail: 'recreated with updated SQL' })
119
136
 
120
- await this.createSubscription(subName, viewName)
121
- actions.push({ type: 'create', resource: 'subscription', name: subName, detail: 'recreated after view update' })
137
+ await this.createSubscription(subName, queryName)
138
+ actions.push({ type: 'create', resource: 'subscription', name: subName, detail: 'recreated after query update' })
122
139
  } else {
123
- // View SQL unchanged
124
- actions.push({ type: 'unchanged', resource: 'view', name: viewName })
140
+ // Query SQL unchanged
141
+ actions.push({ type: 'unchanged', resource: 'query', name: queryName })
125
142
 
126
- // Ensure subscription exists even if the view is unchanged
143
+ // Ensure subscription exists even if the query is unchanged
127
144
  if (!existingSubscriptions.has(subName)) {
128
- await this.createSubscription(subName, viewName)
145
+ await this.createSubscription(subName, queryName)
129
146
  actions.push({ type: 'create', resource: 'subscription', name: subName })
130
147
  } else {
131
148
  actions.push({ type: 'unchanged', resource: 'subscription', name: subName })
@@ -133,19 +150,19 @@ export class DdlManager {
133
150
  }
134
151
  }
135
152
 
136
- // 5. Remove views that are no longer in the config
137
- for (const [existingViewName] of existingViews) {
138
- if (!desiredViewNames.has(existingViewName)) {
139
- const subName = `wavelet_sub_${existingViewName}`
153
+ // 5. Remove queries that are no longer in the config
154
+ for (const [existingMVName] of existingMVs) {
155
+ if (!desiredQueryNames.has(existingMVName)) {
156
+ const subName = `wavelet_sub_${existingMVName}`
140
157
 
141
158
  // Drop subscription first
142
159
  if (existingSubscriptions.has(subName)) {
143
160
  await this.dropSubscription(subName)
144
- actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'view removed from config' })
161
+ actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'query removed from config' })
145
162
  }
146
163
 
147
- await this.dropMaterializedView(existingViewName)
148
- actions.push({ type: 'delete', resource: 'view', name: existingViewName, detail: 'removed from config' })
164
+ await this.dropMaterializedView(existingMVName)
165
+ actions.push({ type: 'delete', resource: 'query', name: existingMVName, detail: 'removed from config' })
149
166
  }
150
167
  }
151
168
 
@@ -154,32 +171,32 @@ export class DdlManager {
154
171
  // Only manage wavelet-prefixed subscriptions
155
172
  if (!existingSubName.startsWith('wavelet_sub_')) continue
156
173
 
157
- const viewName = existingSubName.slice('wavelet_sub_'.length)
158
- if (!desiredViewNames.has(viewName)) {
159
- // Already handled in step 5 if the view existed, but handle dangling subs too
160
- if (!existingViews.has(viewName)) {
174
+ const queryName = existingSubName.slice('wavelet_sub_'.length)
175
+ if (!desiredQueryNames.has(queryName)) {
176
+ // Already handled in step 5 if the MV existed, but handle dangling subs too
177
+ if (!existingMVs.has(queryName)) {
161
178
  await this.dropSubscription(existingSubName)
162
179
  actions.push({ type: 'delete', resource: 'subscription', name: existingSubName, detail: 'orphaned subscription' })
163
180
  }
164
181
  }
165
182
  }
166
183
 
167
- // 7. Remove streams (tables) that are no longer in the config
184
+ // 7. Remove events (tables) that are no longer in the config
168
185
  for (const existingTableName of existingTables) {
169
- if (!desiredStreamNames.has(existingTableName)) {
186
+ if (!desiredEventNames.has(existingTableName)) {
170
187
  // Only drop if no MV depends on it
171
- const hasDependents = await this.tableHasDependentViews(existingTableName)
188
+ const hasDependents = await this.tableHasDependentMVs(existingTableName)
172
189
  if (hasDependents) {
173
190
  console.log(`[ddl-manager] Skipping drop of table "${existingTableName}" - materialized views depend on it`)
174
191
  actions.push({
175
192
  type: 'unchanged',
176
- resource: 'stream',
193
+ resource: 'event',
177
194
  name: existingTableName,
178
- detail: 'kept because dependent views exist',
195
+ detail: 'kept because dependent materialized views exist',
179
196
  })
180
197
  } else {
181
198
  await this.dropTable(existingTableName)
182
- actions.push({ type: 'delete', resource: 'stream', name: existingTableName, detail: 'removed from config' })
199
+ actions.push({ type: 'delete', resource: 'event', name: existingTableName, detail: 'removed from config' })
183
200
  }
184
201
  }
185
202
  }
@@ -196,7 +213,7 @@ export class DdlManager {
196
213
  return new Set(result.rows.map((r: any) => r.table_name))
197
214
  }
198
215
 
199
- private async getExistingViews(): Promise<Map<string, string>> {
216
+ private async getExistingMaterializedViews(): Promise<Map<string, string>> {
200
217
  const result = await this.client!.query(
201
218
  `SELECT name, definition FROM rw_catalog.rw_materialized_views WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public')`
202
219
  )
@@ -207,6 +224,18 @@ export class DdlManager {
207
224
  return views
208
225
  }
209
226
 
227
+ private async getExistingSources(): Promise<Set<string>> {
228
+ try {
229
+ const result = await this.client!.query(
230
+ `SELECT name FROM rw_catalog.rw_tables WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public') AND is_index = false`
231
+ )
232
+ return new Set(result.rows.map((r: any) => r.name))
233
+ } catch {
234
+ // Fallback: if catalog query fails, return empty set
235
+ return new Set()
236
+ }
237
+ }
238
+
210
239
  private async getExistingSubscriptions(): Promise<Set<string>> {
211
240
  const result = await this.client!.query(
212
241
  `SELECT name FROM rw_catalog.rw_subscriptions WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public')`
@@ -214,7 +243,7 @@ export class DdlManager {
214
243
  return new Set(result.rows.map((r: any) => r.name))
215
244
  }
216
245
 
217
- private async tableHasDependentViews(tableName: string): Promise<boolean> {
246
+ private async tableHasDependentMVs(tableName: string): Promise<boolean> {
218
247
  const result = await this.client!.query(
219
248
  `SELECT name FROM rw_catalog.rw_materialized_views WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public') AND definition ILIKE $1`,
220
249
  [`%${tableName}%`]
@@ -224,8 +253,8 @@ export class DdlManager {
224
253
 
225
254
  // ── DDL operations ────────────────────────────────────────────────────
226
255
 
227
- private async createTable(name: string, streamDef: StreamDef): Promise<void> {
228
- const sql = buildCreateTableSql(name, streamDef)
256
+ private async createTable(name: string, eventDef: EventDef): Promise<void> {
257
+ const sql = buildCreateTableSql(name, eventDef)
229
258
  try {
230
259
  await this.client!.query(sql)
231
260
  console.log(`[ddl-manager] Created table: ${name}`)
@@ -304,4 +333,76 @@ export class DdlManager {
304
333
  }
305
334
  }
306
335
  }
336
+
337
+ private async createCdcSource(
338
+ sourceName: string,
339
+ tableName: string,
340
+ source: PostgresCdcSource
341
+ ): Promise<void> {
342
+ const cdcTableName = `${sourceName}_${tableName}`
343
+ const slotName = source.slotName ?? `wavelet_${sourceName}`
344
+ const pubName = source.publicationName ?? `wavelet_${sourceName}_pub`
345
+
346
+ // RisingWave CDC source syntax:
347
+ // CREATE TABLE table_name ( ... ) WITH (
348
+ // connector = 'postgres-cdc',
349
+ // hostname = '...',
350
+ // port = '...',
351
+ // username = '...',
352
+ // password = '...',
353
+ // database.name = '...',
354
+ // table.name = '...',
355
+ // slot.name = '...',
356
+ // publication.name = '...'
357
+ // )
358
+ // We parse the connection string to extract components.
359
+
360
+ const parsed = parsePostgresUrl(source.connection)
361
+
362
+ const esc = (s: string) => s.replace(/'/g, "''")
363
+ try {
364
+ await this.client!.query(`
365
+ CREATE TABLE IF NOT EXISTS ${cdcTableName} (*)
366
+ WITH (
367
+ connector = 'postgres-cdc',
368
+ hostname = '${esc(parsed.host)}',
369
+ port = '${esc(parsed.port)}',
370
+ username = '${esc(parsed.user)}',
371
+ password = '${esc(parsed.password)}',
372
+ database.name = '${esc(parsed.database)}',
373
+ schema.name = '${esc(parsed.schema)}',
374
+ table.name = '${esc(tableName)}',
375
+ slot.name = '${esc(slotName)}',
376
+ publication.name = '${esc(pubName)}'
377
+ )
378
+ `)
379
+ console.log(`[ddl-manager] Created CDC source: ${cdcTableName} (from ${parsed.host}/${parsed.database}.${tableName})`)
380
+ } catch (err: any) {
381
+ if (err.message?.includes('already exists')) {
382
+ console.log(`[ddl-manager] CDC source already exists: ${cdcTableName}`)
383
+ } else {
384
+ throw err
385
+ }
386
+ }
387
+ }
388
+ }
389
+
390
+ function parsePostgresUrl(url: string): {
391
+ host: string
392
+ port: string
393
+ user: string
394
+ password: string
395
+ database: string
396
+ schema: string
397
+ } {
398
+ // Parse postgresql://user:password@host:port/database?schema=xxx
399
+ const u = new URL(url)
400
+ return {
401
+ host: u.hostname,
402
+ port: u.port || '5432',
403
+ user: decodeURIComponent(u.username),
404
+ password: decodeURIComponent(u.password),
405
+ database: u.pathname.replace(/^\//, '') || 'postgres',
406
+ schema: u.searchParams.get('schema') ?? 'public',
407
+ }
307
408
  }
package/src/http-api.ts CHANGED
@@ -1,16 +1,20 @@
1
1
  import type { IncomingMessage, ServerResponse } from 'node:http'
2
2
  import pg from 'pg'
3
- import type { StreamDef, ViewDef, SqlFragment } from '@risingwave/wavelet'
3
+ import type { EventDef, QueryDef, SqlFragment } from '@risingwave/wavelet'
4
+ import type { JwtVerifier, JwtClaims } from './jwt.js'
4
5
 
5
- const { Client } = pg
6
+ const { Pool } = pg
7
+
8
+ const MAX_BODY_SIZE = 10 * 1024 * 1024 // 10MB
6
9
 
7
10
  export class HttpApi {
8
- private client: InstanceType<typeof Client> | null = null
11
+ private pool: InstanceType<typeof Pool> | null = null
9
12
 
10
13
  constructor(
11
14
  private connectionString: string,
12
- private streams: Record<string, StreamDef>,
13
- private views: Record<string, ViewDef | SqlFragment>
15
+ private events: Record<string, EventDef>,
16
+ private queries: Record<string, QueryDef | SqlFragment>,
17
+ private jwt?: JwtVerifier
14
18
  ) {}
15
19
 
16
20
  async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
@@ -28,24 +32,24 @@ export class HttpApi {
28
32
  const url = new URL(req.url ?? '/', `http://${req.headers.host}`)
29
33
 
30
34
  try {
31
- // POST /v1/streams/{name}
32
- const streamMatch = url.pathname.match(/^\/v1\/streams\/([^/]+)$/)
33
- if (streamMatch && req.method === 'POST') {
34
- await this.handleWrite(streamMatch[1], req, res)
35
+ // POST /v1/events/{name}
36
+ const eventMatch = url.pathname.match(/^\/v1\/events\/([^/]+)$/)
37
+ if (eventMatch && req.method === 'POST') {
38
+ await this.handleWrite(eventMatch[1], req, res)
35
39
  return
36
40
  }
37
41
 
38
- // POST /v1/streams/{name}/batch
39
- const batchMatch = url.pathname.match(/^\/v1\/streams\/([^/]+)\/batch$/)
42
+ // POST /v1/events/{name}/batch
43
+ const batchMatch = url.pathname.match(/^\/v1\/events\/([^/]+)\/batch$/)
40
44
  if (batchMatch && req.method === 'POST') {
41
45
  await this.handleBatchWrite(batchMatch[1], req, res)
42
46
  return
43
47
  }
44
48
 
45
- // GET /v1/views/{name}
46
- const viewMatch = url.pathname.match(/^\/v1\/views\/([^/]+)$/)
47
- if (viewMatch && req.method === 'GET') {
48
- await this.handleRead(viewMatch[1], url, res)
49
+ // GET /v1/queries/{name}
50
+ const queryMatch = url.pathname.match(/^\/v1\/queries\/([^/]+)$/)
51
+ if (queryMatch && req.method === 'GET') {
52
+ await this.handleRead(queryMatch[1], url, req, res)
49
53
  return
50
54
  }
51
55
 
@@ -55,15 +59,15 @@ export class HttpApi {
55
59
  return
56
60
  }
57
61
 
58
- // GET /v1/views - list available views
59
- if (url.pathname === '/v1/views' && req.method === 'GET') {
60
- this.json(res, 200, { views: Object.keys(this.views) })
62
+ // GET /v1/queries - list available queries
63
+ if (url.pathname === '/v1/queries' && req.method === 'GET') {
64
+ this.json(res, 200, { queries: Object.keys(this.queries) })
61
65
  return
62
66
  }
63
67
 
64
- // GET /v1/streams - list available streams
65
- if (url.pathname === '/v1/streams' && req.method === 'GET') {
66
- this.json(res, 200, { streams: Object.keys(this.streams) })
68
+ // GET /v1/events - list available events
69
+ if (url.pathname === '/v1/events' && req.method === 'GET') {
70
+ this.json(res, 200, { events: Object.keys(this.events) })
67
71
  return
68
72
  }
69
73
 
@@ -72,11 +76,11 @@ export class HttpApi {
72
76
  message: `${req.method} ${url.pathname} does not match any route.`,
73
77
  routes: [
74
78
  'GET /v1/health',
75
- 'GET /v1/views',
76
- 'GET /v1/views/{name}',
77
- 'GET /v1/streams',
78
- 'POST /v1/streams/{name}',
79
- 'POST /v1/streams/{name}/batch',
79
+ 'GET /v1/queries',
80
+ 'GET /v1/queries/{name}',
81
+ 'GET /v1/events',
82
+ 'POST /v1/events/{name}',
83
+ 'POST /v1/events/{name}/batch',
80
84
  ],
81
85
  })
82
86
  } catch (err: any) {
@@ -85,21 +89,20 @@ export class HttpApi {
85
89
  }
86
90
  }
87
91
 
88
- private async ensureClient(): Promise<InstanceType<typeof Client>> {
89
- if (!this.client) {
90
- this.client = new Client({ connectionString: this.connectionString })
91
- await this.client.connect()
92
+ private ensurePool(): InstanceType<typeof Pool> {
93
+ if (!this.pool) {
94
+ this.pool = new Pool({ connectionString: this.connectionString, max: 10 })
92
95
  }
93
- return this.client
96
+ return this.pool
94
97
  }
95
98
 
96
- private async handleWrite(streamName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
97
- const stream = this.streams[streamName]
98
- if (!stream) {
99
- const available = Object.keys(this.streams)
99
+ private async handleWrite(eventName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
100
+ const eventDef = this.events[eventName]
101
+ if (!eventDef) {
102
+ const available = Object.keys(this.events)
100
103
  this.json(res, 404, {
101
- error: `Stream '${streamName}' not found.`,
102
- available_streams: available,
104
+ error: `Event '${eventName}' not found.`,
105
+ available_events: available,
103
106
  })
104
107
  return
105
108
  }
@@ -107,80 +110,147 @@ export class HttpApi {
107
110
  const body = await this.readBody(req)
108
111
  const data = JSON.parse(body)
109
112
 
110
- const client = await this.ensureClient()
111
- const columns = Object.keys(stream.columns)
113
+ const pool = this.ensurePool()
114
+ const columns = Object.keys(eventDef.columns)
112
115
  const values = columns.map((col) => data[col])
113
116
  const placeholders = columns.map((_, i) => `$${i + 1}`)
114
117
 
115
- await client.query(
116
- `INSERT INTO ${streamName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`,
118
+ await pool.query(
119
+ `INSERT INTO ${eventName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`,
117
120
  values
118
121
  )
119
122
 
120
123
  this.json(res, 200, { ok: true })
121
124
  }
122
125
 
123
- private async handleBatchWrite(streamName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
124
- const stream = this.streams[streamName]
125
- if (!stream) {
126
- const available = Object.keys(this.streams)
126
+ private async handleBatchWrite(eventName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
127
+ const eventDef = this.events[eventName]
128
+ if (!eventDef) {
129
+ const available = Object.keys(this.events)
127
130
  this.json(res, 404, {
128
- error: `Stream '${streamName}' not found.`,
129
- available_streams: available,
131
+ error: `Event '${eventName}' not found.`,
132
+ available_events: available,
130
133
  })
131
134
  return
132
135
  }
133
136
 
134
137
  const body = await this.readBody(req)
135
- const events: any[] = JSON.parse(body)
138
+ const items: any[] = JSON.parse(body)
136
139
 
137
- if (!Array.isArray(events)) {
140
+ if (!Array.isArray(items)) {
138
141
  this.json(res, 400, { error: 'Batch endpoint expects a JSON array.' })
139
142
  return
140
143
  }
141
144
 
142
- const client = await this.ensureClient()
143
- const columns = Object.keys(stream.columns)
145
+ if (items.length === 0) {
146
+ this.json(res, 200, { ok: true, count: 0 })
147
+ return
148
+ }
149
+
150
+ const pool = this.ensurePool()
151
+ const columns = Object.keys(eventDef.columns)
144
152
 
145
- for (const data of events) {
146
- const values = columns.map((col) => data[col])
147
- const placeholders = columns.map((_, i) => `$${i + 1}`)
148
- await client.query(
149
- `INSERT INTO ${streamName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`,
150
- values
151
- )
153
+ // Build a single INSERT with multiple VALUE rows
154
+ const allValues: unknown[] = []
155
+ const rowPlaceholders: string[] = []
156
+
157
+ for (let i = 0; i < items.length; i++) {
158
+ const row = items[i]
159
+ const offset = i * columns.length
160
+ const ph = columns.map((_, j) => `$${offset + j + 1}`)
161
+ rowPlaceholders.push(`(${ph.join(', ')})`)
162
+ for (const col of columns) {
163
+ allValues.push(row[col])
164
+ }
152
165
  }
153
166
 
154
- this.json(res, 200, { ok: true, count: events.length })
167
+ await pool.query(
168
+ `INSERT INTO ${eventName} (${columns.join(', ')}) VALUES ${rowPlaceholders.join(', ')}`,
169
+ allValues
170
+ )
171
+
172
+ this.json(res, 200, { ok: true, count: items.length })
155
173
  }
156
174
 
157
- private async handleRead(viewName: string, url: URL, res: ServerResponse): Promise<void> {
158
- if (!this.views[viewName]) {
159
- const available = Object.keys(this.views)
175
+ private async handleRead(queryName: string, url: URL, req: IncomingMessage, res: ServerResponse): Promise<void> {
176
+ if (!this.queries[queryName]) {
177
+ const available = Object.keys(this.queries)
160
178
  this.json(res, 404, {
161
- error: `View '${viewName}' not found.`,
162
- available_views: available,
179
+ error: `Query '${queryName}' not found.`,
180
+ available_queries: available,
163
181
  })
164
182
  return
165
183
  }
166
184
 
167
- const client = await this.ensureClient()
185
+ // JWT verification for queries with filterBy
186
+ const queryDef = this.queries[queryName]
187
+ const filterBy = this.getFilterBy(queryDef)
188
+
189
+ let claims: JwtClaims | null = null
190
+ if (filterBy && this.jwt?.isConfigured()) {
191
+ const token = url.searchParams.get('token')
192
+ ?? req.headers.authorization?.replace('Bearer ', '')
193
+
194
+ if (!token) {
195
+ this.json(res, 401, { error: 'Authentication required for filtered queries.' })
196
+ return
197
+ }
198
+
199
+ claims = await this.jwt.verify(token)
200
+ }
201
+
202
+ const pool = this.ensurePool()
168
203
 
169
- // Build WHERE clause from query params
204
+ // Build WHERE clause: start with filterBy if applicable
170
205
  const params: string[] = []
171
206
  const values: unknown[] = []
207
+
208
+ if (filterBy && claims) {
209
+ const claimValue = claims[filterBy]
210
+ if (claimValue === undefined) {
211
+ // No matching claim -- return empty result, not all data
212
+ this.json(res, 200, { query: queryName, rows: [] })
213
+ return
214
+ }
215
+ values.push(String(claimValue))
216
+ params.push(`${filterBy} = $${values.length}`)
217
+ }
218
+
219
+ // Add query params as additional filters, validating column names
220
+ const knownColumns = this.getQueryColumns(queryDef)
172
221
  for (const [key, value] of url.searchParams.entries()) {
173
- params.push(`${key} = $${params.length + 1}`)
222
+ if (key === 'token') continue // skip JWT token param
223
+ if (knownColumns && !knownColumns.includes(key)) {
224
+ this.json(res, 400, { error: `Unknown column '${key}'. Known columns: ${knownColumns.join(', ')}` })
225
+ return
226
+ }
227
+ if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
228
+ this.json(res, 400, { error: `Invalid column name: '${key}'` })
229
+ return
230
+ }
174
231
  values.push(value)
232
+ params.push(`${key} = $${values.length}`)
175
233
  }
176
234
 
177
- let sql = `SELECT * FROM ${viewName}`
235
+ let sql = `SELECT * FROM ${queryName}`
178
236
  if (params.length > 0) {
179
237
  sql += ` WHERE ${params.join(' AND ')}`
180
238
  }
181
239
 
182
- const result = await client.query(sql, values)
183
- this.json(res, 200, { view: viewName, rows: result.rows })
240
+ const result = await pool.query(sql, values)
241
+ this.json(res, 200, { query: queryName, rows: result.rows })
242
+ }
243
+
244
+ private getFilterBy(queryDef: QueryDef | SqlFragment): string | undefined {
245
+ if ('_tag' in queryDef && queryDef._tag === 'sql') return undefined
246
+ return (queryDef as QueryDef).filterBy
247
+ }
248
+
249
+ private getQueryColumns(queryDef: QueryDef | SqlFragment): string[] | null {
250
+ if ('_tag' in queryDef && queryDef._tag === 'sql') return null
251
+ const qd = queryDef as QueryDef
252
+ if (qd.columns) return Object.keys(qd.columns)
253
+ return null
184
254
  }
185
255
 
186
256
  private json(res: ServerResponse, status: number, data: unknown): void {
@@ -191,7 +261,16 @@ export class HttpApi {
191
261
  private readBody(req: IncomingMessage): Promise<string> {
192
262
  return new Promise((resolve, reject) => {
193
263
  let body = ''
194
- req.on('data', (chunk) => { body += chunk })
264
+ let size = 0
265
+ req.on('data', (chunk) => {
266
+ size += chunk.length
267
+ if (size > MAX_BODY_SIZE) {
268
+ req.destroy()
269
+ reject(new Error(`Request body exceeds ${MAX_BODY_SIZE / 1024 / 1024}MB limit`))
270
+ return
271
+ }
272
+ body += chunk
273
+ })
195
274
  req.on('end', () => resolve(body))
196
275
  req.on('error', reject)
197
276
  })
package/src/server.ts CHANGED
@@ -2,6 +2,7 @@ import { createServer, type IncomingMessage, type ServerResponse, type Server }
2
2
  import type { WaveletConfig } from '@risingwave/wavelet'
3
3
  import { CursorManager } from './cursor-manager.js'
4
4
  import { WebSocketFanout } from './ws-fanout.js'
5
+ import { WebhookFanout } from './webhook.js'
5
6
  import { HttpApi } from './http-api.js'
6
7
  import { JwtVerifier } from './jwt.js'
7
8
 
@@ -9,14 +10,18 @@ export class WaveletServer {
9
10
  private httpServer: ReturnType<typeof createServer> | null = null
10
11
  private cursorManager: CursorManager
11
12
  private fanout: WebSocketFanout
13
+ private webhooks: WebhookFanout
12
14
  private httpApi: HttpApi
13
15
  private jwt: JwtVerifier
14
16
 
15
17
  constructor(private config: WaveletConfig) {
16
18
  this.jwt = new JwtVerifier(config.jwt)
17
- this.cursorManager = new CursorManager(config.database, config.views ?? {})
18
- this.fanout = new WebSocketFanout(this.cursorManager, this.jwt, config.views ?? {})
19
- this.httpApi = new HttpApi(config.database, config.streams ?? {}, config.views ?? {})
19
+ const queries = config.queries ?? config.views ?? {}
20
+ const events = config.events ?? config.streams ?? {}
21
+ this.cursorManager = new CursorManager(config.database, queries)
22
+ this.fanout = new WebSocketFanout(this.cursorManager, this.jwt, queries)
23
+ this.webhooks = new WebhookFanout(queries, config.jwt?.secret)
24
+ this.httpApi = new HttpApi(config.database, events, queries, this.jwt)
20
25
  }
21
26
 
22
27
  async start(): Promise<void> {
@@ -27,14 +32,15 @@ export class WaveletServer {
27
32
  this.fanout.attach(this.httpServer)
28
33
 
29
34
  await this.cursorManager.initialize()
30
- this.cursorManager.startPolling((viewName, diffs) => {
31
- this.fanout.broadcast(viewName, diffs)
35
+ this.cursorManager.startPolling((queryName, diffs) => {
36
+ this.fanout.broadcast(queryName, diffs)
37
+ this.webhooks.broadcast(queryName, diffs)
32
38
  })
33
39
 
34
40
  await new Promise<void>((resolve) => {
35
41
  this.httpServer!.listen(port, host, () => {
36
42
  console.log(`Wavelet server listening on ${host}:${port}`)
37
- console.log(`WebSocket: ws://${host}:${port}/subscribe/{view}`)
43
+ console.log(`WebSocket: ws://${host}:${port}/subscribe/{query}`)
38
44
  console.log(`HTTP API: http://${host}:${port}/v1/`)
39
45
  resolve()
40
46
  })
@@ -53,8 +59,9 @@ export class WaveletServer {
53
59
  this.fanout.attach(server, prefix)
54
60
 
55
61
  await this.cursorManager.initialize()
56
- this.cursorManager.startPolling((viewName, diffs) => {
57
- this.fanout.broadcast(viewName, diffs)
62
+ this.cursorManager.startPolling((queryName, diffs) => {
63
+ this.fanout.broadcast(queryName, diffs)
64
+ this.webhooks.broadcast(queryName, diffs)
58
65
  })
59
66
 
60
67
  const handleHttp = (req: IncomingMessage, res: ServerResponse) => {
package/src/webhook.ts ADDED
@@ -0,0 +1,67 @@
1
+ import { createHmac } from 'node:crypto'
2
+ import type { QueryDef, SqlFragment } from '@risingwave/wavelet'
3
+ import type { ViewDiff } from './cursor-manager.js'
4
+
5
+ const TIMEOUT_MS = 10000
6
+
7
+ export class WebhookFanout {
8
+ private webhooks: Map<string, string> = new Map() // queryName -> url
9
+
10
+ constructor(
11
+ queries: Record<string, QueryDef | SqlFragment>,
12
+ private signingSecret?: string
13
+ ) {
14
+ for (const [name, def] of Object.entries(queries)) {
15
+ if ('_tag' in def && def._tag === 'sql') continue
16
+ const qd = def as QueryDef
17
+ if (qd.webhook) {
18
+ this.webhooks.set(name, qd.webhook)
19
+ }
20
+ }
21
+ }
22
+
23
+ async broadcast(queryName: string, diff: ViewDiff): Promise<void> {
24
+ const url = this.webhooks.get(queryName)
25
+ if (!url) return
26
+
27
+ const body = JSON.stringify({
28
+ query: queryName,
29
+ cursor: diff.cursor,
30
+ inserted: diff.inserted,
31
+ updated: diff.updated,
32
+ deleted: diff.deleted,
33
+ })
34
+
35
+ const headers: Record<string, string> = {
36
+ 'Content-Type': 'application/json',
37
+ 'User-Agent': 'Wavelet-Webhook/1.0',
38
+ }
39
+
40
+ if (this.signingSecret) {
41
+ const signature = createHmac('sha256', this.signingSecret)
42
+ .update(body)
43
+ .digest('hex')
44
+ headers['X-Wavelet-Signature'] = `sha256=${signature}`
45
+ }
46
+
47
+ try {
48
+ const controller = new AbortController()
49
+ const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS)
50
+
51
+ await fetch(url, {
52
+ method: 'POST',
53
+ headers,
54
+ body,
55
+ signal: controller.signal,
56
+ })
57
+
58
+ clearTimeout(timeout)
59
+ } catch (err: any) {
60
+ if (err.name === 'AbortError') {
61
+ console.error(`[webhook] Timeout sending to ${url} for query ${queryName}`)
62
+ } else {
63
+ console.error(`[webhook] Failed to send to ${url} for query ${queryName}:`, err.message)
64
+ }
65
+ }
66
+ }
67
+ }
package/src/ws-fanout.ts CHANGED
@@ -1,23 +1,23 @@
1
1
  import { WebSocketServer, WebSocket } from 'ws'
2
2
  import type { IncomingMessage, Server } from 'node:http'
3
- import type { ViewDef, SqlFragment } from '@risingwave/wavelet'
3
+ import type { QueryDef, SqlFragment } from '@risingwave/wavelet'
4
4
  import type { CursorManager, ViewDiff } from './cursor-manager.js'
5
5
  import type { JwtVerifier, JwtClaims } from './jwt.js'
6
6
 
7
7
  interface Subscriber {
8
8
  ws: WebSocket
9
- viewName: string
9
+ queryName: string
10
10
  claims: JwtClaims | null
11
11
  }
12
12
 
13
13
  export class WebSocketFanout {
14
14
  private wss: WebSocketServer | null = null
15
- private subscribers: Map<string, Set<Subscriber>> = new Map() // viewName -> subscribers
15
+ private subscribers: Map<string, Set<Subscriber>> = new Map() // queryName -> subscribers
16
16
 
17
17
  constructor(
18
18
  private cursorManager: CursorManager,
19
19
  private jwt: JwtVerifier,
20
- private views: Record<string, ViewDef | SqlFragment>
20
+ private queries: Record<string, QueryDef | SqlFragment>
21
21
  ) {}
22
22
 
23
23
  attach(server: Server, pathPrefix?: string): void {
@@ -54,17 +54,17 @@ export class WebSocketFanout {
54
54
 
55
55
  if (!match) {
56
56
  throw new Error(
57
- `Invalid path: ${url.pathname}. Use /subscribe/{viewName}. ` +
58
- `Available views: ${Object.keys(this.views).join(', ')}`
57
+ `Invalid path: ${url.pathname}. Use /subscribe/{queryName}. ` +
58
+ `Available queries: ${Object.keys(this.queries).join(', ')}`
59
59
  )
60
60
  }
61
61
 
62
- const viewName = match[1]
62
+ const queryName = match[1]
63
63
 
64
- if (!this.views[viewName]) {
65
- const available = Object.keys(this.views)
64
+ if (!this.queries[queryName]) {
65
+ const available = Object.keys(this.queries)
66
66
  throw new Error(
67
- `View '${viewName}' not found. Available views: ${available.join(', ')}`
67
+ `Query '${queryName}' not found. Available queries: ${available.join(', ')}`
68
68
  )
69
69
  }
70
70
 
@@ -80,26 +80,35 @@ export class WebSocketFanout {
80
80
  claims = await this.jwt.verify(token)
81
81
  }
82
82
 
83
- const subscriber: Subscriber = { ws, viewName, claims }
83
+ const subscriber: Subscriber = { ws, queryName, claims }
84
84
 
85
- if (!this.subscribers.has(viewName)) {
86
- this.subscribers.set(viewName, new Set())
85
+ if (!this.subscribers.has(queryName)) {
86
+ this.subscribers.set(queryName, new Set())
87
87
  }
88
- this.subscribers.get(viewName)!.add(subscriber)
88
+ this.subscribers.get(queryName)!.add(subscriber)
89
89
 
90
90
  ws.on('close', () => {
91
- this.subscribers.get(viewName)?.delete(subscriber)
91
+ this.subscribers.get(queryName)?.delete(subscriber)
92
92
  })
93
93
 
94
- ws.send(JSON.stringify({ type: 'connected', view: viewName }))
94
+ // Heartbeat: detect dead connections behind proxies/load balancers
95
+ const pingInterval = setInterval(() => {
96
+ if (ws.readyState === WebSocket.OPEN) {
97
+ ws.ping()
98
+ }
99
+ }, 30000)
100
+ ws.on('close', () => clearInterval(pingInterval))
101
+ ws.on('pong', () => { /* connection alive */ })
102
+
103
+ ws.send(JSON.stringify({ type: 'connected', query: queryName }))
95
104
  }
96
105
 
97
- broadcast(viewName: string, diff: ViewDiff): void {
98
- const subs = this.subscribers.get(viewName)
106
+ broadcast(queryName: string, diff: ViewDiff): void {
107
+ const subs = this.subscribers.get(queryName)
99
108
  if (!subs || subs.size === 0) return
100
109
 
101
- const viewDef = this.views[viewName]
102
- const filterBy = this.getFilterBy(viewDef)
110
+ const queryDef = this.queries[queryName]
111
+ const filterBy = this.getFilterBy(queryDef)
103
112
 
104
113
  for (const sub of subs) {
105
114
  if (sub.ws.readyState !== WebSocket.OPEN) continue
@@ -119,7 +128,7 @@ export class WebSocketFanout {
119
128
 
120
129
  sub.ws.send(JSON.stringify({
121
130
  type: 'diff',
122
- view: viewName,
131
+ query: queryName,
123
132
  cursor: filteredDiff.cursor,
124
133
  inserted: filteredDiff.inserted,
125
134
  updated: filteredDiff.updated,
@@ -130,7 +139,10 @@ export class WebSocketFanout {
130
139
 
131
140
  private filterDiff(diff: ViewDiff, filterBy: string, claims: JwtClaims): ViewDiff {
132
141
  const claimValue = claims[filterBy]
133
- if (claimValue === undefined) return diff
142
+ if (claimValue === undefined) {
143
+ // No matching claim -- return empty diff, not all data
144
+ return { cursor: diff.cursor, inserted: [], updated: [], deleted: [] }
145
+ }
134
146
 
135
147
  const filterFn = (row: Record<string, unknown>) =>
136
148
  String(row[filterBy]) === String(claimValue)
@@ -143,9 +155,9 @@ export class WebSocketFanout {
143
155
  }
144
156
  }
145
157
 
146
- private getFilterBy(viewDef: ViewDef | SqlFragment): string | undefined {
147
- if ('_tag' in viewDef && viewDef._tag === 'sql') return undefined
148
- return (viewDef as ViewDef).filterBy
158
+ private getFilterBy(queryDef: QueryDef | SqlFragment): string | undefined {
159
+ if ('_tag' in queryDef && queryDef._tag === 'sql') return undefined
160
+ return (queryDef as QueryDef).filterBy
149
161
  }
150
162
 
151
163
  closeAll(): void {