@risingwave/wavelet-server 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/__tests__/http-api.test.ts +23 -23
- package/src/__tests__/integration.test.ts +16 -16
- package/src/cursor-manager.ts +29 -29
- package/src/ddl-manager.ts +163 -62
- package/src/http-api.ts +150 -71
- package/src/server.ts +15 -8
- package/src/webhook.ts +67 -0
- package/src/ws-fanout.ts +37 -25
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@risingwave/wavelet-server",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Wavelet server - WebSocket fanout layer for RisingWave",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"pg": "^8.13.0",
|
|
14
14
|
"ws": "^8.18.0",
|
|
15
15
|
"jose": "^6.0.0",
|
|
16
|
-
"@risingwave/wavelet": "0.1
|
|
16
|
+
"@risingwave/wavelet": "0.2.1"
|
|
17
17
|
},
|
|
18
18
|
"devDependencies": {
|
|
19
19
|
"@types/pg": "^8.11.0",
|
|
@@ -36,15 +36,15 @@ function request(server: http.Server, method: string, path: string, body?: unkno
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
describe('HttpApi', () => {
|
|
39
|
-
const
|
|
40
|
-
|
|
39
|
+
const events = {
|
|
40
|
+
game_events: { columns: { user_id: 'string' as const, value: 'int' as const } },
|
|
41
41
|
}
|
|
42
|
-
const
|
|
42
|
+
const queries = {
|
|
43
43
|
leaderboard: { _tag: 'sql' as const, text: 'SELECT 1' },
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
it('returns health check', async () => {
|
|
47
|
-
const api = new HttpApi('postgres://dummy',
|
|
47
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
48
48
|
const server = await createTestServer(api)
|
|
49
49
|
try {
|
|
50
50
|
const res = await request(server, 'GET', '/v1/health')
|
|
@@ -55,32 +55,32 @@ describe('HttpApi', () => {
|
|
|
55
55
|
}
|
|
56
56
|
})
|
|
57
57
|
|
|
58
|
-
it('lists available
|
|
59
|
-
const api = new HttpApi('postgres://dummy',
|
|
58
|
+
it('lists available queries', async () => {
|
|
59
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
60
60
|
const server = await createTestServer(api)
|
|
61
61
|
try {
|
|
62
|
-
const res = await request(server, 'GET', '/v1/
|
|
62
|
+
const res = await request(server, 'GET', '/v1/queries')
|
|
63
63
|
expect(res.status).toBe(200)
|
|
64
|
-
expect(res.data).toEqual({
|
|
64
|
+
expect(res.data).toEqual({ queries: ['leaderboard'] })
|
|
65
65
|
} finally {
|
|
66
66
|
server.close()
|
|
67
67
|
}
|
|
68
68
|
})
|
|
69
69
|
|
|
70
|
-
it('lists available
|
|
71
|
-
const api = new HttpApi('postgres://dummy',
|
|
70
|
+
it('lists available events', async () => {
|
|
71
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
72
72
|
const server = await createTestServer(api)
|
|
73
73
|
try {
|
|
74
|
-
const res = await request(server, 'GET', '/v1/
|
|
74
|
+
const res = await request(server, 'GET', '/v1/events')
|
|
75
75
|
expect(res.status).toBe(200)
|
|
76
|
-
expect(res.data).toEqual({
|
|
76
|
+
expect(res.data).toEqual({ events: ['game_events'] })
|
|
77
77
|
} finally {
|
|
78
78
|
server.close()
|
|
79
79
|
}
|
|
80
80
|
})
|
|
81
81
|
|
|
82
82
|
it('returns 404 for unknown routes with helpful message', async () => {
|
|
83
|
-
const api = new HttpApi('postgres://dummy',
|
|
83
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
84
84
|
const server = await createTestServer(api)
|
|
85
85
|
try {
|
|
86
86
|
const res = await request(server, 'GET', '/v1/nonexistent')
|
|
@@ -94,35 +94,35 @@ describe('HttpApi', () => {
|
|
|
94
94
|
})
|
|
95
95
|
|
|
96
96
|
it('handles CORS preflight', async () => {
|
|
97
|
-
const api = new HttpApi('postgres://dummy',
|
|
97
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
98
98
|
const server = await createTestServer(api)
|
|
99
99
|
try {
|
|
100
|
-
const res = await request(server, 'OPTIONS', '/v1/
|
|
100
|
+
const res = await request(server, 'OPTIONS', '/v1/queries')
|
|
101
101
|
expect(res.status).toBe(204)
|
|
102
102
|
} finally {
|
|
103
103
|
server.close()
|
|
104
104
|
}
|
|
105
105
|
})
|
|
106
106
|
|
|
107
|
-
it('returns 404 for unknown
|
|
108
|
-
const api = new HttpApi('postgres://dummy',
|
|
107
|
+
it('returns 404 for unknown event', async () => {
|
|
108
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
109
109
|
const server = await createTestServer(api)
|
|
110
110
|
try {
|
|
111
|
-
const res = await request(server, 'POST', '/v1/
|
|
111
|
+
const res = await request(server, 'POST', '/v1/events/nonexistent', { x: 1 })
|
|
112
112
|
expect(res.status).toBe(404)
|
|
113
|
-
expect(res.data.
|
|
113
|
+
expect(res.data.available_events).toEqual(['game_events'])
|
|
114
114
|
} finally {
|
|
115
115
|
server.close()
|
|
116
116
|
}
|
|
117
117
|
})
|
|
118
118
|
|
|
119
|
-
it('returns 404 for unknown
|
|
120
|
-
const api = new HttpApi('postgres://dummy',
|
|
119
|
+
it('returns 404 for unknown query', async () => {
|
|
120
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
121
121
|
const server = await createTestServer(api)
|
|
122
122
|
try {
|
|
123
|
-
const res = await request(server, 'GET', '/v1/
|
|
123
|
+
const res = await request(server, 'GET', '/v1/queries/nonexistent')
|
|
124
124
|
expect(res.status).toBe(404)
|
|
125
|
-
expect(res.data.
|
|
125
|
+
expect(res.data.available_queries).toEqual(['leaderboard'])
|
|
126
126
|
} finally {
|
|
127
127
|
server.close()
|
|
128
128
|
}
|
|
@@ -36,7 +36,7 @@ const SUB_NAME = `wavelet_sub_${VIEW_NAME}`
|
|
|
36
36
|
|
|
37
37
|
const testConfig: WaveletConfig = {
|
|
38
38
|
database: DATABASE_URL,
|
|
39
|
-
|
|
39
|
+
events: {
|
|
40
40
|
[STREAM_NAME]: {
|
|
41
41
|
columns: {
|
|
42
42
|
user_id: 'string',
|
|
@@ -44,7 +44,7 @@ const testConfig: WaveletConfig = {
|
|
|
44
44
|
},
|
|
45
45
|
},
|
|
46
46
|
},
|
|
47
|
-
|
|
47
|
+
queries: {
|
|
48
48
|
[VIEW_NAME]: sql`
|
|
49
49
|
SELECT user_id, SUM(value) AS total_value, COUNT(*) AS event_count
|
|
50
50
|
FROM ${STREAM_NAME}
|
|
@@ -75,17 +75,17 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: DDL Manage
|
|
|
75
75
|
await ddl.close()
|
|
76
76
|
})
|
|
77
77
|
|
|
78
|
-
it('creates tables, views, and subscriptions', async () => {
|
|
78
|
+
it('creates tables, materialized views, and subscriptions', async () => {
|
|
79
79
|
const actions = await ddl.sync(testConfig)
|
|
80
80
|
|
|
81
81
|
const creates = actions.filter(a => a.type === 'create')
|
|
82
|
-
expect(creates.length).toBeGreaterThanOrEqual(3) //
|
|
82
|
+
expect(creates.length).toBeGreaterThanOrEqual(3) // event + query + subscription
|
|
83
83
|
|
|
84
|
-
const
|
|
85
|
-
expect(
|
|
84
|
+
const eventAction = creates.find(a => a.resource === 'event' && a.name === STREAM_NAME)
|
|
85
|
+
expect(eventAction).toBeDefined()
|
|
86
86
|
|
|
87
|
-
const
|
|
88
|
-
expect(
|
|
87
|
+
const queryAction = creates.find(a => a.resource === 'query' && a.name === VIEW_NAME)
|
|
88
|
+
expect(queryAction).toBeDefined()
|
|
89
89
|
|
|
90
90
|
const subAction = creates.find(a => a.resource === 'subscription' && a.name === SUB_NAME)
|
|
91
91
|
expect(subAction).toBeDefined()
|
|
@@ -162,15 +162,15 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
162
162
|
expect(data.status).toBe('ok')
|
|
163
163
|
})
|
|
164
164
|
|
|
165
|
-
it('lists
|
|
166
|
-
const res = await fetch(`http://localhost:${port}/v1/
|
|
165
|
+
it('lists queries', async () => {
|
|
166
|
+
const res = await fetch(`http://localhost:${port}/v1/queries`)
|
|
167
167
|
const data = await res.json()
|
|
168
|
-
expect(data.
|
|
168
|
+
expect(data.queries).toContain(VIEW_NAME)
|
|
169
169
|
})
|
|
170
170
|
|
|
171
|
-
it('writes events via HTTP and reads
|
|
171
|
+
it('writes events via HTTP and reads query', async () => {
|
|
172
172
|
// Write events
|
|
173
|
-
await fetch(`http://localhost:${port}/v1/
|
|
173
|
+
await fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
|
|
174
174
|
method: 'POST',
|
|
175
175
|
headers: { 'Content-Type': 'application/json' },
|
|
176
176
|
body: JSON.stringify({ user_id: 'integration_test', value: 42 }),
|
|
@@ -179,8 +179,8 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
179
179
|
// Wait for MV update
|
|
180
180
|
await new Promise(r => setTimeout(r, 2000))
|
|
181
181
|
|
|
182
|
-
// Read
|
|
183
|
-
const res = await fetch(`http://localhost:${port}/v1/
|
|
182
|
+
// Read query
|
|
183
|
+
const res = await fetch(`http://localhost:${port}/v1/queries/${VIEW_NAME}`)
|
|
184
184
|
const data = await res.json()
|
|
185
185
|
expect(data.rows.length).toBeGreaterThan(0)
|
|
186
186
|
})
|
|
@@ -193,7 +193,7 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
193
193
|
const msg = JSON.parse(data.toString())
|
|
194
194
|
if (msg.type === 'connected') {
|
|
195
195
|
// Write event to trigger diff
|
|
196
|
-
fetch(`http://localhost:${port}/v1/
|
|
196
|
+
fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
|
|
197
197
|
method: 'POST',
|
|
198
198
|
headers: { 'Content-Type': 'application/json' },
|
|
199
199
|
body: JSON.stringify({ user_id: 'ws_test_user', value: 7 }),
|
package/src/cursor-manager.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import pg from 'pg'
|
|
2
|
-
import type { SqlFragment,
|
|
2
|
+
import type { SqlFragment, QueryDef } from '@risingwave/wavelet'
|
|
3
3
|
|
|
4
4
|
const { Client } = pg
|
|
5
5
|
|
|
@@ -16,12 +16,12 @@ export interface ViewDiff {
|
|
|
16
16
|
deleted: Record<string, unknown>[]
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
type DiffCallback = (
|
|
19
|
+
type DiffCallback = (queryName: string, diff: ViewDiff) => void
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
22
|
* Manages persistent subscription cursors against RisingWave.
|
|
23
23
|
*
|
|
24
|
-
* Each
|
|
24
|
+
* Each query gets its own dedicated pg connection and a persistent cursor.
|
|
25
25
|
* Uses blocking FETCH (WITH timeout) so there is no polling interval -
|
|
26
26
|
* diffs are dispatched as soon as RisingWave produces them.
|
|
27
27
|
*/
|
|
@@ -29,15 +29,15 @@ export class CursorManager {
|
|
|
29
29
|
// Shared connection for DDL (CREATE SUBSCRIPTION) and ad-hoc queries
|
|
30
30
|
private client: InstanceType<typeof Client> | null = null
|
|
31
31
|
|
|
32
|
-
// Per-
|
|
33
|
-
private
|
|
32
|
+
// Per-query dedicated connections for blocking FETCH
|
|
33
|
+
private queryConnections: Map<string, InstanceType<typeof Client>> = new Map()
|
|
34
34
|
private cursorNames: Map<string, string> = new Map()
|
|
35
35
|
private subscriptions: Map<string, string> = new Map()
|
|
36
36
|
private running = false
|
|
37
37
|
|
|
38
38
|
constructor(
|
|
39
39
|
private connectionString: string,
|
|
40
|
-
private
|
|
40
|
+
private queries: Record<string, QueryDef | SqlFragment>
|
|
41
41
|
) {}
|
|
42
42
|
|
|
43
43
|
async initialize(): Promise<void> {
|
|
@@ -45,13 +45,13 @@ export class CursorManager {
|
|
|
45
45
|
await this.client.connect()
|
|
46
46
|
console.log('Connected to RisingWave')
|
|
47
47
|
|
|
48
|
-
for (const [
|
|
49
|
-
const subName = `wavelet_sub_${
|
|
48
|
+
for (const [queryName] of Object.entries(this.queries)) {
|
|
49
|
+
const subName = `wavelet_sub_${queryName}`
|
|
50
50
|
|
|
51
51
|
// Create subscription if not exists (idempotent)
|
|
52
52
|
try {
|
|
53
53
|
await this.client.query(
|
|
54
|
-
`CREATE SUBSCRIPTION ${subName} FROM ${
|
|
54
|
+
`CREATE SUBSCRIPTION ${subName} FROM ${queryName} WITH (retention = '24h')`
|
|
55
55
|
)
|
|
56
56
|
console.log(`Created subscription: ${subName}`)
|
|
57
57
|
} catch (err: any) {
|
|
@@ -62,31 +62,31 @@ export class CursorManager {
|
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
this.subscriptions.set(
|
|
65
|
+
this.subscriptions.set(queryName, subName)
|
|
66
66
|
|
|
67
|
-
// Create dedicated connection and persistent cursor for this
|
|
67
|
+
// Create dedicated connection and persistent cursor for this query
|
|
68
68
|
const conn = new Client({ connectionString: this.connectionString })
|
|
69
69
|
await conn.connect()
|
|
70
70
|
|
|
71
|
-
const cursorName = `wavelet_cur_${
|
|
71
|
+
const cursorName = `wavelet_cur_${queryName}`
|
|
72
72
|
await conn.query(`DECLARE ${cursorName} SUBSCRIPTION CURSOR FOR ${subName}`)
|
|
73
73
|
console.log(`Opened persistent cursor: ${cursorName}`)
|
|
74
74
|
|
|
75
|
-
this.
|
|
76
|
-
this.cursorNames.set(
|
|
75
|
+
this.queryConnections.set(queryName, conn)
|
|
76
|
+
this.cursorNames.set(queryName, cursorName)
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
|
-
* Start listening for diffs on all
|
|
82
|
-
* Each
|
|
81
|
+
* Start listening for diffs on all queries.
|
|
82
|
+
* Each query runs its own async loop with blocking FETCH.
|
|
83
83
|
* No polling interval - FETCH blocks until data arrives or timeout.
|
|
84
84
|
*/
|
|
85
85
|
startPolling(callback: DiffCallback): void {
|
|
86
86
|
this.running = true
|
|
87
87
|
|
|
88
|
-
for (const [
|
|
89
|
-
this.listenLoop(
|
|
88
|
+
for (const [queryName] of this.subscriptions.entries()) {
|
|
89
|
+
this.listenLoop(queryName, callback)
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
|
|
@@ -94,9 +94,9 @@ export class CursorManager {
|
|
|
94
94
|
this.running = false
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
private async listenLoop(
|
|
98
|
-
const conn = this.
|
|
99
|
-
const cursorName = this.cursorNames.get(
|
|
97
|
+
private async listenLoop(queryName: string, callback: DiffCallback): Promise<void> {
|
|
98
|
+
const conn = this.queryConnections.get(queryName)
|
|
99
|
+
const cursorName = this.cursorNames.get(queryName)
|
|
100
100
|
if (!conn || !cursorName) return
|
|
101
101
|
|
|
102
102
|
while (this.running) {
|
|
@@ -108,12 +108,12 @@ export class CursorManager {
|
|
|
108
108
|
|
|
109
109
|
if (result.rows.length === 0) continue
|
|
110
110
|
|
|
111
|
-
// Got at least one row. Drain any remaining rows
|
|
111
|
+
// Got at least one row. Drain any remaining rows with timeout.
|
|
112
112
|
const allRows = [...result.rows]
|
|
113
113
|
let more = true
|
|
114
114
|
while (more) {
|
|
115
115
|
const batch = await conn.query(
|
|
116
|
-
`FETCH 100 FROM ${cursorName}`
|
|
116
|
+
`FETCH 100 FROM ${cursorName} WITH (timeout = '1s')`
|
|
117
117
|
)
|
|
118
118
|
if (batch.rows.length > 0) {
|
|
119
119
|
allRows.push(...batch.rows)
|
|
@@ -125,11 +125,11 @@ export class CursorManager {
|
|
|
125
125
|
const diff = this.parseDiffs(allRows)
|
|
126
126
|
|
|
127
127
|
if (diff.inserted.length > 0 || diff.updated.length > 0 || diff.deleted.length > 0) {
|
|
128
|
-
callback(
|
|
128
|
+
callback(queryName, diff)
|
|
129
129
|
}
|
|
130
130
|
} catch (err: any) {
|
|
131
131
|
if (!this.running) return
|
|
132
|
-
console.error(`[cursor-manager] Error fetching ${
|
|
132
|
+
console.error(`[cursor-manager] Error fetching ${queryName}:`, err.message)
|
|
133
133
|
// Back off on error, then retry
|
|
134
134
|
await new Promise(r => setTimeout(r, 1000))
|
|
135
135
|
}
|
|
@@ -187,9 +187,9 @@ export class CursorManager {
|
|
|
187
187
|
async close(): Promise<void> {
|
|
188
188
|
this.running = false
|
|
189
189
|
|
|
190
|
-
// Close per-
|
|
191
|
-
for (const [
|
|
192
|
-
const cursorName = this.cursorNames.get(
|
|
190
|
+
// Close per-query connections
|
|
191
|
+
for (const [queryName, conn] of this.queryConnections) {
|
|
192
|
+
const cursorName = this.cursorNames.get(queryName)
|
|
193
193
|
try {
|
|
194
194
|
if (cursorName) await conn.query(`CLOSE ${cursorName}`)
|
|
195
195
|
} catch {}
|
|
@@ -197,7 +197,7 @@ export class CursorManager {
|
|
|
197
197
|
await conn.end()
|
|
198
198
|
} catch {}
|
|
199
199
|
}
|
|
200
|
-
this.
|
|
200
|
+
this.queryConnections.clear()
|
|
201
201
|
this.cursorNames.clear()
|
|
202
202
|
|
|
203
203
|
// Close shared connection
|
package/src/ddl-manager.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import pg from 'pg'
|
|
2
|
-
import type { WaveletConfig,
|
|
2
|
+
import type { WaveletConfig, EventDef, QueryDef, SqlFragment, PostgresCdcSource } from '@risingwave/wavelet'
|
|
3
3
|
|
|
4
4
|
const { Client } = pg
|
|
5
5
|
|
|
6
6
|
export interface DdlAction {
|
|
7
7
|
type: 'create' | 'update' | 'delete' | 'unchanged'
|
|
8
|
-
resource: '
|
|
8
|
+
resource: 'event' | 'source' | 'query' | 'subscription'
|
|
9
9
|
name: string
|
|
10
10
|
detail?: string
|
|
11
11
|
}
|
|
@@ -26,13 +26,13 @@ function normalizeSql(sql: string): string {
|
|
|
26
26
|
return stripped.replace(/\s+/g, ' ').trim().toLowerCase()
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
function
|
|
30
|
-
if ('_tag' in
|
|
31
|
-
return (
|
|
29
|
+
function getQuerySql(queryDef: QueryDef | SqlFragment): string {
|
|
30
|
+
if ('_tag' in queryDef && queryDef._tag === 'sql') return queryDef.text
|
|
31
|
+
return (queryDef as QueryDef).query.text
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
function buildCreateTableSql(name: string,
|
|
35
|
-
const cols = Object.entries(
|
|
34
|
+
function buildCreateTableSql(name: string, eventDef: EventDef): string {
|
|
35
|
+
const cols = Object.entries(eventDef.columns)
|
|
36
36
|
.map(([colName, colType]) => {
|
|
37
37
|
const sqlType = COLUMN_TYPE_MAP[colType]
|
|
38
38
|
if (!sqlType) throw new Error(`Unknown column type "${colType}" for column "${colName}"`)
|
|
@@ -60,7 +60,7 @@ export class DdlManager {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
|
-
* Sync all
|
|
63
|
+
* Sync all events, queries, and subscriptions to match the config.
|
|
64
64
|
* Returns a list of actions taken.
|
|
65
65
|
* Idempotent - safe to call multiple times.
|
|
66
66
|
*/
|
|
@@ -71,61 +71,78 @@ export class DdlManager {
|
|
|
71
71
|
|
|
72
72
|
// 1. Fetch existing state from RisingWave
|
|
73
73
|
const existingTables = await this.getExistingTables()
|
|
74
|
-
const
|
|
74
|
+
const existingMVs = await this.getExistingMaterializedViews()
|
|
75
75
|
const existingSubscriptions = await this.getExistingSubscriptions()
|
|
76
76
|
|
|
77
|
-
const
|
|
78
|
-
const
|
|
77
|
+
const desiredEvents = config.events ?? config.streams ?? {}
|
|
78
|
+
const desiredSources = config.sources ?? {}
|
|
79
|
+
const desiredQueries = config.queries ?? config.views ?? {}
|
|
79
80
|
|
|
80
|
-
// 2. Determine which
|
|
81
|
-
const
|
|
82
|
-
const
|
|
81
|
+
// 2. Determine which events (tables) to create or remove
|
|
82
|
+
const desiredEventNames = new Set(Object.keys(desiredEvents))
|
|
83
|
+
const desiredQueryNames = new Set(Object.keys(desiredQueries))
|
|
83
84
|
|
|
84
|
-
// 3. Sync
|
|
85
|
-
for (const [
|
|
86
|
-
if (existingTables.has(
|
|
87
|
-
actions.push({ type: 'unchanged', resource: '
|
|
85
|
+
// 3. Sync events - create missing tables
|
|
86
|
+
for (const [eventName, eventDef] of Object.entries(desiredEvents)) {
|
|
87
|
+
if (existingTables.has(eventName)) {
|
|
88
|
+
actions.push({ type: 'unchanged', resource: 'event', name: eventName })
|
|
88
89
|
} else {
|
|
89
|
-
await this.createTable(
|
|
90
|
-
actions.push({ type: 'create', resource: '
|
|
90
|
+
await this.createTable(eventName, eventDef)
|
|
91
|
+
actions.push({ type: 'create', resource: 'event', name: eventName })
|
|
91
92
|
}
|
|
92
93
|
}
|
|
93
94
|
|
|
94
|
-
//
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
95
|
+
// 3b. Sync CDC sources - create Postgres CDC tables
|
|
96
|
+
const existingSources = await this.getExistingSources()
|
|
97
|
+
for (const [sourceName, sourceDef] of Object.entries(desiredSources)) {
|
|
98
|
+
if (sourceDef.type === 'postgres') {
|
|
99
|
+
for (const tableName of sourceDef.tables) {
|
|
100
|
+
const cdcTableName = `${sourceName}_${tableName}`
|
|
101
|
+
if (existingSources.has(cdcTableName) || existingTables.has(cdcTableName)) {
|
|
102
|
+
actions.push({ type: 'unchanged', resource: 'source', name: cdcTableName })
|
|
103
|
+
} else {
|
|
104
|
+
await this.createCdcSource(sourceName, tableName, sourceDef)
|
|
105
|
+
actions.push({ type: 'create', resource: 'source', name: cdcTableName })
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// 4. Sync queries - create, update, or leave unchanged
|
|
112
|
+
for (const [queryName, queryDef] of Object.entries(desiredQueries)) {
|
|
113
|
+
const subName = `wavelet_sub_${queryName}`
|
|
114
|
+
const desiredSql = getQuerySql(queryDef)
|
|
115
|
+
const existingSql = existingMVs.get(queryName)
|
|
99
116
|
|
|
100
117
|
if (existingSql === undefined) {
|
|
101
|
-
//
|
|
102
|
-
await this.createMaterializedView(
|
|
103
|
-
actions.push({ type: 'create', resource: '
|
|
118
|
+
// MV does not exist - create MV and subscription
|
|
119
|
+
await this.createMaterializedView(queryName, desiredSql)
|
|
120
|
+
actions.push({ type: 'create', resource: 'query', name: queryName })
|
|
104
121
|
|
|
105
|
-
await this.createSubscription(subName,
|
|
122
|
+
await this.createSubscription(subName, queryName)
|
|
106
123
|
actions.push({ type: 'create', resource: 'subscription', name: subName })
|
|
107
124
|
} else if (normalizeSql(existingSql) !== normalizeSql(desiredSql)) {
|
|
108
|
-
//
|
|
125
|
+
// Query SQL changed - drop subscription, drop MV, recreate
|
|
109
126
|
if (existingSubscriptions.has(subName)) {
|
|
110
127
|
await this.dropSubscription(subName)
|
|
111
|
-
actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'dropped for
|
|
128
|
+
actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'dropped for query update' })
|
|
112
129
|
}
|
|
113
130
|
|
|
114
|
-
await this.dropMaterializedView(
|
|
115
|
-
actions.push({ type: 'delete', resource: '
|
|
131
|
+
await this.dropMaterializedView(queryName)
|
|
132
|
+
actions.push({ type: 'delete', resource: 'query', name: queryName, detail: 'dropped for update' })
|
|
116
133
|
|
|
117
|
-
await this.createMaterializedView(
|
|
118
|
-
actions.push({ type: 'create', resource: '
|
|
134
|
+
await this.createMaterializedView(queryName, desiredSql)
|
|
135
|
+
actions.push({ type: 'create', resource: 'query', name: queryName, detail: 'recreated with updated SQL' })
|
|
119
136
|
|
|
120
|
-
await this.createSubscription(subName,
|
|
121
|
-
actions.push({ type: 'create', resource: 'subscription', name: subName, detail: 'recreated after
|
|
137
|
+
await this.createSubscription(subName, queryName)
|
|
138
|
+
actions.push({ type: 'create', resource: 'subscription', name: subName, detail: 'recreated after query update' })
|
|
122
139
|
} else {
|
|
123
|
-
//
|
|
124
|
-
actions.push({ type: 'unchanged', resource: '
|
|
140
|
+
// Query SQL unchanged
|
|
141
|
+
actions.push({ type: 'unchanged', resource: 'query', name: queryName })
|
|
125
142
|
|
|
126
|
-
// Ensure subscription exists even if the
|
|
143
|
+
// Ensure subscription exists even if the query is unchanged
|
|
127
144
|
if (!existingSubscriptions.has(subName)) {
|
|
128
|
-
await this.createSubscription(subName,
|
|
145
|
+
await this.createSubscription(subName, queryName)
|
|
129
146
|
actions.push({ type: 'create', resource: 'subscription', name: subName })
|
|
130
147
|
} else {
|
|
131
148
|
actions.push({ type: 'unchanged', resource: 'subscription', name: subName })
|
|
@@ -133,19 +150,19 @@ export class DdlManager {
|
|
|
133
150
|
}
|
|
134
151
|
}
|
|
135
152
|
|
|
136
|
-
// 5. Remove
|
|
137
|
-
for (const [
|
|
138
|
-
if (!
|
|
139
|
-
const subName = `wavelet_sub_${
|
|
153
|
+
// 5. Remove queries that are no longer in the config
|
|
154
|
+
for (const [existingMVName] of existingMVs) {
|
|
155
|
+
if (!desiredQueryNames.has(existingMVName)) {
|
|
156
|
+
const subName = `wavelet_sub_${existingMVName}`
|
|
140
157
|
|
|
141
158
|
// Drop subscription first
|
|
142
159
|
if (existingSubscriptions.has(subName)) {
|
|
143
160
|
await this.dropSubscription(subName)
|
|
144
|
-
actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: '
|
|
161
|
+
actions.push({ type: 'delete', resource: 'subscription', name: subName, detail: 'query removed from config' })
|
|
145
162
|
}
|
|
146
163
|
|
|
147
|
-
await this.dropMaterializedView(
|
|
148
|
-
actions.push({ type: 'delete', resource: '
|
|
164
|
+
await this.dropMaterializedView(existingMVName)
|
|
165
|
+
actions.push({ type: 'delete', resource: 'query', name: existingMVName, detail: 'removed from config' })
|
|
149
166
|
}
|
|
150
167
|
}
|
|
151
168
|
|
|
@@ -154,32 +171,32 @@ export class DdlManager {
|
|
|
154
171
|
// Only manage wavelet-prefixed subscriptions
|
|
155
172
|
if (!existingSubName.startsWith('wavelet_sub_')) continue
|
|
156
173
|
|
|
157
|
-
const
|
|
158
|
-
if (!
|
|
159
|
-
// Already handled in step 5 if the
|
|
160
|
-
if (!
|
|
174
|
+
const queryName = existingSubName.slice('wavelet_sub_'.length)
|
|
175
|
+
if (!desiredQueryNames.has(queryName)) {
|
|
176
|
+
// Already handled in step 5 if the MV existed, but handle dangling subs too
|
|
177
|
+
if (!existingMVs.has(queryName)) {
|
|
161
178
|
await this.dropSubscription(existingSubName)
|
|
162
179
|
actions.push({ type: 'delete', resource: 'subscription', name: existingSubName, detail: 'orphaned subscription' })
|
|
163
180
|
}
|
|
164
181
|
}
|
|
165
182
|
}
|
|
166
183
|
|
|
167
|
-
// 7. Remove
|
|
184
|
+
// 7. Remove events (tables) that are no longer in the config
|
|
168
185
|
for (const existingTableName of existingTables) {
|
|
169
|
-
if (!
|
|
186
|
+
if (!desiredEventNames.has(existingTableName)) {
|
|
170
187
|
// Only drop if no MV depends on it
|
|
171
|
-
const hasDependents = await this.
|
|
188
|
+
const hasDependents = await this.tableHasDependentMVs(existingTableName)
|
|
172
189
|
if (hasDependents) {
|
|
173
190
|
console.log(`[ddl-manager] Skipping drop of table "${existingTableName}" - materialized views depend on it`)
|
|
174
191
|
actions.push({
|
|
175
192
|
type: 'unchanged',
|
|
176
|
-
resource: '
|
|
193
|
+
resource: 'event',
|
|
177
194
|
name: existingTableName,
|
|
178
|
-
detail: 'kept because dependent views exist',
|
|
195
|
+
detail: 'kept because dependent materialized views exist',
|
|
179
196
|
})
|
|
180
197
|
} else {
|
|
181
198
|
await this.dropTable(existingTableName)
|
|
182
|
-
actions.push({ type: 'delete', resource: '
|
|
199
|
+
actions.push({ type: 'delete', resource: 'event', name: existingTableName, detail: 'removed from config' })
|
|
183
200
|
}
|
|
184
201
|
}
|
|
185
202
|
}
|
|
@@ -196,7 +213,7 @@ export class DdlManager {
|
|
|
196
213
|
return new Set(result.rows.map((r: any) => r.table_name))
|
|
197
214
|
}
|
|
198
215
|
|
|
199
|
-
private async
|
|
216
|
+
private async getExistingMaterializedViews(): Promise<Map<string, string>> {
|
|
200
217
|
const result = await this.client!.query(
|
|
201
218
|
`SELECT name, definition FROM rw_catalog.rw_materialized_views WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public')`
|
|
202
219
|
)
|
|
@@ -207,6 +224,18 @@ export class DdlManager {
|
|
|
207
224
|
return views
|
|
208
225
|
}
|
|
209
226
|
|
|
227
|
+
private async getExistingSources(): Promise<Set<string>> {
|
|
228
|
+
try {
|
|
229
|
+
const result = await this.client!.query(
|
|
230
|
+
`SELECT name FROM rw_catalog.rw_tables WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public') AND is_index = false`
|
|
231
|
+
)
|
|
232
|
+
return new Set(result.rows.map((r: any) => r.name))
|
|
233
|
+
} catch {
|
|
234
|
+
// Fallback: if catalog query fails, return empty set
|
|
235
|
+
return new Set()
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
210
239
|
private async getExistingSubscriptions(): Promise<Set<string>> {
|
|
211
240
|
const result = await this.client!.query(
|
|
212
241
|
`SELECT name FROM rw_catalog.rw_subscriptions WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public')`
|
|
@@ -214,7 +243,7 @@ export class DdlManager {
|
|
|
214
243
|
return new Set(result.rows.map((r: any) => r.name))
|
|
215
244
|
}
|
|
216
245
|
|
|
217
|
-
private async
|
|
246
|
+
private async tableHasDependentMVs(tableName: string): Promise<boolean> {
|
|
218
247
|
const result = await this.client!.query(
|
|
219
248
|
`SELECT name FROM rw_catalog.rw_materialized_views WHERE schema_id = (SELECT id FROM rw_catalog.rw_schemas WHERE name = 'public') AND definition ILIKE $1`,
|
|
220
249
|
[`%${tableName}%`]
|
|
@@ -224,8 +253,8 @@ export class DdlManager {
|
|
|
224
253
|
|
|
225
254
|
// ── DDL operations ────────────────────────────────────────────────────
|
|
226
255
|
|
|
227
|
-
private async createTable(name: string,
|
|
228
|
-
const sql = buildCreateTableSql(name,
|
|
256
|
+
private async createTable(name: string, eventDef: EventDef): Promise<void> {
|
|
257
|
+
const sql = buildCreateTableSql(name, eventDef)
|
|
229
258
|
try {
|
|
230
259
|
await this.client!.query(sql)
|
|
231
260
|
console.log(`[ddl-manager] Created table: ${name}`)
|
|
@@ -304,4 +333,76 @@ export class DdlManager {
|
|
|
304
333
|
}
|
|
305
334
|
}
|
|
306
335
|
}
|
|
336
|
+
|
|
337
|
+
private async createCdcSource(
|
|
338
|
+
sourceName: string,
|
|
339
|
+
tableName: string,
|
|
340
|
+
source: PostgresCdcSource
|
|
341
|
+
): Promise<void> {
|
|
342
|
+
const cdcTableName = `${sourceName}_${tableName}`
|
|
343
|
+
const slotName = source.slotName ?? `wavelet_${sourceName}`
|
|
344
|
+
const pubName = source.publicationName ?? `wavelet_${sourceName}_pub`
|
|
345
|
+
|
|
346
|
+
// RisingWave CDC source syntax:
|
|
347
|
+
// CREATE TABLE table_name ( ... ) WITH (
|
|
348
|
+
// connector = 'postgres-cdc',
|
|
349
|
+
// hostname = '...',
|
|
350
|
+
// port = '...',
|
|
351
|
+
// username = '...',
|
|
352
|
+
// password = '...',
|
|
353
|
+
// database.name = '...',
|
|
354
|
+
// table.name = '...',
|
|
355
|
+
// slot.name = '...',
|
|
356
|
+
// publication.name = '...'
|
|
357
|
+
// )
|
|
358
|
+
// We parse the connection string to extract components.
|
|
359
|
+
|
|
360
|
+
const parsed = parsePostgresUrl(source.connection)
|
|
361
|
+
|
|
362
|
+
const esc = (s: string) => s.replace(/'/g, "''")
|
|
363
|
+
try {
|
|
364
|
+
await this.client!.query(`
|
|
365
|
+
CREATE TABLE IF NOT EXISTS ${cdcTableName} (*)
|
|
366
|
+
WITH (
|
|
367
|
+
connector = 'postgres-cdc',
|
|
368
|
+
hostname = '${esc(parsed.host)}',
|
|
369
|
+
port = '${esc(parsed.port)}',
|
|
370
|
+
username = '${esc(parsed.user)}',
|
|
371
|
+
password = '${esc(parsed.password)}',
|
|
372
|
+
database.name = '${esc(parsed.database)}',
|
|
373
|
+
schema.name = '${esc(parsed.schema)}',
|
|
374
|
+
table.name = '${esc(tableName)}',
|
|
375
|
+
slot.name = '${esc(slotName)}',
|
|
376
|
+
publication.name = '${esc(pubName)}'
|
|
377
|
+
)
|
|
378
|
+
`)
|
|
379
|
+
console.log(`[ddl-manager] Created CDC source: ${cdcTableName} (from ${parsed.host}/${parsed.database}.${tableName})`)
|
|
380
|
+
} catch (err: any) {
|
|
381
|
+
if (err.message?.includes('already exists')) {
|
|
382
|
+
console.log(`[ddl-manager] CDC source already exists: ${cdcTableName}`)
|
|
383
|
+
} else {
|
|
384
|
+
throw err
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function parsePostgresUrl(url: string): {
|
|
391
|
+
host: string
|
|
392
|
+
port: string
|
|
393
|
+
user: string
|
|
394
|
+
password: string
|
|
395
|
+
database: string
|
|
396
|
+
schema: string
|
|
397
|
+
} {
|
|
398
|
+
// Parse postgresql://user:password@host:port/database?schema=xxx
|
|
399
|
+
const u = new URL(url)
|
|
400
|
+
return {
|
|
401
|
+
host: u.hostname,
|
|
402
|
+
port: u.port || '5432',
|
|
403
|
+
user: decodeURIComponent(u.username),
|
|
404
|
+
password: decodeURIComponent(u.password),
|
|
405
|
+
database: u.pathname.replace(/^\//, '') || 'postgres',
|
|
406
|
+
schema: u.searchParams.get('schema') ?? 'public',
|
|
407
|
+
}
|
|
307
408
|
}
|
package/src/http-api.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
2
2
|
import pg from 'pg'
|
|
3
|
-
import type {
|
|
3
|
+
import type { EventDef, QueryDef, SqlFragment } from '@risingwave/wavelet'
|
|
4
|
+
import type { JwtVerifier, JwtClaims } from './jwt.js'
|
|
4
5
|
|
|
5
|
-
const {
|
|
6
|
+
const { Pool } = pg
|
|
7
|
+
|
|
8
|
+
const MAX_BODY_SIZE = 10 * 1024 * 1024 // 10MB
|
|
6
9
|
|
|
7
10
|
export class HttpApi {
|
|
8
|
-
private
|
|
11
|
+
private pool: InstanceType<typeof Pool> | null = null
|
|
9
12
|
|
|
10
13
|
constructor(
|
|
11
14
|
private connectionString: string,
|
|
12
|
-
private
|
|
13
|
-
private
|
|
15
|
+
private events: Record<string, EventDef>,
|
|
16
|
+
private queries: Record<string, QueryDef | SqlFragment>,
|
|
17
|
+
private jwt?: JwtVerifier
|
|
14
18
|
) {}
|
|
15
19
|
|
|
16
20
|
async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
@@ -28,24 +32,24 @@ export class HttpApi {
|
|
|
28
32
|
const url = new URL(req.url ?? '/', `http://${req.headers.host}`)
|
|
29
33
|
|
|
30
34
|
try {
|
|
31
|
-
// POST /v1/
|
|
32
|
-
const
|
|
33
|
-
if (
|
|
34
|
-
await this.handleWrite(
|
|
35
|
+
// POST /v1/events/{name}
|
|
36
|
+
const eventMatch = url.pathname.match(/^\/v1\/events\/([^/]+)$/)
|
|
37
|
+
if (eventMatch && req.method === 'POST') {
|
|
38
|
+
await this.handleWrite(eventMatch[1], req, res)
|
|
35
39
|
return
|
|
36
40
|
}
|
|
37
41
|
|
|
38
|
-
// POST /v1/
|
|
39
|
-
const batchMatch = url.pathname.match(/^\/v1\/
|
|
42
|
+
// POST /v1/events/{name}/batch
|
|
43
|
+
const batchMatch = url.pathname.match(/^\/v1\/events\/([^/]+)\/batch$/)
|
|
40
44
|
if (batchMatch && req.method === 'POST') {
|
|
41
45
|
await this.handleBatchWrite(batchMatch[1], req, res)
|
|
42
46
|
return
|
|
43
47
|
}
|
|
44
48
|
|
|
45
|
-
// GET /v1/
|
|
46
|
-
const
|
|
47
|
-
if (
|
|
48
|
-
await this.handleRead(
|
|
49
|
+
// GET /v1/queries/{name}
|
|
50
|
+
const queryMatch = url.pathname.match(/^\/v1\/queries\/([^/]+)$/)
|
|
51
|
+
if (queryMatch && req.method === 'GET') {
|
|
52
|
+
await this.handleRead(queryMatch[1], url, req, res)
|
|
49
53
|
return
|
|
50
54
|
}
|
|
51
55
|
|
|
@@ -55,15 +59,15 @@ export class HttpApi {
|
|
|
55
59
|
return
|
|
56
60
|
}
|
|
57
61
|
|
|
58
|
-
// GET /v1/
|
|
59
|
-
if (url.pathname === '/v1/
|
|
60
|
-
this.json(res, 200, {
|
|
62
|
+
// GET /v1/queries - list available queries
|
|
63
|
+
if (url.pathname === '/v1/queries' && req.method === 'GET') {
|
|
64
|
+
this.json(res, 200, { queries: Object.keys(this.queries) })
|
|
61
65
|
return
|
|
62
66
|
}
|
|
63
67
|
|
|
64
|
-
// GET /v1/
|
|
65
|
-
if (url.pathname === '/v1/
|
|
66
|
-
this.json(res, 200, {
|
|
68
|
+
// GET /v1/events - list available events
|
|
69
|
+
if (url.pathname === '/v1/events' && req.method === 'GET') {
|
|
70
|
+
this.json(res, 200, { events: Object.keys(this.events) })
|
|
67
71
|
return
|
|
68
72
|
}
|
|
69
73
|
|
|
@@ -72,11 +76,11 @@ export class HttpApi {
|
|
|
72
76
|
message: `${req.method} ${url.pathname} does not match any route.`,
|
|
73
77
|
routes: [
|
|
74
78
|
'GET /v1/health',
|
|
75
|
-
'GET /v1/
|
|
76
|
-
'GET /v1/
|
|
77
|
-
'GET /v1/
|
|
78
|
-
'POST /v1/
|
|
79
|
-
'POST /v1/
|
|
79
|
+
'GET /v1/queries',
|
|
80
|
+
'GET /v1/queries/{name}',
|
|
81
|
+
'GET /v1/events',
|
|
82
|
+
'POST /v1/events/{name}',
|
|
83
|
+
'POST /v1/events/{name}/batch',
|
|
80
84
|
],
|
|
81
85
|
})
|
|
82
86
|
} catch (err: any) {
|
|
@@ -85,21 +89,20 @@ export class HttpApi {
|
|
|
85
89
|
}
|
|
86
90
|
}
|
|
87
91
|
|
|
88
|
-
private
|
|
89
|
-
if (!this.
|
|
90
|
-
this.
|
|
91
|
-
await this.client.connect()
|
|
92
|
+
private ensurePool(): InstanceType<typeof Pool> {
|
|
93
|
+
if (!this.pool) {
|
|
94
|
+
this.pool = new Pool({ connectionString: this.connectionString, max: 10 })
|
|
92
95
|
}
|
|
93
|
-
return this.
|
|
96
|
+
return this.pool
|
|
94
97
|
}
|
|
95
98
|
|
|
96
|
-
private async handleWrite(
|
|
97
|
-
const
|
|
98
|
-
if (!
|
|
99
|
-
const available = Object.keys(this.
|
|
99
|
+
private async handleWrite(eventName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
100
|
+
const eventDef = this.events[eventName]
|
|
101
|
+
if (!eventDef) {
|
|
102
|
+
const available = Object.keys(this.events)
|
|
100
103
|
this.json(res, 404, {
|
|
101
|
-
error: `
|
|
102
|
-
|
|
104
|
+
error: `Event '${eventName}' not found.`,
|
|
105
|
+
available_events: available,
|
|
103
106
|
})
|
|
104
107
|
return
|
|
105
108
|
}
|
|
@@ -107,80 +110,147 @@ export class HttpApi {
|
|
|
107
110
|
const body = await this.readBody(req)
|
|
108
111
|
const data = JSON.parse(body)
|
|
109
112
|
|
|
110
|
-
const
|
|
111
|
-
const columns = Object.keys(
|
|
113
|
+
const pool = this.ensurePool()
|
|
114
|
+
const columns = Object.keys(eventDef.columns)
|
|
112
115
|
const values = columns.map((col) => data[col])
|
|
113
116
|
const placeholders = columns.map((_, i) => `$${i + 1}`)
|
|
114
117
|
|
|
115
|
-
await
|
|
116
|
-
`INSERT INTO ${
|
|
118
|
+
await pool.query(
|
|
119
|
+
`INSERT INTO ${eventName} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`,
|
|
117
120
|
values
|
|
118
121
|
)
|
|
119
122
|
|
|
120
123
|
this.json(res, 200, { ok: true })
|
|
121
124
|
}
|
|
122
125
|
|
|
123
|
-
private async handleBatchWrite(
|
|
124
|
-
const
|
|
125
|
-
if (!
|
|
126
|
-
const available = Object.keys(this.
|
|
126
|
+
private async handleBatchWrite(eventName: string, req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
127
|
+
const eventDef = this.events[eventName]
|
|
128
|
+
if (!eventDef) {
|
|
129
|
+
const available = Object.keys(this.events)
|
|
127
130
|
this.json(res, 404, {
|
|
128
|
-
error: `
|
|
129
|
-
|
|
131
|
+
error: `Event '${eventName}' not found.`,
|
|
132
|
+
available_events: available,
|
|
130
133
|
})
|
|
131
134
|
return
|
|
132
135
|
}
|
|
133
136
|
|
|
134
137
|
const body = await this.readBody(req)
|
|
135
|
-
const
|
|
138
|
+
const items: any[] = JSON.parse(body)
|
|
136
139
|
|
|
137
|
-
if (!Array.isArray(
|
|
140
|
+
if (!Array.isArray(items)) {
|
|
138
141
|
this.json(res, 400, { error: 'Batch endpoint expects a JSON array.' })
|
|
139
142
|
return
|
|
140
143
|
}
|
|
141
144
|
|
|
142
|
-
|
|
143
|
-
|
|
145
|
+
if (items.length === 0) {
|
|
146
|
+
this.json(res, 200, { ok: true, count: 0 })
|
|
147
|
+
return
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const pool = this.ensurePool()
|
|
151
|
+
const columns = Object.keys(eventDef.columns)
|
|
144
152
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
153
|
+
// Build a single INSERT with multiple VALUE rows
|
|
154
|
+
const allValues: unknown[] = []
|
|
155
|
+
const rowPlaceholders: string[] = []
|
|
156
|
+
|
|
157
|
+
for (let i = 0; i < items.length; i++) {
|
|
158
|
+
const row = items[i]
|
|
159
|
+
const offset = i * columns.length
|
|
160
|
+
const ph = columns.map((_, j) => `$${offset + j + 1}`)
|
|
161
|
+
rowPlaceholders.push(`(${ph.join(', ')})`)
|
|
162
|
+
for (const col of columns) {
|
|
163
|
+
allValues.push(row[col])
|
|
164
|
+
}
|
|
152
165
|
}
|
|
153
166
|
|
|
154
|
-
|
|
167
|
+
await pool.query(
|
|
168
|
+
`INSERT INTO ${eventName} (${columns.join(', ')}) VALUES ${rowPlaceholders.join(', ')}`,
|
|
169
|
+
allValues
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
this.json(res, 200, { ok: true, count: items.length })
|
|
155
173
|
}
|
|
156
174
|
|
|
157
|
-
private async handleRead(
|
|
158
|
-
if (!this.
|
|
159
|
-
const available = Object.keys(this.
|
|
175
|
+
private async handleRead(queryName: string, url: URL, req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
176
|
+
if (!this.queries[queryName]) {
|
|
177
|
+
const available = Object.keys(this.queries)
|
|
160
178
|
this.json(res, 404, {
|
|
161
|
-
error: `
|
|
162
|
-
|
|
179
|
+
error: `Query '${queryName}' not found.`,
|
|
180
|
+
available_queries: available,
|
|
163
181
|
})
|
|
164
182
|
return
|
|
165
183
|
}
|
|
166
184
|
|
|
167
|
-
|
|
185
|
+
// JWT verification for queries with filterBy
|
|
186
|
+
const queryDef = this.queries[queryName]
|
|
187
|
+
const filterBy = this.getFilterBy(queryDef)
|
|
188
|
+
|
|
189
|
+
let claims: JwtClaims | null = null
|
|
190
|
+
if (filterBy && this.jwt?.isConfigured()) {
|
|
191
|
+
const token = url.searchParams.get('token')
|
|
192
|
+
?? req.headers.authorization?.replace('Bearer ', '')
|
|
193
|
+
|
|
194
|
+
if (!token) {
|
|
195
|
+
this.json(res, 401, { error: 'Authentication required for filtered queries.' })
|
|
196
|
+
return
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
claims = await this.jwt.verify(token)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const pool = this.ensurePool()
|
|
168
203
|
|
|
169
|
-
// Build WHERE clause
|
|
204
|
+
// Build WHERE clause: start with filterBy if applicable
|
|
170
205
|
const params: string[] = []
|
|
171
206
|
const values: unknown[] = []
|
|
207
|
+
|
|
208
|
+
if (filterBy && claims) {
|
|
209
|
+
const claimValue = claims[filterBy]
|
|
210
|
+
if (claimValue === undefined) {
|
|
211
|
+
// No matching claim -- return empty result, not all data
|
|
212
|
+
this.json(res, 200, { query: queryName, rows: [] })
|
|
213
|
+
return
|
|
214
|
+
}
|
|
215
|
+
values.push(String(claimValue))
|
|
216
|
+
params.push(`${filterBy} = $${values.length}`)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// Add query params as additional filters, validating column names
|
|
220
|
+
const knownColumns = this.getQueryColumns(queryDef)
|
|
172
221
|
for (const [key, value] of url.searchParams.entries()) {
|
|
173
|
-
|
|
222
|
+
if (key === 'token') continue // skip JWT token param
|
|
223
|
+
if (knownColumns && !knownColumns.includes(key)) {
|
|
224
|
+
this.json(res, 400, { error: `Unknown column '${key}'. Known columns: ${knownColumns.join(', ')}` })
|
|
225
|
+
return
|
|
226
|
+
}
|
|
227
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
228
|
+
this.json(res, 400, { error: `Invalid column name: '${key}'` })
|
|
229
|
+
return
|
|
230
|
+
}
|
|
174
231
|
values.push(value)
|
|
232
|
+
params.push(`${key} = $${values.length}`)
|
|
175
233
|
}
|
|
176
234
|
|
|
177
|
-
let sql = `SELECT * FROM ${
|
|
235
|
+
let sql = `SELECT * FROM ${queryName}`
|
|
178
236
|
if (params.length > 0) {
|
|
179
237
|
sql += ` WHERE ${params.join(' AND ')}`
|
|
180
238
|
}
|
|
181
239
|
|
|
182
|
-
const result = await
|
|
183
|
-
this.json(res, 200, {
|
|
240
|
+
const result = await pool.query(sql, values)
|
|
241
|
+
this.json(res, 200, { query: queryName, rows: result.rows })
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
private getFilterBy(queryDef: QueryDef | SqlFragment): string | undefined {
|
|
245
|
+
if ('_tag' in queryDef && queryDef._tag === 'sql') return undefined
|
|
246
|
+
return (queryDef as QueryDef).filterBy
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private getQueryColumns(queryDef: QueryDef | SqlFragment): string[] | null {
|
|
250
|
+
if ('_tag' in queryDef && queryDef._tag === 'sql') return null
|
|
251
|
+
const qd = queryDef as QueryDef
|
|
252
|
+
if (qd.columns) return Object.keys(qd.columns)
|
|
253
|
+
return null
|
|
184
254
|
}
|
|
185
255
|
|
|
186
256
|
private json(res: ServerResponse, status: number, data: unknown): void {
|
|
@@ -191,7 +261,16 @@ export class HttpApi {
|
|
|
191
261
|
private readBody(req: IncomingMessage): Promise<string> {
|
|
192
262
|
return new Promise((resolve, reject) => {
|
|
193
263
|
let body = ''
|
|
194
|
-
|
|
264
|
+
let size = 0
|
|
265
|
+
req.on('data', (chunk) => {
|
|
266
|
+
size += chunk.length
|
|
267
|
+
if (size > MAX_BODY_SIZE) {
|
|
268
|
+
req.destroy()
|
|
269
|
+
reject(new Error(`Request body exceeds ${MAX_BODY_SIZE / 1024 / 1024}MB limit`))
|
|
270
|
+
return
|
|
271
|
+
}
|
|
272
|
+
body += chunk
|
|
273
|
+
})
|
|
195
274
|
req.on('end', () => resolve(body))
|
|
196
275
|
req.on('error', reject)
|
|
197
276
|
})
|
package/src/server.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { createServer, type IncomingMessage, type ServerResponse, type Server }
|
|
|
2
2
|
import type { WaveletConfig } from '@risingwave/wavelet'
|
|
3
3
|
import { CursorManager } from './cursor-manager.js'
|
|
4
4
|
import { WebSocketFanout } from './ws-fanout.js'
|
|
5
|
+
import { WebhookFanout } from './webhook.js'
|
|
5
6
|
import { HttpApi } from './http-api.js'
|
|
6
7
|
import { JwtVerifier } from './jwt.js'
|
|
7
8
|
|
|
@@ -9,14 +10,18 @@ export class WaveletServer {
|
|
|
9
10
|
private httpServer: ReturnType<typeof createServer> | null = null
|
|
10
11
|
private cursorManager: CursorManager
|
|
11
12
|
private fanout: WebSocketFanout
|
|
13
|
+
private webhooks: WebhookFanout
|
|
12
14
|
private httpApi: HttpApi
|
|
13
15
|
private jwt: JwtVerifier
|
|
14
16
|
|
|
15
17
|
constructor(private config: WaveletConfig) {
|
|
16
18
|
this.jwt = new JwtVerifier(config.jwt)
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
this.
|
|
19
|
+
const queries = config.queries ?? config.views ?? {}
|
|
20
|
+
const events = config.events ?? config.streams ?? {}
|
|
21
|
+
this.cursorManager = new CursorManager(config.database, queries)
|
|
22
|
+
this.fanout = new WebSocketFanout(this.cursorManager, this.jwt, queries)
|
|
23
|
+
this.webhooks = new WebhookFanout(queries, config.jwt?.secret)
|
|
24
|
+
this.httpApi = new HttpApi(config.database, events, queries, this.jwt)
|
|
20
25
|
}
|
|
21
26
|
|
|
22
27
|
async start(): Promise<void> {
|
|
@@ -27,14 +32,15 @@ export class WaveletServer {
|
|
|
27
32
|
this.fanout.attach(this.httpServer)
|
|
28
33
|
|
|
29
34
|
await this.cursorManager.initialize()
|
|
30
|
-
this.cursorManager.startPolling((
|
|
31
|
-
this.fanout.broadcast(
|
|
35
|
+
this.cursorManager.startPolling((queryName, diffs) => {
|
|
36
|
+
this.fanout.broadcast(queryName, diffs)
|
|
37
|
+
this.webhooks.broadcast(queryName, diffs)
|
|
32
38
|
})
|
|
33
39
|
|
|
34
40
|
await new Promise<void>((resolve) => {
|
|
35
41
|
this.httpServer!.listen(port, host, () => {
|
|
36
42
|
console.log(`Wavelet server listening on ${host}:${port}`)
|
|
37
|
-
console.log(`WebSocket: ws://${host}:${port}/subscribe/{
|
|
43
|
+
console.log(`WebSocket: ws://${host}:${port}/subscribe/{query}`)
|
|
38
44
|
console.log(`HTTP API: http://${host}:${port}/v1/`)
|
|
39
45
|
resolve()
|
|
40
46
|
})
|
|
@@ -53,8 +59,9 @@ export class WaveletServer {
|
|
|
53
59
|
this.fanout.attach(server, prefix)
|
|
54
60
|
|
|
55
61
|
await this.cursorManager.initialize()
|
|
56
|
-
this.cursorManager.startPolling((
|
|
57
|
-
this.fanout.broadcast(
|
|
62
|
+
this.cursorManager.startPolling((queryName, diffs) => {
|
|
63
|
+
this.fanout.broadcast(queryName, diffs)
|
|
64
|
+
this.webhooks.broadcast(queryName, diffs)
|
|
58
65
|
})
|
|
59
66
|
|
|
60
67
|
const handleHttp = (req: IncomingMessage, res: ServerResponse) => {
|
package/src/webhook.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { createHmac } from 'node:crypto'
|
|
2
|
+
import type { QueryDef, SqlFragment } from '@risingwave/wavelet'
|
|
3
|
+
import type { ViewDiff } from './cursor-manager.js'
|
|
4
|
+
|
|
5
|
+
const TIMEOUT_MS = 10000
|
|
6
|
+
|
|
7
|
+
export class WebhookFanout {
|
|
8
|
+
private webhooks: Map<string, string> = new Map() // queryName -> url
|
|
9
|
+
|
|
10
|
+
constructor(
|
|
11
|
+
queries: Record<string, QueryDef | SqlFragment>,
|
|
12
|
+
private signingSecret?: string
|
|
13
|
+
) {
|
|
14
|
+
for (const [name, def] of Object.entries(queries)) {
|
|
15
|
+
if ('_tag' in def && def._tag === 'sql') continue
|
|
16
|
+
const qd = def as QueryDef
|
|
17
|
+
if (qd.webhook) {
|
|
18
|
+
this.webhooks.set(name, qd.webhook)
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async broadcast(queryName: string, diff: ViewDiff): Promise<void> {
|
|
24
|
+
const url = this.webhooks.get(queryName)
|
|
25
|
+
if (!url) return
|
|
26
|
+
|
|
27
|
+
const body = JSON.stringify({
|
|
28
|
+
query: queryName,
|
|
29
|
+
cursor: diff.cursor,
|
|
30
|
+
inserted: diff.inserted,
|
|
31
|
+
updated: diff.updated,
|
|
32
|
+
deleted: diff.deleted,
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
const headers: Record<string, string> = {
|
|
36
|
+
'Content-Type': 'application/json',
|
|
37
|
+
'User-Agent': 'Wavelet-Webhook/1.0',
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (this.signingSecret) {
|
|
41
|
+
const signature = createHmac('sha256', this.signingSecret)
|
|
42
|
+
.update(body)
|
|
43
|
+
.digest('hex')
|
|
44
|
+
headers['X-Wavelet-Signature'] = `sha256=${signature}`
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
try {
|
|
48
|
+
const controller = new AbortController()
|
|
49
|
+
const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS)
|
|
50
|
+
|
|
51
|
+
await fetch(url, {
|
|
52
|
+
method: 'POST',
|
|
53
|
+
headers,
|
|
54
|
+
body,
|
|
55
|
+
signal: controller.signal,
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
clearTimeout(timeout)
|
|
59
|
+
} catch (err: any) {
|
|
60
|
+
if (err.name === 'AbortError') {
|
|
61
|
+
console.error(`[webhook] Timeout sending to ${url} for query ${queryName}`)
|
|
62
|
+
} else {
|
|
63
|
+
console.error(`[webhook] Failed to send to ${url} for query ${queryName}:`, err.message)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
package/src/ws-fanout.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
import { WebSocketServer, WebSocket } from 'ws'
|
|
2
2
|
import type { IncomingMessage, Server } from 'node:http'
|
|
3
|
-
import type {
|
|
3
|
+
import type { QueryDef, SqlFragment } from '@risingwave/wavelet'
|
|
4
4
|
import type { CursorManager, ViewDiff } from './cursor-manager.js'
|
|
5
5
|
import type { JwtVerifier, JwtClaims } from './jwt.js'
|
|
6
6
|
|
|
7
7
|
interface Subscriber {
|
|
8
8
|
ws: WebSocket
|
|
9
|
-
|
|
9
|
+
queryName: string
|
|
10
10
|
claims: JwtClaims | null
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
export class WebSocketFanout {
|
|
14
14
|
private wss: WebSocketServer | null = null
|
|
15
|
-
private subscribers: Map<string, Set<Subscriber>> = new Map() //
|
|
15
|
+
private subscribers: Map<string, Set<Subscriber>> = new Map() // queryName -> subscribers
|
|
16
16
|
|
|
17
17
|
constructor(
|
|
18
18
|
private cursorManager: CursorManager,
|
|
19
19
|
private jwt: JwtVerifier,
|
|
20
|
-
private
|
|
20
|
+
private queries: Record<string, QueryDef | SqlFragment>
|
|
21
21
|
) {}
|
|
22
22
|
|
|
23
23
|
attach(server: Server, pathPrefix?: string): void {
|
|
@@ -54,17 +54,17 @@ export class WebSocketFanout {
|
|
|
54
54
|
|
|
55
55
|
if (!match) {
|
|
56
56
|
throw new Error(
|
|
57
|
-
`Invalid path: ${url.pathname}. Use /subscribe/{
|
|
58
|
-
`Available
|
|
57
|
+
`Invalid path: ${url.pathname}. Use /subscribe/{queryName}. ` +
|
|
58
|
+
`Available queries: ${Object.keys(this.queries).join(', ')}`
|
|
59
59
|
)
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
const
|
|
62
|
+
const queryName = match[1]
|
|
63
63
|
|
|
64
|
-
if (!this.
|
|
65
|
-
const available = Object.keys(this.
|
|
64
|
+
if (!this.queries[queryName]) {
|
|
65
|
+
const available = Object.keys(this.queries)
|
|
66
66
|
throw new Error(
|
|
67
|
-
`
|
|
67
|
+
`Query '${queryName}' not found. Available queries: ${available.join(', ')}`
|
|
68
68
|
)
|
|
69
69
|
}
|
|
70
70
|
|
|
@@ -80,26 +80,35 @@ export class WebSocketFanout {
|
|
|
80
80
|
claims = await this.jwt.verify(token)
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
const subscriber: Subscriber = { ws,
|
|
83
|
+
const subscriber: Subscriber = { ws, queryName, claims }
|
|
84
84
|
|
|
85
|
-
if (!this.subscribers.has(
|
|
86
|
-
this.subscribers.set(
|
|
85
|
+
if (!this.subscribers.has(queryName)) {
|
|
86
|
+
this.subscribers.set(queryName, new Set())
|
|
87
87
|
}
|
|
88
|
-
this.subscribers.get(
|
|
88
|
+
this.subscribers.get(queryName)!.add(subscriber)
|
|
89
89
|
|
|
90
90
|
ws.on('close', () => {
|
|
91
|
-
this.subscribers.get(
|
|
91
|
+
this.subscribers.get(queryName)?.delete(subscriber)
|
|
92
92
|
})
|
|
93
93
|
|
|
94
|
-
|
|
94
|
+
// Heartbeat: detect dead connections behind proxies/load balancers
|
|
95
|
+
const pingInterval = setInterval(() => {
|
|
96
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
97
|
+
ws.ping()
|
|
98
|
+
}
|
|
99
|
+
}, 30000)
|
|
100
|
+
ws.on('close', () => clearInterval(pingInterval))
|
|
101
|
+
ws.on('pong', () => { /* connection alive */ })
|
|
102
|
+
|
|
103
|
+
ws.send(JSON.stringify({ type: 'connected', query: queryName }))
|
|
95
104
|
}
|
|
96
105
|
|
|
97
|
-
broadcast(
|
|
98
|
-
const subs = this.subscribers.get(
|
|
106
|
+
broadcast(queryName: string, diff: ViewDiff): void {
|
|
107
|
+
const subs = this.subscribers.get(queryName)
|
|
99
108
|
if (!subs || subs.size === 0) return
|
|
100
109
|
|
|
101
|
-
const
|
|
102
|
-
const filterBy = this.getFilterBy(
|
|
110
|
+
const queryDef = this.queries[queryName]
|
|
111
|
+
const filterBy = this.getFilterBy(queryDef)
|
|
103
112
|
|
|
104
113
|
for (const sub of subs) {
|
|
105
114
|
if (sub.ws.readyState !== WebSocket.OPEN) continue
|
|
@@ -119,7 +128,7 @@ export class WebSocketFanout {
|
|
|
119
128
|
|
|
120
129
|
sub.ws.send(JSON.stringify({
|
|
121
130
|
type: 'diff',
|
|
122
|
-
|
|
131
|
+
query: queryName,
|
|
123
132
|
cursor: filteredDiff.cursor,
|
|
124
133
|
inserted: filteredDiff.inserted,
|
|
125
134
|
updated: filteredDiff.updated,
|
|
@@ -130,7 +139,10 @@ export class WebSocketFanout {
|
|
|
130
139
|
|
|
131
140
|
private filterDiff(diff: ViewDiff, filterBy: string, claims: JwtClaims): ViewDiff {
|
|
132
141
|
const claimValue = claims[filterBy]
|
|
133
|
-
if (claimValue === undefined)
|
|
142
|
+
if (claimValue === undefined) {
|
|
143
|
+
// No matching claim -- return empty diff, not all data
|
|
144
|
+
return { cursor: diff.cursor, inserted: [], updated: [], deleted: [] }
|
|
145
|
+
}
|
|
134
146
|
|
|
135
147
|
const filterFn = (row: Record<string, unknown>) =>
|
|
136
148
|
String(row[filterBy]) === String(claimValue)
|
|
@@ -143,9 +155,9 @@ export class WebSocketFanout {
|
|
|
143
155
|
}
|
|
144
156
|
}
|
|
145
157
|
|
|
146
|
-
private getFilterBy(
|
|
147
|
-
if ('_tag' in
|
|
148
|
-
return (
|
|
158
|
+
private getFilterBy(queryDef: QueryDef | SqlFragment): string | undefined {
|
|
159
|
+
if ('_tag' in queryDef && queryDef._tag === 'sql') return undefined
|
|
160
|
+
return (queryDef as QueryDef).filterBy
|
|
149
161
|
}
|
|
150
162
|
|
|
151
163
|
closeAll(): void {
|