@risingwave/wavelet-server 0.1.4 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -2
- package/src/__tests__/http-api.test.ts +23 -23
- package/src/__tests__/integration.test.ts +49 -17
- package/src/__tests__/ws-fanout.test.ts +143 -0
- package/src/cursor-manager.ts +133 -31
- package/src/ddl-manager.ts +72 -71
- package/src/http-api.ts +150 -71
- package/src/server.ts +15 -8
- package/src/webhook.ts +67 -0
- package/src/ws-fanout.ts +128 -42
package/package.json
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@risingwave/wavelet-server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"description": "Wavelet server - WebSocket fanout layer for RisingWave",
|
|
5
|
+
"homepage": "https://github.com/risingwavelabs/wavelet",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/risingwavelabs/wavelet.git",
|
|
9
|
+
"directory": "packages/server"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/risingwavelabs/wavelet/issues"
|
|
13
|
+
},
|
|
5
14
|
"main": "./dist/index.js",
|
|
6
15
|
"types": "./dist/index.d.ts",
|
|
7
16
|
"scripts": {
|
|
@@ -13,7 +22,7 @@
|
|
|
13
22
|
"pg": "^8.13.0",
|
|
14
23
|
"ws": "^8.18.0",
|
|
15
24
|
"jose": "^6.0.0",
|
|
16
|
-
"@risingwave/wavelet": "0.
|
|
25
|
+
"@risingwave/wavelet": "0.2.4"
|
|
17
26
|
},
|
|
18
27
|
"devDependencies": {
|
|
19
28
|
"@types/pg": "^8.11.0",
|
|
@@ -36,15 +36,15 @@ function request(server: http.Server, method: string, path: string, body?: unkno
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
describe('HttpApi', () => {
|
|
39
|
-
const
|
|
40
|
-
|
|
39
|
+
const events = {
|
|
40
|
+
game_events: { columns: { user_id: 'string' as const, value: 'int' as const } },
|
|
41
41
|
}
|
|
42
|
-
const
|
|
42
|
+
const queries = {
|
|
43
43
|
leaderboard: { _tag: 'sql' as const, text: 'SELECT 1' },
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
it('returns health check', async () => {
|
|
47
|
-
const api = new HttpApi('postgres://dummy',
|
|
47
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
48
48
|
const server = await createTestServer(api)
|
|
49
49
|
try {
|
|
50
50
|
const res = await request(server, 'GET', '/v1/health')
|
|
@@ -55,32 +55,32 @@ describe('HttpApi', () => {
|
|
|
55
55
|
}
|
|
56
56
|
})
|
|
57
57
|
|
|
58
|
-
it('lists available
|
|
59
|
-
const api = new HttpApi('postgres://dummy',
|
|
58
|
+
it('lists available queries', async () => {
|
|
59
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
60
60
|
const server = await createTestServer(api)
|
|
61
61
|
try {
|
|
62
|
-
const res = await request(server, 'GET', '/v1/
|
|
62
|
+
const res = await request(server, 'GET', '/v1/queries')
|
|
63
63
|
expect(res.status).toBe(200)
|
|
64
|
-
expect(res.data).toEqual({
|
|
64
|
+
expect(res.data).toEqual({ queries: ['leaderboard'] })
|
|
65
65
|
} finally {
|
|
66
66
|
server.close()
|
|
67
67
|
}
|
|
68
68
|
})
|
|
69
69
|
|
|
70
|
-
it('lists available
|
|
71
|
-
const api = new HttpApi('postgres://dummy',
|
|
70
|
+
it('lists available events', async () => {
|
|
71
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
72
72
|
const server = await createTestServer(api)
|
|
73
73
|
try {
|
|
74
|
-
const res = await request(server, 'GET', '/v1/
|
|
74
|
+
const res = await request(server, 'GET', '/v1/events')
|
|
75
75
|
expect(res.status).toBe(200)
|
|
76
|
-
expect(res.data).toEqual({
|
|
76
|
+
expect(res.data).toEqual({ events: ['game_events'] })
|
|
77
77
|
} finally {
|
|
78
78
|
server.close()
|
|
79
79
|
}
|
|
80
80
|
})
|
|
81
81
|
|
|
82
82
|
it('returns 404 for unknown routes with helpful message', async () => {
|
|
83
|
-
const api = new HttpApi('postgres://dummy',
|
|
83
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
84
84
|
const server = await createTestServer(api)
|
|
85
85
|
try {
|
|
86
86
|
const res = await request(server, 'GET', '/v1/nonexistent')
|
|
@@ -94,35 +94,35 @@ describe('HttpApi', () => {
|
|
|
94
94
|
})
|
|
95
95
|
|
|
96
96
|
it('handles CORS preflight', async () => {
|
|
97
|
-
const api = new HttpApi('postgres://dummy',
|
|
97
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
98
98
|
const server = await createTestServer(api)
|
|
99
99
|
try {
|
|
100
|
-
const res = await request(server, 'OPTIONS', '/v1/
|
|
100
|
+
const res = await request(server, 'OPTIONS', '/v1/queries')
|
|
101
101
|
expect(res.status).toBe(204)
|
|
102
102
|
} finally {
|
|
103
103
|
server.close()
|
|
104
104
|
}
|
|
105
105
|
})
|
|
106
106
|
|
|
107
|
-
it('returns 404 for unknown
|
|
108
|
-
const api = new HttpApi('postgres://dummy',
|
|
107
|
+
it('returns 404 for unknown event', async () => {
|
|
108
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
109
109
|
const server = await createTestServer(api)
|
|
110
110
|
try {
|
|
111
|
-
const res = await request(server, 'POST', '/v1/
|
|
111
|
+
const res = await request(server, 'POST', '/v1/events/nonexistent', { x: 1 })
|
|
112
112
|
expect(res.status).toBe(404)
|
|
113
|
-
expect(res.data.
|
|
113
|
+
expect(res.data.available_events).toEqual(['game_events'])
|
|
114
114
|
} finally {
|
|
115
115
|
server.close()
|
|
116
116
|
}
|
|
117
117
|
})
|
|
118
118
|
|
|
119
|
-
it('returns 404 for unknown
|
|
120
|
-
const api = new HttpApi('postgres://dummy',
|
|
119
|
+
it('returns 404 for unknown query', async () => {
|
|
120
|
+
const api = new HttpApi('postgres://dummy', events, queries)
|
|
121
121
|
const server = await createTestServer(api)
|
|
122
122
|
try {
|
|
123
|
-
const res = await request(server, 'GET', '/v1/
|
|
123
|
+
const res = await request(server, 'GET', '/v1/queries/nonexistent')
|
|
124
124
|
expect(res.status).toBe(404)
|
|
125
|
-
expect(res.data.
|
|
125
|
+
expect(res.data.available_queries).toEqual(['leaderboard'])
|
|
126
126
|
} finally {
|
|
127
127
|
server.close()
|
|
128
128
|
}
|
|
@@ -36,7 +36,7 @@ const SUB_NAME = `wavelet_sub_${VIEW_NAME}`
|
|
|
36
36
|
|
|
37
37
|
const testConfig: WaveletConfig = {
|
|
38
38
|
database: DATABASE_URL,
|
|
39
|
-
|
|
39
|
+
events: {
|
|
40
40
|
[STREAM_NAME]: {
|
|
41
41
|
columns: {
|
|
42
42
|
user_id: 'string',
|
|
@@ -44,7 +44,7 @@ const testConfig: WaveletConfig = {
|
|
|
44
44
|
},
|
|
45
45
|
},
|
|
46
46
|
},
|
|
47
|
-
|
|
47
|
+
queries: {
|
|
48
48
|
[VIEW_NAME]: sql`
|
|
49
49
|
SELECT user_id, SUM(value) AS total_value, COUNT(*) AS event_count
|
|
50
50
|
FROM ${STREAM_NAME}
|
|
@@ -75,17 +75,17 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: DDL Manage
|
|
|
75
75
|
await ddl.close()
|
|
76
76
|
})
|
|
77
77
|
|
|
78
|
-
it('creates tables, views, and subscriptions', async () => {
|
|
78
|
+
it('creates tables, materialized views, and subscriptions', async () => {
|
|
79
79
|
const actions = await ddl.sync(testConfig)
|
|
80
80
|
|
|
81
81
|
const creates = actions.filter(a => a.type === 'create')
|
|
82
|
-
expect(creates.length).toBeGreaterThanOrEqual(3) //
|
|
82
|
+
expect(creates.length).toBeGreaterThanOrEqual(3) // event + query + subscription
|
|
83
83
|
|
|
84
|
-
const
|
|
85
|
-
expect(
|
|
84
|
+
const eventAction = creates.find(a => a.resource === 'event' && a.name === STREAM_NAME)
|
|
85
|
+
expect(eventAction).toBeDefined()
|
|
86
86
|
|
|
87
|
-
const
|
|
88
|
-
expect(
|
|
87
|
+
const queryAction = creates.find(a => a.resource === 'query' && a.name === VIEW_NAME)
|
|
88
|
+
expect(queryAction).toBeDefined()
|
|
89
89
|
|
|
90
90
|
const subAction = creates.find(a => a.resource === 'subscription' && a.name === SUB_NAME)
|
|
91
91
|
expect(subAction).toBeDefined()
|
|
@@ -162,15 +162,15 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
162
162
|
expect(data.status).toBe('ok')
|
|
163
163
|
})
|
|
164
164
|
|
|
165
|
-
it('lists
|
|
166
|
-
const res = await fetch(`http://localhost:${port}/v1/
|
|
165
|
+
it('lists queries', async () => {
|
|
166
|
+
const res = await fetch(`http://localhost:${port}/v1/queries`)
|
|
167
167
|
const data = await res.json()
|
|
168
|
-
expect(data.
|
|
168
|
+
expect(data.queries).toContain(VIEW_NAME)
|
|
169
169
|
})
|
|
170
170
|
|
|
171
|
-
it('writes events via HTTP and reads
|
|
171
|
+
it('writes events via HTTP and reads query', async () => {
|
|
172
172
|
// Write events
|
|
173
|
-
await fetch(`http://localhost:${port}/v1/
|
|
173
|
+
await fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
|
|
174
174
|
method: 'POST',
|
|
175
175
|
headers: { 'Content-Type': 'application/json' },
|
|
176
176
|
body: JSON.stringify({ user_id: 'integration_test', value: 42 }),
|
|
@@ -179,8 +179,8 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
179
179
|
// Wait for MV update
|
|
180
180
|
await new Promise(r => setTimeout(r, 2000))
|
|
181
181
|
|
|
182
|
-
// Read
|
|
183
|
-
const res = await fetch(`http://localhost:${port}/v1/
|
|
182
|
+
// Read query
|
|
183
|
+
const res = await fetch(`http://localhost:${port}/v1/queries/${VIEW_NAME}`)
|
|
184
184
|
const data = await res.json()
|
|
185
185
|
expect(data.rows.length).toBeGreaterThan(0)
|
|
186
186
|
})
|
|
@@ -188,18 +188,21 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
188
188
|
it('pushes diffs via WebSocket', async () => {
|
|
189
189
|
const diff = await new Promise<any>((resolve, reject) => {
|
|
190
190
|
const ws = new WebSocket(`ws://localhost:${port}/subscribe/${VIEW_NAME}`)
|
|
191
|
+
let sawSnapshot = false
|
|
191
192
|
|
|
192
193
|
ws.on('message', (data: Buffer) => {
|
|
193
194
|
const msg = JSON.parse(data.toString())
|
|
194
|
-
if (msg.type === '
|
|
195
|
+
if (msg.type === 'snapshot') {
|
|
196
|
+
sawSnapshot = true
|
|
195
197
|
// Write event to trigger diff
|
|
196
|
-
fetch(`http://localhost:${port}/v1/
|
|
198
|
+
fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
|
|
197
199
|
method: 'POST',
|
|
198
200
|
headers: { 'Content-Type': 'application/json' },
|
|
199
201
|
body: JSON.stringify({ user_id: 'ws_test_user', value: 7 }),
|
|
200
202
|
})
|
|
201
203
|
}
|
|
202
204
|
if (msg.type === 'diff') {
|
|
205
|
+
expect(sawSnapshot).toBe(true)
|
|
203
206
|
ws.close()
|
|
204
207
|
resolve(msg)
|
|
205
208
|
}
|
|
@@ -214,4 +217,33 @@ describe.runIf(process.env.WAVELET_INTEGRATION === '1')('Integration: Full Serve
|
|
|
214
217
|
// Should have at least one insert or update
|
|
215
218
|
expect(diff.inserted.length + diff.updated.length).toBeGreaterThan(0)
|
|
216
219
|
}, 20000)
|
|
220
|
+
|
|
221
|
+
it('pushes a snapshot immediately on connect', async () => {
|
|
222
|
+
await fetch(`http://localhost:${port}/v1/events/${STREAM_NAME}`, {
|
|
223
|
+
method: 'POST',
|
|
224
|
+
headers: { 'Content-Type': 'application/json' },
|
|
225
|
+
body: JSON.stringify({ user_id: 'snapshot_user', value: 11 }),
|
|
226
|
+
})
|
|
227
|
+
|
|
228
|
+
await new Promise(r => setTimeout(r, 2000))
|
|
229
|
+
|
|
230
|
+
const snapshot = await new Promise<any>((resolve, reject) => {
|
|
231
|
+
const ws = new WebSocket(`ws://localhost:${port}/subscribe/${VIEW_NAME}`)
|
|
232
|
+
|
|
233
|
+
ws.on('message', (data: Buffer) => {
|
|
234
|
+
const msg = JSON.parse(data.toString())
|
|
235
|
+
if (msg.type === 'snapshot') {
|
|
236
|
+
ws.close()
|
|
237
|
+
resolve(msg)
|
|
238
|
+
}
|
|
239
|
+
})
|
|
240
|
+
|
|
241
|
+
ws.on('error', reject)
|
|
242
|
+
setTimeout(() => { ws.close(); reject(new Error('WebSocket snapshot timeout')) }, 15000)
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
expect(snapshot.type).toBe('snapshot')
|
|
246
|
+
expect(Array.isArray(snapshot.rows)).toBe(true)
|
|
247
|
+
expect(snapshot.rows.some((row: any) => row.user_id === 'snapshot_user')).toBe(true)
|
|
248
|
+
}, 20000)
|
|
217
249
|
})
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events'
|
|
2
|
+
import { describe, it, expect, vi } from 'vitest'
|
|
3
|
+
import { WebSocket } from 'ws'
|
|
4
|
+
import { WebSocketFanout } from '../ws-fanout.js'
|
|
5
|
+
import type { BootstrapResult, ViewDiff } from '../cursor-manager.js'
|
|
6
|
+
|
|
7
|
+
class MockSocket extends EventEmitter {
|
|
8
|
+
readyState: number = WebSocket.OPEN
|
|
9
|
+
sent: string[] = []
|
|
10
|
+
|
|
11
|
+
send(message: string): void {
|
|
12
|
+
this.sent.push(message)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
ping(): void {}
|
|
16
|
+
|
|
17
|
+
close(): void {
|
|
18
|
+
this.readyState = WebSocket.CLOSED
|
|
19
|
+
this.emit('close')
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function deferred<T>() {
|
|
24
|
+
let resolve!: (value: T) => void
|
|
25
|
+
let reject!: (reason?: unknown) => void
|
|
26
|
+
const promise = new Promise<T>((res, rej) => {
|
|
27
|
+
resolve = res
|
|
28
|
+
reject = rej
|
|
29
|
+
})
|
|
30
|
+
return { promise, resolve, reject }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe('WebSocketFanout', () => {
|
|
34
|
+
it('drops queued shared diffs that are already covered by bootstrap', async () => {
|
|
35
|
+
const bootstrapDeferred = deferred<BootstrapResult>()
|
|
36
|
+
const cursorManager = {
|
|
37
|
+
bootstrap: vi.fn().mockReturnValue(bootstrapDeferred.promise),
|
|
38
|
+
} as any
|
|
39
|
+
const jwt = {
|
|
40
|
+
isConfigured: () => false,
|
|
41
|
+
} as any
|
|
42
|
+
|
|
43
|
+
const fanout = new WebSocketFanout(cursorManager, jwt, {
|
|
44
|
+
leaderboard: {} as any,
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
const ws = new MockSocket()
|
|
48
|
+
const req = {
|
|
49
|
+
url: '/subscribe/leaderboard',
|
|
50
|
+
headers: { host: 'localhost' },
|
|
51
|
+
} as any
|
|
52
|
+
|
|
53
|
+
const connectionPromise = (fanout as any).handleConnection(ws, req)
|
|
54
|
+
|
|
55
|
+
fanout.broadcast('leaderboard', {
|
|
56
|
+
cursor: '150',
|
|
57
|
+
inserted: [{ player_id: 'alice', score: 10 }],
|
|
58
|
+
updated: [],
|
|
59
|
+
deleted: [],
|
|
60
|
+
})
|
|
61
|
+
fanout.broadcast('leaderboard', {
|
|
62
|
+
cursor: '300',
|
|
63
|
+
inserted: [{ player_id: 'bob', score: 20 }],
|
|
64
|
+
updated: [],
|
|
65
|
+
deleted: [],
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
bootstrapDeferred.resolve({
|
|
69
|
+
snapshotRows: [{ player_id: 'alice', score: 10 }],
|
|
70
|
+
diffs: [{
|
|
71
|
+
cursor: '200',
|
|
72
|
+
inserted: [],
|
|
73
|
+
updated: [{ player_id: 'alice', score: 15 }],
|
|
74
|
+
deleted: [],
|
|
75
|
+
}],
|
|
76
|
+
lastCursor: '200',
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
await connectionPromise
|
|
80
|
+
|
|
81
|
+
const messages = ws.sent.map((message) => JSON.parse(message))
|
|
82
|
+
expect(messages).toEqual([
|
|
83
|
+
{ type: 'connected', query: 'leaderboard' },
|
|
84
|
+
{
|
|
85
|
+
type: 'snapshot',
|
|
86
|
+
query: 'leaderboard',
|
|
87
|
+
rows: [{ player_id: 'alice', score: 10 }],
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
type: 'diff',
|
|
91
|
+
query: 'leaderboard',
|
|
92
|
+
cursor: '200',
|
|
93
|
+
inserted: [],
|
|
94
|
+
updated: [{ player_id: 'alice', score: 15 }],
|
|
95
|
+
deleted: [],
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
type: 'diff',
|
|
99
|
+
query: 'leaderboard',
|
|
100
|
+
cursor: '300',
|
|
101
|
+
inserted: [{ player_id: 'bob', score: 20 }],
|
|
102
|
+
updated: [],
|
|
103
|
+
deleted: [],
|
|
104
|
+
},
|
|
105
|
+
])
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
it('filters snapshot rows with the same claim rule as diffs', async () => {
|
|
109
|
+
const cursorManager = {
|
|
110
|
+
bootstrap: vi.fn().mockResolvedValue({
|
|
111
|
+
snapshotRows: [
|
|
112
|
+
{ user_id: 'u1', total: 10 },
|
|
113
|
+
{ user_id: 'u2', total: 20 },
|
|
114
|
+
],
|
|
115
|
+
diffs: [] as ViewDiff[],
|
|
116
|
+
lastCursor: null,
|
|
117
|
+
}),
|
|
118
|
+
} as any
|
|
119
|
+
const jwt = {
|
|
120
|
+
isConfigured: () => true,
|
|
121
|
+
verify: vi.fn().mockResolvedValue({ user_id: 'u1' }),
|
|
122
|
+
} as any
|
|
123
|
+
|
|
124
|
+
const fanout = new WebSocketFanout(cursorManager, jwt, {
|
|
125
|
+
totals: { filterBy: 'user_id' } as any,
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
const ws = new MockSocket()
|
|
129
|
+
const req = {
|
|
130
|
+
url: '/subscribe/totals?token=test-token',
|
|
131
|
+
headers: { host: 'localhost' },
|
|
132
|
+
} as any
|
|
133
|
+
|
|
134
|
+
await (fanout as any).handleConnection(ws, req)
|
|
135
|
+
|
|
136
|
+
const snapshotMessage = JSON.parse(ws.sent[1])
|
|
137
|
+
expect(snapshotMessage).toEqual({
|
|
138
|
+
type: 'snapshot',
|
|
139
|
+
query: 'totals',
|
|
140
|
+
rows: [{ user_id: 'u1', total: 10 }],
|
|
141
|
+
})
|
|
142
|
+
})
|
|
143
|
+
})
|
package/src/cursor-manager.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import pg from 'pg'
|
|
2
|
-
import type { SqlFragment,
|
|
2
|
+
import type { SqlFragment, QueryDef } from '@risingwave/wavelet'
|
|
3
3
|
|
|
4
4
|
const { Client } = pg
|
|
5
5
|
|
|
@@ -16,12 +16,18 @@ export interface ViewDiff {
|
|
|
16
16
|
deleted: Record<string, unknown>[]
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
export interface BootstrapResult {
|
|
20
|
+
snapshotRows: Record<string, unknown>[]
|
|
21
|
+
diffs: ViewDiff[]
|
|
22
|
+
lastCursor: string | null
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
type DiffCallback = (queryName: string, diff: ViewDiff) => void
|
|
20
26
|
|
|
21
27
|
/**
|
|
22
28
|
* Manages persistent subscription cursors against RisingWave.
|
|
23
29
|
*
|
|
24
|
-
* Each
|
|
30
|
+
* Each query gets its own dedicated pg connection and a persistent cursor.
|
|
25
31
|
* Uses blocking FETCH (WITH timeout) so there is no polling interval -
|
|
26
32
|
* diffs are dispatched as soon as RisingWave produces them.
|
|
27
33
|
*/
|
|
@@ -29,15 +35,15 @@ export class CursorManager {
|
|
|
29
35
|
// Shared connection for DDL (CREATE SUBSCRIPTION) and ad-hoc queries
|
|
30
36
|
private client: InstanceType<typeof Client> | null = null
|
|
31
37
|
|
|
32
|
-
// Per-
|
|
33
|
-
private
|
|
38
|
+
// Per-query dedicated connections for blocking FETCH
|
|
39
|
+
private queryConnections: Map<string, InstanceType<typeof Client>> = new Map()
|
|
34
40
|
private cursorNames: Map<string, string> = new Map()
|
|
35
41
|
private subscriptions: Map<string, string> = new Map()
|
|
36
42
|
private running = false
|
|
37
43
|
|
|
38
44
|
constructor(
|
|
39
45
|
private connectionString: string,
|
|
40
|
-
private
|
|
46
|
+
private queries: Record<string, QueryDef | SqlFragment>
|
|
41
47
|
) {}
|
|
42
48
|
|
|
43
49
|
async initialize(): Promise<void> {
|
|
@@ -45,13 +51,13 @@ export class CursorManager {
|
|
|
45
51
|
await this.client.connect()
|
|
46
52
|
console.log('Connected to RisingWave')
|
|
47
53
|
|
|
48
|
-
for (const [
|
|
49
|
-
const subName = `wavelet_sub_${
|
|
54
|
+
for (const [queryName] of Object.entries(this.queries)) {
|
|
55
|
+
const subName = `wavelet_sub_${queryName}`
|
|
50
56
|
|
|
51
57
|
// Create subscription if not exists (idempotent)
|
|
52
58
|
try {
|
|
53
59
|
await this.client.query(
|
|
54
|
-
`CREATE SUBSCRIPTION ${subName} FROM ${
|
|
60
|
+
`CREATE SUBSCRIPTION ${subName} FROM ${queryName} WITH (retention = '24h')`
|
|
55
61
|
)
|
|
56
62
|
console.log(`Created subscription: ${subName}`)
|
|
57
63
|
} catch (err: any) {
|
|
@@ -62,31 +68,31 @@ export class CursorManager {
|
|
|
62
68
|
}
|
|
63
69
|
}
|
|
64
70
|
|
|
65
|
-
this.subscriptions.set(
|
|
71
|
+
this.subscriptions.set(queryName, subName)
|
|
66
72
|
|
|
67
|
-
// Create dedicated connection and persistent cursor for this
|
|
73
|
+
// Create dedicated connection and persistent cursor for this query
|
|
68
74
|
const conn = new Client({ connectionString: this.connectionString })
|
|
69
75
|
await conn.connect()
|
|
70
76
|
|
|
71
|
-
const cursorName = `wavelet_cur_${
|
|
77
|
+
const cursorName = `wavelet_cur_${queryName}`
|
|
72
78
|
await conn.query(`DECLARE ${cursorName} SUBSCRIPTION CURSOR FOR ${subName}`)
|
|
73
79
|
console.log(`Opened persistent cursor: ${cursorName}`)
|
|
74
80
|
|
|
75
|
-
this.
|
|
76
|
-
this.cursorNames.set(
|
|
81
|
+
this.queryConnections.set(queryName, conn)
|
|
82
|
+
this.cursorNames.set(queryName, cursorName)
|
|
77
83
|
}
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
/**
|
|
81
|
-
* Start listening for diffs on all
|
|
82
|
-
* Each
|
|
87
|
+
* Start listening for diffs on all queries.
|
|
88
|
+
* Each query runs its own async loop with blocking FETCH.
|
|
83
89
|
* No polling interval - FETCH blocks until data arrives or timeout.
|
|
84
90
|
*/
|
|
85
91
|
startPolling(callback: DiffCallback): void {
|
|
86
92
|
this.running = true
|
|
87
93
|
|
|
88
|
-
for (const [
|
|
89
|
-
this.listenLoop(
|
|
94
|
+
for (const [queryName] of this.subscriptions.entries()) {
|
|
95
|
+
this.listenLoop(queryName, callback)
|
|
90
96
|
}
|
|
91
97
|
}
|
|
92
98
|
|
|
@@ -94,9 +100,9 @@ export class CursorManager {
|
|
|
94
100
|
this.running = false
|
|
95
101
|
}
|
|
96
102
|
|
|
97
|
-
private async listenLoop(
|
|
98
|
-
const conn = this.
|
|
99
|
-
const cursorName = this.cursorNames.get(
|
|
103
|
+
private async listenLoop(queryName: string, callback: DiffCallback): Promise<void> {
|
|
104
|
+
const conn = this.queryConnections.get(queryName)
|
|
105
|
+
const cursorName = this.cursorNames.get(queryName)
|
|
100
106
|
if (!conn || !cursorName) return
|
|
101
107
|
|
|
102
108
|
while (this.running) {
|
|
@@ -108,12 +114,12 @@ export class CursorManager {
|
|
|
108
114
|
|
|
109
115
|
if (result.rows.length === 0) continue
|
|
110
116
|
|
|
111
|
-
// Got at least one row. Drain any remaining rows
|
|
117
|
+
// Got at least one row. Drain any remaining rows with timeout.
|
|
112
118
|
const allRows = [...result.rows]
|
|
113
119
|
let more = true
|
|
114
120
|
while (more) {
|
|
115
121
|
const batch = await conn.query(
|
|
116
|
-
`FETCH 100 FROM ${cursorName}`
|
|
122
|
+
`FETCH 100 FROM ${cursorName} WITH (timeout = '1s')`
|
|
117
123
|
)
|
|
118
124
|
if (batch.rows.length > 0) {
|
|
119
125
|
allRows.push(...batch.rows)
|
|
@@ -122,14 +128,17 @@ export class CursorManager {
|
|
|
122
128
|
}
|
|
123
129
|
}
|
|
124
130
|
|
|
125
|
-
const
|
|
131
|
+
const diffs = this.parseDiffBatches(allRows)
|
|
126
132
|
|
|
127
|
-
|
|
128
|
-
|
|
133
|
+
for (const diff of diffs) {
|
|
134
|
+
if (diff.inserted.length === 0 && diff.updated.length === 0 && diff.deleted.length === 0) {
|
|
135
|
+
continue
|
|
136
|
+
}
|
|
137
|
+
callback(queryName, diff)
|
|
129
138
|
}
|
|
130
139
|
} catch (err: any) {
|
|
131
140
|
if (!this.running) return
|
|
132
|
-
console.error(`[cursor-manager] Error fetching ${
|
|
141
|
+
console.error(`[cursor-manager] Error fetching ${queryName}:`, err.message)
|
|
133
142
|
// Back off on error, then retry
|
|
134
143
|
await new Promise(r => setTimeout(r, 1000))
|
|
135
144
|
}
|
|
@@ -173,6 +182,88 @@ export class CursorManager {
|
|
|
173
182
|
return diff
|
|
174
183
|
}
|
|
175
184
|
|
|
185
|
+
parseDiffBatches(rows: any[]): ViewDiff[] {
|
|
186
|
+
const diffs: ViewDiff[] = []
|
|
187
|
+
let currentRows: any[] = []
|
|
188
|
+
let currentCursor: string | null = null
|
|
189
|
+
|
|
190
|
+
for (const row of rows) {
|
|
191
|
+
const cursor = this.normalizeCursor(row.rw_timestamp)
|
|
192
|
+
if (!cursor) continue
|
|
193
|
+
|
|
194
|
+
if (currentCursor !== null && cursor !== currentCursor) {
|
|
195
|
+
diffs.push(this.parseDiffs(currentRows))
|
|
196
|
+
currentRows = []
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
currentCursor = cursor
|
|
200
|
+
currentRows.push(row)
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (currentRows.length > 0) {
|
|
204
|
+
diffs.push(this.parseDiffs(currentRows))
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return diffs
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async bootstrap(queryName: string): Promise<BootstrapResult> {
|
|
211
|
+
const subName = this.subscriptions.get(queryName)
|
|
212
|
+
if (!subName) {
|
|
213
|
+
throw new Error(
|
|
214
|
+
`Subscription for query '${queryName}' is not initialized. Start the server before accepting WebSocket clients.`
|
|
215
|
+
)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const conn = new Client({ connectionString: this.connectionString })
|
|
219
|
+
await conn.connect()
|
|
220
|
+
|
|
221
|
+
const cursorName = `wavelet_boot_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
|
|
222
|
+
|
|
223
|
+
try {
|
|
224
|
+
await conn.query(`DECLARE ${cursorName} SUBSCRIPTION CURSOR FOR ${subName} FULL`)
|
|
225
|
+
|
|
226
|
+
const snapshotRows: Record<string, unknown>[] = []
|
|
227
|
+
const incrementalRows: any[] = []
|
|
228
|
+
let readingSnapshot = true
|
|
229
|
+
|
|
230
|
+
while (readingSnapshot) {
|
|
231
|
+
const result = await conn.query(`FETCH 1000 FROM ${cursorName}`)
|
|
232
|
+
if (result.rows.length === 0) break
|
|
233
|
+
|
|
234
|
+
let firstIncrementalIndex = result.rows.findIndex((row) => this.normalizeCursor(row.rw_timestamp) !== null)
|
|
235
|
+
if (firstIncrementalIndex === -1) firstIncrementalIndex = result.rows.length
|
|
236
|
+
|
|
237
|
+
for (const row of result.rows.slice(0, firstIncrementalIndex)) {
|
|
238
|
+
snapshotRows.push(this.stripSubscriptionMetadata(row))
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
if (firstIncrementalIndex < result.rows.length) {
|
|
242
|
+
incrementalRows.push(...result.rows.slice(firstIncrementalIndex))
|
|
243
|
+
readingSnapshot = false
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
while (true) {
|
|
248
|
+
const result = await conn.query(`FETCH 1000 FROM ${cursorName}`)
|
|
249
|
+
if (result.rows.length === 0) break
|
|
250
|
+
incrementalRows.push(...result.rows)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const diffs = this.parseDiffBatches(incrementalRows)
|
|
254
|
+
const lastCursor = diffs.length > 0 ? diffs[diffs.length - 1].cursor : null
|
|
255
|
+
|
|
256
|
+
return { snapshotRows, diffs, lastCursor }
|
|
257
|
+
} finally {
|
|
258
|
+
try {
|
|
259
|
+
await conn.query(`CLOSE ${cursorName}`)
|
|
260
|
+
} catch {}
|
|
261
|
+
try {
|
|
262
|
+
await conn.end()
|
|
263
|
+
} catch {}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
176
267
|
async query(sql: string): Promise<any[]> {
|
|
177
268
|
if (!this.client) throw new Error('Not connected')
|
|
178
269
|
const result = await this.client.query(sql)
|
|
@@ -187,9 +278,9 @@ export class CursorManager {
|
|
|
187
278
|
async close(): Promise<void> {
|
|
188
279
|
this.running = false
|
|
189
280
|
|
|
190
|
-
// Close per-
|
|
191
|
-
for (const [
|
|
192
|
-
const cursorName = this.cursorNames.get(
|
|
281
|
+
// Close per-query connections
|
|
282
|
+
for (const [queryName, conn] of this.queryConnections) {
|
|
283
|
+
const cursorName = this.cursorNames.get(queryName)
|
|
193
284
|
try {
|
|
194
285
|
if (cursorName) await conn.query(`CLOSE ${cursorName}`)
|
|
195
286
|
} catch {}
|
|
@@ -197,7 +288,7 @@ export class CursorManager {
|
|
|
197
288
|
await conn.end()
|
|
198
289
|
} catch {}
|
|
199
290
|
}
|
|
200
|
-
this.
|
|
291
|
+
this.queryConnections.clear()
|
|
201
292
|
this.cursorNames.clear()
|
|
202
293
|
|
|
203
294
|
// Close shared connection
|
|
@@ -206,4 +297,15 @@ export class CursorManager {
|
|
|
206
297
|
} catch {}
|
|
207
298
|
this.client = null
|
|
208
299
|
}
|
|
300
|
+
|
|
301
|
+
private stripSubscriptionMetadata(row: Record<string, unknown>): Record<string, unknown> {
|
|
302
|
+
const { op: _op, rw_timestamp: _rwTimestamp, ...data } = row
|
|
303
|
+
return data
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
private normalizeCursor(value: unknown): string | null {
|
|
307
|
+
if (value === null || value === undefined) return null
|
|
308
|
+
const cursor = String(value).trim()
|
|
309
|
+
return cursor === '' ? null : cursor
|
|
310
|
+
}
|
|
209
311
|
}
|