@dxos/feed 0.0.0 → 0.8.4-main.05e74ebcff
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +6 -2
- package/dist/lib/neutral/chunk-YYASRWLE.mjs +880 -0
- package/dist/lib/neutral/chunk-YYASRWLE.mjs.map +7 -0
- package/dist/lib/neutral/index.mjs +13 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/testing/index.mjs +174 -0
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/errors.d.ts +39 -0
- package/dist/types/src/errors.d.ts.map +1 -0
- package/dist/types/src/feed-store.d.ts +121 -0
- package/dist/types/src/feed-store.d.ts.map +1 -0
- package/dist/types/src/feed-store.test.d.ts +2 -0
- package/dist/types/src/feed-store.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +5 -0
- package/dist/types/src/index.d.ts.map +1 -0
- package/dist/types/src/sync-client.d.ts +54 -0
- package/dist/types/src/sync-client.d.ts.map +1 -0
- package/dist/types/src/sync-server.d.ts +28 -0
- package/dist/types/src/sync-server.d.ts.map +1 -0
- package/dist/types/src/sync.test.d.ts +2 -0
- package/dist/types/src/sync.test.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +2 -0
- package/dist/types/src/testing/index.d.ts.map +1 -0
- package/dist/types/src/testing/test-builder.d.ts +78 -0
- package/dist/types/src/testing/test-builder.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +24 -22
- package/src/errors.ts +20 -0
- package/src/feed-store.test.ts +397 -40
- package/src/feed-store.ts +508 -142
- package/src/index.ts +3 -1
- package/src/sync-client.ts +280 -0
- package/src/sync-server.ts +150 -0
- package/src/sync.test.ts +151 -91
- package/src/testing/index.ts +1 -1
- package/src/testing/test-builder.ts +223 -0
- package/src/protocol.ts +0 -110
package/src/feed-store.ts
CHANGED
|
@@ -7,73 +7,112 @@ import type * as SqlError from '@effect/sql/SqlError';
|
|
|
7
7
|
import * as Effect from 'effect/Effect';
|
|
8
8
|
|
|
9
9
|
import { Event } from '@dxos/async';
|
|
10
|
-
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
10
|
+
import { assertArgument } from '@dxos/invariant';
|
|
11
|
+
import { type SpaceId } from '@dxos/keys';
|
|
12
|
+
import { FeedProtocol } from '@dxos/protocols';
|
|
13
|
+
import { SqlTransaction } from '@dxos/sql-sqlite';
|
|
14
|
+
|
|
15
|
+
import { PositionConflictError } from './errors';
|
|
16
|
+
|
|
17
|
+
type AppendRequest = FeedProtocol.AppendRequest;
|
|
18
|
+
type AppendResponse = FeedProtocol.AppendResponse;
|
|
19
|
+
type Block = FeedProtocol.Block;
|
|
20
|
+
const FeedCursor = FeedProtocol.FeedCursor;
|
|
21
|
+
type FeedCursor = FeedProtocol.FeedCursor;
|
|
22
|
+
const isWellKnownNamespace = FeedProtocol.isWellKnownNamespace;
|
|
23
|
+
type QueryRequest = FeedProtocol.QueryRequest;
|
|
24
|
+
type QueryResponse = FeedProtocol.QueryResponse;
|
|
25
|
+
type SubscribeRequest = FeedProtocol.SubscribeRequest;
|
|
26
|
+
type SubscribeResponse = FeedProtocol.SubscribeResponse;
|
|
21
27
|
|
|
22
28
|
export interface FeedStoreOptions {
|
|
29
|
+
/**
|
|
30
|
+
* The actor ID of the local user.
|
|
31
|
+
*/
|
|
23
32
|
localActorId: string;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Whether to assign positions to appended blocks.
|
|
36
|
+
* Only a single peer (usually the server) can assign positions.
|
|
37
|
+
*/
|
|
24
38
|
assignPositions: boolean;
|
|
25
39
|
}
|
|
26
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Persistent storage for feed metadata, blocks, subscriptions, and sync state.
|
|
43
|
+
*
|
|
44
|
+
*/
|
|
27
45
|
export class FeedStore {
|
|
28
|
-
|
|
46
|
+
readonly #options: FeedStoreOptions;
|
|
29
47
|
|
|
48
|
+
constructor(options: FeedStoreOptions) {
|
|
49
|
+
this.#options = options;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Emits after successful block append operations.
|
|
54
|
+
*/
|
|
30
55
|
readonly onNewBlocks = new Event<void>();
|
|
31
56
|
|
|
32
|
-
|
|
33
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Creates required feed store tables and indexes if they do not exist.
|
|
59
|
+
*/
|
|
60
|
+
migrate = Effect.fn('FeedStore.migrate')(() =>
|
|
61
|
+
Effect.gen(function* () {
|
|
62
|
+
const sql = yield* SqlClient.SqlClient;
|
|
34
63
|
|
|
35
|
-
|
|
36
|
-
|
|
64
|
+
// Feeds Table
|
|
65
|
+
yield* sql`CREATE TABLE IF NOT EXISTS feeds (
|
|
37
66
|
feedPrivateId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
38
67
|
spaceId TEXT NOT NULL,
|
|
39
68
|
feedId TEXT NOT NULL,
|
|
40
69
|
feedNamespace TEXT
|
|
41
70
|
)`;
|
|
42
|
-
|
|
71
|
+
yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_feeds_spaceId_feedId ON feeds(spaceId, feedId)`;
|
|
43
72
|
|
|
44
|
-
|
|
45
|
-
|
|
73
|
+
// Blocks Table
|
|
74
|
+
yield* sql`CREATE TABLE IF NOT EXISTS blocks (
|
|
46
75
|
insertionId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
47
76
|
feedPrivateId INTEGER NOT NULL,
|
|
48
77
|
position INTEGER,
|
|
49
78
|
sequence INTEGER NOT NULL,
|
|
50
79
|
actorId TEXT NOT NULL,
|
|
51
|
-
|
|
52
|
-
|
|
80
|
+
prevSequence INTEGER,
|
|
81
|
+
prevActorId TEXT,
|
|
53
82
|
timestamp INTEGER NOT NULL,
|
|
54
83
|
data BLOB NOT NULL,
|
|
55
84
|
FOREIGN KEY(feedPrivateId) REFERENCES feeds(feedPrivateId)
|
|
56
85
|
)`;
|
|
57
|
-
|
|
58
|
-
|
|
86
|
+
yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_position ON blocks(feedPrivateId, position)`;
|
|
87
|
+
yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_sequence_actorId ON blocks(feedPrivateId, sequence, actorId)`;
|
|
59
88
|
|
|
60
|
-
|
|
61
|
-
|
|
89
|
+
// Subscriptions Table
|
|
90
|
+
yield* sql`CREATE TABLE IF NOT EXISTS subscriptions (
|
|
62
91
|
subscriptionId TEXT PRIMARY KEY,
|
|
63
92
|
expiresAt INTEGER NOT NULL,
|
|
64
93
|
feedPrivateIds TEXT NOT NULL -- JSON array
|
|
65
94
|
)`;
|
|
66
95
|
|
|
67
|
-
|
|
68
|
-
|
|
96
|
+
// Cursor Tokens Table
|
|
97
|
+
yield* sql`CREATE TABLE IF NOT EXISTS cursor_tokens (
|
|
69
98
|
spaceId TEXT PRIMARY KEY,
|
|
70
99
|
token TEXT NOT NULL
|
|
71
100
|
)`;
|
|
72
|
-
});
|
|
73
101
|
|
|
74
|
-
|
|
102
|
+
// Sync State Table.
|
|
103
|
+
yield* sql`CREATE TABLE IF NOT EXISTS sync_state (
|
|
104
|
+
spaceId TEXT NOT NULL,
|
|
105
|
+
feedNamespace TEXT NOT NULL,
|
|
106
|
+
lastPulledPosition INTEGER NOT NULL DEFAULT -1,
|
|
107
|
+
PRIMARY KEY (spaceId, feedNamespace)
|
|
108
|
+
)`;
|
|
109
|
+
}).pipe(Effect.withSpan('FeedStore.migrate')),
|
|
110
|
+
);
|
|
75
111
|
|
|
76
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Ensures a feed row exists and returns its internal feed ID.
|
|
114
|
+
*/
|
|
115
|
+
#ensureFeed = Effect.fn('Feed.ensureFeed')(
|
|
77
116
|
(
|
|
78
117
|
spaceId: string,
|
|
79
118
|
feedId: string,
|
|
@@ -85,30 +124,38 @@ export class FeedStore {
|
|
|
85
124
|
const rows = yield* sql<{ feedPrivateId: number }>`
|
|
86
125
|
SELECT feedPrivateId FROM feeds WHERE spaceId = ${spaceId} AND feedId = ${feedId}
|
|
87
126
|
`;
|
|
88
|
-
if (rows.length > 0)
|
|
127
|
+
if (rows.length > 0) {
|
|
128
|
+
return rows[0].feedPrivateId;
|
|
129
|
+
}
|
|
89
130
|
|
|
90
131
|
const newRows = yield* sql<{ feedPrivateId: number }>`
|
|
91
132
|
INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
|
|
92
133
|
`;
|
|
93
134
|
return newRows[0].feedPrivateId;
|
|
94
|
-
}),
|
|
135
|
+
}).pipe(Effect.withSpan('FeedStore.ensureFeed')),
|
|
95
136
|
);
|
|
96
137
|
|
|
97
|
-
|
|
138
|
+
/**
|
|
139
|
+
* Ensures cursor token exists for a space and returns it.
|
|
140
|
+
*/
|
|
141
|
+
#ensureCursorToken = Effect.fn('Feed.ensureCursorToken')(
|
|
98
142
|
(spaceId: string): Effect.Effect<string, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
99
143
|
Effect.gen(this, function* () {
|
|
100
144
|
const sql = yield* SqlClient.SqlClient;
|
|
101
145
|
const rows = yield* sql<{ token: string }>`SELECT token FROM cursor_tokens WHERE spaceId = ${spaceId}`;
|
|
102
|
-
if (rows.length > 0)
|
|
146
|
+
if (rows.length > 0) {
|
|
147
|
+
return rows[0].token;
|
|
148
|
+
}
|
|
103
149
|
|
|
104
150
|
const token = crypto.randomUUID().replace(/-/g, '').slice(0, 6);
|
|
105
151
|
yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
|
|
106
152
|
return token;
|
|
107
|
-
}),
|
|
153
|
+
}).pipe(Effect.withSpan('FeedStore.ensureCursorToken')),
|
|
108
154
|
);
|
|
109
155
|
|
|
110
|
-
|
|
111
|
-
|
|
156
|
+
/**
|
|
157
|
+
* Queries feed blocks by feed IDs or subscription with cursor/position pagination.
|
|
158
|
+
*/
|
|
112
159
|
query = Effect.fn('Feed.query')(
|
|
113
160
|
(request: QueryRequest): Effect.Effect<QueryResponse, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
114
161
|
Effect.gen(this, function* () {
|
|
@@ -121,6 +168,15 @@ export class FeedStore {
|
|
|
121
168
|
return yield* Effect.die(new Error('spaceId is required'));
|
|
122
169
|
}
|
|
123
170
|
|
|
171
|
+
if (
|
|
172
|
+
(request.position !== undefined ? 1 : 0) +
|
|
173
|
+
(request.cursor !== undefined ? 1 : 0) +
|
|
174
|
+
(request.unpositionedOnly === true ? 1 : 0) >
|
|
175
|
+
1
|
|
176
|
+
) {
|
|
177
|
+
return yield* Effect.die(new Error('Only one of position, cursor, or unpositionedOnly can be used'));
|
|
178
|
+
}
|
|
179
|
+
|
|
124
180
|
if (request.cursor) {
|
|
125
181
|
const { token, insertionId } = decodeCursor(request.cursor as FeedCursor);
|
|
126
182
|
if (!token || insertionId === undefined || isNaN(insertionId)) {
|
|
@@ -130,8 +186,8 @@ export class FeedStore {
|
|
|
130
186
|
cursorInsertionId = insertionId;
|
|
131
187
|
}
|
|
132
188
|
|
|
133
|
-
// Validate Token if cursor used
|
|
134
|
-
const validCursorToken = yield* this
|
|
189
|
+
// Validate Token if cursor used.
|
|
190
|
+
const validCursorToken = yield* this.#ensureCursorToken(request.spaceId);
|
|
135
191
|
if (request.cursor && cursorToken !== validCursorToken) {
|
|
136
192
|
return yield* Effect.die(new Error(`Cursor token mismatch`));
|
|
137
193
|
}
|
|
@@ -140,7 +196,7 @@ export class FeedStore {
|
|
|
140
196
|
// If spaceId is not provided in request (e.g. feedIds query), we can't easily validate token unless we look up spaceId for feedIds.
|
|
141
197
|
// Ideally spaceId should be required for token validation.
|
|
142
198
|
|
|
143
|
-
/*
|
|
199
|
+
/*
|
|
144
200
|
Logic:
|
|
145
201
|
1. If `cursor` is present, it's `token|insertionId`.
|
|
146
202
|
2. If `position` is present, it's `position` (legacy/manual).
|
|
@@ -150,8 +206,8 @@ export class FeedStore {
|
|
|
150
206
|
|
|
151
207
|
const position = request.position ?? -1;
|
|
152
208
|
|
|
153
|
-
// Resolve
|
|
154
|
-
if ('subscriptionId' in request.query) {
|
|
209
|
+
// Resolve subscriptions or feed IDs.
|
|
210
|
+
if (request.query && 'subscriptionId' in request.query) {
|
|
155
211
|
const rows = yield* sql<{ feedPrivateIds: string; expiresAt: number }>`
|
|
156
212
|
SELECT feedPrivateIds, expiresAt FROM subscriptions WHERE subscriptionId = ${request.query.subscriptionId}
|
|
157
213
|
`;
|
|
@@ -167,50 +223,57 @@ export class FeedStore {
|
|
|
167
223
|
}
|
|
168
224
|
}
|
|
169
225
|
}
|
|
170
|
-
} else if ('feedIds' in request.query) {
|
|
171
|
-
feedIds = [...
|
|
226
|
+
} else if (request.query && 'feedIds' in request.query) {
|
|
227
|
+
feedIds = [...request.query.feedIds];
|
|
172
228
|
} else {
|
|
173
229
|
feedIds = undefined;
|
|
174
230
|
}
|
|
175
231
|
|
|
176
232
|
if (feedIds !== undefined && feedIds.length === 0) {
|
|
177
|
-
return {
|
|
233
|
+
return {
|
|
234
|
+
requestId: request.requestId,
|
|
235
|
+
blocks: [],
|
|
236
|
+
nextCursor: encodeCursor(validCursorToken, -1),
|
|
237
|
+
hasMore: false,
|
|
238
|
+
};
|
|
178
239
|
}
|
|
179
240
|
|
|
180
|
-
// Fetch
|
|
241
|
+
// Fetch blocks.
|
|
181
242
|
const query = sql<Block>`
|
|
182
|
-
SELECT blocks.*, feeds.feedId
|
|
243
|
+
SELECT blocks.*, feeds.feedId, feeds.feedNamespace
|
|
183
244
|
FROM blocks
|
|
184
245
|
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
185
246
|
WHERE 1=1
|
|
186
247
|
${feedIds !== undefined ? sql`AND feeds.feedId IN ${sql.in(feedIds)}` : sql``}
|
|
187
248
|
${request.spaceId ? sql`AND feeds.spaceId = ${request.spaceId}` : sql``}
|
|
188
|
-
${
|
|
189
|
-
'feedNamespace' in request.query && request.query.feedNamespace
|
|
190
|
-
? sql`AND feeds.feedNamespace = ${request.query.feedNamespace}`
|
|
191
|
-
: sql``
|
|
192
|
-
}
|
|
249
|
+
${sql`AND feeds.feedNamespace = ${request.feedNamespace}`}
|
|
193
250
|
`;
|
|
194
251
|
|
|
195
|
-
// Add filter based on cursor or position
|
|
252
|
+
// Add filter based on cursor or position.
|
|
196
253
|
const filter = request.cursor
|
|
197
254
|
? sql`AND blocks.insertionId > ${cursorInsertionId}`
|
|
198
|
-
:
|
|
255
|
+
: request.unpositionedOnly
|
|
256
|
+
? sql`AND blocks.position IS NULL`
|
|
257
|
+
: sql`AND (blocks.position > ${position} OR blocks.position IS NULL)`;
|
|
199
258
|
|
|
200
259
|
const orderBy = request.cursor
|
|
201
260
|
? sql`ORDER BY blocks.insertionId ASC`
|
|
202
261
|
: sql`ORDER BY blocks.position ASC NULLS LAST`;
|
|
203
262
|
|
|
263
|
+
const requestLimit = request.limit;
|
|
264
|
+
const queryLimit = requestLimit != null ? requestLimit + 1 : undefined;
|
|
204
265
|
const rows = yield* sql<Block>`
|
|
205
266
|
${query}
|
|
206
267
|
${filter}
|
|
207
268
|
${orderBy}
|
|
208
|
-
${
|
|
269
|
+
${queryLimit != null ? sql`LIMIT ${queryLimit}` : sql``}
|
|
209
270
|
`;
|
|
210
271
|
|
|
211
|
-
const
|
|
272
|
+
const hasMore = requestLimit != null && rows.length > requestLimit;
|
|
273
|
+
const slice = hasMore ? rows.slice(0, requestLimit) : rows;
|
|
274
|
+
const blocks = slice.map((row) => ({
|
|
212
275
|
...row,
|
|
213
|
-
// Have to buffer otherwise we get empty Uint8Array.
|
|
276
|
+
// Have to clone buffer otherwise we get empty Uint8Array.
|
|
214
277
|
data: new Uint8Array(row.data),
|
|
215
278
|
}));
|
|
216
279
|
|
|
@@ -222,10 +285,13 @@ export class FeedStore {
|
|
|
222
285
|
}
|
|
223
286
|
}
|
|
224
287
|
|
|
225
|
-
return { requestId: request.requestId, blocks, nextCursor } satisfies QueryResponse;
|
|
226
|
-
}),
|
|
288
|
+
return { requestId: request.requestId, blocks, nextCursor, hasMore } satisfies QueryResponse;
|
|
289
|
+
}).pipe(Effect.withSpan('FeedStore.query')),
|
|
227
290
|
);
|
|
228
291
|
|
|
292
|
+
/**
|
|
293
|
+
* Creates a subscription and stores the resolved internal feed IDs.
|
|
294
|
+
*/
|
|
229
295
|
subscribe = Effect.fn('Feed.subscribe')(
|
|
230
296
|
(request: SubscribeRequest): Effect.Effect<SubscribeResponse, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
231
297
|
Effect.gen(this, function* () {
|
|
@@ -241,7 +307,7 @@ export class FeedStore {
|
|
|
241
307
|
|
|
242
308
|
const feedPrivateIds = yield* Effect.forEach(
|
|
243
309
|
request.feedIds,
|
|
244
|
-
(feedId) => this
|
|
310
|
+
(feedId) => this.#ensureFeed(request.spaceId!, feedId),
|
|
245
311
|
{ concurrency: 'unbounded' },
|
|
246
312
|
);
|
|
247
313
|
|
|
@@ -255,119 +321,419 @@ export class FeedStore {
|
|
|
255
321
|
subscriptionId,
|
|
256
322
|
expiresAt,
|
|
257
323
|
};
|
|
258
|
-
}),
|
|
324
|
+
}).pipe(Effect.withSpan('FeedStore.subscribe')),
|
|
259
325
|
);
|
|
260
326
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
327
|
+
/**
|
|
328
|
+
* Get the last pulled position for the given space and namespace.
|
|
329
|
+
* Returns -1 if no sync state exists yet.
|
|
330
|
+
*/
|
|
331
|
+
getSyncState = (opts: {
|
|
332
|
+
spaceId: SpaceId;
|
|
333
|
+
feedNamespace: string;
|
|
334
|
+
}): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
335
|
+
Effect.gen(this, function* () {
|
|
336
|
+
const sql = yield* SqlClient.SqlClient;
|
|
337
|
+
const rows = yield* sql<{ lastPulledPosition: number }>`
|
|
338
|
+
SELECT lastPulledPosition FROM sync_state
|
|
339
|
+
WHERE spaceId = ${opts.spaceId} AND feedNamespace = ${opts.feedNamespace}
|
|
340
|
+
`;
|
|
341
|
+
return rows[0]?.lastPulledPosition ?? -1;
|
|
342
|
+
}).pipe(Effect.withSpan('FeedStore.getSyncState'));
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Update the last pulled position for the given space and namespace.
|
|
346
|
+
*/
|
|
347
|
+
setSyncState = (opts: {
|
|
348
|
+
spaceId: SpaceId;
|
|
349
|
+
feedNamespace: string;
|
|
350
|
+
lastPulledPosition: number;
|
|
351
|
+
}): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
352
|
+
Effect.gen(this, function* () {
|
|
353
|
+
const sql = yield* SqlClient.SqlClient;
|
|
354
|
+
yield* sql`
|
|
355
|
+
INSERT INTO sync_state (spaceId, feedNamespace, lastPulledPosition)
|
|
356
|
+
VALUES (${opts.spaceId}, ${opts.feedNamespace}, ${opts.lastPulledPosition})
|
|
357
|
+
ON CONFLICT (spaceId, feedNamespace) DO UPDATE SET lastPulledPosition = ${opts.lastPulledPosition}
|
|
358
|
+
`;
|
|
359
|
+
}).pipe(Effect.withSpan('FeedStore.setSyncState'));
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Returns the number of stored blocks for a single feed in a space/namespace.
|
|
363
|
+
* Intended as a low-level primitive for callers (for example Cloudflare Worker code)
|
|
364
|
+
* that need to make retention decisions under constrained storage resources.
|
|
365
|
+
*/
|
|
366
|
+
countBlocks = (opts: {
|
|
367
|
+
spaceId: SpaceId;
|
|
368
|
+
feedNamespace: string;
|
|
369
|
+
feedId: string;
|
|
370
|
+
}): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
371
|
+
Effect.gen(this, function* () {
|
|
372
|
+
const sql = yield* SqlClient.SqlClient;
|
|
373
|
+
const rows = yield* sql<{ count: number }>`
|
|
374
|
+
SELECT COUNT(*) AS count
|
|
375
|
+
FROM blocks
|
|
376
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
377
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
378
|
+
AND feeds.feedNamespace = ${opts.feedNamespace}
|
|
379
|
+
AND feeds.feedId = ${opts.feedId}
|
|
380
|
+
`;
|
|
381
|
+
return rows[0]?.count ?? 0;
|
|
382
|
+
}).pipe(Effect.withSpan('FeedStore.countBlocks'));
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Deletes the oldest blocks for a single feed in a space/namespace.
|
|
386
|
+
* This API intentionally does not enforce any retention policy (such as max size);
|
|
387
|
+
* callers decide when and how much to prune, which is useful for constrained
|
|
388
|
+
* environments like Cloudflare Workers.
|
|
389
|
+
*
|
|
390
|
+
* @returns Number of deleted rows.
|
|
391
|
+
*/
|
|
392
|
+
deleteOldestBlocks = (opts: {
|
|
393
|
+
spaceId: SpaceId;
|
|
394
|
+
feedNamespace: string;
|
|
395
|
+
feedId: string;
|
|
396
|
+
count: number;
|
|
397
|
+
}): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
398
|
+
Effect.gen(this, function* () {
|
|
399
|
+
const sql = yield* SqlClient.SqlClient;
|
|
400
|
+
if (opts.count <= 0) {
|
|
401
|
+
return 0;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const deletedRows = yield* sql<{ insertionId: number }>`
|
|
405
|
+
DELETE FROM blocks
|
|
406
|
+
WHERE insertionId IN (
|
|
407
|
+
SELECT blocks.insertionId
|
|
408
|
+
FROM blocks
|
|
409
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
410
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
411
|
+
AND feeds.feedNamespace = ${opts.feedNamespace}
|
|
412
|
+
AND feeds.feedId = ${opts.feedId}
|
|
413
|
+
ORDER BY blocks.insertionId ASC
|
|
414
|
+
LIMIT ${opts.count}
|
|
415
|
+
)
|
|
416
|
+
RETURNING insertionId
|
|
417
|
+
`;
|
|
418
|
+
|
|
419
|
+
return deletedRows.length;
|
|
420
|
+
}).pipe(Effect.withSpan('FeedStore.deleteOldestBlocks'));
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Appends blocks for a space/namespace and optionally assigns global positions.
|
|
424
|
+
*/
|
|
425
|
+
append = (
|
|
426
|
+
request: AppendRequest,
|
|
427
|
+
): Effect.Effect<AppendResponse, SqlError.SqlError, SqlClient.SqlClient | SqlTransaction.SqlTransaction> =>
|
|
428
|
+
Effect.gen(this, function* () {
|
|
429
|
+
if (!request.spaceId) {
|
|
430
|
+
return yield* Effect.die(new Error('spaceId required for append'));
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
assertArgument(
|
|
434
|
+
isWellKnownNamespace(request.feedNamespace),
|
|
435
|
+
'request.feedNamespace',
|
|
436
|
+
'specified well-known namespace',
|
|
437
|
+
);
|
|
438
|
+
|
|
439
|
+
// Validate all blocks upfront.
|
|
440
|
+
for (const block of request.blocks) {
|
|
441
|
+
assertArgument(block.feedId, 'block.feedId', 'feedId is required');
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// Wrap in transaction to ensure atomicity when assigning positions.
|
|
445
|
+
const sqlTransaction = yield* SqlTransaction.SqlTransaction;
|
|
446
|
+
const positions = yield* sqlTransaction.withTransaction(
|
|
447
|
+
Effect.gen(this, function* () {
|
|
448
|
+
const sql = yield* SqlClient.SqlClient;
|
|
449
|
+
|
|
450
|
+
// 1. Collect unique feed IDs and batch #ensureFeed calls.
|
|
451
|
+
const feedKeys = new Map<string, { feedId: string }>();
|
|
452
|
+
for (const block of request.blocks) {
|
|
453
|
+
const key = block.feedId!;
|
|
454
|
+
if (!feedKeys.has(key)) {
|
|
455
|
+
feedKeys.set(key, { feedId: block.feedId! });
|
|
456
|
+
}
|
|
457
|
+
}
|
|
274
458
|
|
|
275
|
-
|
|
276
|
-
|
|
459
|
+
const feedPrivateIds = new Map<string, number>();
|
|
460
|
+
yield* Effect.forEach(
|
|
461
|
+
[...feedKeys.entries()],
|
|
462
|
+
([key, { feedId }]) =>
|
|
463
|
+
Effect.gen(this, function* () {
|
|
464
|
+
const id = yield* this.#ensureFeed(request.spaceId!, feedId, request.feedNamespace);
|
|
465
|
+
feedPrivateIds.set(key, id);
|
|
466
|
+
}),
|
|
467
|
+
{ concurrency: 'unbounded' },
|
|
468
|
+
);
|
|
469
|
+
|
|
470
|
+
// 2. Get max position per namespace ONCE (not per block).
|
|
471
|
+
const maxPositions = new Map<string, number>();
|
|
472
|
+
if (this.#options.assignPositions) {
|
|
277
473
|
const maxPosResult = yield* sql<{ maxPos: number | null }>`
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
positions.push(nextPos);
|
|
474
|
+
SELECT MAX(position) as maxPos
|
|
475
|
+
FROM blocks
|
|
476
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
477
|
+
WHERE feeds.spaceId = ${request.spaceId} AND feeds.feedNamespace = ${request.feedNamespace}
|
|
478
|
+
`;
|
|
479
|
+
maxPositions.set(request.feedNamespace, maxPosResult[0]?.maxPos ?? -1);
|
|
285
480
|
}
|
|
286
481
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
482
|
+
// 3. Insert all blocks and compute positions.
|
|
483
|
+
const positions: number[] = [];
|
|
484
|
+
for (const block of request.blocks) {
|
|
485
|
+
const key = block.feedId!;
|
|
486
|
+
const feedPrivateId = feedPrivateIds.get(key)!;
|
|
487
|
+
|
|
488
|
+
let positionToInsert: number | null = null;
|
|
489
|
+
if (this.#options.assignPositions) {
|
|
490
|
+
const currentMax = maxPositions.get(request.feedNamespace)!;
|
|
491
|
+
positionToInsert = currentMax + 1;
|
|
492
|
+
maxPositions.set(request.feedNamespace, positionToInsert); // Increment for next block in same namespace.
|
|
493
|
+
positions.push(positionToInsert);
|
|
494
|
+
} else if (block.position != null) {
|
|
495
|
+
positionToInsert = block.position;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
yield* sql`
|
|
499
|
+
INSERT INTO blocks (
|
|
500
|
+
feedPrivateId, position, sequence, actorId,
|
|
501
|
+
prevSequence, prevActorId, timestamp, data
|
|
502
|
+
) VALUES (
|
|
503
|
+
${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
|
|
504
|
+
${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
|
|
505
|
+
) ON CONFLICT DO NOTHING
|
|
295
506
|
`;
|
|
296
|
-
|
|
507
|
+
}
|
|
297
508
|
|
|
298
|
-
|
|
509
|
+
return positions;
|
|
510
|
+
}),
|
|
511
|
+
);
|
|
299
512
|
|
|
300
|
-
|
|
301
|
-
}),
|
|
302
|
-
);
|
|
513
|
+
this.onNewBlocks.emit();
|
|
303
514
|
|
|
515
|
+
return { requestId: request.requestId, positions };
|
|
516
|
+
}).pipe(Effect.withSpan('FeedStore.append'));
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Creates local blocks with sequential predecessors and appends grouped batches.
|
|
520
|
+
*/
|
|
304
521
|
appendLocal = Effect.fn('Feed.appendLocal')(
|
|
305
522
|
(
|
|
306
523
|
messages: { spaceId: string; feedId: string; feedNamespace: string; data: Uint8Array }[],
|
|
307
|
-
): Effect.Effect<Block[], SqlError.SqlError, SqlClient.SqlClient> =>
|
|
524
|
+
): Effect.Effect<Block[], SqlError.SqlError, SqlClient.SqlClient | SqlTransaction.SqlTransaction> =>
|
|
308
525
|
Effect.gen(this, function* () {
|
|
309
526
|
const sql = yield* SqlClient.SqlClient;
|
|
310
|
-
const blocks: Block[] = [];
|
|
311
|
-
|
|
312
|
-
// Group by spaceId for efficient processing? Or just iterate.
|
|
313
|
-
// Assuming small batch.
|
|
314
527
|
|
|
528
|
+
// 1. Collect unique feeds and ensure they exist.
|
|
529
|
+
type FeedKey = string; // `${spaceId}|${feedNamespace}|${feedId}`
|
|
530
|
+
const feedKeys = new Map<FeedKey, { spaceId: string; feedId: string; feedNamespace: string }>();
|
|
315
531
|
for (const msg of messages) {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
532
|
+
const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
|
|
533
|
+
if (!feedKeys.has(key)) {
|
|
534
|
+
feedKeys.set(key, { spaceId: msg.spaceId, feedId: msg.feedId, feedNamespace: msg.feedNamespace });
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// Batch ensure feeds and get their private IDs.
|
|
539
|
+
const feedPrivateIds = new Map<FeedKey, number>();
|
|
540
|
+
yield* Effect.forEach(
|
|
541
|
+
[...feedKeys.entries()],
|
|
542
|
+
([key, { spaceId, feedId, feedNamespace }]) =>
|
|
543
|
+
Effect.gen(this, function* () {
|
|
544
|
+
const id = yield* this.#ensureFeed(spaceId, feedId, feedNamespace);
|
|
545
|
+
feedPrivateIds.set(key, id);
|
|
546
|
+
}),
|
|
547
|
+
{ concurrency: 'unbounded' },
|
|
548
|
+
);
|
|
549
|
+
|
|
550
|
+
// 2. Get last sequence for each unique feed (batch query).
|
|
551
|
+
const lastSeqs = new Map<FeedKey, { sequence: number; actorId: string } | null>();
|
|
552
|
+
for (const [key] of feedKeys) {
|
|
553
|
+
const feedPrivateId = feedPrivateIds.get(key)!;
|
|
335
554
|
const lastBlockResult = yield* sql<{ sequence: number; actorId: string }>`
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
555
|
+
SELECT sequence, actorId FROM blocks
|
|
556
|
+
WHERE feedPrivateId = ${feedPrivateId}
|
|
557
|
+
ORDER BY sequence DESC
|
|
558
|
+
LIMIT 1
|
|
559
|
+
`;
|
|
560
|
+
lastSeqs.set(key, lastBlockResult[0] ?? null);
|
|
561
|
+
}
|
|
341
562
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
563
|
+
// 3. Build all blocks with correct sequences.
|
|
564
|
+
// Track in-flight sequences per feed to handle multiple messages to same feed.
|
|
565
|
+
const currentSeqs = new Map<FeedKey, { sequence: number; actorId: string }>();
|
|
566
|
+
const blocks: Block[] = [];
|
|
567
|
+
const blocksBySpaceNamespace = new Map<string, { spaceId: string; feedNamespace: string; blocks: Block[] }>();
|
|
568
|
+
|
|
569
|
+
for (const msg of messages) {
|
|
570
|
+
const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
|
|
571
|
+
|
|
572
|
+
// Determine predecessor: either from in-flight blocks or from DB.
|
|
573
|
+
let sequence: number;
|
|
574
|
+
let prevSequence: number | null;
|
|
575
|
+
let prevActorId: string | null;
|
|
576
|
+
|
|
577
|
+
const inFlight = currentSeqs.get(key);
|
|
578
|
+
if (inFlight) {
|
|
579
|
+
// We've already added blocks for this feed - continue from last in-flight.
|
|
580
|
+
sequence = inFlight.sequence + 1;
|
|
581
|
+
prevSequence = inFlight.sequence;
|
|
582
|
+
prevActorId = inFlight.actorId;
|
|
583
|
+
} else {
|
|
584
|
+
// First block for this feed - use DB state.
|
|
585
|
+
const lastBlock = lastSeqs.get(key);
|
|
586
|
+
sequence = (lastBlock?.sequence ?? -1) + 1;
|
|
587
|
+
prevSequence = lastBlock?.sequence ?? null;
|
|
588
|
+
prevActorId = lastBlock?.actorId ?? null;
|
|
589
|
+
}
|
|
346
590
|
|
|
347
591
|
const block: Block = {
|
|
348
|
-
feedId:
|
|
349
|
-
actorId: this.
|
|
592
|
+
feedId: msg.feedId,
|
|
593
|
+
actorId: this.#options.localActorId,
|
|
350
594
|
sequence,
|
|
351
|
-
|
|
352
|
-
|
|
595
|
+
prevActorId,
|
|
596
|
+
prevSequence,
|
|
353
597
|
timestamp: Date.now(),
|
|
354
598
|
data: msg.data,
|
|
355
|
-
position: null, //
|
|
599
|
+
position: null, // Assigned by append.
|
|
356
600
|
};
|
|
601
|
+
|
|
357
602
|
blocks.push(block);
|
|
358
603
|
|
|
359
|
-
//
|
|
360
|
-
|
|
604
|
+
// Update in-flight tracking.
|
|
605
|
+
currentSeqs.set(key, { sequence, actorId: this.#options.localActorId });
|
|
606
|
+
|
|
607
|
+
// Group by (spaceId, feedNamespace).
|
|
608
|
+
const spaceNamespaceKey = `${msg.spaceId}|${msg.feedNamespace}`;
|
|
609
|
+
if (!blocksBySpaceNamespace.has(spaceNamespaceKey)) {
|
|
610
|
+
blocksBySpaceNamespace.set(spaceNamespaceKey, {
|
|
611
|
+
spaceId: msg.spaceId,
|
|
612
|
+
feedNamespace: msg.feedNamespace,
|
|
613
|
+
blocks: [],
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
blocksBySpaceNamespace.get(spaceNamespaceKey)!.blocks.push(block);
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
// 4. Call append once per (spaceId, namespace) batch and assign returned positions.
|
|
620
|
+
const positionByBlock = new Map<Block, number | null>();
|
|
621
|
+
for (const { spaceId, feedNamespace, blocks: batchBlocks } of blocksBySpaceNamespace.values()) {
|
|
622
|
+
const { positions } = yield* this.append({
|
|
361
623
|
requestId: 'local-append',
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
feedId: msg.feedId,
|
|
624
|
+
blocks: batchBlocks,
|
|
625
|
+
spaceId,
|
|
626
|
+
feedNamespace,
|
|
366
627
|
});
|
|
628
|
+
for (let i = 0; i < batchBlocks.length; i++) {
|
|
629
|
+
positionByBlock.set(batchBlocks[i], positions[i] ?? null);
|
|
630
|
+
}
|
|
367
631
|
}
|
|
368
|
-
|
|
369
|
-
|
|
632
|
+
|
|
633
|
+
return blocks.map((block) => ({
|
|
634
|
+
...block,
|
|
635
|
+
position: positionByBlock.get(block) ?? block.position,
|
|
636
|
+
}));
|
|
637
|
+
}).pipe(Effect.withSpan('FeedStore.appendLocal')),
|
|
370
638
|
);
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Sets positions for existing blocks while preventing conflicting reassignments.
|
|
642
|
+
*/
|
|
643
|
+
setPosition = (request: {
|
|
644
|
+
spaceId: string;
|
|
645
|
+
blocks: (Pick<Block, 'feedId' | 'actorId' | 'sequence' | 'position'> & { feedNamespace: string })[];
|
|
646
|
+
}): Effect.Effect<void, SqlError.SqlError | PositionConflictError, SqlClient.SqlClient> =>
|
|
647
|
+
Effect.gen(this, function* () {
|
|
648
|
+
const sql = yield* SqlClient.SqlClient;
|
|
649
|
+
for (const block of request.blocks) {
|
|
650
|
+
// Fold the conflict check into the UPDATE itself: only write when the row is
|
|
651
|
+
// unset or already at the requested position. RETURNING tells us whether the
|
|
652
|
+
// write took effect, so we avoid the per-block SELECT round-trip on the
|
|
653
|
+
// common path. Conflicts (rare) take the slow path below.
|
|
654
|
+
const updated = yield* sql<{ position: number | null }>`
|
|
655
|
+
UPDATE blocks SET position = ${block.position}
|
|
656
|
+
WHERE feedPrivateId = (
|
|
657
|
+
SELECT feedPrivateId FROM feeds
|
|
658
|
+
WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
|
|
659
|
+
)
|
|
660
|
+
AND actorId = ${block.actorId} AND sequence = ${block.sequence}
|
|
661
|
+
AND (position IS NULL OR position = ${block.position})
|
|
662
|
+
RETURNING position
|
|
663
|
+
`;
|
|
664
|
+
if (updated.length > 0) {
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
const existing = yield* sql<{ position: number | null }>`
|
|
668
|
+
SELECT position FROM blocks
|
|
669
|
+
WHERE feedPrivateId = (
|
|
670
|
+
SELECT feedPrivateId FROM feeds
|
|
671
|
+
WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
|
|
672
|
+
)
|
|
673
|
+
AND actorId = ${block.actorId} AND sequence = ${block.sequence}
|
|
674
|
+
`;
|
|
675
|
+
const current = existing[0];
|
|
676
|
+
if (current?.position != null && current.position !== block.position) {
|
|
677
|
+
yield* Effect.fail(
|
|
678
|
+
new PositionConflictError({
|
|
679
|
+
feedId: block.feedId,
|
|
680
|
+
actorId: block.actorId,
|
|
681
|
+
sequence: block.sequence,
|
|
682
|
+
currentPosition: current.position,
|
|
683
|
+
requestedPosition: block.position,
|
|
684
|
+
}),
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}).pipe(Effect.withSpan('FeedStore.setPosition'));
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Gets all feeds and their blocks for a space, organized by feed ID and namespace.
|
|
692
|
+
* Used for space archive export.
|
|
693
|
+
*/
|
|
694
|
+
getAllFeedsForSpace = (opts: {
|
|
695
|
+
spaceId: SpaceId;
|
|
696
|
+
}): Effect.Effect<
|
|
697
|
+
Array<{
|
|
698
|
+
feedId: string;
|
|
699
|
+
feedNamespace: string;
|
|
700
|
+
blocks: Block[];
|
|
701
|
+
}>,
|
|
702
|
+
SqlError.SqlError,
|
|
703
|
+
SqlClient.SqlClient
|
|
704
|
+
> =>
|
|
705
|
+
Effect.gen(this, function* () {
|
|
706
|
+
const sql = yield* SqlClient.SqlClient;
|
|
707
|
+
|
|
708
|
+
const feeds = yield* sql<{ feedId: string; feedNamespace: string }>`
|
|
709
|
+
SELECT DISTINCT feedId, feedNamespace FROM feeds WHERE spaceId = ${opts.spaceId}
|
|
710
|
+
`;
|
|
711
|
+
|
|
712
|
+
const result: Array<{ feedId: string; feedNamespace: string; blocks: Block[] }> = [];
|
|
713
|
+
|
|
714
|
+
for (const feed of feeds) {
|
|
715
|
+
const blocks = yield* sql<Block>`
|
|
716
|
+
SELECT blocks.*, feeds.feedId, feeds.feedNamespace
|
|
717
|
+
FROM blocks
|
|
718
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
719
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
720
|
+
AND feeds.feedId = ${feed.feedId}
|
|
721
|
+
AND feeds.feedNamespace = ${feed.feedNamespace}
|
|
722
|
+
ORDER BY blocks.sequence ASC
|
|
723
|
+
`;
|
|
724
|
+
|
|
725
|
+
result.push({
|
|
726
|
+
feedId: feed.feedId,
|
|
727
|
+
feedNamespace: feed.feedNamespace,
|
|
728
|
+
blocks: blocks.map((row) => ({
|
|
729
|
+
...row,
|
|
730
|
+
data: new Uint8Array(row.data),
|
|
731
|
+
})),
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
return result;
|
|
736
|
+
}).pipe(Effect.withSpan('FeedStore.getAllFeedsForSpace'));
|
|
371
737
|
}
|
|
372
738
|
|
|
373
739
|
const encodeCursor = (token: string, insertionId: number) => FeedCursor.make(`${token}|${insertionId}`);
|