@anabranch/eventlog 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -106
- package/esm/adapter.d.ts +83 -62
- package/esm/adapter.d.ts.map +1 -1
- package/esm/adapter.js +0 -9
- package/esm/errors.d.ts +2 -7
- package/esm/errors.d.ts.map +1 -1
- package/esm/errors.js +9 -21
- package/esm/eventlog.d.ts +74 -92
- package/esm/eventlog.d.ts.map +1 -1
- package/esm/eventlog.js +93 -113
- package/esm/in-memory.d.ts +15 -34
- package/esm/in-memory.d.ts.map +1 -1
- package/esm/in-memory.js +139 -167
- package/esm/index.d.ts +8 -8
- package/esm/index.d.ts.map +1 -1
- package/esm/index.js +4 -4
- package/package.json +1 -1
package/esm/eventlog.d.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type { AppendOptions, ConsumeOptions,
|
|
3
|
-
import { EventLogAppendFailed, EventLogCloseFailed, EventLogCommitCursorFailed, EventLogConnectionFailed,
|
|
1
|
+
import { Channel, Task } from 'anabranch';
|
|
2
|
+
import type { AppendOptions, ConsumeOptions, EventBatch, EventLogAdapter, EventLogConnector } from './adapter.js';
|
|
3
|
+
import { EventLogAppendFailed, EventLogCloseFailed, EventLogCommitCursorFailed, EventLogConnectionFailed, EventLogConsumeFailed, EventLogGetCursorFailed } from './errors.js';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Event log wrapper with Task/Stream semantics for event-sourced systems.
|
|
6
|
+
*
|
|
7
|
+
* Provides high-level methods for appending events, consuming streams,
|
|
8
|
+
* and managing cursors. All operations return Tasks for composable error
|
|
9
|
+
* handling.
|
|
6
10
|
*
|
|
7
11
|
* @example Basic usage
|
|
8
12
|
* ```ts
|
|
@@ -12,35 +16,34 @@ import { EventLogAppendFailed, EventLogCloseFailed, EventLogCommitCursorFailed,
|
|
|
12
16
|
* const log = await EventLog.connect(connector).run();
|
|
13
17
|
*
|
|
14
18
|
* // Append an event
|
|
15
|
-
* const eventId = await log.append("users", {
|
|
16
|
-
*
|
|
17
|
-
* // Get a specific event
|
|
18
|
-
* const event = await log.get("users", 0).run();
|
|
19
|
-
*
|
|
20
|
-
* // List events
|
|
21
|
-
* const events = await log.list("users").run();
|
|
22
|
-
*
|
|
23
|
-
* await log.close().run();
|
|
24
|
-
* ```
|
|
19
|
+
* const eventId = await log.append("users", { userId: 123 }).run();
|
|
25
20
|
*
|
|
26
|
-
*
|
|
27
|
-
* ```ts
|
|
21
|
+
* // Consume events as a stream
|
|
28
22
|
* const { successes, errors } = await log
|
|
29
|
-
* .consume("users", "my-
|
|
23
|
+
* .consume("users", "my-processor")
|
|
30
24
|
* .withConcurrency(5)
|
|
31
25
|
* .map(async (batch) => {
|
|
32
26
|
* for (const event of batch.events) {
|
|
33
|
-
* await
|
|
27
|
+
* await processEvent(event.data);
|
|
34
28
|
* }
|
|
35
|
-
* // Explicitly commit after successful processing!
|
|
36
|
-
* await log.commit(batch.topic, batch.consumerGroup, batch.cursor).run();
|
|
37
29
|
* })
|
|
38
30
|
* .partition();
|
|
31
|
+
*
|
|
32
|
+
* await log.close().run();
|
|
33
|
+
* ```
|
|
34
|
+
*
|
|
35
|
+
* @example Manual cursor management
|
|
36
|
+
* ```ts
|
|
37
|
+
* // Get current cursor position
|
|
38
|
+
* const cursor = await log.getCommittedCursor("users", "my-processor").run();
|
|
39
|
+
*
|
|
40
|
+
* // Save cursor after processing
|
|
41
|
+
* await log.commit("users", "my-processor", batch.cursor).run();
|
|
39
42
|
* ```
|
|
40
43
|
*/
|
|
41
|
-
export declare class EventLog {
|
|
44
|
+
export declare class EventLog<Cursor = string> {
|
|
42
45
|
private readonly adapter;
|
|
43
|
-
constructor(adapter: EventLogAdapter);
|
|
46
|
+
constructor(adapter: EventLogAdapter<Cursor>);
|
|
44
47
|
/**
|
|
45
48
|
* Connect to an event log via a connector.
|
|
46
49
|
*
|
|
@@ -49,9 +52,11 @@ export declare class EventLog {
|
|
|
49
52
|
* const log = await EventLog.connect(createInMemory()).run();
|
|
50
53
|
* ```
|
|
51
54
|
*/
|
|
52
|
-
static connect(connector: EventLogConnector): Task<EventLog
|
|
55
|
+
static connect<Cursor = string>(connector: EventLogConnector<Cursor>): Task<EventLog<Cursor>, EventLogConnectionFailed>;
|
|
53
56
|
/**
|
|
54
|
-
*
|
|
57
|
+
* Close the event log connection.
|
|
58
|
+
*
|
|
59
|
+
* After closing, no further operations can be performed on this instance.
|
|
55
60
|
*
|
|
56
61
|
* @example
|
|
57
62
|
* ```ts
|
|
@@ -62,110 +67,87 @@ export declare class EventLog {
|
|
|
62
67
|
/**
|
|
63
68
|
* Append an event to a topic.
|
|
64
69
|
*
|
|
65
|
-
*
|
|
66
|
-
* ```ts
|
|
67
|
-
* const eventId = await log.append("users", { action: "created", userId: 123 }).run();
|
|
68
|
-
* ```
|
|
70
|
+
* Returns the event ID which can be used for logging or correlation.
|
|
69
71
|
*
|
|
70
|
-
* @example
|
|
72
|
+
* @example
|
|
71
73
|
* ```ts
|
|
72
|
-
* const eventId = await log.append("
|
|
73
|
-
*
|
|
74
|
+
* const eventId = await log.append("users", { action: "created" }).run();
|
|
75
|
+
*
|
|
76
|
+
* // With options
|
|
77
|
+
* await log.append("orders", order, {
|
|
78
|
+
* partitionKey: order.userId,
|
|
79
|
+
* metadata: { source: "checkout" },
|
|
74
80
|
* }).run();
|
|
75
81
|
* ```
|
|
76
82
|
*/
|
|
77
83
|
append<T>(topic: string, data: T, options?: AppendOptions): Task<string, EventLogAppendFailed>;
|
|
78
84
|
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
* @example
|
|
82
|
-
* ```ts
|
|
83
|
-
* const event = await log.get("users", 0).run();
|
|
84
|
-
* if (event) {
|
|
85
|
-
* console.log(event.data);
|
|
86
|
-
* }
|
|
87
|
-
* ```
|
|
88
|
-
*/
|
|
89
|
-
get<T>(topic: string, sequenceNumber: number): Task<Event<T> | null, EventLogGetFailed>;
|
|
90
|
-
/**
|
|
91
|
-
* List events in a topic with optional filtering and pagination.
|
|
85
|
+
* Consume events from a topic as a stream.
|
|
92
86
|
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
* ```
|
|
87
|
+
* Returns a Channel that yields batches of events. Each batch includes
|
|
88
|
+
* a cursor that can be committed to mark progress. Use stream methods
|
|
89
|
+
* like `withConcurrency()`, `map()`, and `partition()` for processing.
|
|
97
90
|
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* fromSequenceNumber: 100,
|
|
102
|
-
* limit: 50,
|
|
103
|
-
* }).run();
|
|
104
|
-
* ```
|
|
91
|
+
* Batches are delivered asynchronously as they become available. Use
|
|
92
|
+
* `take()` to limit iterations or pass an AbortSignal in options to
|
|
93
|
+
* cancel consumption.
|
|
105
94
|
*
|
|
106
|
-
* @example
|
|
95
|
+
* @example
|
|
107
96
|
* ```ts
|
|
108
|
-
* const
|
|
109
|
-
* partitionKey: "user-123",
|
|
110
|
-
* }).run();
|
|
111
|
-
* ```
|
|
112
|
-
*/
|
|
113
|
-
list<T>(topic: string, options?: ListOptions): Task<Event<T>[], EventLogListFailed>;
|
|
114
|
-
/**
|
|
115
|
-
* Consume events from a topic as a Source for streaming.
|
|
97
|
+
* const ac = new AbortController();
|
|
116
98
|
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* data loss when using concurrent processing.
|
|
120
|
-
*
|
|
121
|
-
* @example Basic consumption
|
|
122
|
-
* ```ts
|
|
123
|
-
* const { successes, errors } = await log
|
|
124
|
-
* .consume("users", "processor-1")
|
|
125
|
-
* .withConcurrency(5)
|
|
99
|
+
* await log.consume("users", "processor-1", { signal: ac.signal })
|
|
100
|
+
* .withConcurrency(10)
|
|
126
101
|
* .map(async (batch) => {
|
|
127
102
|
* for (const event of batch.events) {
|
|
128
|
-
* await
|
|
103
|
+
* await processUser(event.data);
|
|
129
104
|
* }
|
|
130
|
-
* await
|
|
105
|
+
* await batch.commit(); // Mark progress
|
|
131
106
|
* })
|
|
132
107
|
* .partition();
|
|
108
|
+
*
|
|
109
|
+
* ac.abort(); // Stop consumption
|
|
133
110
|
* ```
|
|
134
111
|
*
|
|
135
|
-
* @example
|
|
112
|
+
* @example Resume from a saved cursor
|
|
136
113
|
* ```ts
|
|
137
|
-
* const
|
|
138
|
-
* const
|
|
139
|
-
* .consume("users", "processor-1", { cursor: lastCursor })
|
|
140
|
-
* .tap(async (batch) => {
|
|
141
|
-
* for (const event of batch.events) {
|
|
142
|
-
* console.log(event);
|
|
143
|
-
* }
|
|
144
|
-
* })
|
|
145
|
-
* .partition();
|
|
114
|
+
* const cursor = await log.getCommittedCursor("users", "processor-1").run();
|
|
115
|
+
* const stream = log.consume("users", "processor-1", { cursor });
|
|
146
116
|
* ```
|
|
147
117
|
*/
|
|
148
|
-
consume<T>(topic: string, consumerGroup: string, options?: ConsumeOptions):
|
|
118
|
+
consume<T>(topic: string, consumerGroup: string, options?: ConsumeOptions<Cursor>): Channel<EventBatch<T, Cursor>, EventLogConsumeFailed>;
|
|
149
119
|
/**
|
|
150
|
-
* Commit a cursor
|
|
120
|
+
* Commit a cursor to mark progress for a consumer group.
|
|
121
|
+
*
|
|
122
|
+
* This is for administrative use cases where you can't commit in-band, preferably when you're
|
|
123
|
+
* not actively consuming events. For example, you might want to skip ahead after a downtime or reset to the beginning for reprocessing.
|
|
124
|
+
* Do prefer to commit in-band, i.e. after processing each batch, by calling `batch.commit()`.
|
|
125
|
+
*
|
|
126
|
+
* After processing events, commit the cursor to resume from that position
|
|
127
|
+
* on the next run. Cursors are obtained from `batch.cursor` in the consume
|
|
128
|
+
* stream or from `getCommittedCursor()`.
|
|
151
129
|
*
|
|
152
130
|
* @example
|
|
153
131
|
* ```ts
|
|
154
|
-
* await log.commit("users", "processor-1", cursor).run();
|
|
132
|
+
* await log.commit("users", "processor-1", batch.cursor).run();
|
|
155
133
|
* ```
|
|
156
134
|
*/
|
|
157
|
-
commit(topic: string, consumerGroup: string, cursor:
|
|
135
|
+
commit(topic: string, consumerGroup: string, cursor: Cursor): Task<void, EventLogCommitCursorFailed>;
|
|
158
136
|
/**
|
|
159
|
-
* Get the committed cursor
|
|
137
|
+
* Get the last committed cursor for a consumer group.
|
|
138
|
+
*
|
|
139
|
+
* Returns null if no cursor has been committed yet. Use this to resume
|
|
140
|
+
* consumption from the last processed position.
|
|
160
141
|
*
|
|
161
142
|
* @example
|
|
162
143
|
* ```ts
|
|
163
144
|
* const cursor = await log.getCommittedCursor("users", "processor-1").run();
|
|
164
145
|
* if (cursor) {
|
|
165
|
-
*
|
|
146
|
+
* // Resume from saved position
|
|
147
|
+
* const stream = log.consume("users", "processor-1", { cursor });
|
|
166
148
|
* }
|
|
167
149
|
* ```
|
|
168
150
|
*/
|
|
169
|
-
getCommittedCursor(topic: string, consumerGroup: string): Task<
|
|
151
|
+
getCommittedCursor(topic: string, consumerGroup: string): Task<Cursor | null, EventLogGetCursorFailed>;
|
|
170
152
|
}
|
|
171
153
|
//# sourceMappingURL=eventlog.d.ts.map
|
package/esm/eventlog.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eventlog.d.ts","sourceRoot":"","sources":["../src/eventlog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"eventlog.d.ts","sourceRoot":"","sources":["../src/eventlog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,KAAK,EACV,aAAa,EACb,cAAc,EACd,UAAU,EACV,eAAe,EACf,iBAAiB,EAClB,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,0BAA0B,EAC1B,wBAAwB,EACxB,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,aAAa,CAAA;AAEpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,QAAQ,CAAC,MAAM,GAAG,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC;IAE7D;;;;;;;OAOG;IACH,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,EAC5B,SAAS,EAAE,iBAAiB,CAAC,MAAM,CAAC,GACnC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,wBAAwB,CAAC;IAanD;;;;;;;;;OASG;IACH,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC;IAaxC;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,CAAC,EACN,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,CAAC,EACP,OAAO,CAAC,EAAE,aAAa,GACtB,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC;IAcrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;IACH,OAAO,CAAC,CAAC,EACP,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,GAC/B,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAsDxD;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CACJ,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,MAAM,GACb,IAAI,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAezC;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAChB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,GACpB,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,uBAAuB,CAAC;CAchD"}
|
package/esm/eventlog.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { EventLogAppendFailed, EventLogCloseFailed, EventLogCommitCursorFailed, EventLogConnectionFailed,
|
|
1
|
+
import { Channel, Task } from 'anabranch';
|
|
2
|
+
import { EventLogAppendFailed, EventLogCloseFailed, EventLogCommitCursorFailed, EventLogConnectionFailed, EventLogConsumeFailed, EventLogGetCursorFailed, } from './errors.js';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Event log wrapper with Task/Stream semantics for event-sourced systems.
|
|
5
|
+
*
|
|
6
|
+
* Provides high-level methods for appending events, consuming streams,
|
|
7
|
+
* and managing cursors. All operations return Tasks for composable error
|
|
8
|
+
* handling.
|
|
5
9
|
*
|
|
6
10
|
* @example Basic usage
|
|
7
11
|
* ```ts
|
|
@@ -11,30 +15,29 @@ import { EventLogAppendFailed, EventLogCloseFailed, EventLogCommitCursorFailed,
|
|
|
11
15
|
* const log = await EventLog.connect(connector).run();
|
|
12
16
|
*
|
|
13
17
|
* // Append an event
|
|
14
|
-
* const eventId = await log.append("users", {
|
|
15
|
-
*
|
|
16
|
-
* // Get a specific event
|
|
17
|
-
* const event = await log.get("users", 0).run();
|
|
18
|
-
*
|
|
19
|
-
* // List events
|
|
20
|
-
* const events = await log.list("users").run();
|
|
21
|
-
*
|
|
22
|
-
* await log.close().run();
|
|
23
|
-
* ```
|
|
18
|
+
* const eventId = await log.append("users", { userId: 123 }).run();
|
|
24
19
|
*
|
|
25
|
-
*
|
|
26
|
-
* ```ts
|
|
20
|
+
* // Consume events as a stream
|
|
27
21
|
* const { successes, errors } = await log
|
|
28
|
-
* .consume("users", "my-
|
|
22
|
+
* .consume("users", "my-processor")
|
|
29
23
|
* .withConcurrency(5)
|
|
30
24
|
* .map(async (batch) => {
|
|
31
25
|
* for (const event of batch.events) {
|
|
32
|
-
* await
|
|
26
|
+
* await processEvent(event.data);
|
|
33
27
|
* }
|
|
34
|
-
* // Explicitly commit after successful processing!
|
|
35
|
-
* await log.commit(batch.topic, batch.consumerGroup, batch.cursor).run();
|
|
36
28
|
* })
|
|
37
29
|
* .partition();
|
|
30
|
+
*
|
|
31
|
+
* await log.close().run();
|
|
32
|
+
* ```
|
|
33
|
+
*
|
|
34
|
+
* @example Manual cursor management
|
|
35
|
+
* ```ts
|
|
36
|
+
* // Get current cursor position
|
|
37
|
+
* const cursor = await log.getCommittedCursor("users", "my-processor").run();
|
|
38
|
+
*
|
|
39
|
+
* // Save cursor after processing
|
|
40
|
+
* await log.commit("users", "my-processor", batch.cursor).run();
|
|
38
41
|
* ```
|
|
39
42
|
*/
|
|
40
43
|
export class EventLog {
|
|
@@ -65,7 +68,9 @@ export class EventLog {
|
|
|
65
68
|
});
|
|
66
69
|
}
|
|
67
70
|
/**
|
|
68
|
-
*
|
|
71
|
+
* Close the event log connection.
|
|
72
|
+
*
|
|
73
|
+
* After closing, no further operations can be performed on this instance.
|
|
69
74
|
*
|
|
70
75
|
* @example
|
|
71
76
|
* ```ts
|
|
@@ -85,15 +90,16 @@ export class EventLog {
|
|
|
85
90
|
/**
|
|
86
91
|
* Append an event to a topic.
|
|
87
92
|
*
|
|
88
|
-
*
|
|
89
|
-
* ```ts
|
|
90
|
-
* const eventId = await log.append("users", { action: "created", userId: 123 }).run();
|
|
91
|
-
* ```
|
|
93
|
+
* Returns the event ID which can be used for logging or correlation.
|
|
92
94
|
*
|
|
93
|
-
* @example
|
|
95
|
+
* @example
|
|
94
96
|
* ```ts
|
|
95
|
-
* const eventId = await log.append("
|
|
96
|
-
*
|
|
97
|
+
* const eventId = await log.append("users", { action: "created" }).run();
|
|
98
|
+
*
|
|
99
|
+
* // With options
|
|
100
|
+
* await log.append("orders", order, {
|
|
101
|
+
* partitionKey: order.userId,
|
|
102
|
+
* metadata: { source: "checkout" },
|
|
97
103
|
* }).run();
|
|
98
104
|
* ```
|
|
99
105
|
*/
|
|
@@ -108,112 +114,82 @@ export class EventLog {
|
|
|
108
114
|
});
|
|
109
115
|
}
|
|
110
116
|
/**
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* @example
|
|
114
|
-
* ```ts
|
|
115
|
-
* const event = await log.get("users", 0).run();
|
|
116
|
-
* if (event) {
|
|
117
|
-
* console.log(event.data);
|
|
118
|
-
* }
|
|
119
|
-
* ```
|
|
120
|
-
*/
|
|
121
|
-
get(topic, sequenceNumber) {
|
|
122
|
-
return Task.of(async () => {
|
|
123
|
-
try {
|
|
124
|
-
return await this.adapter.get(topic, sequenceNumber);
|
|
125
|
-
}
|
|
126
|
-
catch (error) {
|
|
127
|
-
throw new EventLogGetFailed(topic, sequenceNumber, error instanceof Error ? error.message : String(error), error);
|
|
128
|
-
}
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* List events in a topic with optional filtering and pagination.
|
|
117
|
+
* Consume events from a topic as a stream.
|
|
133
118
|
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
* ```
|
|
119
|
+
* Returns a Channel that yields batches of events. Each batch includes
|
|
120
|
+
* a cursor that can be committed to mark progress. Use stream methods
|
|
121
|
+
* like `withConcurrency()`, `map()`, and `partition()` for processing.
|
|
138
122
|
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
* fromSequenceNumber: 100,
|
|
143
|
-
* limit: 50,
|
|
144
|
-
* }).run();
|
|
145
|
-
* ```
|
|
123
|
+
* Batches are delivered asynchronously as they become available. Use
|
|
124
|
+
* `take()` to limit iterations or pass an AbortSignal in options to
|
|
125
|
+
* cancel consumption.
|
|
146
126
|
*
|
|
147
|
-
* @example
|
|
127
|
+
* @example
|
|
148
128
|
* ```ts
|
|
149
|
-
* const
|
|
150
|
-
* partitionKey: "user-123",
|
|
151
|
-
* }).run();
|
|
152
|
-
* ```
|
|
153
|
-
*/
|
|
154
|
-
list(topic, options) {
|
|
155
|
-
return Task.of(async () => {
|
|
156
|
-
try {
|
|
157
|
-
return await this.adapter.list(topic, options);
|
|
158
|
-
}
|
|
159
|
-
catch (error) {
|
|
160
|
-
throw new EventLogListFailed(topic, error instanceof Error ? error.message : String(error), error);
|
|
161
|
-
}
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
/**
|
|
165
|
-
* Consume events from a topic as a Source for streaming.
|
|
166
|
-
*
|
|
167
|
-
* Note: You must manually commit the cursor after processing to guarantee
|
|
168
|
-
* at-least-once delivery. Auto-commit is intentionally omitted to prevent
|
|
169
|
-
* data loss when using concurrent processing.
|
|
129
|
+
* const ac = new AbortController();
|
|
170
130
|
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
* const { successes, errors } = await log
|
|
174
|
-
* .consume("users", "processor-1")
|
|
175
|
-
* .withConcurrency(5)
|
|
131
|
+
* await log.consume("users", "processor-1", { signal: ac.signal })
|
|
132
|
+
* .withConcurrency(10)
|
|
176
133
|
* .map(async (batch) => {
|
|
177
134
|
* for (const event of batch.events) {
|
|
178
|
-
* await
|
|
135
|
+
* await processUser(event.data);
|
|
179
136
|
* }
|
|
180
|
-
* await
|
|
137
|
+
* await batch.commit(); // Mark progress
|
|
181
138
|
* })
|
|
182
139
|
* .partition();
|
|
140
|
+
*
|
|
141
|
+
* ac.abort(); // Stop consumption
|
|
183
142
|
* ```
|
|
184
143
|
*
|
|
185
|
-
* @example
|
|
144
|
+
* @example Resume from a saved cursor
|
|
186
145
|
* ```ts
|
|
187
|
-
* const
|
|
188
|
-
* const
|
|
189
|
-
* .consume("users", "processor-1", { cursor: lastCursor })
|
|
190
|
-
* .tap(async (batch) => {
|
|
191
|
-
* for (const event of batch.events) {
|
|
192
|
-
* console.log(event);
|
|
193
|
-
* }
|
|
194
|
-
* })
|
|
195
|
-
* .partition();
|
|
146
|
+
* const cursor = await log.getCommittedCursor("users", "processor-1").run();
|
|
147
|
+
* const stream = log.consume("users", "processor-1", { cursor });
|
|
196
148
|
* ```
|
|
197
149
|
*/
|
|
198
150
|
consume(topic, consumerGroup, options) {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
151
|
+
if (options?.bufferSize !== undefined) {
|
|
152
|
+
if (options.bufferSize <= 0 || !Number.isInteger(options.bufferSize)) {
|
|
153
|
+
throw new Error('bufferSize must be a positive integer');
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (options?.batchSize !== undefined) {
|
|
157
|
+
if (options.batchSize <= 0 || !Number.isInteger(options.batchSize)) {
|
|
158
|
+
throw new Error('batchSize must be a positive integer');
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const channel = new Channel({
|
|
162
|
+
onDrop: (batch) => {
|
|
163
|
+
channel.fail(new EventLogConsumeFailed(topic, consumerGroup, `Batch dropped due to full buffer (events ${batch.events
|
|
164
|
+
.map((e) => e.id)
|
|
165
|
+
.join(',')})`));
|
|
166
|
+
},
|
|
167
|
+
onClose: () => close(),
|
|
168
|
+
bufferSize: options?.bufferSize ?? Infinity,
|
|
169
|
+
signal: options?.signal,
|
|
209
170
|
});
|
|
171
|
+
const { close } = this.adapter.consume(topic, consumerGroup, async (batch) => {
|
|
172
|
+
await channel.waitForCapacity();
|
|
173
|
+
channel.send(batch);
|
|
174
|
+
}, (error) => {
|
|
175
|
+
channel.fail(new EventLogConsumeFailed(topic, consumerGroup, error instanceof Error ? error.message : String(error)));
|
|
176
|
+
}, options);
|
|
177
|
+
return channel;
|
|
210
178
|
}
|
|
211
179
|
/**
|
|
212
|
-
* Commit a cursor
|
|
180
|
+
* Commit a cursor to mark progress for a consumer group.
|
|
181
|
+
*
|
|
182
|
+
* This is for administrative use cases where you can't commit in-band, preferably when you're
|
|
183
|
+
* not actively consuming events. For example, you might want to skip ahead after a downtime or reset to the beginning for reprocessing.
|
|
184
|
+
* Do prefer to commit in-band, i.e. after processing each batch, by calling `batch.commit()`.
|
|
185
|
+
*
|
|
186
|
+
* After processing events, commit the cursor to resume from that position
|
|
187
|
+
* on the next run. Cursors are obtained from `batch.cursor` in the consume
|
|
188
|
+
* stream or from `getCommittedCursor()`.
|
|
213
189
|
*
|
|
214
190
|
* @example
|
|
215
191
|
* ```ts
|
|
216
|
-
* await log.commit("users", "processor-1", cursor).run();
|
|
192
|
+
* await log.commit("users", "processor-1", batch.cursor).run();
|
|
217
193
|
* ```
|
|
218
194
|
*/
|
|
219
195
|
commit(topic, consumerGroup, cursor) {
|
|
@@ -227,13 +203,17 @@ export class EventLog {
|
|
|
227
203
|
});
|
|
228
204
|
}
|
|
229
205
|
/**
|
|
230
|
-
* Get the committed cursor
|
|
206
|
+
* Get the last committed cursor for a consumer group.
|
|
207
|
+
*
|
|
208
|
+
* Returns null if no cursor has been committed yet. Use this to resume
|
|
209
|
+
* consumption from the last processed position.
|
|
231
210
|
*
|
|
232
211
|
* @example
|
|
233
212
|
* ```ts
|
|
234
213
|
* const cursor = await log.getCommittedCursor("users", "processor-1").run();
|
|
235
214
|
* if (cursor) {
|
|
236
|
-
*
|
|
215
|
+
* // Resume from saved position
|
|
216
|
+
* const stream = log.consume("users", "processor-1", { cursor });
|
|
237
217
|
* }
|
|
238
218
|
* ```
|
|
239
219
|
*/
|
package/esm/in-memory.d.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import type { EventLogAdapter, EventLogConnector, EventLogOptions } from
|
|
1
|
+
import type { EventLogAdapter, EventLogConnector, EventLogOptions } from './adapter.js';
|
|
2
|
+
export declare function createInMemory(options?: InMemoryOptions): InMemoryConnector;
|
|
3
|
+
/** Configuration options for in-memory event log. */
|
|
4
|
+
export interface InMemoryOptions extends EventLogOptions {
|
|
5
|
+
}
|
|
2
6
|
/**
|
|
3
|
-
* Creates an in-memory event log connector
|
|
7
|
+
* Creates an in-memory event log connector for testing and development.
|
|
4
8
|
*
|
|
5
|
-
* Events are stored in memory
|
|
6
|
-
*
|
|
9
|
+
* Events are stored in memory and lost when the process exits. Ideal for
|
|
10
|
+
* unit tests, prototyping, and development environments.
|
|
7
11
|
*
|
|
8
12
|
* @example Basic usage
|
|
9
13
|
* ```ts
|
|
@@ -12,42 +16,19 @@ import type { EventLogAdapter, EventLogConnector, EventLogOptions } from "./adap
|
|
|
12
16
|
* const connector = createInMemory();
|
|
13
17
|
* const log = await EventLog.connect(connector).run();
|
|
14
18
|
*
|
|
15
|
-
*
|
|
16
|
-
* const eventId = await log.append("users", { action: "created", userId: 123 }).run();
|
|
19
|
+
* await log.append("users", { userId: 123 }).run();
|
|
17
20
|
*
|
|
18
|
-
* //
|
|
19
|
-
*
|
|
21
|
+
* // After testing, clean up
|
|
22
|
+
* await connector.end();
|
|
20
23
|
* ```
|
|
21
24
|
*
|
|
22
|
-
* @example With partition key
|
|
25
|
+
* @example With custom partition key
|
|
23
26
|
* ```ts
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* })
|
|
27
|
-
* ```
|
|
28
|
-
*
|
|
29
|
-
* @example Consuming events
|
|
30
|
-
* ```ts
|
|
31
|
-
* const connector = createInMemory();
|
|
32
|
-
* const log = await EventLog.connect(connector).run();
|
|
33
|
-
*
|
|
34
|
-
* // Append some events first
|
|
35
|
-
* await log.append("notifications", { type: "email" }).run();
|
|
36
|
-
* await log.append("notifications", { type: "sms" }).run();
|
|
37
|
-
*
|
|
38
|
-
* // Consume events
|
|
39
|
-
* for await (const batch of log.consume("notifications", "my-consumer-group")) {
|
|
40
|
-
* for (const event of batch.events) {
|
|
41
|
-
* console.log(event.data);
|
|
42
|
-
* }
|
|
43
|
-
* }
|
|
27
|
+
* const connector = createInMemory({
|
|
28
|
+
* defaultPartitionKey: "user-events",
|
|
29
|
+
* });
|
|
44
30
|
* ```
|
|
45
31
|
*/
|
|
46
|
-
export declare function createInMemory(options?: InMemoryOptions): InMemoryConnector;
|
|
47
|
-
/** In-memory event log connector options. */
|
|
48
|
-
export interface InMemoryOptions extends EventLogOptions {
|
|
49
|
-
}
|
|
50
|
-
/** In-memory event log connector. */
|
|
51
32
|
export interface InMemoryConnector extends EventLogConnector {
|
|
52
33
|
connect(): Promise<EventLogAdapter>;
|
|
53
34
|
end(): Promise<void>;
|
package/esm/in-memory.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"in-memory.d.ts","sourceRoot":"","sources":["../src/in-memory.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"in-memory.d.ts","sourceRoot":"","sources":["../src/in-memory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAKV,eAAe,EACf,iBAAiB,EACjB,eAAe,EAChB,MAAM,cAAc,CAAA;AAsBrB,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,iBAAiB,CA4Q3E;AAED,qDAAqD;AACrD,MAAM,WAAW,eAAgB,SAAQ,eAAe;CAAG;AAE3D;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,iBAAkB,SAAQ,iBAAiB;IAC1D,OAAO,IAAI,OAAO,CAAC,eAAe,CAAC,CAAA;IACnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACrB"}
|