@ddd-ts/event-sourcing-firestore 0.0.46 → 0.0.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/firestore.projected-stream.storage-layer.d.ts +2 -0
- package/dist/firestore.projected-stream.storage-layer.d.ts.map +1 -1
- package/dist/firestore.projected-stream.storage-layer.js +55 -10
- package/dist/firestore.projected-stream.storage-layer.mjs +56 -11
- package/dist/projection/cases/replay-api.spec.d.ts +2 -0
- package/dist/projection/cases/replay-api.spec.d.ts.map +1 -0
- package/dist/projection/cases/tiedcursor.spec.d.ts +2 -0
- package/dist/projection/cases/tiedcursor.spec.d.ts.map +1 -0
- package/dist/projection/firestore.projector.d.ts +76 -11
- package/dist/projection/firestore.projector.d.ts.map +1 -1
- package/dist/projection/firestore.projector.js +164 -47
- package/dist/projection/firestore.projector.mjs +164 -47
- package/package.json +10 -10
|
@@ -27,5 +27,7 @@ export declare class FirestoreProjectedStreamStorageLayer implements ProjectedSt
|
|
|
27
27
|
get(cursor: Cursor): Promise<ISerializedFact | undefined>;
|
|
28
28
|
getCursor(savedChange: ISerializedSavedChange): Promise<Cursor | undefined>;
|
|
29
29
|
slice(projectedStream: ProjectedStream, shard: string, startAfter?: Cursor, endAt?: Cursor, limit?: number): Promise<ISerializedFact[]>;
|
|
30
|
+
firstAtOrAfter(projectedStream: ProjectedStream, shard: string, from: MicrosecondTimestamp): Promise<ISerializedFact | undefined>;
|
|
31
|
+
latest(projectedStream: ProjectedStream, shard: string): Promise<ISerializedFact | undefined>;
|
|
30
32
|
}
|
|
31
33
|
//# sourceMappingURL=firestore.projected-stream.storage-layer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"firestore.projected-stream.storage-layer.d.ts","sourceRoot":"","sources":["../src/firestore.projected-stream.storage-layer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,UAAU,EACV,eAAe,EACf,KAAK,2BAA2B,EAChC,YAAY,EACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,
|
|
1
|
+
{"version":3,"file":"firestore.projected-stream.storage-layer.d.ts","sourceRoot":"","sources":["../src/firestore.projected-stream.storage-layer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,UAAU,EACV,eAAe,EACf,KAAK,2BAA2B,EAChC,YAAY,EACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAEL,MAAM,EACN,SAAS,EAET,SAAS,EACV,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,qBAAa,yBAAyB;IACpC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU;CAM7C;AAED,qBAAa,2BAA2B;IACtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY;CAOjD;AAED,qBAAa,oCACX,YAAW,2BAA2B;IAGpC,OAAO,CAAC,QAAQ,CAAC,SAAS;aACV,SAAS;gBADR,SAAS,EAAE,SAAS,EACrB,SAAS,mDAAyB;IAG7C,IAAI,CACT,eAAe,EAAE,eAAe,EAChC,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,EACnB,KAAK,CAAC,EAAE,MAAM;;;;;;;;;;IAoDT,sBAAsB,CAAC,YAAY,EAAE,oBAAoB;IAM1D,GAAG,CAAC,MAAM,EAAE,MAAM;IAkBlB,SAAS,CACb,WAAW,EAAE,sBAAsB,GAClC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAcxB,KAAK,CACT,eAAe,EAAE,eAAe,EAChC,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,EACnB,KAAK,CAAC,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,MAAM;IA2DV,cAAc,CAClB,eAAe,EAAE,eAAe,EAChC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAoCjC,MAAM,CACV,eAAe,EAAE,eAAe,EAChC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;CAmCxC"}
|
|
@@ -22,20 +22,20 @@ var FirestoreProjectedStreamStorageLayer = class {
|
|
|
22
22
|
this.converter = converter;
|
|
23
23
|
}
|
|
24
24
|
async *read(projectedStream, shard, startAfter, endAt) {
|
|
25
|
-
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision");
|
|
25
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision").orderBy(firebase_admin_firestore.FieldPath.documentId());
|
|
26
26
|
const filters = projectedStream.sources.map((source) => {
|
|
27
27
|
if (source instanceof _ddd_ts_core.LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
28
28
|
if (source instanceof _ddd_ts_core.StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
29
29
|
throw new Error("Unknown source type");
|
|
30
30
|
});
|
|
31
31
|
query = query.where(firebase_admin_firestore.Filter.or(...filters));
|
|
32
|
-
if (startAfter) {
|
|
32
|
+
if (startAfter && !startAfter.isMin()) {
|
|
33
33
|
const ts = this.microsecondToTimestamp(startAfter.occurredAt);
|
|
34
|
-
query = query.startAfter(ts, startAfter.revision);
|
|
34
|
+
query = query.startAfter(ts, startAfter.revision, this.firestore.doc(startAfter.ref));
|
|
35
35
|
}
|
|
36
|
-
if (endAt) {
|
|
36
|
+
if (endAt && !endAt.isMax()) {
|
|
37
37
|
const ts = this.microsecondToTimestamp(endAt.occurredAt);
|
|
38
|
-
query = query.endAt(ts, endAt.revision);
|
|
38
|
+
query = query.endAt(ts, endAt.revision, this.firestore.doc(endAt.ref));
|
|
39
39
|
}
|
|
40
40
|
for await (const doc of query.stream()) {
|
|
41
41
|
const data = this.converter.fromFirestore(doc);
|
|
@@ -83,20 +83,20 @@ var FirestoreProjectedStreamStorageLayer = class {
|
|
|
83
83
|
});
|
|
84
84
|
}
|
|
85
85
|
async slice(projectedStream, shard, startAfter, endAt, limit) {
|
|
86
|
-
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision");
|
|
86
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision").orderBy(firebase_admin_firestore.FieldPath.documentId());
|
|
87
87
|
const filters = projectedStream.sources.map((source) => {
|
|
88
88
|
if (source instanceof _ddd_ts_core.LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
89
89
|
if (source instanceof _ddd_ts_core.StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
90
90
|
throw new Error("Unknown source type");
|
|
91
91
|
});
|
|
92
92
|
query = query.where(firebase_admin_firestore.Filter.or(...filters));
|
|
93
|
-
if (startAfter) {
|
|
93
|
+
if (startAfter && !startAfter.isMin()) {
|
|
94
94
|
const ts = this.microsecondToTimestamp(startAfter.occurredAt);
|
|
95
|
-
query = query.startAfter(ts, startAfter.revision);
|
|
95
|
+
query = query.startAfter(ts, startAfter.revision, this.firestore.doc(startAfter.ref));
|
|
96
96
|
}
|
|
97
|
-
if (endAt) {
|
|
97
|
+
if (endAt && !endAt.isMax()) {
|
|
98
98
|
const ts = this.microsecondToTimestamp(endAt.occurredAt);
|
|
99
|
-
query = query.endAt(ts, endAt.revision);
|
|
99
|
+
query = query.endAt(ts, endAt.revision, this.firestore.doc(endAt.ref));
|
|
100
100
|
}
|
|
101
101
|
if (limit) query = query.limit(limit);
|
|
102
102
|
return (await query.get()).docs.map((doc) => {
|
|
@@ -113,6 +113,51 @@ var FirestoreProjectedStreamStorageLayer = class {
|
|
|
113
113
|
};
|
|
114
114
|
});
|
|
115
115
|
}
|
|
116
|
+
async firstAtOrAfter(projectedStream, shard, from) {
|
|
117
|
+
const ts = this.microsecondToTimestamp(from);
|
|
118
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision").orderBy(firebase_admin_firestore.FieldPath.documentId());
|
|
119
|
+
const filters = projectedStream.sources.map((source) => {
|
|
120
|
+
if (source instanceof _ddd_ts_core.LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
121
|
+
if (source instanceof _ddd_ts_core.StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
122
|
+
throw new Error("Unknown source type");
|
|
123
|
+
});
|
|
124
|
+
query = query.where(firebase_admin_firestore.Filter.or(...filters)).startAt(ts).limit(1);
|
|
125
|
+
const doc = (await query.get()).docs[0];
|
|
126
|
+
if (!doc) return void 0;
|
|
127
|
+
const data = this.converter.fromFirestore(doc);
|
|
128
|
+
return {
|
|
129
|
+
id: data.eventId,
|
|
130
|
+
ref: doc.ref.path,
|
|
131
|
+
revision: data.revision,
|
|
132
|
+
name: data.name,
|
|
133
|
+
$name: data.name,
|
|
134
|
+
payload: data.payload,
|
|
135
|
+
occurredAt: data.occurredAt,
|
|
136
|
+
version: data.version ?? 1
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
async latest(projectedStream, shard) {
|
|
140
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt", "desc").orderBy("revision", "desc").orderBy(firebase_admin_firestore.FieldPath.documentId(), "desc").limit(1);
|
|
141
|
+
const filters = projectedStream.sources.map((source) => {
|
|
142
|
+
if (source instanceof _ddd_ts_core.LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
143
|
+
if (source instanceof _ddd_ts_core.StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
144
|
+
throw new Error("Unknown source type");
|
|
145
|
+
});
|
|
146
|
+
query = query.where(firebase_admin_firestore.Filter.or(...filters));
|
|
147
|
+
const doc = (await query.get()).docs[0];
|
|
148
|
+
if (!doc) return void 0;
|
|
149
|
+
const data = this.converter.fromFirestore(doc);
|
|
150
|
+
return {
|
|
151
|
+
id: data.eventId,
|
|
152
|
+
ref: doc.ref.path,
|
|
153
|
+
revision: data.revision,
|
|
154
|
+
name: data.name,
|
|
155
|
+
$name: data.name,
|
|
156
|
+
payload: data.payload,
|
|
157
|
+
occurredAt: data.occurredAt,
|
|
158
|
+
version: data.version ?? 1
|
|
159
|
+
};
|
|
160
|
+
}
|
|
116
161
|
};
|
|
117
162
|
|
|
118
163
|
//#endregion
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { LakeSource, StreamSource } from "@ddd-ts/core";
|
|
2
2
|
import { DefaultConverter } from "@ddd-ts/store-firestore";
|
|
3
|
-
import { Filter, Timestamp } from "firebase-admin/firestore";
|
|
3
|
+
import { FieldPath, Filter, Timestamp } from "firebase-admin/firestore";
|
|
4
4
|
import { MicrosecondTimestamp } from "@ddd-ts/shape";
|
|
5
5
|
import { Cursor as Cursor$1 } from "@ddd-ts/core/dist/components/cursor";
|
|
6
6
|
|
|
@@ -21,20 +21,20 @@ var FirestoreProjectedStreamStorageLayer = class {
|
|
|
21
21
|
this.converter = converter;
|
|
22
22
|
}
|
|
23
23
|
async *read(projectedStream, shard, startAfter, endAt) {
|
|
24
|
-
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision");
|
|
24
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision").orderBy(FieldPath.documentId());
|
|
25
25
|
const filters = projectedStream.sources.map((source) => {
|
|
26
26
|
if (source instanceof LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
27
27
|
if (source instanceof StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
28
28
|
throw new Error("Unknown source type");
|
|
29
29
|
});
|
|
30
30
|
query = query.where(Filter.or(...filters));
|
|
31
|
-
if (startAfter) {
|
|
31
|
+
if (startAfter && !startAfter.isMin()) {
|
|
32
32
|
const ts = this.microsecondToTimestamp(startAfter.occurredAt);
|
|
33
|
-
query = query.startAfter(ts, startAfter.revision);
|
|
33
|
+
query = query.startAfter(ts, startAfter.revision, this.firestore.doc(startAfter.ref));
|
|
34
34
|
}
|
|
35
|
-
if (endAt) {
|
|
35
|
+
if (endAt && !endAt.isMax()) {
|
|
36
36
|
const ts = this.microsecondToTimestamp(endAt.occurredAt);
|
|
37
|
-
query = query.endAt(ts, endAt.revision);
|
|
37
|
+
query = query.endAt(ts, endAt.revision, this.firestore.doc(endAt.ref));
|
|
38
38
|
}
|
|
39
39
|
for await (const doc of query.stream()) {
|
|
40
40
|
const data = this.converter.fromFirestore(doc);
|
|
@@ -82,20 +82,20 @@ var FirestoreProjectedStreamStorageLayer = class {
|
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
84
|
async slice(projectedStream, shard, startAfter, endAt, limit) {
|
|
85
|
-
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision");
|
|
85
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision").orderBy(FieldPath.documentId());
|
|
86
86
|
const filters = projectedStream.sources.map((source) => {
|
|
87
87
|
if (source instanceof LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
88
88
|
if (source instanceof StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
89
89
|
throw new Error("Unknown source type");
|
|
90
90
|
});
|
|
91
91
|
query = query.where(Filter.or(...filters));
|
|
92
|
-
if (startAfter) {
|
|
92
|
+
if (startAfter && !startAfter.isMin()) {
|
|
93
93
|
const ts = this.microsecondToTimestamp(startAfter.occurredAt);
|
|
94
|
-
query = query.startAfter(ts, startAfter.revision);
|
|
94
|
+
query = query.startAfter(ts, startAfter.revision, this.firestore.doc(startAfter.ref));
|
|
95
95
|
}
|
|
96
|
-
if (endAt) {
|
|
96
|
+
if (endAt && !endAt.isMax()) {
|
|
97
97
|
const ts = this.microsecondToTimestamp(endAt.occurredAt);
|
|
98
|
-
query = query.endAt(ts, endAt.revision);
|
|
98
|
+
query = query.endAt(ts, endAt.revision, this.firestore.doc(endAt.ref));
|
|
99
99
|
}
|
|
100
100
|
if (limit) query = query.limit(limit);
|
|
101
101
|
return (await query.get()).docs.map((doc) => {
|
|
@@ -112,6 +112,51 @@ var FirestoreProjectedStreamStorageLayer = class {
|
|
|
112
112
|
};
|
|
113
113
|
});
|
|
114
114
|
}
|
|
115
|
+
async firstAtOrAfter(projectedStream, shard, from) {
|
|
116
|
+
const ts = this.microsecondToTimestamp(from);
|
|
117
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt").orderBy("revision").orderBy(FieldPath.documentId());
|
|
118
|
+
const filters = projectedStream.sources.map((source) => {
|
|
119
|
+
if (source instanceof LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
120
|
+
if (source instanceof StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
121
|
+
throw new Error("Unknown source type");
|
|
122
|
+
});
|
|
123
|
+
query = query.where(Filter.or(...filters)).startAt(ts).limit(1);
|
|
124
|
+
const doc = (await query.get()).docs[0];
|
|
125
|
+
if (!doc) return void 0;
|
|
126
|
+
const data = this.converter.fromFirestore(doc);
|
|
127
|
+
return {
|
|
128
|
+
id: data.eventId,
|
|
129
|
+
ref: doc.ref.path,
|
|
130
|
+
revision: data.revision,
|
|
131
|
+
name: data.name,
|
|
132
|
+
$name: data.name,
|
|
133
|
+
payload: data.payload,
|
|
134
|
+
occurredAt: data.occurredAt,
|
|
135
|
+
version: data.version ?? 1
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
async latest(projectedStream, shard) {
|
|
139
|
+
let query = this.firestore.collectionGroup("events").orderBy("occurredAt", "desc").orderBy("revision", "desc").orderBy(FieldPath.documentId(), "desc").limit(1);
|
|
140
|
+
const filters = projectedStream.sources.map((source) => {
|
|
141
|
+
if (source instanceof LakeSource) return new FirestoreLakeSourceFilter().filter(shard, source);
|
|
142
|
+
if (source instanceof StreamSource) return new FirestoreStreamSourceFilter().filter(shard, source);
|
|
143
|
+
throw new Error("Unknown source type");
|
|
144
|
+
});
|
|
145
|
+
query = query.where(Filter.or(...filters));
|
|
146
|
+
const doc = (await query.get()).docs[0];
|
|
147
|
+
if (!doc) return void 0;
|
|
148
|
+
const data = this.converter.fromFirestore(doc);
|
|
149
|
+
return {
|
|
150
|
+
id: data.eventId,
|
|
151
|
+
ref: doc.ref.path,
|
|
152
|
+
revision: data.revision,
|
|
153
|
+
name: data.name,
|
|
154
|
+
$name: data.name,
|
|
155
|
+
payload: data.payload,
|
|
156
|
+
occurredAt: data.occurredAt,
|
|
157
|
+
version: data.version ?? 1
|
|
158
|
+
};
|
|
159
|
+
}
|
|
115
160
|
};
|
|
116
161
|
|
|
117
162
|
//#endregion
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay-api.spec.d.ts","sourceRoot":"","sources":["../../../src/projection/cases/replay-api.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tiedcursor.spec.d.ts","sourceRoot":"","sources":["../../../src/projection/cases/tiedcursor.spec.ts"],"names":[],"mappings":""}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CheckpointId, Cursor, ESProjection, EventId, type IEsEvent, type IFact, type ISavedChange, Lock, ProjectedStreamReader, type Serialized } from "@ddd-ts/core";
|
|
1
|
+
import { CheckpointId, Cursor, ESProjection, EventId, type IEsEvent, type IFact, type ISavedChange, Lock, ProjectedStream, ProjectedStreamReader, type Serialized } from "@ddd-ts/core";
|
|
2
2
|
import { Mapping, MicrosecondTimestamp } from "@ddd-ts/shape";
|
|
3
3
|
import { DefaultConverter, FirestoreTransaction } from "@ddd-ts/store-firestore";
|
|
4
4
|
import { Firestore, WriteBatch } from "firebase-admin/firestore";
|
|
@@ -18,6 +18,23 @@ interface FirestoreProjectorConfig {
|
|
|
18
18
|
enqueue: {
|
|
19
19
|
batchSize: number;
|
|
20
20
|
};
|
|
21
|
+
/**
|
|
22
|
+
* Controls retention of processed queue tasks.
|
|
23
|
+
*
|
|
24
|
+
* - `false` (default): never delete processed tasks. Preserves the queue
|
|
25
|
+
* as a durable audit trail for replay diagnostics.
|
|
26
|
+
* - `{ olderThan }`: delete processed tasks older than `olderThan` after
|
|
27
|
+
* each successful event handling.
|
|
28
|
+
*/
|
|
29
|
+
cleanup?: false | {
|
|
30
|
+
olderThan: MicrosecondTimestamp;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* Optional pre-handle filter. Returning `false` causes `handle()` to
|
|
34
|
+
* return without enqueuing or processing the event. Useful for feature
|
|
35
|
+
* flags or per-shard gating without subclassing the projector.
|
|
36
|
+
*/
|
|
37
|
+
shouldHandle?: (event: ISavedChange<IEsEvent>) => Promise<boolean> | boolean;
|
|
21
38
|
logger?: ProjectorLogger;
|
|
22
39
|
/** @deprecated Use `logger.error` instead */
|
|
23
40
|
onProcessError: (error: Error) => void;
|
|
@@ -40,11 +57,53 @@ export declare class FirestoreProjector {
|
|
|
40
57
|
handle(savedChange: ISavedChange<IEsEvent>): Promise<void>;
|
|
41
58
|
private handleOne;
|
|
42
59
|
private getCursor;
|
|
43
|
-
|
|
44
|
-
|
|
60
|
+
attempt(source: ProjectedStream, checkpointId: CheckpointId, target: Cursor): Promise<readonly ["DEFERRED", any] | readonly ["OK", "Target event already processed"] | readonly ["FAILURE", any] | readonly ["OK", "Target event processed successfully"]>;
|
|
61
|
+
getQueueHead(checkpointId: CheckpointId): Promise<Cursor | undefined>;
|
|
45
62
|
private readSourceStream;
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
enqueue(source: ProjectedStream, head: Cursor | undefined, checkpointId: CheckpointId, target: Cursor): Promise<readonly ["OK", "Tasks enqueued successfully"] | readonly ["DEFERRED", any]>;
|
|
64
|
+
enqueueOne(checkpointId: CheckpointId, target: Cursor): Promise<readonly ["OK", "Tasks enqueued successfully"] | readonly ["DEFERRED", any]>;
|
|
65
|
+
/**
|
|
66
|
+
* Enqueue all source events between the current queue head (exclusive)
|
|
67
|
+
* and `until` (inclusive). Defaults to Cursor.MAX, i.e. "to the end of
|
|
68
|
+
* the stream".
|
|
69
|
+
*/
|
|
70
|
+
enqueueUpTo(checkpointId: CheckpointId, until?: Cursor): Promise<readonly ["OK", "Tasks enqueued successfully"] | readonly ["DEFERRED", any]>;
|
|
71
|
+
/**
|
|
72
|
+
* Enqueue a single event by cursor. With `idempotent: true`, deletes any
|
|
73
|
+
* existing queued doc with the same eventId before re-creating, so a
|
|
74
|
+
* re-trigger after a partial replay doesn't fail with ALREADY_EXISTS.
|
|
75
|
+
*/
|
|
76
|
+
enqueueCursor(checkpointId: CheckpointId, cursor: Cursor, opts?: {
|
|
77
|
+
idempotent?: boolean;
|
|
78
|
+
}): Promise<readonly ["OK", "Tasks enqueued successfully"] | readonly ["DEFERRED", any]>;
|
|
79
|
+
/**
|
|
80
|
+
* Drain the queue to completion: repeatedly call attempt() with the
|
|
81
|
+
* current head as target until no unprocessed tasks remain. Yields to
|
|
82
|
+
* the event loop between iterations.
|
|
83
|
+
*/
|
|
84
|
+
dequeue(checkpointId: CheckpointId): Promise<void>;
|
|
85
|
+
/**
|
|
86
|
+
* Enqueue + drain in a loop until both phases are quiet. New events
|
|
87
|
+
* arriving mid-drain are picked up by the next enqueueUpTo cycle.
|
|
88
|
+
*/
|
|
89
|
+
catchup(checkpointId: CheckpointId, opts?: {
|
|
90
|
+
deadlineMs?: number;
|
|
91
|
+
}): Promise<void>;
|
|
92
|
+
/**
|
|
93
|
+
* Replay a checkpoint. Blocks the queue head, wipes existing tasks
|
|
94
|
+
* (except the blocker), seeds the start cursor, unblocks, and runs
|
|
95
|
+
* catchup forward. Concurrent listen() calls are safe — events
|
|
96
|
+
* arriving mid-replay are coalesced or re-enqueued by catchup.
|
|
97
|
+
*
|
|
98
|
+
* Provide either `from` (a Cursor) or `fromDate` (a wall-clock Date).
|
|
99
|
+
* If neither is given, replays from the first event in the stream.
|
|
100
|
+
*/
|
|
101
|
+
play(checkpointId: CheckpointId, opts?: {
|
|
102
|
+
from?: Cursor;
|
|
103
|
+
fromDate?: Date;
|
|
104
|
+
}): Promise<void>;
|
|
105
|
+
playFrom(checkpointId: CheckpointId, fromDate: Date): Promise<void>;
|
|
106
|
+
private resolvePlayStart;
|
|
48
107
|
private checkIsProcessed;
|
|
49
108
|
private getUnprocessed;
|
|
50
109
|
private claimTasks;
|
|
@@ -58,6 +117,7 @@ export declare class FirestoreQueueStore {
|
|
|
58
117
|
db: Firestore;
|
|
59
118
|
converter: DefaultConverter<FirebaseFirestore.DocumentData>;
|
|
60
119
|
collection: FirebaseFirestore.CollectionReference;
|
|
120
|
+
static readonly BLOCKER_EVENT_ID: EventId;
|
|
61
121
|
constructor(db: Firestore);
|
|
62
122
|
private timestampToMicroseconds;
|
|
63
123
|
private microsecondsToTimestamp;
|
|
@@ -82,8 +142,13 @@ export declare class FirestoreQueueStore {
|
|
|
82
142
|
batchWriter?: WriteBatch;
|
|
83
143
|
}): Promise<void>;
|
|
84
144
|
getTailCursor(id: CheckpointId): Promise<Cursor | undefined>;
|
|
85
|
-
cleanup(id: CheckpointId): Promise<void>;
|
|
86
|
-
flush(id: CheckpointId
|
|
145
|
+
cleanup(id: CheckpointId, olderThan?: MicrosecondTimestamp): Promise<void>;
|
|
146
|
+
flush(id: CheckpointId, opts?: {
|
|
147
|
+
except?: EventId[];
|
|
148
|
+
}): Promise<void>;
|
|
149
|
+
hasUnprocessed(checkpointId: CheckpointId): Promise<boolean>;
|
|
150
|
+
block(checkpointId: CheckpointId, blockerId?: EventId): Promise<void>;
|
|
151
|
+
unblock(checkpointId: CheckpointId, blockerId?: EventId): Promise<void>;
|
|
87
152
|
/**
|
|
88
153
|
* This method adds a fake processed event to the queue.
|
|
89
154
|
* It is useful for initializing the tail cursor of a new projection, at the
|
|
@@ -105,16 +170,17 @@ declare const Task_base: import("@ddd-ts/shape").IDict<{
|
|
|
105
170
|
readonly revision: NumberConstructor;
|
|
106
171
|
readonly attempts: NumberConstructor;
|
|
107
172
|
readonly processed: BooleanConstructor;
|
|
108
|
-
|
|
173
|
+
readonly claimer: import("@ddd-ts/shape").IOptional<StringConstructor, typeof import("@ddd-ts/shape").Empty>;
|
|
109
174
|
/** @deprecated */ readonly claimedAt: import("@ddd-ts/shape").IOptional<typeof MicrosecondTimestamp, typeof import("@ddd-ts/shape").Empty>;
|
|
110
175
|
readonly claimsMetadata: Mapping<[{
|
|
111
176
|
claimedAt: typeof MicrosecondTimestamp;
|
|
112
177
|
processedAt: import("@ddd-ts/shape").IOptional<typeof MicrosecondTimestamp, typeof import("@ddd-ts/shape").Empty>;
|
|
113
178
|
}], typeof import("@ddd-ts/shape").Empty>;
|
|
114
|
-
readonly claimIds: [StringConstructor];
|
|
179
|
+
/** @deprecated */ readonly claimIds: [StringConstructor];
|
|
115
180
|
readonly lock: typeof Lock;
|
|
116
181
|
readonly skipAfter: NumberConstructor;
|
|
117
182
|
readonly remaining: NumberConstructor;
|
|
183
|
+
readonly hasRemaining: BooleanConstructor;
|
|
118
184
|
readonly isolateAfter: NumberConstructor;
|
|
119
185
|
readonly claimTimeout: NumberConstructor;
|
|
120
186
|
readonly lastUpdateTime: import("@ddd-ts/shape").IOptional<typeof MicrosecondTimestamp, typeof import("@ddd-ts/shape").Empty>;
|
|
@@ -128,12 +194,11 @@ export declare class Task<Stored extends boolean> extends Task_base {
|
|
|
128
194
|
skipAfter: number;
|
|
129
195
|
isolateAfter: number;
|
|
130
196
|
}): Task<false>;
|
|
131
|
-
get currentClaimId(): string | undefined;
|
|
132
197
|
get isProcessing(): boolean;
|
|
133
198
|
get isProcessed(): boolean;
|
|
134
199
|
get shouldSkip(): boolean;
|
|
135
200
|
get shouldIsolate(): boolean;
|
|
136
|
-
|
|
201
|
+
hasTimedOut(): boolean | undefined;
|
|
137
202
|
static deserializeWithLastUpdateTime(data: Omit<Serialized<Task<true>>, "lastUpdateTime">, timestamp?: MicrosecondTimestamp): Task<true>;
|
|
138
203
|
static batch(tasks: Task<true>[]): Task<true>[];
|
|
139
204
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"firestore.projector.d.ts","sourceRoot":"","sources":["../../src/projection/firestore.projector.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,OAAO,EACP,KAAK,QAAQ,EACb,KAAK,KAAK,EACV,KAAK,YAAY,EACjB,IAAI,
|
|
1
|
+
{"version":3,"file":"firestore.projector.d.ts","sourceRoot":"","sources":["../../src/projection/firestore.projector.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,OAAO,EACP,KAAK,QAAQ,EACb,KAAK,KAAK,EACV,KAAK,YAAY,EACjB,IAAI,EACJ,eAAe,EACf,qBAAqB,EACrB,KAAK,UAAU,EAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAmB,MAAM,eAAe,CAAC;AAC/E,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAEL,SAAS,EAET,UAAU,EACX,MAAM,0BAA0B,CAAC;AAmBlC,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAChE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACjE;AASD,UAAU,wBAAwB;IAChC,KAAK,EAAE;QACL,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,KAAK,GAAG;QAAE,SAAS,EAAE,oBAAoB,CAAA;KAAE,CAAC;IACtD;;;;OAIG;IACH,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC7E,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,6CAA6C;IAC7C,cAAc,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACvC,6CAA6C;IAC7C,cAAc,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CACxC;AAED,qBAAa,kBAAkB;aAIX,UAAU,EAAE,YAAY,CAAC,QAAQ,CAAC;aAClC,MAAM,EAAE,qBAAqB,CAAC,QAAQ,CAAC;aACvC,KAAK,EAAE,mBAAmB;IACnC,MAAM,EAAE,wBAAwB;IANzC,QAAQ,UAAQ;gBAGE,UAAU,EAAE,YAAY,CAAC,QAAQ,CAAC,EAClC,MAAM,EAAE,qBAAqB,CAAC,QAAQ,CAAC,EACvC,KAAK,EAAE,mBAAmB,EACnC,MAAM,GAAE,wBAUd;IAGH,OAAO,KAAK,MAAM,GAEjB;IAEM,OAAO;IAwBd,OAAO,CAAC,qBAAqB,CAGf;IACd,OAAO,CAAC,eAAe,CAAkC;IAMzD,OAAO,CAAC,cAAc,CAGR;IAEd,OAAO,CAAC,mBAAmB;IASrB,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;YAmHlC,SAAS;YAwDT,SAAS;IAKjB,OAAO,CACX,MAAM,EAAE,eAAe,EACvB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM;IA8DV,YAAY,CAAC,YAAY,EAAE,YAAY;YAK/B,gBAAgB;IAaxB,OAAO,CACX,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM;IAoCV,UAAU,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM;IAkB3D;;;;OAIG;IACG,WAAW,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,GAAE,MAAmB;IAMxE;;;;OAIG;IACG,aAAa,CACjB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAO;IAQrC;;;;OAIG;IACG,OAAO,CAAC,YAAY,EAAE,YAAY;IAUxC;;;OAGG;IACG,OAAO,CACX,YAAY,EAAE,YAAY,EAC1B,IAAI,GAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAO;IAgBpC;;;;;;;;OAQG;IACG,IAAI,CACR,YAAY,EAAE,YAAY,EAC1B,IAAI,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,IAAI,CAAA;KAAO;IAsBzC,QAAQ,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI;YAI3C,gBAAgB;YAsBhB,gBAAgB;YAKhB,cAAc;YAKd,UAAU;YAcV,aAAa;YA+Db,kBAAkB;CAyBjC;AAED,qBAAa,oBAAqB,SAAQ,KAAK;;CAK9C;AAED,qBAAa,mBAAmB;IASX,EAAE,EAAE,SAAS;IARhC,SAAS,mDAA0B;IACnC,UAAU,EAAE,iBAAiB,CAAC,mBAAmB,CAAC;IAKlD,MAAM,CAAC,QAAQ,CAAC,gBAAgB,UAA4B;gBAEzC,EAAE,EAAE,SAAS;IAMhC,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,uBAAuB;IAQzB,OAAO,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE;IA0BxD,KAAK,CACT,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,SAAS,EAClB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE;IAmCf,IAAI,CAAC,YAAY,EAAE,YAAY;IA0B/B,WAAW,CAAC,YAAY,EAAE,YAAY;IAmCtC,OAAO,CACX,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,SAAS,GACjB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAiBlB,OAAO,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE;IAmB7D;;;;;OAKG;IACG,WAAW,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM;IAgB5D,UAAU,CAAC,EAAE,EAAE,YAAY;IAI3B,KAAK,CAAC,EAAE,EAAE,YAAY;IAItB,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO;IAInC,SAAS,CACb,SAAS,EAAE,SAAS,EACpB,EAAE,EAAE,YAAY,EAChB,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,oBAAoB,CAAC;QACnC,WAAW,CAAC,EAAE,UAAU,CAAC;KACrB;IA6BF,aAAa,CAAC,EAAE,EAAE,YAAY;IA2B9B,OAAO,CACX,EAAE,EAAE,YAAY,EAChB,SAAS,GAAE,oBAAwD;IA6B/D,KAAK,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,GAAE;QAAE,MAAM,CAAC,EAAE,OAAO,EAAE,CAAA;KAAO;IAWzD,cAAc,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;IAS5D,KAAK,CACT,YAAY,EAAE,YAAY,EAC1B,SAAS,GAAE,OAA8C;IA+BrD,OAAO,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,GAAE,OAA8C;IAK3D;;;;;;;;;OASG;IACG,IAAI,CAAC,YAAY,EAAE,YAAY;CAwCtC;AAED,qBAAa,SAAU,SAAQ,OAAO;CAAG;;;;;;;;;IAavC,kBAAkB;;;;;IAOlB,kBAAkB;;;;;;;;;AAnBpB,qBAAa,IAAI,CAAC,MAAM,SAAS,OAAO,CAAE,SAAQ,SA2BhD;IACQ,cAAc,EAAE,MAAM,SAAS,IAAI,GACvC,oBAAoB,GACpB,SAAS,CAAC;IAEd,IAAI,MAAM,WAOT;IAED,MAAM,CAAC,GAAG,CACR,IAAI,EAAE,KAAK,EACX,MAAM,EAAE;QACN,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;KACtB,GACA,IAAI,CAAC,KAAK,CAAC;IAuBd,IAAI,YAAY,YAEf;IAED,IAAI,WAAW,YAEd;IAED,IAAI,UAAU,YAEb;IAED,IAAI,aAAa,YAEhB;IAED,WAAW;IAaX,MAAM,CAAC,6BAA6B,CAClC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,gBAAgB,CAAC,EACpD,SAAS,CAAC,EAAE,oBAAoB,GAC/B,IAAI,CAAC,IAAI,CAAC;IAab,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE;CA0DjC"}
|
|
@@ -71,6 +71,9 @@ var FirestoreProjector = class {
|
|
|
71
71
|
for (const eventId of eventIds) pending.delete(eventId.serialize());
|
|
72
72
|
}
|
|
73
73
|
async handle(savedChange) {
|
|
74
|
+
if (this.config.shouldHandle) {
|
|
75
|
+
if (!await this.config.shouldHandle(savedChange)) return;
|
|
76
|
+
}
|
|
74
77
|
const key = this.projection.getCheckpointId(savedChange).serialize();
|
|
75
78
|
const cursor = await this.getCursor(savedChange);
|
|
76
79
|
if (!cursor) throw new Error(`Cursor not found for event ${savedChange.id.serialize()}`);
|
|
@@ -170,7 +173,7 @@ var FirestoreProjector = class {
|
|
|
170
173
|
continue;
|
|
171
174
|
}
|
|
172
175
|
if (status === Status.SUCCESS) {
|
|
173
|
-
await this.queue.cleanup(checkpointId);
|
|
176
|
+
if (this.config.cleanup) await this.queue.cleanup(checkpointId, this.config.cleanup.olderThan);
|
|
174
177
|
const durationMs = Date.now() - startedAt;
|
|
175
178
|
this.logger.info(`processed: Checkpoint<${checkpointKey}> caught up to Event<${eventId}> in ${durationMs}ms`, {
|
|
176
179
|
checkpointId: checkpointKey,
|
|
@@ -254,6 +257,90 @@ var FirestoreProjector = class {
|
|
|
254
257
|
if (status === Status.SUCCESS) this.prunePendingCursors(checkpointId, [task.id]);
|
|
255
258
|
return result;
|
|
256
259
|
}
|
|
260
|
+
/**
|
|
261
|
+
* Enqueue all source events between the current queue head (exclusive)
|
|
262
|
+
* and `until` (inclusive). Defaults to Cursor.MAX, i.e. "to the end of
|
|
263
|
+
* the stream".
|
|
264
|
+
*/
|
|
265
|
+
async enqueueUpTo(checkpointId, until = _ddd_ts_core.Cursor.MAX) {
|
|
266
|
+
const source = this.projection.getSource();
|
|
267
|
+
const head = await this.getQueueHead(checkpointId);
|
|
268
|
+
return this.enqueue(source, head, checkpointId, until);
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Enqueue a single event by cursor. With `idempotent: true`, deletes any
|
|
272
|
+
* existing queued doc with the same eventId before re-creating, so a
|
|
273
|
+
* re-trigger after a partial replay doesn't fail with ALREADY_EXISTS.
|
|
274
|
+
*/
|
|
275
|
+
async enqueueCursor(checkpointId, cursor, opts = {}) {
|
|
276
|
+
if (opts.idempotent) await this.queue.queued(checkpointId, cursor.eventId).delete();
|
|
277
|
+
return this.enqueueOne(checkpointId, cursor);
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Drain the queue to completion: repeatedly call attempt() with the
|
|
281
|
+
* current head as target until no unprocessed tasks remain. Yields to
|
|
282
|
+
* the event loop between iterations.
|
|
283
|
+
*/
|
|
284
|
+
async dequeue(checkpointId) {
|
|
285
|
+
const source = this.projection.getSource();
|
|
286
|
+
while (await this.queue.hasUnprocessed(checkpointId)) {
|
|
287
|
+
const head = await this.getQueueHead(checkpointId);
|
|
288
|
+
if (!head) return;
|
|
289
|
+
await this.attempt(source, checkpointId, head);
|
|
290
|
+
await wait(10);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Enqueue + drain in a loop until both phases are quiet. New events
|
|
295
|
+
* arriving mid-drain are picked up by the next enqueueUpTo cycle.
|
|
296
|
+
*/
|
|
297
|
+
async catchup(checkpointId, opts = {}) {
|
|
298
|
+
const deadline = opts.deadlineMs ? Date.now() + opts.deadlineMs : void 0;
|
|
299
|
+
while (true) {
|
|
300
|
+
await this.enqueueUpTo(checkpointId);
|
|
301
|
+
if (!await this.queue.hasUnprocessed(checkpointId)) return;
|
|
302
|
+
await this.dequeue(checkpointId);
|
|
303
|
+
if (deadline !== void 0 && Date.now() > deadline) throw new Error(`catchup: deadline of ${opts.deadlineMs}ms exceeded for checkpoint ${checkpointId.serialize()}`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Replay a checkpoint. Blocks the queue head, wipes existing tasks
|
|
308
|
+
* (except the blocker), seeds the start cursor, unblocks, and runs
|
|
309
|
+
* catchup forward. Concurrent listen() calls are safe — events
|
|
310
|
+
* arriving mid-replay are coalesced or re-enqueued by catchup.
|
|
311
|
+
*
|
|
312
|
+
* Provide either `from` (a Cursor) or `fromDate` (a wall-clock Date).
|
|
313
|
+
* If neither is given, replays from the first event in the stream.
|
|
314
|
+
*/
|
|
315
|
+
async play(checkpointId, opts = {}) {
|
|
316
|
+
const start = await this.resolvePlayStart(checkpointId, opts);
|
|
317
|
+
if (!start) throw new Error(`play: no events found for checkpoint ${checkpointId.serialize()}`);
|
|
318
|
+
await this.queue.block(checkpointId);
|
|
319
|
+
try {
|
|
320
|
+
await this.queue.flush(checkpointId, { except: [FirestoreQueueStore.BLOCKER_EVENT_ID] });
|
|
321
|
+
await this.enqueueCursor(checkpointId, start, { idempotent: true });
|
|
322
|
+
} finally {
|
|
323
|
+
await this.queue.unblock(checkpointId);
|
|
324
|
+
}
|
|
325
|
+
await this.catchup(checkpointId);
|
|
326
|
+
}
|
|
327
|
+
async playFrom(checkpointId, fromDate) {
|
|
328
|
+
return this.play(checkpointId, { fromDate });
|
|
329
|
+
}
|
|
330
|
+
async resolvePlayStart(checkpointId, opts) {
|
|
331
|
+
if (opts.from) return opts.from;
|
|
332
|
+
const source = this.projection.getSource();
|
|
333
|
+
const shard = checkpointId.shard();
|
|
334
|
+
if (opts.fromDate) {
|
|
335
|
+
const fromMicros = _ddd_ts_shape.MicrosecondTimestamp.deserialize(opts.fromDate);
|
|
336
|
+
const fact = await this.reader.firstAtOrAfter(source, shard, fromMicros);
|
|
337
|
+
if (!fact) return void 0;
|
|
338
|
+
return _ddd_ts_core.Cursor.from(fact);
|
|
339
|
+
}
|
|
340
|
+
const fact = await this.reader.first(source, shard);
|
|
341
|
+
if (!fact) return void 0;
|
|
342
|
+
return _ddd_ts_core.Cursor.from(fact);
|
|
343
|
+
}
|
|
257
344
|
async checkIsProcessed(checkpointId, cursor) {
|
|
258
345
|
return await this.queue.isProcessed(checkpointId, cursor);
|
|
259
346
|
}
|
|
@@ -301,7 +388,7 @@ var FirestoreProjector = class {
|
|
|
301
388
|
for (const event of events) {
|
|
302
389
|
const task = claimedTasksMap.get(event.id.serialize());
|
|
303
390
|
if (!task) throw new Error(`Task not found for event ${event.id.serialize()} in claimer ${claimer.serialize()}`);
|
|
304
|
-
if (task.
|
|
391
|
+
if (task.claimer !== claimer.serialize()) throw new Error(`Task ${task.id.serialize()} claimer mismatch: expected ${claimer.serialize()}, found ${task.claimer}`);
|
|
305
392
|
}
|
|
306
393
|
}
|
|
307
394
|
};
|
|
@@ -311,9 +398,10 @@ var AlreadyEnqueuedError = class extends Error {
|
|
|
311
398
|
this.name = "AlreadyEnqueuedError";
|
|
312
399
|
}
|
|
313
400
|
};
|
|
314
|
-
var FirestoreQueueStore = class {
|
|
401
|
+
var FirestoreQueueStore = class FirestoreQueueStore {
|
|
315
402
|
converter = new _ddd_ts_store_firestore.DefaultConverter();
|
|
316
403
|
collection;
|
|
404
|
+
static BLOCKER_EVENT_ID = new _ddd_ts_core.EventId("_blocker_");
|
|
317
405
|
constructor(db) {
|
|
318
406
|
this.db = db;
|
|
319
407
|
this.collection = db.collection("checkpoints");
|
|
@@ -331,8 +419,8 @@ var FirestoreQueueStore = class {
|
|
|
331
419
|
for (const task of tasks) {
|
|
332
420
|
const ref = this.queued(checkpointId, task.id);
|
|
333
421
|
batch.create(ref, {
|
|
334
|
-
|
|
335
|
-
|
|
422
|
+
...this.converter.toFirestore(task.serialize()),
|
|
423
|
+
ref: task.cursor.ref
|
|
336
424
|
});
|
|
337
425
|
}
|
|
338
426
|
try {
|
|
@@ -346,21 +434,22 @@ var FirestoreQueueStore = class {
|
|
|
346
434
|
async claim(checkpointId, claimer, tasks) {
|
|
347
435
|
const batch = this.collection.firestore.batch();
|
|
348
436
|
for (const task of tasks) {
|
|
349
|
-
if (task.
|
|
437
|
+
if (task.claimer !== void 0) throw new Error(`Task ${task.id.serialize()} is already claimed by ${task.claimer}`);
|
|
350
438
|
const ref = this.queued(checkpointId, task.id);
|
|
351
439
|
batch.update(ref, {
|
|
352
440
|
claimer: claimer.serialize(),
|
|
353
441
|
claimedAt: firebase_admin_firestore.FieldValue.serverTimestamp(),
|
|
354
442
|
[`claimsMetadata.${claimer.serialize()}`]: { claimedAt: firebase_admin_firestore.FieldValue.serverTimestamp() },
|
|
355
443
|
claimIds: firebase_admin_firestore.FieldValue.arrayUnion(claimer.serialize()),
|
|
356
|
-
attempts:
|
|
357
|
-
remaining:
|
|
444
|
+
attempts: task.attempts + 1,
|
|
445
|
+
remaining: task.remaining - 1,
|
|
446
|
+
hasRemaining: task.remaining > 1
|
|
358
447
|
}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
|
|
359
448
|
}
|
|
360
449
|
await batch.commit();
|
|
361
450
|
}
|
|
362
451
|
async head(checkpointId) {
|
|
363
|
-
const headDoc = (await this.queue(checkpointId).orderBy("occurredAt", "desc").orderBy("revision", "desc").limit(1).get()).docs[0];
|
|
452
|
+
const headDoc = (await this.queue(checkpointId).orderBy("occurredAt", "desc").orderBy("revision", "desc").orderBy("ref", "desc").limit(1).get()).docs[0];
|
|
364
453
|
if (!headDoc) return;
|
|
365
454
|
const headData = this.converter.fromFirestoreSnapshot(headDoc);
|
|
366
455
|
if (!headData) return;
|
|
@@ -372,37 +461,27 @@ var FirestoreQueueStore = class {
|
|
|
372
461
|
}) : void 0;
|
|
373
462
|
}
|
|
374
463
|
async unprocessed(checkpointId) {
|
|
375
|
-
const tasks = (await this.queue(checkpointId).where("processed", "==", false).where("
|
|
464
|
+
const tasks = (await this.queue(checkpointId).where("processed", "==", false).where("hasRemaining", "==", true).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc").limit(100).get()).docs.map((doc) => {
|
|
376
465
|
const data = this.converter.fromFirestoreSnapshot(doc);
|
|
377
466
|
const timestamp = doc.updateTime ? this.timestampToMicroseconds(doc.updateTime) : void 0;
|
|
378
467
|
return Task.deserializeWithLastUpdateTime(data, timestamp);
|
|
379
468
|
});
|
|
380
|
-
const expiredTasks =
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
const batch = this.collection.firestore.batch();
|
|
388
|
-
for (const task of expiredTasks) {
|
|
389
|
-
const ref = this.queued(checkpointId, task.id);
|
|
390
|
-
batch.update(ref, {
|
|
391
|
-
claimer: firebase_admin_firestore.FieldValue.delete(),
|
|
392
|
-
claimedAt: firebase_admin_firestore.FieldValue.delete(),
|
|
393
|
-
claimIds: task.claimIds
|
|
394
|
-
}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
|
|
395
|
-
}
|
|
396
|
-
await batch.commit();
|
|
469
|
+
const expiredTasks = tasks.filter((task) => task.hasTimedOut());
|
|
470
|
+
if (expiredTasks.length === 0) return tasks;
|
|
471
|
+
await this.unclaim(checkpointId, expiredTasks);
|
|
472
|
+
for (const task of expiredTasks) {
|
|
473
|
+
task.claimIds = task.claimIds.filter((id) => id !== task.claimer);
|
|
474
|
+
task.claimer = void 0;
|
|
475
|
+
task.claimedAt = void 0;
|
|
397
476
|
}
|
|
398
477
|
return tasks;
|
|
399
478
|
}
|
|
400
479
|
async claimed(checkpointId, claimer) {
|
|
401
|
-
return (await this.queue(checkpointId).where("
|
|
480
|
+
return (await this.queue(checkpointId).where("claimer", "==", claimer.serialize()).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc").get()).docs.map((doc) => {
|
|
402
481
|
const data = this.converter.fromFirestoreSnapshot(doc);
|
|
403
482
|
const timestamp = doc.updateTime ? this.timestampToMicroseconds(doc.updateTime) : void 0;
|
|
404
483
|
return Task.deserializeWithLastUpdateTime(data, timestamp);
|
|
405
|
-
})
|
|
484
|
+
});
|
|
406
485
|
}
|
|
407
486
|
async unclaim(checkpointId, tasks) {
|
|
408
487
|
const batch = this.collection.firestore.batch();
|
|
@@ -411,7 +490,7 @@ var FirestoreQueueStore = class {
|
|
|
411
490
|
batch.update(ref, {
|
|
412
491
|
claimer: firebase_admin_firestore.FieldValue.delete(),
|
|
413
492
|
claimedAt: firebase_admin_firestore.FieldValue.delete(),
|
|
414
|
-
claimIds: firebase_admin_firestore.FieldValue.arrayRemove(task.
|
|
493
|
+
claimIds: firebase_admin_firestore.FieldValue.arrayRemove(task.claimer)
|
|
415
494
|
}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
|
|
416
495
|
}
|
|
417
496
|
await batch.commit();
|
|
@@ -461,7 +540,7 @@ var FirestoreQueueStore = class {
|
|
|
461
540
|
})));
|
|
462
541
|
}
|
|
463
542
|
async getTailCursor(id) {
|
|
464
|
-
const tailDoc = (await this.queue(id).where("
|
|
543
|
+
const tailDoc = (await this.queue(id).where("hasRemaining", "==", true).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc").limit(1).get()).docs[0];
|
|
465
544
|
if (!tailDoc) return;
|
|
466
545
|
const tailData = this.converter.fromFirestoreSnapshot(tailDoc);
|
|
467
546
|
if (!tailData) return;
|
|
@@ -472,9 +551,9 @@ var FirestoreQueueStore = class {
|
|
|
472
551
|
eventId: tailData.id
|
|
473
552
|
}) : void 0;
|
|
474
553
|
}
|
|
475
|
-
async cleanup(id) {
|
|
476
|
-
const
|
|
477
|
-
const query = this.queue(id).where("
|
|
554
|
+
async cleanup(id, olderThan = _ddd_ts_shape.MicrosecondTimestamp.WEEK.mult(4)) {
|
|
555
|
+
const threshold = _ddd_ts_shape.MicrosecondTimestamp.now().sub(olderThan);
|
|
556
|
+
const query = this.queue(id).where("hasRemaining", "==", true).where("occurredAt", "<", threshold.serialize()).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc");
|
|
478
557
|
const TRAIL = 1;
|
|
479
558
|
const snapshot = await query.get();
|
|
480
559
|
if (snapshot.size < TRAIL) return;
|
|
@@ -486,12 +565,50 @@ var FirestoreQueueStore = class {
|
|
|
486
565
|
for (const queued of cleaning) batch.delete(queued.ref);
|
|
487
566
|
await batch.commit();
|
|
488
567
|
}
|
|
489
|
-
async flush(id) {
|
|
568
|
+
async flush(id, opts = {}) {
|
|
569
|
+
const exclude = new Set((opts.except ?? []).map((e) => e.serialize()));
|
|
490
570
|
const stream = this.queue(id).stream();
|
|
491
571
|
const writer = this.collection.firestore.bulkWriter();
|
|
492
|
-
for await (const queued of stream)
|
|
572
|
+
for await (const queued of stream) {
|
|
573
|
+
if (exclude.has(queued.id)) continue;
|
|
574
|
+
writer.delete(queued.ref);
|
|
575
|
+
}
|
|
493
576
|
await writer.close();
|
|
494
577
|
}
|
|
578
|
+
async hasUnprocessed(checkpointId) {
|
|
579
|
+
return !(await this.queue(checkpointId).where("processed", "==", false).where("hasRemaining", "==", true).limit(1).get()).empty;
|
|
580
|
+
}
|
|
581
|
+
async block(checkpointId, blockerId = FirestoreQueueStore.BLOCKER_EVENT_ID) {
|
|
582
|
+
const task = new Task({
|
|
583
|
+
id: blockerId,
|
|
584
|
+
ref: _ddd_ts_core.Cursor.MAX.ref,
|
|
585
|
+
occurredAt: _ddd_ts_core.Cursor.MAX.occurredAt,
|
|
586
|
+
revision: _ddd_ts_core.Cursor.MAX.revision,
|
|
587
|
+
attempts: 0,
|
|
588
|
+
processed: true,
|
|
589
|
+
claimer: void 0,
|
|
590
|
+
claimedAt: void 0,
|
|
591
|
+
claimsMetadata: {},
|
|
592
|
+
claimIds: [],
|
|
593
|
+
lock: new _ddd_ts_core.Lock({}),
|
|
594
|
+
remaining: 1,
|
|
595
|
+
claimTimeout: 0,
|
|
596
|
+
skipAfter: 0,
|
|
597
|
+
isolateAfter: 0,
|
|
598
|
+
lastUpdateTime: void 0,
|
|
599
|
+
hasRemaining: true
|
|
600
|
+
});
|
|
601
|
+
await this.queued(checkpointId, blockerId).delete();
|
|
602
|
+
const serialized = task.serialize();
|
|
603
|
+
const converted = this.converter.toFirestore(serialized);
|
|
604
|
+
await this.queued(checkpointId, blockerId).create({
|
|
605
|
+
...converted,
|
|
606
|
+
ref: _ddd_ts_core.Cursor.MAX.ref
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
async unblock(checkpointId, blockerId = FirestoreQueueStore.BLOCKER_EVENT_ID) {
|
|
610
|
+
await this.queued(checkpointId, blockerId).delete();
|
|
611
|
+
}
|
|
495
612
|
/**
|
|
496
613
|
* This method adds a fake processed event to the queue.
|
|
497
614
|
* It is useful for initializing the tail cursor of a new projection, at the
|
|
@@ -525,7 +642,8 @@ var FirestoreQueueStore = class {
|
|
|
525
642
|
claimTimeout: 0,
|
|
526
643
|
skipAfter: 0,
|
|
527
644
|
isolateAfter: 0,
|
|
528
|
-
lastUpdateTime: void 0
|
|
645
|
+
lastUpdateTime: void 0,
|
|
646
|
+
hasRemaining: true
|
|
529
647
|
});
|
|
530
648
|
try {
|
|
531
649
|
const serialized = task.serialize();
|
|
@@ -554,6 +672,7 @@ var Task = class Task extends (0, _ddd_ts_shape.Shape)({
|
|
|
554
672
|
lock: _ddd_ts_core.Lock,
|
|
555
673
|
skipAfter: Number,
|
|
556
674
|
remaining: Number,
|
|
675
|
+
hasRemaining: Boolean,
|
|
557
676
|
isolateAfter: Number,
|
|
558
677
|
claimTimeout: Number,
|
|
559
678
|
lastUpdateTime: (0, _ddd_ts_shape.Optional)(_ddd_ts_shape.MicrosecondTimestamp)
|
|
@@ -580,17 +699,15 @@ var Task = class Task extends (0, _ddd_ts_shape.Shape)({
|
|
|
580
699
|
skipAfter: config.skipAfter,
|
|
581
700
|
isolateAfter: config.isolateAfter,
|
|
582
701
|
remaining: config.skipAfter,
|
|
702
|
+
hasRemaining: config.skipAfter > 0,
|
|
583
703
|
ref: fact.ref,
|
|
584
704
|
revision: fact.revision,
|
|
585
705
|
occurredAt: fact.occurredAt,
|
|
586
706
|
lastUpdateTime: void 0
|
|
587
707
|
});
|
|
588
708
|
}
|
|
589
|
-
get currentClaimId() {
|
|
590
|
-
return this.claimIds.at(-1);
|
|
591
|
-
}
|
|
592
709
|
get isProcessing() {
|
|
593
|
-
return this.
|
|
710
|
+
return this.claimer !== void 0;
|
|
594
711
|
}
|
|
595
712
|
get isProcessed() {
|
|
596
713
|
return !!this.processed;
|
|
@@ -601,19 +718,19 @@ var Task = class Task extends (0, _ddd_ts_shape.Shape)({
|
|
|
601
718
|
get shouldIsolate() {
|
|
602
719
|
return this.attempts > this.isolateAfter;
|
|
603
720
|
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
const claimInfo = this.claimsMetadata[claimer];
|
|
721
|
+
hasTimedOut() {
|
|
722
|
+
if (!this.claimer) return;
|
|
723
|
+
const claimInfo = this.claimsMetadata[this.claimer];
|
|
608
724
|
if (!claimInfo || !claimInfo.claimedAt) return;
|
|
609
|
-
|
|
725
|
+
return _ddd_ts_shape.MicrosecondTimestamp.now().micros - claimInfo.claimedAt.micros > BigInt(this.claimTimeout) * 1000n;
|
|
610
726
|
}
|
|
611
727
|
static deserializeWithLastUpdateTime(data, timestamp) {
|
|
612
728
|
return Task.deserialize({
|
|
613
729
|
...data,
|
|
614
730
|
lastUpdateTime: timestamp,
|
|
615
731
|
claimIds: data.claimIds || [],
|
|
616
|
-
claimsMetadata: data.claimsMetadata || {}
|
|
732
|
+
claimsMetadata: data.claimsMetadata || {},
|
|
733
|
+
hasRemaining: data.hasRemaining ?? data.remaining > 0
|
|
617
734
|
});
|
|
618
735
|
}
|
|
619
736
|
static batch(tasks) {
|
|
@@ -70,6 +70,9 @@ var FirestoreProjector = class {
|
|
|
70
70
|
for (const eventId of eventIds) pending.delete(eventId.serialize());
|
|
71
71
|
}
|
|
72
72
|
async handle(savedChange) {
|
|
73
|
+
if (this.config.shouldHandle) {
|
|
74
|
+
if (!await this.config.shouldHandle(savedChange)) return;
|
|
75
|
+
}
|
|
73
76
|
const key = this.projection.getCheckpointId(savedChange).serialize();
|
|
74
77
|
const cursor = await this.getCursor(savedChange);
|
|
75
78
|
if (!cursor) throw new Error(`Cursor not found for event ${savedChange.id.serialize()}`);
|
|
@@ -169,7 +172,7 @@ var FirestoreProjector = class {
|
|
|
169
172
|
continue;
|
|
170
173
|
}
|
|
171
174
|
if (status === Status.SUCCESS) {
|
|
172
|
-
await this.queue.cleanup(checkpointId);
|
|
175
|
+
if (this.config.cleanup) await this.queue.cleanup(checkpointId, this.config.cleanup.olderThan);
|
|
173
176
|
const durationMs = Date.now() - startedAt;
|
|
174
177
|
this.logger.info(`processed: Checkpoint<${checkpointKey}> caught up to Event<${eventId}> in ${durationMs}ms`, {
|
|
175
178
|
checkpointId: checkpointKey,
|
|
@@ -253,6 +256,90 @@ var FirestoreProjector = class {
|
|
|
253
256
|
if (status === Status.SUCCESS) this.prunePendingCursors(checkpointId, [task.id]);
|
|
254
257
|
return result;
|
|
255
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* Enqueue all source events between the current queue head (exclusive)
|
|
261
|
+
* and `until` (inclusive). Defaults to Cursor.MAX, i.e. "to the end of
|
|
262
|
+
* the stream".
|
|
263
|
+
*/
|
|
264
|
+
async enqueueUpTo(checkpointId, until = Cursor.MAX) {
|
|
265
|
+
const source = this.projection.getSource();
|
|
266
|
+
const head = await this.getQueueHead(checkpointId);
|
|
267
|
+
return this.enqueue(source, head, checkpointId, until);
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Enqueue a single event by cursor. With `idempotent: true`, deletes any
|
|
271
|
+
* existing queued doc with the same eventId before re-creating, so a
|
|
272
|
+
* re-trigger after a partial replay doesn't fail with ALREADY_EXISTS.
|
|
273
|
+
*/
|
|
274
|
+
async enqueueCursor(checkpointId, cursor, opts = {}) {
|
|
275
|
+
if (opts.idempotent) await this.queue.queued(checkpointId, cursor.eventId).delete();
|
|
276
|
+
return this.enqueueOne(checkpointId, cursor);
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Drain the queue to completion: repeatedly call attempt() with the
|
|
280
|
+
* current head as target until no unprocessed tasks remain. Yields to
|
|
281
|
+
* the event loop between iterations.
|
|
282
|
+
*/
|
|
283
|
+
async dequeue(checkpointId) {
|
|
284
|
+
const source = this.projection.getSource();
|
|
285
|
+
while (await this.queue.hasUnprocessed(checkpointId)) {
|
|
286
|
+
const head = await this.getQueueHead(checkpointId);
|
|
287
|
+
if (!head) return;
|
|
288
|
+
await this.attempt(source, checkpointId, head);
|
|
289
|
+
await wait(10);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Enqueue + drain in a loop until both phases are quiet. New events
|
|
294
|
+
* arriving mid-drain are picked up by the next enqueueUpTo cycle.
|
|
295
|
+
*/
|
|
296
|
+
async catchup(checkpointId, opts = {}) {
|
|
297
|
+
const deadline = opts.deadlineMs ? Date.now() + opts.deadlineMs : void 0;
|
|
298
|
+
while (true) {
|
|
299
|
+
await this.enqueueUpTo(checkpointId);
|
|
300
|
+
if (!await this.queue.hasUnprocessed(checkpointId)) return;
|
|
301
|
+
await this.dequeue(checkpointId);
|
|
302
|
+
if (deadline !== void 0 && Date.now() > deadline) throw new Error(`catchup: deadline of ${opts.deadlineMs}ms exceeded for checkpoint ${checkpointId.serialize()}`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Replay a checkpoint. Blocks the queue head, wipes existing tasks
|
|
307
|
+
* (except the blocker), seeds the start cursor, unblocks, and runs
|
|
308
|
+
* catchup forward. Concurrent listen() calls are safe — events
|
|
309
|
+
* arriving mid-replay are coalesced or re-enqueued by catchup.
|
|
310
|
+
*
|
|
311
|
+
* Provide either `from` (a Cursor) or `fromDate` (a wall-clock Date).
|
|
312
|
+
* If neither is given, replays from the first event in the stream.
|
|
313
|
+
*/
|
|
314
|
+
async play(checkpointId, opts = {}) {
|
|
315
|
+
const start = await this.resolvePlayStart(checkpointId, opts);
|
|
316
|
+
if (!start) throw new Error(`play: no events found for checkpoint ${checkpointId.serialize()}`);
|
|
317
|
+
await this.queue.block(checkpointId);
|
|
318
|
+
try {
|
|
319
|
+
await this.queue.flush(checkpointId, { except: [FirestoreQueueStore.BLOCKER_EVENT_ID] });
|
|
320
|
+
await this.enqueueCursor(checkpointId, start, { idempotent: true });
|
|
321
|
+
} finally {
|
|
322
|
+
await this.queue.unblock(checkpointId);
|
|
323
|
+
}
|
|
324
|
+
await this.catchup(checkpointId);
|
|
325
|
+
}
|
|
326
|
+
async playFrom(checkpointId, fromDate) {
|
|
327
|
+
return this.play(checkpointId, { fromDate });
|
|
328
|
+
}
|
|
329
|
+
async resolvePlayStart(checkpointId, opts) {
|
|
330
|
+
if (opts.from) return opts.from;
|
|
331
|
+
const source = this.projection.getSource();
|
|
332
|
+
const shard = checkpointId.shard();
|
|
333
|
+
if (opts.fromDate) {
|
|
334
|
+
const fromMicros = MicrosecondTimestamp.deserialize(opts.fromDate);
|
|
335
|
+
const fact = await this.reader.firstAtOrAfter(source, shard, fromMicros);
|
|
336
|
+
if (!fact) return void 0;
|
|
337
|
+
return Cursor.from(fact);
|
|
338
|
+
}
|
|
339
|
+
const fact = await this.reader.first(source, shard);
|
|
340
|
+
if (!fact) return void 0;
|
|
341
|
+
return Cursor.from(fact);
|
|
342
|
+
}
|
|
256
343
|
async checkIsProcessed(checkpointId, cursor) {
|
|
257
344
|
return await this.queue.isProcessed(checkpointId, cursor);
|
|
258
345
|
}
|
|
@@ -300,7 +387,7 @@ var FirestoreProjector = class {
|
|
|
300
387
|
for (const event of events) {
|
|
301
388
|
const task = claimedTasksMap.get(event.id.serialize());
|
|
302
389
|
if (!task) throw new Error(`Task not found for event ${event.id.serialize()} in claimer ${claimer.serialize()}`);
|
|
303
|
-
if (task.
|
|
390
|
+
if (task.claimer !== claimer.serialize()) throw new Error(`Task ${task.id.serialize()} claimer mismatch: expected ${claimer.serialize()}, found ${task.claimer}`);
|
|
304
391
|
}
|
|
305
392
|
}
|
|
306
393
|
};
|
|
@@ -310,9 +397,10 @@ var AlreadyEnqueuedError = class extends Error {
|
|
|
310
397
|
this.name = "AlreadyEnqueuedError";
|
|
311
398
|
}
|
|
312
399
|
};
|
|
313
|
-
var FirestoreQueueStore = class {
|
|
400
|
+
var FirestoreQueueStore = class FirestoreQueueStore {
|
|
314
401
|
converter = new DefaultConverter();
|
|
315
402
|
collection;
|
|
403
|
+
static BLOCKER_EVENT_ID = new EventId("_blocker_");
|
|
316
404
|
constructor(db) {
|
|
317
405
|
this.db = db;
|
|
318
406
|
this.collection = db.collection("checkpoints");
|
|
@@ -330,8 +418,8 @@ var FirestoreQueueStore = class {
|
|
|
330
418
|
for (const task of tasks) {
|
|
331
419
|
const ref = this.queued(checkpointId, task.id);
|
|
332
420
|
batch.create(ref, {
|
|
333
|
-
|
|
334
|
-
|
|
421
|
+
...this.converter.toFirestore(task.serialize()),
|
|
422
|
+
ref: task.cursor.ref
|
|
335
423
|
});
|
|
336
424
|
}
|
|
337
425
|
try {
|
|
@@ -345,21 +433,22 @@ var FirestoreQueueStore = class {
|
|
|
345
433
|
async claim(checkpointId, claimer, tasks) {
|
|
346
434
|
const batch = this.collection.firestore.batch();
|
|
347
435
|
for (const task of tasks) {
|
|
348
|
-
if (task.
|
|
436
|
+
if (task.claimer !== void 0) throw new Error(`Task ${task.id.serialize()} is already claimed by ${task.claimer}`);
|
|
349
437
|
const ref = this.queued(checkpointId, task.id);
|
|
350
438
|
batch.update(ref, {
|
|
351
439
|
claimer: claimer.serialize(),
|
|
352
440
|
claimedAt: FieldValue.serverTimestamp(),
|
|
353
441
|
[`claimsMetadata.${claimer.serialize()}`]: { claimedAt: FieldValue.serverTimestamp() },
|
|
354
442
|
claimIds: FieldValue.arrayUnion(claimer.serialize()),
|
|
355
|
-
attempts:
|
|
356
|
-
remaining:
|
|
443
|
+
attempts: task.attempts + 1,
|
|
444
|
+
remaining: task.remaining - 1,
|
|
445
|
+
hasRemaining: task.remaining > 1
|
|
357
446
|
}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
|
|
358
447
|
}
|
|
359
448
|
await batch.commit();
|
|
360
449
|
}
|
|
361
450
|
async head(checkpointId) {
|
|
362
|
-
const headDoc = (await this.queue(checkpointId).orderBy("occurredAt", "desc").orderBy("revision", "desc").limit(1).get()).docs[0];
|
|
451
|
+
const headDoc = (await this.queue(checkpointId).orderBy("occurredAt", "desc").orderBy("revision", "desc").orderBy("ref", "desc").limit(1).get()).docs[0];
|
|
363
452
|
if (!headDoc) return;
|
|
364
453
|
const headData = this.converter.fromFirestoreSnapshot(headDoc);
|
|
365
454
|
if (!headData) return;
|
|
@@ -371,37 +460,27 @@ var FirestoreQueueStore = class {
|
|
|
371
460
|
}) : void 0;
|
|
372
461
|
}
|
|
373
462
|
async unprocessed(checkpointId) {
|
|
374
|
-
const tasks = (await this.queue(checkpointId).where("processed", "==", false).where("
|
|
463
|
+
const tasks = (await this.queue(checkpointId).where("processed", "==", false).where("hasRemaining", "==", true).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc").limit(100).get()).docs.map((doc) => {
|
|
375
464
|
const data = this.converter.fromFirestoreSnapshot(doc);
|
|
376
465
|
const timestamp = doc.updateTime ? this.timestampToMicroseconds(doc.updateTime) : void 0;
|
|
377
466
|
return Task.deserializeWithLastUpdateTime(data, timestamp);
|
|
378
467
|
});
|
|
379
|
-
const expiredTasks =
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
const batch = this.collection.firestore.batch();
|
|
387
|
-
for (const task of expiredTasks) {
|
|
388
|
-
const ref = this.queued(checkpointId, task.id);
|
|
389
|
-
batch.update(ref, {
|
|
390
|
-
claimer: FieldValue.delete(),
|
|
391
|
-
claimedAt: FieldValue.delete(),
|
|
392
|
-
claimIds: task.claimIds
|
|
393
|
-
}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
|
|
394
|
-
}
|
|
395
|
-
await batch.commit();
|
|
468
|
+
const expiredTasks = tasks.filter((task) => task.hasTimedOut());
|
|
469
|
+
if (expiredTasks.length === 0) return tasks;
|
|
470
|
+
await this.unclaim(checkpointId, expiredTasks);
|
|
471
|
+
for (const task of expiredTasks) {
|
|
472
|
+
task.claimIds = task.claimIds.filter((id) => id !== task.claimer);
|
|
473
|
+
task.claimer = void 0;
|
|
474
|
+
task.claimedAt = void 0;
|
|
396
475
|
}
|
|
397
476
|
return tasks;
|
|
398
477
|
}
|
|
399
478
|
async claimed(checkpointId, claimer) {
|
|
400
|
-
return (await this.queue(checkpointId).where("
|
|
479
|
+
return (await this.queue(checkpointId).where("claimer", "==", claimer.serialize()).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc").get()).docs.map((doc) => {
|
|
401
480
|
const data = this.converter.fromFirestoreSnapshot(doc);
|
|
402
481
|
const timestamp = doc.updateTime ? this.timestampToMicroseconds(doc.updateTime) : void 0;
|
|
403
482
|
return Task.deserializeWithLastUpdateTime(data, timestamp);
|
|
404
|
-
})
|
|
483
|
+
});
|
|
405
484
|
}
|
|
406
485
|
async unclaim(checkpointId, tasks) {
|
|
407
486
|
const batch = this.collection.firestore.batch();
|
|
@@ -410,7 +489,7 @@ var FirestoreQueueStore = class {
|
|
|
410
489
|
batch.update(ref, {
|
|
411
490
|
claimer: FieldValue.delete(),
|
|
412
491
|
claimedAt: FieldValue.delete(),
|
|
413
|
-
claimIds: FieldValue.arrayRemove(task.
|
|
492
|
+
claimIds: FieldValue.arrayRemove(task.claimer)
|
|
414
493
|
}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
|
|
415
494
|
}
|
|
416
495
|
await batch.commit();
|
|
@@ -460,7 +539,7 @@ var FirestoreQueueStore = class {
|
|
|
460
539
|
})));
|
|
461
540
|
}
|
|
462
541
|
async getTailCursor(id) {
|
|
463
|
-
const tailDoc = (await this.queue(id).where("
|
|
542
|
+
const tailDoc = (await this.queue(id).where("hasRemaining", "==", true).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc").limit(1).get()).docs[0];
|
|
464
543
|
if (!tailDoc) return;
|
|
465
544
|
const tailData = this.converter.fromFirestoreSnapshot(tailDoc);
|
|
466
545
|
if (!tailData) return;
|
|
@@ -471,9 +550,9 @@ var FirestoreQueueStore = class {
|
|
|
471
550
|
eventId: tailData.id
|
|
472
551
|
}) : void 0;
|
|
473
552
|
}
|
|
474
|
-
async cleanup(id) {
|
|
475
|
-
const
|
|
476
|
-
const query = this.queue(id).where("
|
|
553
|
+
async cleanup(id, olderThan = MicrosecondTimestamp.WEEK.mult(4)) {
|
|
554
|
+
const threshold = MicrosecondTimestamp.now().sub(olderThan);
|
|
555
|
+
const query = this.queue(id).where("hasRemaining", "==", true).where("occurredAt", "<", threshold.serialize()).orderBy("occurredAt", "asc").orderBy("revision", "asc").orderBy("ref", "asc");
|
|
477
556
|
const TRAIL = 1;
|
|
478
557
|
const snapshot = await query.get();
|
|
479
558
|
if (snapshot.size < TRAIL) return;
|
|
@@ -485,12 +564,50 @@ var FirestoreQueueStore = class {
|
|
|
485
564
|
for (const queued of cleaning) batch.delete(queued.ref);
|
|
486
565
|
await batch.commit();
|
|
487
566
|
}
|
|
488
|
-
async flush(id) {
|
|
567
|
+
async flush(id, opts = {}) {
|
|
568
|
+
const exclude = new Set((opts.except ?? []).map((e) => e.serialize()));
|
|
489
569
|
const stream = this.queue(id).stream();
|
|
490
570
|
const writer = this.collection.firestore.bulkWriter();
|
|
491
|
-
for await (const queued of stream)
|
|
571
|
+
for await (const queued of stream) {
|
|
572
|
+
if (exclude.has(queued.id)) continue;
|
|
573
|
+
writer.delete(queued.ref);
|
|
574
|
+
}
|
|
492
575
|
await writer.close();
|
|
493
576
|
}
|
|
577
|
+
async hasUnprocessed(checkpointId) {
|
|
578
|
+
return !(await this.queue(checkpointId).where("processed", "==", false).where("hasRemaining", "==", true).limit(1).get()).empty;
|
|
579
|
+
}
|
|
580
|
+
async block(checkpointId, blockerId = FirestoreQueueStore.BLOCKER_EVENT_ID) {
|
|
581
|
+
const task = new Task({
|
|
582
|
+
id: blockerId,
|
|
583
|
+
ref: Cursor.MAX.ref,
|
|
584
|
+
occurredAt: Cursor.MAX.occurredAt,
|
|
585
|
+
revision: Cursor.MAX.revision,
|
|
586
|
+
attempts: 0,
|
|
587
|
+
processed: true,
|
|
588
|
+
claimer: void 0,
|
|
589
|
+
claimedAt: void 0,
|
|
590
|
+
claimsMetadata: {},
|
|
591
|
+
claimIds: [],
|
|
592
|
+
lock: new Lock({}),
|
|
593
|
+
remaining: 1,
|
|
594
|
+
claimTimeout: 0,
|
|
595
|
+
skipAfter: 0,
|
|
596
|
+
isolateAfter: 0,
|
|
597
|
+
lastUpdateTime: void 0,
|
|
598
|
+
hasRemaining: true
|
|
599
|
+
});
|
|
600
|
+
await this.queued(checkpointId, blockerId).delete();
|
|
601
|
+
const serialized = task.serialize();
|
|
602
|
+
const converted = this.converter.toFirestore(serialized);
|
|
603
|
+
await this.queued(checkpointId, blockerId).create({
|
|
604
|
+
...converted,
|
|
605
|
+
ref: Cursor.MAX.ref
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
async unblock(checkpointId, blockerId = FirestoreQueueStore.BLOCKER_EVENT_ID) {
|
|
609
|
+
await this.queued(checkpointId, blockerId).delete();
|
|
610
|
+
}
|
|
494
611
|
/**
|
|
495
612
|
* This method adds a fake processed event to the queue.
|
|
496
613
|
* It is useful for initializing the tail cursor of a new projection, at the
|
|
@@ -524,7 +641,8 @@ var FirestoreQueueStore = class {
|
|
|
524
641
|
claimTimeout: 0,
|
|
525
642
|
skipAfter: 0,
|
|
526
643
|
isolateAfter: 0,
|
|
527
|
-
lastUpdateTime: void 0
|
|
644
|
+
lastUpdateTime: void 0,
|
|
645
|
+
hasRemaining: true
|
|
528
646
|
});
|
|
529
647
|
try {
|
|
530
648
|
const serialized = task.serialize();
|
|
@@ -553,6 +671,7 @@ var Task = class Task extends Shape({
|
|
|
553
671
|
lock: Lock,
|
|
554
672
|
skipAfter: Number,
|
|
555
673
|
remaining: Number,
|
|
674
|
+
hasRemaining: Boolean,
|
|
556
675
|
isolateAfter: Number,
|
|
557
676
|
claimTimeout: Number,
|
|
558
677
|
lastUpdateTime: Optional(MicrosecondTimestamp)
|
|
@@ -579,17 +698,15 @@ var Task = class Task extends Shape({
|
|
|
579
698
|
skipAfter: config.skipAfter,
|
|
580
699
|
isolateAfter: config.isolateAfter,
|
|
581
700
|
remaining: config.skipAfter,
|
|
701
|
+
hasRemaining: config.skipAfter > 0,
|
|
582
702
|
ref: fact.ref,
|
|
583
703
|
revision: fact.revision,
|
|
584
704
|
occurredAt: fact.occurredAt,
|
|
585
705
|
lastUpdateTime: void 0
|
|
586
706
|
});
|
|
587
707
|
}
|
|
588
|
-
get currentClaimId() {
|
|
589
|
-
return this.claimIds.at(-1);
|
|
590
|
-
}
|
|
591
708
|
get isProcessing() {
|
|
592
|
-
return this.
|
|
709
|
+
return this.claimer !== void 0;
|
|
593
710
|
}
|
|
594
711
|
get isProcessed() {
|
|
595
712
|
return !!this.processed;
|
|
@@ -600,19 +717,19 @@ var Task = class Task extends Shape({
|
|
|
600
717
|
get shouldIsolate() {
|
|
601
718
|
return this.attempts > this.isolateAfter;
|
|
602
719
|
}
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
const claimInfo = this.claimsMetadata[claimer];
|
|
720
|
+
hasTimedOut() {
|
|
721
|
+
if (!this.claimer) return;
|
|
722
|
+
const claimInfo = this.claimsMetadata[this.claimer];
|
|
607
723
|
if (!claimInfo || !claimInfo.claimedAt) return;
|
|
608
|
-
|
|
724
|
+
return MicrosecondTimestamp.now().micros - claimInfo.claimedAt.micros > BigInt(this.claimTimeout) * 1000n;
|
|
609
725
|
}
|
|
610
726
|
static deserializeWithLastUpdateTime(data, timestamp) {
|
|
611
727
|
return Task.deserialize({
|
|
612
728
|
...data,
|
|
613
729
|
lastUpdateTime: timestamp,
|
|
614
730
|
claimIds: data.claimIds || [],
|
|
615
|
-
claimsMetadata: data.claimsMetadata || {}
|
|
731
|
+
claimsMetadata: data.claimsMetadata || {},
|
|
732
|
+
hasRemaining: data.hasRemaining ?? data.remaining > 0
|
|
616
733
|
});
|
|
617
734
|
}
|
|
618
735
|
static batch(tasks) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ddd-ts/event-sourcing-firestore",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.48",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -10,19 +10,19 @@
|
|
|
10
10
|
"dist"
|
|
11
11
|
],
|
|
12
12
|
"dependencies": {
|
|
13
|
-
"@ddd-ts/core": "0.0.
|
|
14
|
-
"@ddd-ts/shape": "0.0.
|
|
15
|
-
"@ddd-ts/store-firestore": "0.0.
|
|
16
|
-
"@ddd-ts/traits": "0.0.
|
|
17
|
-
"@ddd-ts/types": "0.0.
|
|
13
|
+
"@ddd-ts/core": "0.0.48",
|
|
14
|
+
"@ddd-ts/shape": "0.0.48",
|
|
15
|
+
"@ddd-ts/store-firestore": "0.0.48",
|
|
16
|
+
"@ddd-ts/traits": "0.0.48",
|
|
17
|
+
"@ddd-ts/types": "0.0.48",
|
|
18
18
|
"@opentelemetry/api": "1.6.0",
|
|
19
19
|
"firebase-admin": "^13.2.0"
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
22
|
-
"@ddd-ts/shape": "0.0.
|
|
23
|
-
"@ddd-ts/tests": "0.0.
|
|
24
|
-
"@ddd-ts/tools": "0.0.
|
|
25
|
-
"@ddd-ts/types": "0.0.
|
|
22
|
+
"@ddd-ts/shape": "0.0.48",
|
|
23
|
+
"@ddd-ts/tests": "0.0.48",
|
|
24
|
+
"@ddd-ts/tools": "0.0.48",
|
|
25
|
+
"@ddd-ts/types": "0.0.48",
|
|
26
26
|
"@types/jest": "^29.5.1"
|
|
27
27
|
},
|
|
28
28
|
"exports": {
|