@event-driven-io/emmett-sqlite 0.34.0 → 0.35.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +552 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +152 -8
- package/dist/index.d.ts +152 -8
- package/dist/index.js +544 -70
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -6,19 +6,35 @@ var isSQLiteError = (error) => {
|
|
|
6
6
|
}
|
|
7
7
|
return false;
|
|
8
8
|
};
|
|
9
|
+
var InMemorySharedCacheSQLiteDatabase = "file::memory:?cache=shared";
|
|
9
10
|
var InMemorySQLiteDatabase = ":memory:";
|
|
10
11
|
var sqliteConnection = (options) => {
|
|
11
|
-
const
|
|
12
|
+
const fileName = options.fileName ?? InMemorySQLiteDatabase;
|
|
13
|
+
let db;
|
|
14
|
+
if (fileName.startsWith("file:")) {
|
|
15
|
+
db = new sqlite3.Database(
|
|
16
|
+
fileName,
|
|
17
|
+
sqlite3.OPEN_URI | sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE
|
|
18
|
+
);
|
|
19
|
+
} else {
|
|
20
|
+
db = new sqlite3.Database(fileName);
|
|
21
|
+
}
|
|
22
|
+
db.run("PRAGMA journal_mode = WAL;");
|
|
23
|
+
let transactionNesting = 0;
|
|
12
24
|
return {
|
|
13
25
|
close: () => db.close(),
|
|
14
26
|
command: (sql2, params) => new Promise((resolve, reject) => {
|
|
15
|
-
db.run(
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
27
|
+
db.run(
|
|
28
|
+
sql2,
|
|
29
|
+
params ?? [],
|
|
30
|
+
function(err) {
|
|
31
|
+
if (err) {
|
|
32
|
+
reject(err);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
resolve(this);
|
|
19
36
|
}
|
|
20
|
-
|
|
21
|
-
});
|
|
37
|
+
);
|
|
22
38
|
}),
|
|
23
39
|
query: (sql2, params) => new Promise((resolve, reject) => {
|
|
24
40
|
db.all(sql2, params ?? [], (err, result) => {
|
|
@@ -37,11 +53,53 @@ var sqliteConnection = (options) => {
|
|
|
37
53
|
}
|
|
38
54
|
resolve(result);
|
|
39
55
|
});
|
|
40
|
-
})
|
|
56
|
+
}),
|
|
57
|
+
withTransaction: async (fn) => {
|
|
58
|
+
try {
|
|
59
|
+
if (transactionNesting++ == 0) {
|
|
60
|
+
await beginTransaction(db);
|
|
61
|
+
}
|
|
62
|
+
const result = await fn();
|
|
63
|
+
if (transactionNesting === 1) await commitTransaction(db);
|
|
64
|
+
transactionNesting--;
|
|
65
|
+
return result;
|
|
66
|
+
} catch (err) {
|
|
67
|
+
console.log(err);
|
|
68
|
+
if (--transactionNesting === 0) await rollbackTransaction(db);
|
|
69
|
+
throw err;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
41
72
|
};
|
|
42
73
|
};
|
|
74
|
+
var beginTransaction = (db) => new Promise((resolve, reject) => {
|
|
75
|
+
db.run("BEGIN IMMEDIATE TRANSACTION", (err) => {
|
|
76
|
+
if (err) {
|
|
77
|
+
reject(err);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
resolve();
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
var commitTransaction = (db) => new Promise((resolve, reject) => {
|
|
84
|
+
db.run("COMMIT", (err) => {
|
|
85
|
+
if (err) {
|
|
86
|
+
reject(err);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
resolve();
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
var rollbackTransaction = (db) => new Promise((resolve, reject) => {
|
|
93
|
+
db.run("ROLLBACK", (err) => {
|
|
94
|
+
if (err) {
|
|
95
|
+
reject(err);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
resolve();
|
|
99
|
+
});
|
|
100
|
+
});
|
|
43
101
|
|
|
44
|
-
// ../emmett/dist/chunk-
|
|
102
|
+
// ../emmett/dist/chunk-BQYVGGNE.js
|
|
45
103
|
var isNumber = (val) => typeof val === "number" && val === val;
|
|
46
104
|
var isString = (val) => typeof val === "string";
|
|
47
105
|
var EmmettError = class _EmmettError extends Error {
|
|
@@ -73,6 +131,7 @@ import { v4 as uuid2 } from "uuid";
|
|
|
73
131
|
import { v4 as uuid } from "uuid";
|
|
74
132
|
import { TransformStream as TransformStream2 } from "web-streams-polyfill";
|
|
75
133
|
import retry from "async-retry";
|
|
134
|
+
import { v7 as uuid4 } from "uuid";
|
|
76
135
|
import { ReadableStream } from "web-streams-polyfill";
|
|
77
136
|
import "web-streams-polyfill";
|
|
78
137
|
import { TransformStream as TransformStream3 } from "web-streams-polyfill";
|
|
@@ -324,7 +383,7 @@ var streamTransformations = {
|
|
|
324
383
|
var { retry: retry2 } = streamTransformations;
|
|
325
384
|
|
|
326
385
|
// src/eventStore/schema/appendToStream.ts
|
|
327
|
-
import { v4 as
|
|
386
|
+
import { v4 as uuid5 } from "uuid";
|
|
328
387
|
|
|
329
388
|
// src/eventStore/schema/typing.ts
|
|
330
389
|
var emmettPrefix = "emt";
|
|
@@ -353,6 +412,9 @@ var messagesTable = {
|
|
|
353
412
|
isArchived: columns.isArchived
|
|
354
413
|
}
|
|
355
414
|
};
|
|
415
|
+
var subscriptionsTable = {
|
|
416
|
+
name: `${emmettPrefix}_subscriptions`
|
|
417
|
+
};
|
|
356
418
|
|
|
357
419
|
// src/eventStore/schema/appendToStream.ts
|
|
358
420
|
var appendToStream = async (db, streamName, streamType, messages, options) => {
|
|
@@ -366,15 +428,14 @@ var appendToStream = async (db, streamName, streamType, messages, options) => {
|
|
|
366
428
|
kind: m.kind ?? "Event",
|
|
367
429
|
metadata: {
|
|
368
430
|
streamName,
|
|
369
|
-
messageId:
|
|
431
|
+
messageId: uuid5(),
|
|
370
432
|
streamPosition: BigInt(i + 1),
|
|
371
433
|
..."metadata" in m ? m.metadata ?? {} : {}
|
|
372
434
|
}
|
|
373
435
|
})
|
|
374
436
|
);
|
|
375
437
|
let result;
|
|
376
|
-
await db.
|
|
377
|
-
try {
|
|
438
|
+
return await db.withTransaction(async () => {
|
|
378
439
|
result = await appendToStreamRaw(
|
|
379
440
|
db,
|
|
380
441
|
streamName,
|
|
@@ -385,17 +446,9 @@ var appendToStream = async (db, streamName, streamType, messages, options) => {
|
|
|
385
446
|
}
|
|
386
447
|
);
|
|
387
448
|
if (options?.onBeforeCommit)
|
|
388
|
-
await options.onBeforeCommit(messagesToAppend, {
|
|
389
|
-
} catch (err) {
|
|
390
|
-
await db.command(`ROLLBACK`);
|
|
391
|
-
throw err;
|
|
392
|
-
}
|
|
393
|
-
if (result.success == null || !result.success) {
|
|
394
|
-
await db.command(`ROLLBACK`);
|
|
449
|
+
await options.onBeforeCommit(messagesToAppend, { db });
|
|
395
450
|
return result;
|
|
396
|
-
}
|
|
397
|
-
await db.command(`COMMIT`);
|
|
398
|
-
return result;
|
|
451
|
+
});
|
|
399
452
|
};
|
|
400
453
|
var toExpectedVersion = (expected) => {
|
|
401
454
|
if (expected === void 0) return null;
|
|
@@ -472,11 +525,13 @@ var appendToStreamRaw = async (db, streamId, streamType, messages, options) => {
|
|
|
472
525
|
streamId,
|
|
473
526
|
options?.partition?.toString() ?? defaultTag
|
|
474
527
|
);
|
|
475
|
-
const
|
|
476
|
-
if (
|
|
528
|
+
const returningIds = await db.query(sqlString, values);
|
|
529
|
+
if (returningIds.length === 0 || !returningIds[returningIds.length - 1]?.global_position) {
|
|
477
530
|
throw new Error("Could not find global position");
|
|
478
531
|
}
|
|
479
|
-
globalPosition = BigInt(
|
|
532
|
+
globalPosition = BigInt(
|
|
533
|
+
returningIds[returningIds.length - 1].global_position
|
|
534
|
+
);
|
|
480
535
|
} catch (err) {
|
|
481
536
|
if (isSQLiteError(err) && isOptimisticConcurrencyError(err)) {
|
|
482
537
|
return {
|
|
@@ -553,16 +608,383 @@ var buildMessageInsertQuery = (messages, expectedStreamVersion, streamId, partit
|
|
|
553
608
|
return { sqlString, values: query.values };
|
|
554
609
|
};
|
|
555
610
|
|
|
611
|
+
// src/eventStore/schema/tables.ts
|
|
612
|
+
var sql = (sql2) => sql2;
|
|
613
|
+
var streamsTableSQL = sql(
|
|
614
|
+
`CREATE TABLE IF NOT EXISTS ${streamsTable.name}(
|
|
615
|
+
stream_id TEXT NOT NULL,
|
|
616
|
+
stream_position BIGINT NOT NULL DEFAULT 0,
|
|
617
|
+
partition TEXT NOT NULL DEFAULT '${globalTag}',
|
|
618
|
+
stream_type TEXT NOT NULL,
|
|
619
|
+
stream_metadata JSONB NOT NULL,
|
|
620
|
+
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
621
|
+
PRIMARY KEY (stream_id, stream_position, partition, is_archived),
|
|
622
|
+
UNIQUE (stream_id, partition, is_archived)
|
|
623
|
+
);`
|
|
624
|
+
);
|
|
625
|
+
var messagesTableSQL = sql(
|
|
626
|
+
`CREATE TABLE IF NOT EXISTS ${messagesTable.name}(
|
|
627
|
+
stream_id TEXT NOT NULL,
|
|
628
|
+
stream_position BIGINT NOT NULL,
|
|
629
|
+
partition TEXT NOT NULL DEFAULT '${globalTag}',
|
|
630
|
+
message_kind CHAR(1) NOT NULL DEFAULT 'E',
|
|
631
|
+
message_data JSONB NOT NULL,
|
|
632
|
+
message_metadata JSONB NOT NULL,
|
|
633
|
+
message_schema_version TEXT NOT NULL,
|
|
634
|
+
message_type TEXT NOT NULL,
|
|
635
|
+
message_id TEXT NOT NULL,
|
|
636
|
+
is_archived BOOLEAN NOT NULL DEFAULT FALSE,
|
|
637
|
+
global_position INTEGER PRIMARY KEY,
|
|
638
|
+
created DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
639
|
+
UNIQUE (stream_id, stream_position, partition, is_archived)
|
|
640
|
+
);
|
|
641
|
+
`
|
|
642
|
+
);
|
|
643
|
+
var subscriptionsTableSQL = sql(
|
|
644
|
+
`
|
|
645
|
+
CREATE TABLE IF NOT EXISTS ${subscriptionsTable.name}(
|
|
646
|
+
subscription_id TEXT NOT NULL,
|
|
647
|
+
version INTEGER NOT NULL DEFAULT 1,
|
|
648
|
+
partition TEXT NOT NULL DEFAULT '${globalTag}',
|
|
649
|
+
last_processed_position BIGINT NOT NULL,
|
|
650
|
+
PRIMARY KEY (subscription_id, partition, version)
|
|
651
|
+
);
|
|
652
|
+
`
|
|
653
|
+
);
|
|
654
|
+
var schemaSQL = [
|
|
655
|
+
streamsTableSQL,
|
|
656
|
+
messagesTableSQL,
|
|
657
|
+
subscriptionsTableSQL
|
|
658
|
+
];
|
|
659
|
+
var createEventStoreSchema = async (db) => {
|
|
660
|
+
for (const sql2 of schemaSQL) {
|
|
661
|
+
await db.command(sql2);
|
|
662
|
+
}
|
|
663
|
+
};
|
|
664
|
+
|
|
665
|
+
// src/eventStore/schema/utils.ts
|
|
666
|
+
var singleOrNull = async (getResult) => {
|
|
667
|
+
const result = await getResult;
|
|
668
|
+
if (result.length > 1) throw new Error("Query had more than one result");
|
|
669
|
+
return result.length > 0 ? result[0] ?? null : null;
|
|
670
|
+
};
|
|
671
|
+
|
|
672
|
+
// src/eventStore/schema/readLastMessageGlobalPosition.ts
|
|
673
|
+
var readLastMessageGlobalPosition = async (db, options) => {
|
|
674
|
+
const result = await singleOrNull(
|
|
675
|
+
db.query(
|
|
676
|
+
sql(
|
|
677
|
+
`SELECT global_position
|
|
678
|
+
FROM ${messagesTable.name}
|
|
679
|
+
WHERE partition = ? AND is_archived = FALSE
|
|
680
|
+
ORDER BY global_position
|
|
681
|
+
LIMIT 1`
|
|
682
|
+
),
|
|
683
|
+
[options?.partition ?? defaultTag]
|
|
684
|
+
)
|
|
685
|
+
);
|
|
686
|
+
return {
|
|
687
|
+
currentGlobalPosition: result !== null ? BigInt(result.global_position) : null
|
|
688
|
+
};
|
|
689
|
+
};
|
|
690
|
+
|
|
691
|
+
// src/eventStore/schema/readMessagesBatch.ts
|
|
692
|
+
var readMessagesBatch = async (db, options) => {
|
|
693
|
+
const from = "from" in options ? options.from : "after" in options ? options.after + 1n : 0n;
|
|
694
|
+
const batchSize = options && "batchSize" in options ? options.batchSize : options.to - options.from;
|
|
695
|
+
const fromCondition = from !== -0n ? `AND global_position >= ${from}` : "";
|
|
696
|
+
const toCondition = "to" in options ? `AND global_position <= ${options.to}` : "";
|
|
697
|
+
const limitCondition = "batchSize" in options ? `LIMIT ${options.batchSize}` : "";
|
|
698
|
+
const events = (await db.query(
|
|
699
|
+
sql(
|
|
700
|
+
`SELECT stream_id, stream_position, global_position, message_data, message_metadata, message_schema_version, message_type, message_id
|
|
701
|
+
FROM ${messagesTable.name}
|
|
702
|
+
WHERE partition = ? AND is_archived = FALSE ${fromCondition} ${toCondition}
|
|
703
|
+
ORDER BY global_position
|
|
704
|
+
${limitCondition}`
|
|
705
|
+
),
|
|
706
|
+
[options?.partition ?? defaultTag]
|
|
707
|
+
)).map((row) => {
|
|
708
|
+
const rawEvent = {
|
|
709
|
+
type: row.message_type,
|
|
710
|
+
data: JSONParser.parse(row.message_data),
|
|
711
|
+
metadata: JSONParser.parse(row.message_metadata)
|
|
712
|
+
};
|
|
713
|
+
const metadata = {
|
|
714
|
+
..."metadata" in rawEvent ? rawEvent.metadata ?? {} : {},
|
|
715
|
+
messageId: row.message_id,
|
|
716
|
+
streamName: row.stream_id,
|
|
717
|
+
streamPosition: BigInt(row.stream_position),
|
|
718
|
+
globalPosition: BigInt(row.global_position)
|
|
719
|
+
};
|
|
720
|
+
return {
|
|
721
|
+
...rawEvent,
|
|
722
|
+
kind: "Event",
|
|
723
|
+
metadata
|
|
724
|
+
};
|
|
725
|
+
});
|
|
726
|
+
return events.length > 0 ? {
|
|
727
|
+
currentGlobalPosition: events[events.length - 1].metadata.globalPosition,
|
|
728
|
+
messages: events,
|
|
729
|
+
areEventsLeft: events.length === batchSize
|
|
730
|
+
} : {
|
|
731
|
+
currentGlobalPosition: "from" in options ? options.from : "after" in options ? options.after : 0n,
|
|
732
|
+
messages: [],
|
|
733
|
+
areEventsLeft: false
|
|
734
|
+
};
|
|
735
|
+
};
|
|
736
|
+
|
|
737
|
+
// src/eventStore/schema/readProcessorCheckpoint.ts
|
|
738
|
+
var readProcessorCheckpoint = async (db, options) => {
|
|
739
|
+
const result = await singleOrNull(
|
|
740
|
+
db.query(
|
|
741
|
+
sql(
|
|
742
|
+
`SELECT last_processed_position
|
|
743
|
+
FROM ${subscriptionsTable.name}
|
|
744
|
+
WHERE partition = ? AND subscription_id = ?
|
|
745
|
+
LIMIT 1`
|
|
746
|
+
),
|
|
747
|
+
[options?.partition ?? defaultTag, options.processorId]
|
|
748
|
+
)
|
|
749
|
+
);
|
|
750
|
+
return {
|
|
751
|
+
lastProcessedPosition: result !== null ? BigInt(result.last_processed_position) : null
|
|
752
|
+
};
|
|
753
|
+
};
|
|
754
|
+
|
|
755
|
+
// src/eventStore/consumers/messageBatchProcessing/index.ts
|
|
756
|
+
var DefaultSQLiteEventStoreProcessorBatchSize = 100;
|
|
757
|
+
var DefaultSQLiteEventStoreProcessorPullingFrequencyInMs = 50;
|
|
758
|
+
var sqliteEventStoreMessageBatchPuller = ({
|
|
759
|
+
db,
|
|
760
|
+
batchSize,
|
|
761
|
+
eachBatch,
|
|
762
|
+
pullingFrequencyInMs
|
|
763
|
+
}) => {
|
|
764
|
+
let isRunning = false;
|
|
765
|
+
let start;
|
|
766
|
+
const pullMessages = async (options) => {
|
|
767
|
+
const after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(db)).currentGlobalPosition ?? 0n : options.startFrom.globalPosition;
|
|
768
|
+
const readMessagesOptions = {
|
|
769
|
+
after,
|
|
770
|
+
batchSize
|
|
771
|
+
};
|
|
772
|
+
let waitTime = 100;
|
|
773
|
+
do {
|
|
774
|
+
const { messages, currentGlobalPosition, areEventsLeft } = await readMessagesBatch(db, readMessagesOptions);
|
|
775
|
+
if (messages.length > 0) {
|
|
776
|
+
const result = await eachBatch({ messages });
|
|
777
|
+
if (result && result.type === "STOP") {
|
|
778
|
+
isRunning = false;
|
|
779
|
+
break;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
readMessagesOptions.after = currentGlobalPosition;
|
|
783
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
784
|
+
if (!areEventsLeft) {
|
|
785
|
+
waitTime = Math.min(waitTime * 2, 1e3);
|
|
786
|
+
} else {
|
|
787
|
+
waitTime = pullingFrequencyInMs;
|
|
788
|
+
}
|
|
789
|
+
} while (isRunning);
|
|
790
|
+
};
|
|
791
|
+
return {
|
|
792
|
+
get isRunning() {
|
|
793
|
+
return isRunning;
|
|
794
|
+
},
|
|
795
|
+
start: (options) => {
|
|
796
|
+
if (isRunning) return start;
|
|
797
|
+
start = (async () => {
|
|
798
|
+
isRunning = true;
|
|
799
|
+
return pullMessages(options);
|
|
800
|
+
})();
|
|
801
|
+
return start;
|
|
802
|
+
},
|
|
803
|
+
stop: async () => {
|
|
804
|
+
if (!isRunning) return;
|
|
805
|
+
isRunning = false;
|
|
806
|
+
await start;
|
|
807
|
+
}
|
|
808
|
+
};
|
|
809
|
+
};
|
|
810
|
+
var zipSQLiteEventStoreMessageBatchPullerStartFrom = (options) => {
|
|
811
|
+
if (options.length === 0 || options.some((o) => o === void 0 || o === "BEGINNING"))
|
|
812
|
+
return "BEGINNING";
|
|
813
|
+
if (options.every((o) => o === "END")) return "END";
|
|
814
|
+
return options.filter((o) => o !== void 0 && o !== "BEGINNING" && o !== "END").sort((a, b) => a > b ? 1 : -1)[0];
|
|
815
|
+
};
|
|
816
|
+
|
|
817
|
+
// src/eventStore/consumers/sqliteProcessor.ts
|
|
818
|
+
var genericSQLiteProcessor = (options) => {
|
|
819
|
+
const { eachMessage } = options;
|
|
820
|
+
let isActive = true;
|
|
821
|
+
const getDb = (context) => {
|
|
822
|
+
const fileName = context.fileName ?? options.connectionOptions?.fileName;
|
|
823
|
+
if (!fileName)
|
|
824
|
+
throw new EmmettError(
|
|
825
|
+
`SQLite processor '${options.processorId}' is missing file name. Ensure that you passed it through options`
|
|
826
|
+
);
|
|
827
|
+
const db = context.db ?? options.connectionOptions?.db ?? sqliteConnection({ fileName });
|
|
828
|
+
return { db, fileName };
|
|
829
|
+
};
|
|
830
|
+
return {
|
|
831
|
+
id: options.processorId,
|
|
832
|
+
start: async (db) => {
|
|
833
|
+
isActive = true;
|
|
834
|
+
if (options.startFrom !== "CURRENT") return options.startFrom;
|
|
835
|
+
const { lastProcessedPosition } = await readProcessorCheckpoint(db, {
|
|
836
|
+
processorId: options.processorId,
|
|
837
|
+
partition: options.partition
|
|
838
|
+
});
|
|
839
|
+
if (lastProcessedPosition === null) return "BEGINNING";
|
|
840
|
+
return { globalPosition: lastProcessedPosition };
|
|
841
|
+
},
|
|
842
|
+
get isActive() {
|
|
843
|
+
return isActive;
|
|
844
|
+
},
|
|
845
|
+
handle: async ({ messages }, context) => {
|
|
846
|
+
if (!isActive) return;
|
|
847
|
+
const { db, fileName } = getDb(context);
|
|
848
|
+
return db.withTransaction(async () => {
|
|
849
|
+
let result = void 0;
|
|
850
|
+
let lastProcessedPosition = null;
|
|
851
|
+
for (const message of messages) {
|
|
852
|
+
const typedMessage = message;
|
|
853
|
+
const messageProcessingResult = await eachMessage(typedMessage, {
|
|
854
|
+
db,
|
|
855
|
+
fileName
|
|
856
|
+
});
|
|
857
|
+
await storeProcessorCheckpoint(db, {
|
|
858
|
+
processorId: options.processorId,
|
|
859
|
+
version: options.version,
|
|
860
|
+
lastProcessedPosition,
|
|
861
|
+
newPosition: typedMessage.metadata.globalPosition,
|
|
862
|
+
partition: options.partition
|
|
863
|
+
});
|
|
864
|
+
lastProcessedPosition = typedMessage.metadata.globalPosition;
|
|
865
|
+
if (messageProcessingResult && messageProcessingResult.type === "STOP") {
|
|
866
|
+
isActive = false;
|
|
867
|
+
result = messageProcessingResult;
|
|
868
|
+
break;
|
|
869
|
+
}
|
|
870
|
+
if (options.stopAfter && options.stopAfter(typedMessage)) {
|
|
871
|
+
isActive = false;
|
|
872
|
+
result = { type: "STOP", reason: "Stop condition reached" };
|
|
873
|
+
break;
|
|
874
|
+
}
|
|
875
|
+
if (messageProcessingResult && messageProcessingResult.type === "SKIP")
|
|
876
|
+
continue;
|
|
877
|
+
}
|
|
878
|
+
return result;
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
};
|
|
882
|
+
};
|
|
883
|
+
var sqliteProjectionProcessor = (options) => {
|
|
884
|
+
const projection2 = options.projection;
|
|
885
|
+
return genericSQLiteProcessor({
|
|
886
|
+
processorId: options.processorId ?? `projection:${projection2.name}`,
|
|
887
|
+
eachMessage: async (event, context) => {
|
|
888
|
+
if (!projection2.canHandle.includes(event.type)) return;
|
|
889
|
+
await projection2.handle([event], context);
|
|
890
|
+
},
|
|
891
|
+
...options
|
|
892
|
+
});
|
|
893
|
+
};
|
|
894
|
+
var sqliteProcessor = (options) => {
|
|
895
|
+
if ("projection" in options) {
|
|
896
|
+
return sqliteProjectionProcessor(options);
|
|
897
|
+
}
|
|
898
|
+
return genericSQLiteProcessor(options);
|
|
899
|
+
};
|
|
900
|
+
|
|
901
|
+
// src/eventStore/consumers/sqliteEventStoreConsumer.ts
|
|
902
|
+
var sqliteEventStoreConsumer = (options) => {
|
|
903
|
+
let isRunning = false;
|
|
904
|
+
const { pulling } = options;
|
|
905
|
+
const processors = options.processors ?? [];
|
|
906
|
+
let start;
|
|
907
|
+
let currentMessagePuller;
|
|
908
|
+
const db = options.db ?? sqliteConnection({ fileName: options.fileName });
|
|
909
|
+
const eachBatch = async (messagesBatch) => {
|
|
910
|
+
const activeProcessors = processors.filter((s) => s.isActive);
|
|
911
|
+
if (activeProcessors.length === 0)
|
|
912
|
+
return {
|
|
913
|
+
type: "STOP",
|
|
914
|
+
reason: "No active processors"
|
|
915
|
+
};
|
|
916
|
+
const result = await Promise.allSettled(
|
|
917
|
+
activeProcessors.map((s) => {
|
|
918
|
+
return s.handle(messagesBatch, { db, fileName: options.fileName });
|
|
919
|
+
})
|
|
920
|
+
);
|
|
921
|
+
return result.some(
|
|
922
|
+
(r) => r.status === "fulfilled" && r.value?.type !== "STOP"
|
|
923
|
+
) ? void 0 : {
|
|
924
|
+
type: "STOP"
|
|
925
|
+
};
|
|
926
|
+
};
|
|
927
|
+
const messagePooler = currentMessagePuller = sqliteEventStoreMessageBatchPuller({
|
|
928
|
+
db,
|
|
929
|
+
eachBatch,
|
|
930
|
+
batchSize: pulling?.batchSize ?? DefaultSQLiteEventStoreProcessorBatchSize,
|
|
931
|
+
pullingFrequencyInMs: pulling?.pullingFrequencyInMs ?? DefaultSQLiteEventStoreProcessorPullingFrequencyInMs
|
|
932
|
+
});
|
|
933
|
+
const stop = async () => {
|
|
934
|
+
if (!isRunning) return;
|
|
935
|
+
isRunning = false;
|
|
936
|
+
if (currentMessagePuller) {
|
|
937
|
+
await currentMessagePuller.stop();
|
|
938
|
+
currentMessagePuller = void 0;
|
|
939
|
+
}
|
|
940
|
+
await start;
|
|
941
|
+
};
|
|
942
|
+
return {
|
|
943
|
+
processors,
|
|
944
|
+
get isRunning() {
|
|
945
|
+
return isRunning;
|
|
946
|
+
},
|
|
947
|
+
processor: (options2) => {
|
|
948
|
+
const processor = sqliteProcessor(options2);
|
|
949
|
+
processors.push(processor);
|
|
950
|
+
return processor;
|
|
951
|
+
},
|
|
952
|
+
start: () => {
|
|
953
|
+
if (isRunning) return start;
|
|
954
|
+
start = (async () => {
|
|
955
|
+
if (processors.length === 0)
|
|
956
|
+
return Promise.reject(
|
|
957
|
+
new EmmettError(
|
|
958
|
+
"Cannot start consumer without at least a single processor"
|
|
959
|
+
)
|
|
960
|
+
);
|
|
961
|
+
isRunning = true;
|
|
962
|
+
const startFrom = zipSQLiteEventStoreMessageBatchPullerStartFrom(
|
|
963
|
+
await Promise.all(processors.map((o) => o.start(db)))
|
|
964
|
+
);
|
|
965
|
+
return messagePooler.start({ startFrom });
|
|
966
|
+
})();
|
|
967
|
+
return start;
|
|
968
|
+
},
|
|
969
|
+
stop,
|
|
970
|
+
close: async () => {
|
|
971
|
+
await stop();
|
|
972
|
+
db.close();
|
|
973
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
974
|
+
}
|
|
975
|
+
};
|
|
976
|
+
};
|
|
977
|
+
|
|
556
978
|
// src/eventStore/projections/index.ts
|
|
557
979
|
var handleProjections = async (options) => {
|
|
558
|
-
const { projections: allProjections, events,
|
|
980
|
+
const { projections: allProjections, events, db } = options;
|
|
559
981
|
const eventTypes = events.map((e) => e.type);
|
|
560
982
|
const projections = allProjections.filter(
|
|
561
983
|
(p) => p.canHandle.some((type) => eventTypes.includes(type))
|
|
562
984
|
);
|
|
563
985
|
for (const projection2 of projections) {
|
|
564
986
|
await projection2.handle(events, {
|
|
565
|
-
|
|
987
|
+
db
|
|
566
988
|
});
|
|
567
989
|
}
|
|
568
990
|
};
|
|
@@ -574,7 +996,7 @@ var getSQLiteEventStore = (options) => {
|
|
|
574
996
|
let autoGenerateSchema = false;
|
|
575
997
|
let database;
|
|
576
998
|
const fileName = options.fileName ?? InMemorySQLiteDatabase;
|
|
577
|
-
const isInMemory = fileName === InMemorySQLiteDatabase;
|
|
999
|
+
const isInMemory = fileName === InMemorySQLiteDatabase || fileName === InMemorySharedCacheSQLiteDatabase;
|
|
578
1000
|
const inlineProjections = (options.projections ?? []).filter(({ type }) => type === "inline").map(({ projection: projection2 }) => projection2);
|
|
579
1001
|
const onBeforeCommitHook = options.hooks?.onBeforeCommit;
|
|
580
1002
|
const createConnection = () => {
|
|
@@ -608,10 +1030,10 @@ var getSQLiteEventStore = (options) => {
|
|
|
608
1030
|
if (options) {
|
|
609
1031
|
autoGenerateSchema = options.schema?.autoMigration === void 0 || options.schema?.autoMigration !== "None";
|
|
610
1032
|
}
|
|
611
|
-
const ensureSchemaExists = async (
|
|
1033
|
+
const ensureSchemaExists = async (db) => {
|
|
612
1034
|
if (!autoGenerateSchema) return Promise.resolve();
|
|
613
1035
|
if (!schemaMigrated) {
|
|
614
|
-
await createEventStoreSchema(
|
|
1036
|
+
await createEventStoreSchema(db);
|
|
615
1037
|
schemaMigrated = true;
|
|
616
1038
|
}
|
|
617
1039
|
return Promise.resolve();
|
|
@@ -678,7 +1100,12 @@ var getSQLiteEventStore = (options) => {
|
|
|
678
1100
|
lastEventGlobalPosition: appendResult.lastGlobalPosition,
|
|
679
1101
|
createdNewStream: appendResult.nextStreamPosition >= BigInt(events.length)
|
|
680
1102
|
};
|
|
681
|
-
}
|
|
1103
|
+
},
|
|
1104
|
+
consumer: (options2) => sqliteEventStoreConsumer({
|
|
1105
|
+
...options2 ?? {},
|
|
1106
|
+
fileName,
|
|
1107
|
+
db: database ?? void 0
|
|
1108
|
+
})
|
|
682
1109
|
};
|
|
683
1110
|
};
|
|
684
1111
|
|
|
@@ -725,46 +1152,87 @@ var readStream = async (db, streamId, options) => {
|
|
|
725
1152
|
};
|
|
726
1153
|
};
|
|
727
1154
|
|
|
728
|
-
// src/eventStore/schema/
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
)
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
1155
|
+
// src/eventStore/schema/storeProcessorCheckpoint.ts
|
|
1156
|
+
async function storeSubscriptionCheckpointSQLite(db, processorId, version, position, checkPosition, partition) {
|
|
1157
|
+
if (checkPosition !== null) {
|
|
1158
|
+
const updateResult = await db.command(
|
|
1159
|
+
sql(`
|
|
1160
|
+
UPDATE ${subscriptionsTable.name}
|
|
1161
|
+
SET last_processed_position = ?
|
|
1162
|
+
WHERE subscription_id = ?
|
|
1163
|
+
AND last_processed_position = ?
|
|
1164
|
+
AND partition = ?
|
|
1165
|
+
`),
|
|
1166
|
+
[position.toString(), processorId, checkPosition.toString(), partition]
|
|
1167
|
+
);
|
|
1168
|
+
if (updateResult.changes > 0) {
|
|
1169
|
+
return 1;
|
|
1170
|
+
} else {
|
|
1171
|
+
const current_position = await singleOrNull(
|
|
1172
|
+
db.query(
|
|
1173
|
+
sql(
|
|
1174
|
+
`SELECT last_processed_position FROM ${subscriptionsTable.name}
|
|
1175
|
+
WHERE subscription_id = ? AND partition = ?`
|
|
1176
|
+
),
|
|
1177
|
+
[processorId, partition]
|
|
1178
|
+
)
|
|
1179
|
+
);
|
|
1180
|
+
if (current_position?.last_processed_position === position) {
|
|
1181
|
+
return 0;
|
|
1182
|
+
} else if (position !== null && current_position !== null && current_position?.last_processed_position > position) {
|
|
1183
|
+
return 2;
|
|
1184
|
+
} else {
|
|
1185
|
+
return 2;
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
} else {
|
|
1189
|
+
try {
|
|
1190
|
+
await db.command(
|
|
1191
|
+
sql(
|
|
1192
|
+
`INSERT INTO ${subscriptionsTable.name} (subscription_id, version, last_processed_position, partition) VALUES (?, ?, ?, ?)`
|
|
1193
|
+
),
|
|
1194
|
+
[processorId, version, position.toString(), partition]
|
|
1195
|
+
);
|
|
1196
|
+
return 1;
|
|
1197
|
+
} catch (err) {
|
|
1198
|
+
if (!(isSQLiteError(err) && (err.errno === 19 || err.errno === 2067))) {
|
|
1199
|
+
throw err;
|
|
1200
|
+
}
|
|
1201
|
+
const current = await singleOrNull(
|
|
1202
|
+
db.query(
|
|
1203
|
+
sql(
|
|
1204
|
+
`SELECT last_processed_position FROM ${subscriptionsTable.name} WHERE subscription_id = ? AND partition = ?`
|
|
1205
|
+
),
|
|
1206
|
+
[processorId, partition]
|
|
1207
|
+
)
|
|
1208
|
+
);
|
|
1209
|
+
if (current?.last_processed_position === position) {
|
|
1210
|
+
return 0;
|
|
1211
|
+
} else {
|
|
1212
|
+
return 2;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
764
1215
|
}
|
|
765
|
-
}
|
|
1216
|
+
}
|
|
1217
|
+
async function storeProcessorCheckpoint(db, options) {
|
|
1218
|
+
try {
|
|
1219
|
+
const result = await storeSubscriptionCheckpointSQLite(
|
|
1220
|
+
db,
|
|
1221
|
+
options.processorId,
|
|
1222
|
+
options.version ?? 1,
|
|
1223
|
+
options.newPosition,
|
|
1224
|
+
options.lastProcessedPosition,
|
|
1225
|
+
options.partition ?? defaultTag
|
|
1226
|
+
);
|
|
1227
|
+
return result === 1 ? { success: true, newPosition: options.newPosition } : { success: false, reason: result === 0 ? "IGNORED" : "MISMATCH" };
|
|
1228
|
+
} catch (error) {
|
|
1229
|
+
console.log(error);
|
|
1230
|
+
throw error;
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
766
1233
|
export {
|
|
767
1234
|
InMemorySQLiteDatabase,
|
|
1235
|
+
InMemorySharedCacheSQLiteDatabase,
|
|
768
1236
|
SQLiteEventStoreDefaultStreamVersion,
|
|
769
1237
|
appendToStream,
|
|
770
1238
|
createEventStoreSchema,
|
|
@@ -776,11 +1244,17 @@ export {
|
|
|
776
1244
|
isSQLiteError,
|
|
777
1245
|
messagesTable,
|
|
778
1246
|
messagesTableSQL,
|
|
1247
|
+
readLastMessageGlobalPosition,
|
|
1248
|
+
readMessagesBatch,
|
|
1249
|
+
readProcessorCheckpoint,
|
|
779
1250
|
readStream,
|
|
780
1251
|
schemaSQL,
|
|
781
1252
|
sql,
|
|
782
1253
|
sqliteConnection,
|
|
1254
|
+
storeProcessorCheckpoint,
|
|
783
1255
|
streamsTable,
|
|
784
|
-
streamsTableSQL
|
|
1256
|
+
streamsTableSQL,
|
|
1257
|
+
subscriptionsTable,
|
|
1258
|
+
subscriptionsTableSQL
|
|
785
1259
|
};
|
|
786
1260
|
//# sourceMappingURL=index.js.map
|