@event-driven-io/emmett-postgresql 0.42.0-rc.2 → 0.42.1-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +354 -234
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +11 -4
- package/dist/index.d.ts +11 -4
- package/dist/index.js +313 -193
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -657,19 +657,39 @@ var reactor = (options) => {
|
|
|
657
657
|
type,
|
|
658
658
|
init,
|
|
659
659
|
start: async (startOptions) => {
|
|
660
|
-
if (isActive)
|
|
660
|
+
if (isActive) {
|
|
661
|
+
console.log(
|
|
662
|
+
`Processor ${processorId} with instance id ${instanceId} is already active. Start request ignored.`
|
|
663
|
+
);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
console.log(
|
|
667
|
+
`Starting processor ${processorId} with instance id ${instanceId}`
|
|
668
|
+
);
|
|
661
669
|
await init(startOptions);
|
|
662
670
|
isActive = true;
|
|
663
671
|
closeSignal = onShutdown(() => close({}));
|
|
664
|
-
if (lastCheckpoint !== null)
|
|
672
|
+
if (lastCheckpoint !== null) {
|
|
673
|
+
console.log(
|
|
674
|
+
`Processor ${processorId} started with instance id ${instanceId}, checkpoint: ${JSONParser.stringify(lastCheckpoint)}`
|
|
675
|
+
);
|
|
665
676
|
return {
|
|
666
677
|
lastCheckpoint
|
|
667
678
|
};
|
|
679
|
+
}
|
|
668
680
|
return await processingScope(async (context) => {
|
|
669
681
|
if (hooks.onStart) {
|
|
682
|
+
console.log(
|
|
683
|
+
`Executing onStart hook for processor ${processorId} with instance id ${instanceId}`
|
|
684
|
+
);
|
|
670
685
|
await hooks.onStart(context);
|
|
671
686
|
}
|
|
672
|
-
if (startFrom && startFrom !== "CURRENT")
|
|
687
|
+
if (startFrom && startFrom !== "CURRENT") {
|
|
688
|
+
console.log(
|
|
689
|
+
`Processor ${processorId} with instance id ${instanceId} starting from: ${JSONParser.stringify(startFrom)}`
|
|
690
|
+
);
|
|
691
|
+
return startFrom;
|
|
692
|
+
}
|
|
673
693
|
if (checkpoints) {
|
|
674
694
|
const readResult = await checkpoints?.read(
|
|
675
695
|
{
|
|
@@ -680,7 +700,15 @@ var reactor = (options) => {
|
|
|
680
700
|
);
|
|
681
701
|
lastCheckpoint = readResult.lastCheckpoint;
|
|
682
702
|
}
|
|
683
|
-
if (lastCheckpoint === null)
|
|
703
|
+
if (lastCheckpoint === null) {
|
|
704
|
+
console.log(
|
|
705
|
+
`Processor ${processorId} with instance id ${instanceId} starting from: BEGINNING`
|
|
706
|
+
);
|
|
707
|
+
return "BEGINNING";
|
|
708
|
+
}
|
|
709
|
+
console.log(
|
|
710
|
+
`Checkpoint read for processor ${processorId} with instance id ${instanceId}: ${JSONParser.stringify(lastCheckpoint)}`
|
|
711
|
+
);
|
|
684
712
|
return {
|
|
685
713
|
lastCheckpoint
|
|
686
714
|
};
|
|
@@ -692,48 +720,64 @@ var reactor = (options) => {
|
|
|
692
720
|
},
|
|
693
721
|
handle: async (messages, partialContext) => {
|
|
694
722
|
if (!isActive) return Promise.resolve();
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
{
|
|
710
|
-
processorId,
|
|
711
|
-
version,
|
|
712
|
-
message: upcasted,
|
|
713
|
-
lastCheckpoint,
|
|
714
|
-
partition
|
|
715
|
-
},
|
|
723
|
+
try {
|
|
724
|
+
return await processingScope(async (context) => {
|
|
725
|
+
let result = void 0;
|
|
726
|
+
for (const message2 of messages) {
|
|
727
|
+
if (wasMessageHandled(message2, lastCheckpoint)) continue;
|
|
728
|
+
const upcasted = upcastRecordedMessage(
|
|
729
|
+
// TODO: Make it smarter
|
|
730
|
+
message2,
|
|
731
|
+
options.messageOptions?.schema?.versioning
|
|
732
|
+
);
|
|
733
|
+
if (canHandle !== void 0 && !canHandle.includes(upcasted.type))
|
|
734
|
+
continue;
|
|
735
|
+
const messageProcessingResult = await eachMessage(
|
|
736
|
+
upcasted,
|
|
716
737
|
context
|
|
717
738
|
);
|
|
718
|
-
if (
|
|
719
|
-
|
|
739
|
+
if (checkpoints) {
|
|
740
|
+
const storeCheckpointResult = await checkpoints.store(
|
|
741
|
+
{
|
|
742
|
+
processorId,
|
|
743
|
+
version,
|
|
744
|
+
message: upcasted,
|
|
745
|
+
lastCheckpoint,
|
|
746
|
+
partition
|
|
747
|
+
},
|
|
748
|
+
context
|
|
749
|
+
);
|
|
750
|
+
if (storeCheckpointResult.success) {
|
|
751
|
+
lastCheckpoint = storeCheckpointResult.newCheckpoint;
|
|
752
|
+
}
|
|
720
753
|
}
|
|
754
|
+
if (messageProcessingResult && messageProcessingResult.type === "STOP") {
|
|
755
|
+
isActive = false;
|
|
756
|
+
result = messageProcessingResult;
|
|
757
|
+
break;
|
|
758
|
+
}
|
|
759
|
+
if (stopAfter && stopAfter(upcasted)) {
|
|
760
|
+
isActive = false;
|
|
761
|
+
result = { type: "STOP", reason: "Stop condition reached" };
|
|
762
|
+
break;
|
|
763
|
+
}
|
|
764
|
+
if (messageProcessingResult && messageProcessingResult.type === "SKIP")
|
|
765
|
+
continue;
|
|
721
766
|
}
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
}, partialContext);
|
|
767
|
+
return result;
|
|
768
|
+
}, partialContext);
|
|
769
|
+
} catch (error2) {
|
|
770
|
+
console.log(
|
|
771
|
+
`Error during message processing for processor ${processorId} with instance id ${instanceId}. Stopping the processor.`,
|
|
772
|
+
error2
|
|
773
|
+
);
|
|
774
|
+
isActive = false;
|
|
775
|
+
return {
|
|
776
|
+
type: "STOP",
|
|
777
|
+
error: error2,
|
|
778
|
+
reason: "Error during message processing"
|
|
779
|
+
};
|
|
780
|
+
}
|
|
737
781
|
}
|
|
738
782
|
};
|
|
739
783
|
};
|
|
@@ -828,32 +872,40 @@ var postgreSQLEventStoreMessageBatchPuller = ({
|
|
|
828
872
|
let isRunning = false;
|
|
829
873
|
let start;
|
|
830
874
|
const pullMessages = async (options) => {
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
875
|
+
try {
|
|
876
|
+
const after = options.startFrom === "BEGINNING" ? 0n : options.startFrom === "END" ? (await readLastMessageGlobalPosition(executor)).currentGlobalPosition ?? 0n : options.startFrom.lastCheckpoint;
|
|
877
|
+
const readMessagesOptions = {
|
|
878
|
+
after,
|
|
879
|
+
batchSize
|
|
880
|
+
};
|
|
881
|
+
let waitTime = 100;
|
|
882
|
+
while (isRunning && !signal?.aborted) {
|
|
883
|
+
const { messages, currentGlobalPosition, areMessagesLeft } = await readMessagesBatch(executor, readMessagesOptions);
|
|
884
|
+
if (messages.length > 0) {
|
|
885
|
+
const result = await eachBatch(messages);
|
|
886
|
+
if (result && result.type === "STOP") {
|
|
887
|
+
isRunning = false;
|
|
888
|
+
break;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
readMessagesOptions.after = currentGlobalPosition;
|
|
892
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
893
|
+
if (stopWhen?.noMessagesLeft === true && !areMessagesLeft) {
|
|
894
|
+
console.log(
|
|
895
|
+
`No messages left to process after reaching global position ${currentGlobalPosition}. Stopping the puller.`
|
|
896
|
+
);
|
|
842
897
|
isRunning = false;
|
|
843
898
|
break;
|
|
844
899
|
}
|
|
900
|
+
if (!areMessagesLeft) {
|
|
901
|
+
waitTime = Math.min(waitTime * 2, 1e3);
|
|
902
|
+
} else {
|
|
903
|
+
waitTime = pullingFrequencyInMs;
|
|
904
|
+
}
|
|
845
905
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
isRunning = false;
|
|
850
|
-
break;
|
|
851
|
-
}
|
|
852
|
-
if (!areMessagesLeft) {
|
|
853
|
-
waitTime = Math.min(waitTime * 2, 1e3);
|
|
854
|
-
} else {
|
|
855
|
-
waitTime = pullingFrequencyInMs;
|
|
856
|
-
}
|
|
906
|
+
} catch (error) {
|
|
907
|
+
console.log("Error occurred during message pulling:", error);
|
|
908
|
+
throw error;
|
|
857
909
|
}
|
|
858
910
|
};
|
|
859
911
|
return {
|
|
@@ -883,7 +935,10 @@ var zipPostgreSQLEventStoreMessageBatchPullerStartFrom = (options) => {
|
|
|
883
935
|
};
|
|
884
936
|
|
|
885
937
|
// src/eventStore/consumers/postgreSQLEventStoreConsumer.ts
|
|
886
|
-
import {
|
|
938
|
+
import {
|
|
939
|
+
dumbo as dumbo6,
|
|
940
|
+
JSONSerializer as JSONSerializer2
|
|
941
|
+
} from "@event-driven-io/dumbo";
|
|
887
942
|
import { v7 as uuid7 } from "uuid";
|
|
888
943
|
|
|
889
944
|
// src/eventStore/consumers/postgreSQLProcessor.ts
|
|
@@ -892,10 +947,10 @@ import {
|
|
|
892
947
|
} from "@event-driven-io/dumbo";
|
|
893
948
|
import "pg";
|
|
894
949
|
|
|
895
|
-
// src/eventStore/projections/locks/
|
|
950
|
+
// src/eventStore/projections/locks/tryAcquireProjectionLock.ts
|
|
896
951
|
import { single } from "@event-driven-io/dumbo";
|
|
897
952
|
|
|
898
|
-
// src/eventStore/schema/
|
|
953
|
+
// src/eventStore/schema/projections/projectionsLocks.ts
|
|
899
954
|
import { sql as sql3 } from "@event-driven-io/dumbo";
|
|
900
955
|
|
|
901
956
|
// src/eventStore/schema/createFunctionIfDoesNotExist.ts
|
|
@@ -911,7 +966,97 @@ END $$;
|
|
|
911
966
|
`
|
|
912
967
|
);
|
|
913
968
|
|
|
969
|
+
// src/eventStore/schema/projections/projectionsLocks.ts
|
|
970
|
+
var tryAcquireProjectionLockSQL = createFunctionIfDoesNotExistSQL(
|
|
971
|
+
"emt_try_acquire_projection_lock",
|
|
972
|
+
`
|
|
973
|
+
CREATE OR REPLACE FUNCTION emt_try_acquire_projection_lock(
|
|
974
|
+
p_lock_key BIGINT,
|
|
975
|
+
p_partition TEXT,
|
|
976
|
+
p_name TEXT,
|
|
977
|
+
p_version INT
|
|
978
|
+
)
|
|
979
|
+
RETURNS TABLE (acquired BOOLEAN, is_active BOOLEAN)
|
|
980
|
+
LANGUAGE plpgsql
|
|
981
|
+
AS $emt_try_acquire_projection_lock$
|
|
982
|
+
BEGIN
|
|
983
|
+
RETURN QUERY
|
|
984
|
+
WITH lock_check AS (
|
|
985
|
+
SELECT pg_try_advisory_xact_lock_shared(p_lock_key) AS acquired
|
|
986
|
+
),
|
|
987
|
+
status_check AS (
|
|
988
|
+
SELECT status = 'active' AS is_active
|
|
989
|
+
FROM ${projectionsTable.name}
|
|
990
|
+
WHERE partition = p_partition AND name = p_name AND version = p_version
|
|
991
|
+
)
|
|
992
|
+
SELECT
|
|
993
|
+
COALESCE((SELECT lc.acquired FROM lock_check lc), false),
|
|
994
|
+
COALESCE((SELECT sc.is_active FROM status_check sc), true);
|
|
995
|
+
END;
|
|
996
|
+
$emt_try_acquire_projection_lock$;
|
|
997
|
+
`
|
|
998
|
+
);
|
|
999
|
+
var callTryAcquireProjectionLock = (params) => sql3(
|
|
1000
|
+
`SELECT * FROM emt_try_acquire_projection_lock(%s::BIGINT, %L, %L, %s);`,
|
|
1001
|
+
params.lockKey,
|
|
1002
|
+
params.partition,
|
|
1003
|
+
params.name,
|
|
1004
|
+
params.version
|
|
1005
|
+
);
|
|
1006
|
+
|
|
1007
|
+
// src/eventStore/projections/locks/tryAcquireProjectionLock.ts
|
|
1008
|
+
var tryAcquireProjectionLock = async (execute, {
|
|
1009
|
+
lockKey,
|
|
1010
|
+
projectionName,
|
|
1011
|
+
partition,
|
|
1012
|
+
version
|
|
1013
|
+
}) => {
|
|
1014
|
+
const lockKeyBigInt = isBigint(lockKey) ? lockKey : await hashText(lockKey);
|
|
1015
|
+
const { acquired, is_active } = await single(
|
|
1016
|
+
execute.query(
|
|
1017
|
+
callTryAcquireProjectionLock({
|
|
1018
|
+
lockKey: lockKeyBigInt.toString(),
|
|
1019
|
+
partition,
|
|
1020
|
+
name: projectionName,
|
|
1021
|
+
version
|
|
1022
|
+
})
|
|
1023
|
+
)
|
|
1024
|
+
);
|
|
1025
|
+
return acquired === true && is_active === true;
|
|
1026
|
+
};
|
|
1027
|
+
|
|
1028
|
+
// src/eventStore/projections/locks/postgreSQLProjectionLock.ts
|
|
1029
|
+
var postgreSQLProjectionLock = (options) => {
|
|
1030
|
+
let acquired = false;
|
|
1031
|
+
const lockKey = options.lockKey ?? toProjectionLockKey(options);
|
|
1032
|
+
return {
|
|
1033
|
+
tryAcquire: async (context) => {
|
|
1034
|
+
if (acquired) {
|
|
1035
|
+
return true;
|
|
1036
|
+
}
|
|
1037
|
+
acquired = await tryAcquireProjectionLock(context.execute, {
|
|
1038
|
+
...options,
|
|
1039
|
+
lockKey
|
|
1040
|
+
});
|
|
1041
|
+
return acquired;
|
|
1042
|
+
},
|
|
1043
|
+
release: (_context) => {
|
|
1044
|
+
if (!acquired) return;
|
|
1045
|
+
acquired = false;
|
|
1046
|
+
}
|
|
1047
|
+
};
|
|
1048
|
+
};
|
|
1049
|
+
var toProjectionLockKey = ({
|
|
1050
|
+
projectionName,
|
|
1051
|
+
partition,
|
|
1052
|
+
version
|
|
1053
|
+
}) => `${partition}:${projectionName}:${version}`;
|
|
1054
|
+
|
|
1055
|
+
// src/eventStore/projections/locks/tryAcquireProcessorLock.ts
|
|
1056
|
+
import { single as single2 } from "@event-driven-io/dumbo";
|
|
1057
|
+
|
|
914
1058
|
// src/eventStore/schema/processors/processorsLocks.ts
|
|
1059
|
+
import { sql as sql4 } from "@event-driven-io/dumbo";
|
|
915
1060
|
var tryAcquireProcessorLockSQL = createFunctionIfDoesNotExistSQL(
|
|
916
1061
|
"emt_try_acquire_processor_lock",
|
|
917
1062
|
`
|
|
@@ -1026,7 +1171,7 @@ END;
|
|
|
1026
1171
|
$emt_release_processor_lock$;
|
|
1027
1172
|
`
|
|
1028
1173
|
);
|
|
1029
|
-
var callTryAcquireProcessorLock = (params) =>
|
|
1174
|
+
var callTryAcquireProcessorLock = (params) => sql4(
|
|
1030
1175
|
`SELECT * FROM emt_try_acquire_processor_lock(%s::BIGINT, %L, %s, %L, %L, %L, %L, %L, %s);`,
|
|
1031
1176
|
params.lockKey,
|
|
1032
1177
|
params.processorId,
|
|
@@ -1038,7 +1183,7 @@ var callTryAcquireProcessorLock = (params) => sql3(
|
|
|
1038
1183
|
params.projectionKind,
|
|
1039
1184
|
params.lockTimeoutSeconds
|
|
1040
1185
|
);
|
|
1041
|
-
var callReleaseProcessorLock = (params) =>
|
|
1186
|
+
var callReleaseProcessorLock = (params) => sql4(
|
|
1042
1187
|
`SELECT emt_release_processor_lock(%s::BIGINT, %L, %L, %s, %L, %L) as result;`,
|
|
1043
1188
|
params.lockKey,
|
|
1044
1189
|
params.processorId,
|
|
@@ -1052,7 +1197,7 @@ var callReleaseProcessorLock = (params) => sql3(
|
|
|
1052
1197
|
var PROCESSOR_LOCK_DEFAULT_TIMEOUT_SECONDS = 300;
|
|
1053
1198
|
var tryAcquireProcessorLock = async (execute, options) => {
|
|
1054
1199
|
const lockKeyBigInt = isBigint(options.lockKey) ? options.lockKey : await hashText(options.lockKey);
|
|
1055
|
-
const { acquired, checkpoint } = await
|
|
1200
|
+
const { acquired, checkpoint } = await single2(
|
|
1056
1201
|
execute.command(
|
|
1057
1202
|
callTryAcquireProcessorLock({
|
|
1058
1203
|
lockKey: lockKeyBigInt.toString(),
|
|
@@ -1067,10 +1212,13 @@ var tryAcquireProcessorLock = async (execute, options) => {
|
|
|
1067
1212
|
})
|
|
1068
1213
|
)
|
|
1069
1214
|
);
|
|
1215
|
+
console.log(
|
|
1216
|
+
`Lock acquisition attempt for processor '${options.processorId}' with processor instance '${options.processorInstanceId}' and lock '${options.lockKey}' resulted in: ${acquired}`
|
|
1217
|
+
);
|
|
1070
1218
|
return acquired ? { acquired: true, checkpoint } : { acquired: false };
|
|
1071
1219
|
};
|
|
1072
1220
|
var tryAcquireProcessorLockWithRetry = async (execute, options) => {
|
|
1073
|
-
const policy = options.
|
|
1221
|
+
const policy = options.lockAcquisitionPolicy ?? DefaultPostgreSQLProcessorLockPolicy;
|
|
1074
1222
|
if (policy.type === "retry") {
|
|
1075
1223
|
return asyncRetry(() => tryAcquireProcessorLock(execute, options), {
|
|
1076
1224
|
retries: policy.retries - 1,
|
|
@@ -1083,7 +1231,7 @@ var tryAcquireProcessorLockWithRetry = async (execute, options) => {
|
|
|
1083
1231
|
};
|
|
1084
1232
|
var releaseProcessorLock = async (execute, options) => {
|
|
1085
1233
|
const lockKeyBigInt = isBigint(options.lockKey) ? options.lockKey : await hashText(options.lockKey);
|
|
1086
|
-
const { result } = await
|
|
1234
|
+
const { result } = await single2(
|
|
1087
1235
|
execute.command(
|
|
1088
1236
|
callReleaseProcessorLock({
|
|
1089
1237
|
lockKey: lockKeyBigInt.toString(),
|
|
@@ -1095,98 +1243,11 @@ var releaseProcessorLock = async (execute, options) => {
|
|
|
1095
1243
|
})
|
|
1096
1244
|
)
|
|
1097
1245
|
);
|
|
1098
|
-
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
|
-
// src/eventStore/projections/locks/tryAcquireProjectionLock.ts
|
|
1102
|
-
import { single as single2 } from "@event-driven-io/dumbo";
|
|
1103
|
-
|
|
1104
|
-
// src/eventStore/schema/projections/projectionsLocks.ts
|
|
1105
|
-
import { sql as sql4 } from "@event-driven-io/dumbo";
|
|
1106
|
-
var tryAcquireProjectionLockSQL = createFunctionIfDoesNotExistSQL(
|
|
1107
|
-
"emt_try_acquire_projection_lock",
|
|
1108
|
-
`
|
|
1109
|
-
CREATE OR REPLACE FUNCTION emt_try_acquire_projection_lock(
|
|
1110
|
-
p_lock_key BIGINT,
|
|
1111
|
-
p_partition TEXT,
|
|
1112
|
-
p_name TEXT,
|
|
1113
|
-
p_version INT
|
|
1114
|
-
)
|
|
1115
|
-
RETURNS TABLE (acquired BOOLEAN, is_active BOOLEAN)
|
|
1116
|
-
LANGUAGE plpgsql
|
|
1117
|
-
AS $emt_try_acquire_projection_lock$
|
|
1118
|
-
BEGIN
|
|
1119
|
-
RETURN QUERY
|
|
1120
|
-
WITH lock_check AS (
|
|
1121
|
-
SELECT pg_try_advisory_xact_lock_shared(p_lock_key) AS acquired
|
|
1122
|
-
),
|
|
1123
|
-
status_check AS (
|
|
1124
|
-
SELECT status = 'active' AS is_active
|
|
1125
|
-
FROM ${projectionsTable.name}
|
|
1126
|
-
WHERE partition = p_partition AND name = p_name AND version = p_version
|
|
1127
|
-
)
|
|
1128
|
-
SELECT
|
|
1129
|
-
COALESCE((SELECT lc.acquired FROM lock_check lc), false),
|
|
1130
|
-
COALESCE((SELECT sc.is_active FROM status_check sc), true);
|
|
1131
|
-
END;
|
|
1132
|
-
$emt_try_acquire_projection_lock$;
|
|
1133
|
-
`
|
|
1134
|
-
);
|
|
1135
|
-
var callTryAcquireProjectionLock = (params) => sql4(
|
|
1136
|
-
`SELECT * FROM emt_try_acquire_projection_lock(%s::BIGINT, %L, %L, %s);`,
|
|
1137
|
-
params.lockKey,
|
|
1138
|
-
params.partition,
|
|
1139
|
-
params.name,
|
|
1140
|
-
params.version
|
|
1141
|
-
);
|
|
1142
|
-
|
|
1143
|
-
// src/eventStore/projections/locks/tryAcquireProjectionLock.ts
|
|
1144
|
-
var tryAcquireProjectionLock = async (execute, {
|
|
1145
|
-
lockKey,
|
|
1146
|
-
projectionName,
|
|
1147
|
-
partition,
|
|
1148
|
-
version
|
|
1149
|
-
}) => {
|
|
1150
|
-
const lockKeyBigInt = isBigint(lockKey) ? lockKey : await hashText(lockKey);
|
|
1151
|
-
const { acquired, is_active } = await single2(
|
|
1152
|
-
execute.query(
|
|
1153
|
-
callTryAcquireProjectionLock({
|
|
1154
|
-
lockKey: lockKeyBigInt.toString(),
|
|
1155
|
-
partition,
|
|
1156
|
-
name: projectionName,
|
|
1157
|
-
version
|
|
1158
|
-
})
|
|
1159
|
-
)
|
|
1246
|
+
console.log(
|
|
1247
|
+
`Lock for processor '${options.processorId}' with processor instance '${options.processorInstanceId}' and lock '${options.lockKey}' released: ${result}`
|
|
1160
1248
|
);
|
|
1161
|
-
return
|
|
1162
|
-
};
|
|
1163
|
-
|
|
1164
|
-
// src/eventStore/projections/locks/postgreSQLProjectionLock.ts
|
|
1165
|
-
var postgreSQLProjectionLock = (options) => {
|
|
1166
|
-
let acquired = false;
|
|
1167
|
-
const lockKey = options.lockKey ?? toProjectionLockKey(options);
|
|
1168
|
-
return {
|
|
1169
|
-
tryAcquire: async (context) => {
|
|
1170
|
-
if (acquired) {
|
|
1171
|
-
return true;
|
|
1172
|
-
}
|
|
1173
|
-
acquired = await tryAcquireProjectionLock(context.execute, {
|
|
1174
|
-
...options,
|
|
1175
|
-
lockKey
|
|
1176
|
-
});
|
|
1177
|
-
return acquired;
|
|
1178
|
-
},
|
|
1179
|
-
release: (_context) => {
|
|
1180
|
-
if (!acquired) return;
|
|
1181
|
-
acquired = false;
|
|
1182
|
-
}
|
|
1183
|
-
};
|
|
1249
|
+
return result;
|
|
1184
1250
|
};
|
|
1185
|
-
var toProjectionLockKey = ({
|
|
1186
|
-
projectionName,
|
|
1187
|
-
partition,
|
|
1188
|
-
version
|
|
1189
|
-
}) => `${partition}:${projectionName}:${version}`;
|
|
1190
1251
|
|
|
1191
1252
|
// src/eventStore/projections/locks/postgreSQLProcessorLock.ts
|
|
1192
1253
|
var DefaultPostgreSQLProcessorLockPolicy = {
|
|
@@ -1198,13 +1259,19 @@ var postgreSQLProcessorLock = (options) => {
|
|
|
1198
1259
|
return {
|
|
1199
1260
|
tryAcquire: async (context) => {
|
|
1200
1261
|
if (acquired) {
|
|
1262
|
+
console.log(
|
|
1263
|
+
`Lock for processor '${options.processorId}' is already acquired by this instance. Reusing the lock.`
|
|
1264
|
+
);
|
|
1201
1265
|
return true;
|
|
1202
1266
|
}
|
|
1203
1267
|
const result = await tryAcquireProcessorLockWithRetry(context.execute, {
|
|
1204
1268
|
...options,
|
|
1205
1269
|
lockKey
|
|
1206
1270
|
});
|
|
1207
|
-
if (!result.acquired && options.
|
|
1271
|
+
if (!result.acquired && options.lockAcquisitionPolicy?.type !== "skip") {
|
|
1272
|
+
console.log(
|
|
1273
|
+
`Failed to acquire lock for processor '${options.processorId}' with policy '${options.lockAcquisitionPolicy?.type}'.`
|
|
1274
|
+
);
|
|
1208
1275
|
throw new EmmettError(
|
|
1209
1276
|
`Failed to acquire lock for processor '${options.processorId}'`
|
|
1210
1277
|
);
|
|
@@ -1213,7 +1280,12 @@ var postgreSQLProcessorLock = (options) => {
|
|
|
1213
1280
|
return acquired;
|
|
1214
1281
|
},
|
|
1215
1282
|
release: async (context) => {
|
|
1216
|
-
if (!acquired)
|
|
1283
|
+
if (!acquired) {
|
|
1284
|
+
console.log(
|
|
1285
|
+
`Lock for processor '${options.processorId}' is not acquired by this instance. Skipping release.`
|
|
1286
|
+
);
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1217
1289
|
const { projection: projection2, ...releaseOptions } = options;
|
|
1218
1290
|
await releaseProcessorLock(context.execute, {
|
|
1219
1291
|
...releaseOptions,
|
|
@@ -2673,7 +2745,7 @@ BEGIN
|
|
|
2673
2745
|
END $$;
|
|
2674
2746
|
`);
|
|
2675
2747
|
var migration_0_42_0_FromSubscriptionsToProcessors = sqlMigration2(
|
|
2676
|
-
"emt:postgresql:eventstore:0.42.
|
|
2748
|
+
"emt:postgresql:eventstore:0.42.1-alpha.1:from-subscriptions-to-processors",
|
|
2677
2749
|
[migration_0_42_0_FromSubscriptionsToProcessorsSQL]
|
|
2678
2750
|
);
|
|
2679
2751
|
var migration_0_42_0_2_AddProcessorProjectionFunctionsSQL = rawSql3(`
|
|
@@ -2950,7 +3022,7 @@ END;
|
|
|
2950
3022
|
$emt_deactivate_projection$;
|
|
2951
3023
|
`);
|
|
2952
3024
|
var migration_0_42_0_2_AddProcessorProjectionFunctions = sqlMigration2(
|
|
2953
|
-
"emt:postgresql:eventstore:0.42.
|
|
3025
|
+
"emt:postgresql:eventstore:0.42.1-alpha.1-2:add-processor-projection-functions",
|
|
2954
3026
|
[migration_0_42_0_2_AddProcessorProjectionFunctionsSQL]
|
|
2955
3027
|
);
|
|
2956
3028
|
|
|
@@ -4723,8 +4795,7 @@ var postgreSQLProjector = (options) => {
|
|
|
4723
4795
|
processorInstanceId = getProcessorInstanceId(processorId),
|
|
4724
4796
|
version = defaultProcessorVersion,
|
|
4725
4797
|
partition = defaultProcessorPartition,
|
|
4726
|
-
|
|
4727
|
-
lockTimeoutSeconds
|
|
4798
|
+
lock
|
|
4728
4799
|
} = options;
|
|
4729
4800
|
const { pool, connectionString, close } = getProcessorPool(options);
|
|
4730
4801
|
const processorLock = postgreSQLProcessorLock({
|
|
@@ -4738,8 +4809,8 @@ var postgreSQLProjector = (options) => {
|
|
|
4738
4809
|
version: options.projection.version ?? version,
|
|
4739
4810
|
handlingType: "async"
|
|
4740
4811
|
} : void 0,
|
|
4741
|
-
|
|
4742
|
-
lockTimeoutSeconds
|
|
4812
|
+
lockAcquisitionPolicy: lock?.acquisitionPolicy ?? DefaultPostgreSQLProcessorLockPolicy,
|
|
4813
|
+
lockTimeoutSeconds: lock?.timeoutSeconds
|
|
4743
4814
|
});
|
|
4744
4815
|
const hooks = wrapHooksWithProcessorLocks(
|
|
4745
4816
|
{
|
|
@@ -4791,7 +4862,7 @@ var postgreSQLReactor = (options) => {
|
|
|
4791
4862
|
processorInstanceId = getProcessorInstanceId(processorId),
|
|
4792
4863
|
version = defaultProcessorVersion,
|
|
4793
4864
|
partition = defaultProcessorPartition,
|
|
4794
|
-
|
|
4865
|
+
lock
|
|
4795
4866
|
} = options;
|
|
4796
4867
|
const { pool, connectionString, close } = getProcessorPool(options);
|
|
4797
4868
|
const processorLock = postgreSQLProcessorLock({
|
|
@@ -4800,7 +4871,8 @@ var postgreSQLReactor = (options) => {
|
|
|
4800
4871
|
partition,
|
|
4801
4872
|
processorInstanceId,
|
|
4802
4873
|
projection: void 0,
|
|
4803
|
-
|
|
4874
|
+
lockAcquisitionPolicy: lock?.acquisitionPolicy ?? DefaultPostgreSQLProcessorLockPolicy,
|
|
4875
|
+
lockTimeoutSeconds: lock?.timeoutSeconds
|
|
4804
4876
|
});
|
|
4805
4877
|
const hooks = wrapHooksWithProcessorLocks(
|
|
4806
4878
|
{
|
|
@@ -4856,12 +4928,20 @@ var postgreSQLEventStoreConsumer = (options) => {
|
|
|
4856
4928
|
};
|
|
4857
4929
|
const result = await Promise.allSettled(
|
|
4858
4930
|
activeProcessors.map(async (s) => {
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4931
|
+
try {
|
|
4932
|
+
return await s.handle(messagesBatch, {
|
|
4933
|
+
connection: {
|
|
4934
|
+
connectionString: options.connectionString,
|
|
4935
|
+
pool
|
|
4936
|
+
}
|
|
4937
|
+
});
|
|
4938
|
+
} catch (error) {
|
|
4939
|
+
console.log(
|
|
4940
|
+
`Error during message batch processing for processor: ${s.id}`,
|
|
4941
|
+
error
|
|
4942
|
+
);
|
|
4943
|
+
throw error;
|
|
4944
|
+
}
|
|
4865
4945
|
})
|
|
4866
4946
|
);
|
|
4867
4947
|
return result.some(
|
|
@@ -4897,7 +4977,24 @@ var postgreSQLEventStoreConsumer = (options) => {
|
|
|
4897
4977
|
const postgresProcessors = processors;
|
|
4898
4978
|
for (const processor of postgresProcessors) {
|
|
4899
4979
|
if (processor.init) {
|
|
4900
|
-
|
|
4980
|
+
try {
|
|
4981
|
+
await processor.init(processorContext);
|
|
4982
|
+
} catch (error) {
|
|
4983
|
+
console.log(
|
|
4984
|
+
`Error during processor initialization for processor: ${processor.id}. Stopping it.`,
|
|
4985
|
+
error
|
|
4986
|
+
);
|
|
4987
|
+
await processor.close(processorContext).catch((closeError) => {
|
|
4988
|
+
console.log(
|
|
4989
|
+
`Error during processor cleanup after failed initialization for processor: ${processor.id}`,
|
|
4990
|
+
closeError
|
|
4991
|
+
);
|
|
4992
|
+
});
|
|
4993
|
+
console.log(
|
|
4994
|
+
`Processor ${processor.id} stopped successfully after failed initialization.`
|
|
4995
|
+
);
|
|
4996
|
+
throw error;
|
|
4997
|
+
}
|
|
4901
4998
|
}
|
|
4902
4999
|
}
|
|
4903
5000
|
isInitialized = true;
|
|
@@ -4926,11 +5023,20 @@ var postgreSQLEventStoreConsumer = (options) => {
|
|
|
4926
5023
|
return processor;
|
|
4927
5024
|
},
|
|
4928
5025
|
start: () => {
|
|
4929
|
-
if (isRunning)
|
|
4930
|
-
|
|
5026
|
+
if (isRunning) {
|
|
5027
|
+
console.log(
|
|
5028
|
+
"Consumer is already running. Returning the existing start promise."
|
|
5029
|
+
);
|
|
5030
|
+
return start;
|
|
5031
|
+
}
|
|
5032
|
+
if (processors.length === 0) {
|
|
5033
|
+
console.log(
|
|
5034
|
+
"Cannot start consumer without at least a single processor"
|
|
5035
|
+
);
|
|
4931
5036
|
throw new EmmettError(
|
|
4932
5037
|
"Cannot start consumer without at least a single processor"
|
|
4933
5038
|
);
|
|
5039
|
+
}
|
|
4934
5040
|
isRunning = true;
|
|
4935
5041
|
abortController = new AbortController();
|
|
4936
5042
|
messagePuller = postgreSQLEventStoreMessageBatchPuller({
|
|
@@ -4944,22 +5050,36 @@ var postgreSQLEventStoreConsumer = (options) => {
|
|
|
4944
5050
|
start = (async () => {
|
|
4945
5051
|
if (!isRunning) return;
|
|
4946
5052
|
if (!isInitialized) {
|
|
5053
|
+
console.log("Initializing consumer before starting message pulling.");
|
|
4947
5054
|
await init();
|
|
4948
5055
|
}
|
|
4949
5056
|
const startFrom = zipPostgreSQLEventStoreMessageBatchPullerStartFrom(
|
|
4950
5057
|
await Promise.all(
|
|
4951
5058
|
processors.map(async (o) => {
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
5059
|
+
try {
|
|
5060
|
+
const result = await o.start({
|
|
5061
|
+
execute: pool.execute,
|
|
5062
|
+
connection: {
|
|
5063
|
+
connectionString: options.connectionString,
|
|
5064
|
+
pool
|
|
5065
|
+
}
|
|
5066
|
+
});
|
|
5067
|
+
return result;
|
|
5068
|
+
} catch (error) {
|
|
5069
|
+
console.log(
|
|
5070
|
+
`Error during processor start position retrieval for processor: ${o.id}. Stopping it.`,
|
|
5071
|
+
error
|
|
5072
|
+
);
|
|
5073
|
+
throw error;
|
|
5074
|
+
}
|
|
4960
5075
|
})
|
|
4961
5076
|
)
|
|
4962
5077
|
);
|
|
5078
|
+
console.log(
|
|
5079
|
+
`Starting message pulling with start position: ${JSONSerializer2.serialize(
|
|
5080
|
+
startFrom
|
|
5081
|
+
)}. Waiting for messages...`
|
|
5082
|
+
);
|
|
4963
5083
|
await messagePuller.start({ startFrom });
|
|
4964
5084
|
await stopProcessors();
|
|
4965
5085
|
isRunning = false;
|
|
@@ -4986,9 +5106,9 @@ var rebuildPostgreSQLProjections = (options) => {
|
|
|
4986
5106
|
...options,
|
|
4987
5107
|
stopWhen: { noMessagesLeft: true }
|
|
4988
5108
|
});
|
|
5109
|
+
const lock = { acquisitionPolicy: defaultRebuildLockPolicy, ...options.lock };
|
|
4989
5110
|
const projections = "projections" in options ? options.projections.map(
|
|
4990
5111
|
(p) => "projection" in p ? {
|
|
4991
|
-
lockPolicy: defaultRebuildLockPolicy,
|
|
4992
5112
|
truncateOnStart: true,
|
|
4993
5113
|
processorId: getProjectorId({
|
|
4994
5114
|
projectionName: p.projection.name ?? unknownTag2
|
|
@@ -4999,14 +5119,14 @@ var rebuildPostgreSQLProjections = (options) => {
|
|
|
4999
5119
|
processorId: getProjectorId({
|
|
5000
5120
|
projectionName: p.name ?? unknownTag2
|
|
5001
5121
|
}),
|
|
5002
|
-
truncateOnStart: true
|
|
5003
|
-
lockPolicy: defaultRebuildLockPolicy
|
|
5122
|
+
truncateOnStart: true
|
|
5004
5123
|
}
|
|
5005
5124
|
) : [options];
|
|
5006
5125
|
for (const projectionDefinition of projections) {
|
|
5007
5126
|
consumer.projector({
|
|
5008
5127
|
...projectionDefinition,
|
|
5009
|
-
truncateOnStart: projectionDefinition.truncateOnStart ?? true
|
|
5128
|
+
truncateOnStart: projectionDefinition.truncateOnStart ?? true,
|
|
5129
|
+
lock
|
|
5010
5130
|
});
|
|
5011
5131
|
}
|
|
5012
5132
|
return consumer;
|