@open-e2ee/signal-protocol-sdk 13.5.1 → 14.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +329 -0
- package/README.md +2 -2
- package/SECURITY.md +2 -2
- package/device/README.md +4 -3
- package/dist/client/client.d.ts +20 -55
- package/dist/client/client.d.ts.map +1 -1
- package/dist/client/client.js +133 -165
- package/dist/client/client.js.map +1 -1
- package/dist/client/content-adapter.d.ts +35 -6
- package/dist/client/content-adapter.d.ts.map +1 -1
- package/dist/client/content-adapter.js +28 -0
- package/dist/client/content-adapter.js.map +1 -1
- package/dist/client/event-hooks.d.ts +22 -11
- package/dist/client/event-hooks.d.ts.map +1 -1
- package/dist/client/event-hooks.js.map +1 -1
- package/dist/client/group-fetcher.d.ts.map +1 -1
- package/dist/client/group-fetcher.js +43 -0
- package/dist/client/group-fetcher.js.map +1 -1
- package/dist/client/group-key-sync.d.ts +65 -0
- package/dist/client/group-key-sync.d.ts.map +1 -0
- package/dist/client/group-key-sync.js +286 -0
- package/dist/client/group-key-sync.js.map +1 -0
- package/dist/client/group-outbox.d.ts +20 -6
- package/dist/client/group-outbox.d.ts.map +1 -1
- package/dist/client/group-outbox.js +153 -80
- package/dist/client/group-outbox.js.map +1 -1
- package/dist/client/group-receive-authorization.d.ts +16 -9
- package/dist/client/group-receive-authorization.d.ts.map +1 -1
- package/dist/client/group-receive-authorization.js +25 -16
- package/dist/client/group-receive-authorization.js.map +1 -1
- package/dist/client/hosted.d.ts +7 -7
- package/dist/client/hosted.js +7 -7
- package/dist/client/index.d.ts +1 -1
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js.map +1 -1
- package/dist/client/receive-queue.d.ts +3 -4
- package/dist/client/receive-queue.d.ts.map +1 -1
- package/dist/client/receive-queue.js +24 -32
- package/dist/client/receive-queue.js.map +1 -1
- package/dist/client/relay-subscription.d.ts.map +1 -1
- package/dist/client/relay-subscription.js +4 -9
- package/dist/client/relay-subscription.js.map +1 -1
- package/dist/client/retry-request-envelope.d.ts +4 -2
- package/dist/client/retry-request-envelope.d.ts.map +1 -1
- package/dist/client/retry-request-envelope.js +6 -4
- package/dist/client/retry-request-envelope.js.map +1 -1
- package/dist/client/retry-request-lifecycle.d.ts +7 -8
- package/dist/client/retry-request-lifecycle.d.ts.map +1 -1
- package/dist/client/retry-request-lifecycle.js +10 -7
- package/dist/client/retry-request-lifecycle.js.map +1 -1
- package/dist/client/retry-terminal-recovery.d.ts +14 -15
- package/dist/client/retry-terminal-recovery.d.ts.map +1 -1
- package/dist/client/retry-terminal-recovery.js +16 -17
- package/dist/client/retry-terminal-recovery.js.map +1 -1
- package/dist/client/retry-utils.d.ts +22 -0
- package/dist/client/retry-utils.d.ts.map +1 -1
- package/dist/client/retry-utils.js +30 -0
- package/dist/client/retry-utils.js.map +1 -1
- package/dist/client/send-deadline.js +1 -1
- package/dist/client/send-deadline.js.map +1 -1
- package/dist/client/send-timestamp.d.ts +71 -14
- package/dist/client/send-timestamp.d.ts.map +1 -1
- package/dist/client/send-timestamp.js +87 -19
- package/dist/client/send-timestamp.js.map +1 -1
- package/dist/client/signal-service-cipher.d.ts +125 -13
- package/dist/client/signal-service-cipher.d.ts.map +1 -1
- package/dist/client/signal-service-cipher.js +413 -102
- package/dist/client/signal-service-cipher.js.map +1 -1
- package/dist/client/types.d.ts +21 -16
- package/dist/client/types.d.ts.map +1 -1
- package/dist/client/types.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/internal/groups/key-sync-ledger.d.ts +65 -0
- package/dist/internal/groups/key-sync-ledger.d.ts.map +1 -0
- package/dist/internal/groups/key-sync-ledger.js +104 -0
- package/dist/internal/groups/key-sync-ledger.js.map +1 -0
- package/dist/internal/groups/receive-record.d.ts +4 -1
- package/dist/internal/groups/receive-record.d.ts.map +1 -1
- package/dist/internal/groups/receive-record.js.map +1 -1
- package/dist/internal/session/cipher.d.ts.map +1 -1
- package/dist/internal/session/cipher.js +14 -0
- package/dist/internal/session/cipher.js.map +1 -1
- package/dist/local/store/conformance/cases/durability.d.ts.map +1 -1
- package/dist/local/store/conformance/cases/durability.js +6 -0
- package/dist/local/store/conformance/cases/durability.js.map +1 -1
- package/dist/local/store/conformance/records.d.ts.map +1 -1
- package/dist/local/store/conformance/records.js +3 -0
- package/dist/local/store/conformance/records.js.map +1 -1
- package/dist/local/store/expo/adapter.js +2 -2
- package/dist/local/store/expo/adapter.js.map +1 -1
- package/dist/local/store/key-value/adapter.d.ts +6 -5
- package/dist/local/store/key-value/adapter.d.ts.map +1 -1
- package/dist/local/store/key-value/adapter.js +21 -19
- package/dist/local/store/key-value/adapter.js.map +1 -1
- package/dist/local/store/key-value/receive-work.d.ts +0 -5
- package/dist/local/store/key-value/receive-work.d.ts.map +1 -1
- package/dist/local/store/key-value/receive-work.js +0 -16
- package/dist/local/store/key-value/receive-work.js.map +1 -1
- package/dist/local/store/node/adapter.js +2 -2
- package/dist/local/store/node/adapter.js.map +1 -1
- package/dist/local/store/react-native/adapter.js +2 -2
- package/dist/local/store/react-native/adapter.js.map +1 -1
- package/dist/local/store/receive-work.d.ts +32 -32
- package/dist/local/store/receive-work.d.ts.map +1 -1
- package/dist/local/store/receive-work.js +112 -65
- package/dist/local/store/receive-work.js.map +1 -1
- package/dist/local/store/received-content.d.ts +0 -5
- package/dist/local/store/received-content.d.ts.map +1 -1
- package/dist/local/store/received-content.js +0 -17
- package/dist/local/store/received-content.js.map +1 -1
- package/dist/local/store/reliability.d.ts +191 -53
- package/dist/local/store/reliability.d.ts.map +1 -1
- package/dist/local/store/reliability.js +517 -37
- package/dist/local/store/reliability.js.map +1 -1
- package/dist/local/store/sqlite/key-storage/metadata.d.ts +1 -8
- package/dist/local/store/sqlite/key-storage/metadata.d.ts.map +1 -1
- package/dist/local/store/sqlite/key-storage/metadata.js +1 -18
- package/dist/local/store/sqlite/key-storage/metadata.js.map +1 -1
- package/dist/local/store/sqlite/migrations.d.ts.map +1 -1
- package/dist/local/store/sqlite/migrations.js +8 -0
- package/dist/local/store/sqlite/migrations.js.map +1 -1
- package/dist/local/store/sqlite/store.d.ts +5 -5
- package/dist/local/store/sqlite/store.d.ts.map +1 -1
- package/dist/local/store/sqlite/store.js +29 -9
- package/dist/local/store/sqlite/store.js.map +1 -1
- package/dist/local/store/web/adapter.d.ts +11 -10
- package/dist/local/store/web/adapter.d.ts.map +1 -1
- package/dist/local/store/web/adapter.js +69 -51
- package/dist/local/store/web/adapter.js.map +1 -1
- package/dist/local/store/web-sqlite/adapter.js +2 -2
- package/dist/local/store/web-sqlite/adapter.js.map +1 -1
- package/dist/types/api.d.ts +50 -71
- package/dist/types/api.d.ts.map +1 -1
- package/dist/types/errors.d.ts +52 -18
- package/dist/types/errors.d.ts.map +1 -1
- package/dist/types/errors.js +58 -17
- package/dist/types/errors.js.map +1 -1
- package/docs/ASSURANCE.md +5 -5
- package/docs/DEVIATIONS.md +126 -9
- package/docs/ERROR_HANDLING.md +38 -23
- package/docs/GETTING_STARTED.md +17 -0
- package/local/store/README.md +32 -8
- package/package.json +1 -1
|
@@ -1,13 +1,17 @@
|
|
|
1
|
+
import { InvalidSendTimestampError } from '../../types/errors.js';
|
|
2
|
+
import { defaultSignalProtocolLogger } from '../../logger.js';
|
|
1
3
|
import { createKeyedLocks, withKeyedLock } from '../../utils/keyed-lock.js';
|
|
2
4
|
import { originLockScope, withOriginLock } from '../../utils/origin-lock.js';
|
|
3
5
|
const OUTBOX_PREFIX = 'reliability:exact-ciphertext-outbox:v2';
|
|
4
6
|
const INCOMING_PREFIX = 'reliability:processed-envelopes:v2';
|
|
5
|
-
const RETRY_REQUESTED_PREFIX = 'reliability:retry-requested:
|
|
7
|
+
const RETRY_REQUESTED_PREFIX = 'reliability:retry-requested:v3';
|
|
6
8
|
const RETRY_RESEND_PREFIX = 'reliability:retry-resend:v1';
|
|
7
9
|
const RETRY_FAMILY_PREFIX = 'reliability:retry-family:v1';
|
|
8
10
|
const SENT_TIMESTAMP_PREFIX = 'reliability:sent-timestamps:v1';
|
|
9
11
|
const DELIVERY_STATUS_PREFIX = 'reliability:delivery-status:v1';
|
|
10
|
-
const
|
|
12
|
+
const RECEIVED_TIMESTAMP_V2_PREFIX = 'reliability:received-timestamps:v2';
|
|
13
|
+
/** The prefix of the SDK 13.x received-message records. Only the rewrite step reads it. */
|
|
14
|
+
const RECEIVED_TIMESTAMP_V1_PREFIX = 'reliability:received-timestamps:v1';
|
|
11
15
|
const DAY_MS = 24 * 60 * 60 * 1_000;
|
|
12
16
|
export const RELIABILITY_RECORD_TTL_MS = 30 * DAY_MS;
|
|
13
17
|
/**
|
|
@@ -29,7 +33,7 @@ export const RETRY_REQUESTER_RECORD_TTL_MS = 2 * RELIABILITY_RECORD_TTL_MS;
|
|
|
29
33
|
* The time from the first failure of a retry request to its end: 14 days,
|
|
30
34
|
* the default retention of resendable content of a sender,
|
|
31
35
|
* `RESENDABLE_CONTENT_RETENTION_MS`. A request whose content did not come
|
|
32
|
-
* at this time ends
|
|
36
|
+
* at this time ends with the class that `deadlineTerminalClass` gives.
|
|
33
37
|
*/
|
|
34
38
|
export const RETRY_REQUEST_DEADLINE_MS = 14 * DAY_MS;
|
|
35
39
|
/**
|
|
@@ -42,6 +46,7 @@ const mutationTails = new WeakMap();
|
|
|
42
46
|
const outgoingIntentLocks = createKeyedLocks();
|
|
43
47
|
const outgoingIntentEndReports = new WeakMap();
|
|
44
48
|
const incomingEnvelopeLocks = createKeyedLocks();
|
|
49
|
+
const sendTimestampLocks = createKeyedLocks();
|
|
45
50
|
const LEDGER_PREFIXES = {
|
|
46
51
|
outbox: OUTBOX_PREFIX,
|
|
47
52
|
incoming: INCOMING_PREFIX,
|
|
@@ -50,7 +55,7 @@ const LEDGER_PREFIXES = {
|
|
|
50
55
|
'retry-resend': RETRY_RESEND_PREFIX,
|
|
51
56
|
'sent-timestamp': SENT_TIMESTAMP_PREFIX,
|
|
52
57
|
'delivery-status': DELIVERY_STATUS_PREFIX,
|
|
53
|
-
'received-timestamp':
|
|
58
|
+
'received-timestamp': RECEIVED_TIMESTAMP_V2_PREFIX,
|
|
54
59
|
};
|
|
55
60
|
const LEDGER_TTL_MS = {
|
|
56
61
|
outbox: RELIABILITY_RECORD_TTL_MS,
|
|
@@ -409,6 +414,7 @@ export function outgoingMessageIntentResult(intent) {
|
|
|
409
414
|
return undefined;
|
|
410
415
|
return {
|
|
411
416
|
...cloneReliabilityValue(intent.result),
|
|
417
|
+
clientTimestamp: intent.clientTimestamp,
|
|
412
418
|
...(intent.syncPending && { syncPending: true }),
|
|
413
419
|
};
|
|
414
420
|
}
|
|
@@ -888,8 +894,7 @@ export async function storeOutgoingMessageIntent(store, intent, now = Date.now()
|
|
|
888
894
|
throw new Error('A clientMessageId cannot identify two different encrypted sends');
|
|
889
895
|
}
|
|
890
896
|
if (!retainedExisting) {
|
|
891
|
-
await store.setMetadata(key, JSON.stringify(intent));
|
|
892
|
-
await indexSentTimestamp(store, intent, now);
|
|
897
|
+
await indexSentTimestamp(store, intent, now, () => store.setMetadata(key, JSON.stringify(intent)));
|
|
893
898
|
}
|
|
894
899
|
else {
|
|
895
900
|
// A failed index write after the intent write leaves the retained
|
|
@@ -903,18 +908,222 @@ export async function storeOutgoingMessageIntent(store, intent, now = Date.now()
|
|
|
903
908
|
/**
|
|
904
909
|
* Index the client message ID of an intent under its client timestamp,
|
|
905
910
|
* the value by which an E2EE receipt names the message. The index expires
|
|
906
|
-
* with the intent. A second call for the same intent changes no entry.
|
|
911
|
+
* with the intent. A second call for the same intent changes no entry. A new
|
|
912
|
+
* send reserved its timestamp with `reserveSendTimestamp` before this call,
|
|
913
|
+
* so the entry already holds the ID, and this call deletes the pending
|
|
914
|
+
* marker of the reservation. Every read and write of the entry, its bucket,
|
|
915
|
+
* its manifest and the marker runs under the send timestamp lock, so a
|
|
916
|
+
* reservation in another tab does not write over it.
|
|
917
|
+
*
|
|
918
|
+
* With `writeIntent`, the call writes a new intent under the same lock.
|
|
919
|
+
* When the intent has no pending marker for its timestamp, and the entry
|
|
920
|
+
* holds a different ID and not this one, a later send took the timestamp
|
|
921
|
+
* of this send as an orphan. The call then throws `InvalidSendTimestampError`
|
|
922
|
+
* with the reason `already_used` and writes no intent, so the device never
|
|
923
|
+
* posts two messages with one timestamp.
|
|
924
|
+
*
|
|
925
|
+
* @throws {InvalidSendTimestampError} when a later send took the timestamp
|
|
926
|
+
* of the new intent
|
|
927
|
+
*/
|
|
928
|
+
async function indexSentTimestamp(store, intent, now, writeIntent) {
|
|
929
|
+
await withSendTimestampLock(store, async () => {
|
|
930
|
+
const retainedDays = await pruneExpiredBuckets(store, 'sent-timestamp', now);
|
|
931
|
+
const id = String(intent.clientTimestamp);
|
|
932
|
+
const key = recordKey('sent-timestamp', id);
|
|
933
|
+
const clientMessageIds = parseJson(await store.getMetadata(key), []);
|
|
934
|
+
const held = clientMessageIds.includes(intent.clientMessageId);
|
|
935
|
+
const pending = await pendingSendTimestamp(store, intent.clientMessageId);
|
|
936
|
+
const ownsPending = pending?.timestamp === intent.clientTimestamp;
|
|
937
|
+
if (writeIntent) {
|
|
938
|
+
if (!ownsPending && !held && clientMessageIds.length > 0) {
|
|
939
|
+
throw new InvalidSendTimestampError(intent.clientTimestamp, 'already_used');
|
|
940
|
+
}
|
|
941
|
+
await writeIntent();
|
|
942
|
+
}
|
|
943
|
+
if (!held) {
|
|
944
|
+
clientMessageIds.push(intent.clientMessageId);
|
|
945
|
+
await store.setMetadata(key, JSON.stringify(clientMessageIds));
|
|
946
|
+
}
|
|
947
|
+
if (ownsPending)
|
|
948
|
+
await store.deleteMetadata(pendingSendTimestampKey(intent.clientMessageId));
|
|
949
|
+
await indexRecord(store, 'sent-timestamp', id, intent.createdAt, retainedDays);
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
/** The key of the pending marker of one client message ID. */
|
|
953
|
+
function pendingSendTimestampKey(clientMessageId) {
|
|
954
|
+
return `${SENT_TIMESTAMP_PREFIX}:pending:${encodeURIComponent(clientMessageId)}`;
|
|
955
|
+
}
|
|
956
|
+
async function pendingSendTimestamp(store, clientMessageId) {
|
|
957
|
+
return parseJson(await store.getMetadata(pendingSendTimestampKey(clientMessageId)), null);
|
|
958
|
+
}
|
|
959
|
+
/**
|
|
960
|
+
* The state of a different ID in the entry of a requested timestamp.
|
|
961
|
+
*
|
|
962
|
+
* - `committed`: a retained intent with this timestamp owns the ID, or the
|
|
963
|
+
* ID has no pending marker for this timestamp.
|
|
964
|
+
* - `in-flight`: the pending marker is younger than the grace period.
|
|
965
|
+
* - `orphan`: the pending marker is as old as the grace period or older, and
|
|
966
|
+
* no intent with this timestamp exists. The reservation does not count.
|
|
967
|
+
*/
|
|
968
|
+
async function sendTimestampHolderState(store, pending, clientMessageId, timestamp, grace, now) {
|
|
969
|
+
if (pending?.timestamp !== timestamp)
|
|
970
|
+
return 'committed';
|
|
971
|
+
const intent = await getOutgoingMessageIntent(store, clientMessageId, now);
|
|
972
|
+
if (intent?.clientTimestamp === timestamp)
|
|
973
|
+
return 'committed';
|
|
974
|
+
return now - pending.reservedAt < grace ? 'in-flight' : 'orphan';
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* Run `operation` under the send timestamp lock of one store. When
|
|
978
|
+
* `navigator.locks.request` exists and the store sets `storeIdentity`, the
|
|
979
|
+
* lock is the Web Lock `open-e2ee:send-timestamp:<store identity>`, so the
|
|
980
|
+
* tabs, workers and threads of one origin that share the store share it.
|
|
981
|
+
* Otherwise it is an in-process lock keyed by the store object.
|
|
982
|
+
*
|
|
983
|
+
* The lock is a leaf lock: `operation` reads and writes store metadata only,
|
|
984
|
+
* and takes no other lock of the SDK. A caller can hold the intent lock or
|
|
985
|
+
* the mutation queue of the store when it takes this lock, never the reverse.
|
|
907
986
|
*/
|
|
908
|
-
|
|
909
|
-
const
|
|
910
|
-
|
|
987
|
+
function withSendTimestampLock(store, operation) {
|
|
988
|
+
const scope = originLockScope(store);
|
|
989
|
+
if (scope !== null)
|
|
990
|
+
return withOriginLock(`open-e2ee:send-timestamp:${scope}`, operation);
|
|
991
|
+
return withKeyedLock(sendTimestampLocks, store, 'send-timestamp', operation);
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* Reserve the client timestamp of a new send, so that each send timestamp is
|
|
995
|
+
* unique per sender device. The reservation is the ID of the intent in the
|
|
996
|
+
* sent-timestamp entry of the timestamp, the entry that a receipt reads, and
|
|
997
|
+
* a pending marker of the ID with the time of the reservation. A new send
|
|
998
|
+
* reserves its timestamp before it writes its intent and before it posts
|
|
999
|
+
* anything. The intent write deletes the marker. The entry expires as the
|
|
1000
|
+
* entry of an intent does, and a new send accepts a caller timestamp at most
|
|
1001
|
+
* 24 hours old, so the entries hold every timestamp that a new send can
|
|
1002
|
+
* repeat.
|
|
1003
|
+
*
|
|
1004
|
+
* An entry that holds the ID is a reuse, so a retry of the same intent gets
|
|
1005
|
+
* its timestamp again, and a pending reuse gets a new reservation time. An
|
|
1006
|
+
* entry that a 13.x send wrote can hold several IDs. Each other ID in the
|
|
1007
|
+
* entry is committed, in flight or an orphan (see `sendTimestampHolderState`).
|
|
1008
|
+
* A committed or an in-flight ID makes the timestamp taken. The call removes
|
|
1009
|
+
* each orphan from the entry and deletes its marker, so a reservation that a
|
|
1010
|
+
* crash left before its intent write holds its timestamp only for the grace
|
|
1011
|
+
* period.
|
|
1012
|
+
*
|
|
1013
|
+
* A new reservation of the ID at another timestamp also removes the ID from
|
|
1014
|
+
* the entry of its earlier pending marker, unless a stored intent of the ID
|
|
1015
|
+
* has that timestamp. A send that stopped before its intent write and that a
|
|
1016
|
+
* retry with no timestamp reserves again leaves that earlier reservation.
|
|
1017
|
+
*
|
|
1018
|
+
* The read, the check and the write run under the send timestamp lock, so
|
|
1019
|
+
* two tabs on one store do not reserve one timestamp.
|
|
1020
|
+
*/
|
|
1021
|
+
export async function reserveSendTimestamp(store, request, now = Date.now()) {
|
|
1022
|
+
return serializeMutation(store, () => withSendTimestampLock(store, async () => {
|
|
1023
|
+
let retainedDays = await pruneExpiredBuckets(store, 'sent-timestamp', now);
|
|
1024
|
+
const markerKey = pendingSendTimestampKey(request.clientMessageId);
|
|
1025
|
+
let timestamp = request.timestamp;
|
|
1026
|
+
for (;;) {
|
|
1027
|
+
const id = String(timestamp);
|
|
1028
|
+
const key = recordKey('sent-timestamp', id);
|
|
1029
|
+
const clientMessageIds = parseJson(await store.getMetadata(key), []);
|
|
1030
|
+
if (clientMessageIds.includes(request.clientMessageId)) {
|
|
1031
|
+
const pending = await pendingSendTimestamp(store, request.clientMessageId);
|
|
1032
|
+
if (pending?.timestamp === timestamp && pending.reservedAt < now) {
|
|
1033
|
+
await store.setMetadata(markerKey, JSON.stringify({ reservedAt: now, timestamp }));
|
|
1034
|
+
}
|
|
1035
|
+
return { kind: 'reused', timestamp, reservedAt: now };
|
|
1036
|
+
}
|
|
1037
|
+
const orphans = [];
|
|
1038
|
+
let taken = false;
|
|
1039
|
+
for (const holder of clientMessageIds) {
|
|
1040
|
+
const pending = await pendingSendTimestamp(store, holder);
|
|
1041
|
+
const state = await sendTimestampHolderState(store, pending, holder, timestamp, request.grace, now);
|
|
1042
|
+
if (state !== 'orphan') {
|
|
1043
|
+
taken = true;
|
|
1044
|
+
break;
|
|
1045
|
+
}
|
|
1046
|
+
orphans.push({ clientMessageId: holder, reservedAt: pending.reservedAt });
|
|
1047
|
+
}
|
|
1048
|
+
if (!taken) {
|
|
1049
|
+
// The new marker writes over the earlier marker of this ID, so the
|
|
1050
|
+
// earlier reservation goes first. A stored intent with its
|
|
1051
|
+
// timestamp owns it, and a replay of the intent reuses it.
|
|
1052
|
+
let unindexed = false;
|
|
1053
|
+
const earlier = await pendingSendTimestamp(store, request.clientMessageId);
|
|
1054
|
+
if (earlier !== null && earlier.timestamp !== timestamp) {
|
|
1055
|
+
const intent = await getOutgoingMessageIntent(store, request.clientMessageId, now);
|
|
1056
|
+
if (intent?.clientTimestamp !== earlier.timestamp) {
|
|
1057
|
+
unindexed = await removeSentTimestampId(store, request.clientMessageId, earlier);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
// An ID is never in an entry without its marker before its intent
|
|
1061
|
+
// write: the marker goes before the entry that adds the ID, and the
|
|
1062
|
+
// entry that removes the orphans goes before their marker deletes.
|
|
1063
|
+
// A stop between the writes leaves only a stale marker.
|
|
1064
|
+
await store.setMetadata(markerKey, JSON.stringify({ reservedAt: now, timestamp }));
|
|
1065
|
+
await store.setMetadata(key, JSON.stringify([request.clientMessageId]));
|
|
1066
|
+
for (const orphan of orphans) {
|
|
1067
|
+
await store.deleteMetadata(pendingSendTimestampKey(orphan.clientMessageId));
|
|
1068
|
+
await unindexRecord(store, 'sent-timestamp', id, orphan.reservedAt);
|
|
1069
|
+
}
|
|
1070
|
+
if (unindexed || orphans.length > 0) {
|
|
1071
|
+
// The removal can delete a bucket and its day in the manifest.
|
|
1072
|
+
const manifest = parseJson(await store.getMetadata(manifestKey('sent-timestamp')), []);
|
|
1073
|
+
retainedDays = retainedDays.filter((day) => manifest.includes(day));
|
|
1074
|
+
}
|
|
1075
|
+
await indexRecord(store, 'sent-timestamp', id, now, retainedDays);
|
|
1076
|
+
return { kind: 'reserved', timestamp, reservedAt: now };
|
|
1077
|
+
}
|
|
1078
|
+
if (request.origin === 'caller')
|
|
1079
|
+
return { kind: 'used', timestamp };
|
|
1080
|
+
timestamp += 1;
|
|
1081
|
+
if (timestamp > request.latest)
|
|
1082
|
+
return { kind: 'past-latest', timestamp };
|
|
1083
|
+
}
|
|
1084
|
+
}));
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Remove one ID from the sent-timestamp entry of `reservation.timestamp`.
|
|
1088
|
+
* The call deletes the entry and its index record when no ID remains.
|
|
1089
|
+
*
|
|
1090
|
+
* @returns True when the call removed the index record of the entry.
|
|
1091
|
+
*/
|
|
1092
|
+
async function removeSentTimestampId(store, clientMessageId, reservation) {
|
|
1093
|
+
const id = String(reservation.timestamp);
|
|
911
1094
|
const key = recordKey('sent-timestamp', id);
|
|
912
1095
|
const clientMessageIds = parseJson(await store.getMetadata(key), []);
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
1096
|
+
const remaining = clientMessageIds.filter((candidate) => candidate !== clientMessageId);
|
|
1097
|
+
if (remaining.length === clientMessageIds.length)
|
|
1098
|
+
return false;
|
|
1099
|
+
if (remaining.length > 0) {
|
|
1100
|
+
await store.setMetadata(key, JSON.stringify(remaining));
|
|
1101
|
+
return false;
|
|
916
1102
|
}
|
|
917
|
-
await
|
|
1103
|
+
await store.deleteMetadata(key);
|
|
1104
|
+
await unindexRecord(store, 'sent-timestamp', id, reservation.reservedAt);
|
|
1105
|
+
return true;
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Remove the reservation of a send that failed before its intent write. A
|
|
1109
|
+
* retained intent with this ID and timestamp owns the reservation, so the
|
|
1110
|
+
* call then changes nothing. The call removes the ID from the entry, deletes
|
|
1111
|
+
* the entry when no ID remains, and then deletes the pending marker of the
|
|
1112
|
+
* reservation. The marker delete is the last write: when an earlier write
|
|
1113
|
+
* fails, the marker stays, and the ID is an orphan after the grace period.
|
|
1114
|
+
* Without the marker, the ID would be committed for the life of the entry.
|
|
1115
|
+
*/
|
|
1116
|
+
export async function releaseSendTimestamp(store, reservation, now = Date.now()) {
|
|
1117
|
+
await serializeMutation(store, () => withSendTimestampLock(store, async () => {
|
|
1118
|
+
const intent = await getOutgoingMessageIntent(store, reservation.clientMessageId, now);
|
|
1119
|
+
if (intent?.clientTimestamp === reservation.timestamp)
|
|
1120
|
+
return;
|
|
1121
|
+
await removeSentTimestampId(store, reservation.clientMessageId, reservation);
|
|
1122
|
+
const pending = await pendingSendTimestamp(store, reservation.clientMessageId);
|
|
1123
|
+
if (pending?.timestamp === reservation.timestamp) {
|
|
1124
|
+
await store.deleteMetadata(pendingSendTimestampKey(reservation.clientMessageId));
|
|
1125
|
+
}
|
|
1126
|
+
}));
|
|
918
1127
|
}
|
|
919
1128
|
/**
|
|
920
1129
|
* The client message IDs of the intents with this client timestamp. More
|
|
@@ -924,9 +1133,12 @@ async function indexSentTimestamp(store, intent, now) {
|
|
|
924
1133
|
export async function getSentClientMessageIds(store, clientTimestamp) {
|
|
925
1134
|
return parseJson(await store.getMetadata(recordKey('sent-timestamp', String(clientTimestamp))), []);
|
|
926
1135
|
}
|
|
927
|
-
/**
|
|
1136
|
+
/**
|
|
1137
|
+
* The record ID of one received message. The ID reads only the sender user
|
|
1138
|
+
* and the timestamp, so an SDK 13.x key with more fields gives the same ID.
|
|
1139
|
+
*/
|
|
928
1140
|
function receivedMessageId(key) {
|
|
929
|
-
return JSON.stringify([key.senderUserId, key.
|
|
1141
|
+
return JSON.stringify([key.senderUserId, key.timestamp]);
|
|
930
1142
|
}
|
|
931
1143
|
/**
|
|
932
1144
|
* True when the local device processed a message with this key. The record
|
|
@@ -1476,25 +1688,59 @@ export function retryRequestPostingDeadlineAt(record) {
|
|
|
1476
1688
|
const firstFailureAt = record.receivedAt ?? record.requestedAt;
|
|
1477
1689
|
return Math.min(record.deadlineAt, firstFailureAt + RETRY_REQUEST_POSTING_WINDOW_MS);
|
|
1478
1690
|
}
|
|
1691
|
+
/**
|
|
1692
|
+
* The terminal class of a retry request at its deadline, `deadlineAt`. The
|
|
1693
|
+
* class follows the acceptance state of the latest attempt, the stored
|
|
1694
|
+
* attempt:
|
|
1695
|
+
*
|
|
1696
|
+
* - The Relay answered the latest attempt, `answered`, or holds it, with no
|
|
1697
|
+
* `nextAttemptAt`: `request-expired`. The session does not reset.
|
|
1698
|
+
* - The Relay did not accept the latest attempt, which is not answered and
|
|
1699
|
+
* has a `nextAttemptAt`: `posting-deadline`. The posting deadline,
|
|
1700
|
+
* `retryRequestPostingDeadlineAt`, is at most `deadlineAt`, so it has
|
|
1701
|
+
* passed. The session resets, as at each `posting-deadline` end.
|
|
1702
|
+
*
|
|
1703
|
+
* Both writers of the deadline end use it: the scheduler pass and the post
|
|
1704
|
+
* of a retry request.
|
|
1705
|
+
*/
|
|
1706
|
+
export function deadlineTerminalClass(record) {
|
|
1707
|
+
if (record.answered || record.nextAttemptAt === undefined)
|
|
1708
|
+
return 'request-expired';
|
|
1709
|
+
return 'posting-deadline';
|
|
1710
|
+
}
|
|
1479
1711
|
/** The shortest time between two session reset null messages to one device. */
|
|
1480
1712
|
export const RETRY_TERMINAL_NULL_MESSAGE_INTERVAL_MS = 60 * 60 * 1_000;
|
|
1481
1713
|
const RETRY_TERMINAL_CLASSES = new Set([
|
|
1482
1714
|
'missing-sender',
|
|
1483
1715
|
'missing-prekey',
|
|
1484
1716
|
'revoked-authority',
|
|
1485
|
-
'expired
|
|
1717
|
+
'request-expired',
|
|
1486
1718
|
'posting-deadline',
|
|
1487
1719
|
'attempts-exhausted',
|
|
1488
1720
|
'deletion',
|
|
1489
1721
|
'abuse-allowance-exhausted',
|
|
1490
1722
|
'membership-refused',
|
|
1491
1723
|
]);
|
|
1724
|
+
/**
|
|
1725
|
+
* The SDK 13.x name of the class `request-expired`. The dual reader reads a
|
|
1726
|
+
* stored `expired-content` terminal as `request-expired`, so a record that the
|
|
1727
|
+
* migration did not move yet stays readable. Remove the dual reader only when
|
|
1728
|
+
* both conditions are true: the next major SDK version ships, and at least 60
|
|
1729
|
+
* days, `RETRY_REQUESTER_RECORD_TTL_MS`, have passed since the publish day of
|
|
1730
|
+
* SDK 14.0.0.
|
|
1731
|
+
*/
|
|
1732
|
+
const EXPIRED_CONTENT_CLASS = 'expired-content';
|
|
1733
|
+
/** The class of a stored terminal: the dual reader reads `expired-content` as `request-expired`. */
|
|
1734
|
+
function readRetryTerminalClass(value) {
|
|
1735
|
+
return value === EXPIRED_CONTENT_CLASS ? 'request-expired' : value;
|
|
1736
|
+
}
|
|
1492
1737
|
function isRetryTerminal(value) {
|
|
1493
1738
|
if (value === null || typeof value !== 'object')
|
|
1494
1739
|
return false;
|
|
1495
1740
|
const terminal = value;
|
|
1496
|
-
|
|
1497
|
-
|
|
1741
|
+
const terminalClass = readRetryTerminalClass(terminal.class);
|
|
1742
|
+
return (typeof terminalClass === 'string' &&
|
|
1743
|
+
RETRY_TERMINAL_CLASSES.has(terminalClass) &&
|
|
1498
1744
|
Number.isFinite(terminal.at) &&
|
|
1499
1745
|
typeof terminal.emitted === 'boolean');
|
|
1500
1746
|
}
|
|
@@ -1522,9 +1768,27 @@ function isRetryTerminalRecovery(value) {
|
|
|
1522
1768
|
* not the given fingerprint.
|
|
1523
1769
|
*/
|
|
1524
1770
|
export async function getRetryRequest(store, failedEnvelopeId, now = Date.now(), fingerprint) {
|
|
1525
|
-
const record =
|
|
1771
|
+
const record = parseRetryRequest(await store.getMetadata(recordKey('retry-requested', failedEnvelopeId)), failedEnvelopeId);
|
|
1772
|
+
if (!record)
|
|
1773
|
+
return null;
|
|
1774
|
+
if (record.requestedAt < now - RETRY_REQUESTER_RECORD_TTL_MS)
|
|
1775
|
+
return null;
|
|
1776
|
+
if (fingerprint !== undefined && fingerprint !== record.fingerprint) {
|
|
1777
|
+
throw new RetryRequestedEnvelopeChangedError();
|
|
1778
|
+
}
|
|
1779
|
+
return record;
|
|
1780
|
+
}
|
|
1781
|
+
/**
|
|
1782
|
+
* Parse a stored retry request record of one failed envelope, with the dual
|
|
1783
|
+
* reader: a stored `expired-content` terminal reads as `request-expired`.
|
|
1784
|
+
*
|
|
1785
|
+
* @throws Error When the record is corrupt.
|
|
1786
|
+
*/
|
|
1787
|
+
function parseRetryRequest(value, failedEnvelopeId) {
|
|
1788
|
+
const record = parseJson(value, null);
|
|
1526
1789
|
if (!record)
|
|
1527
1790
|
return null;
|
|
1791
|
+
const terminalClass = readRetryTerminalClass(record.terminal?.class);
|
|
1528
1792
|
if (!Number.isFinite(record.requestedAt) ||
|
|
1529
1793
|
record.failedEnvelopeId !== failedEnvelopeId ||
|
|
1530
1794
|
!isRetryId(record.retryId) ||
|
|
@@ -1547,17 +1811,14 @@ export async function getRetryRequest(store, failedEnvelopeId, now = Date.now(),
|
|
|
1547
1811
|
(typeof record.failedSender?.userId !== 'string' ||
|
|
1548
1812
|
!Number.isSafeInteger(record.failedSender.deviceId))) ||
|
|
1549
1813
|
(record.recovery !== undefined &&
|
|
1550
|
-
((
|
|
1551
|
-
record.terminal?.class !== 'posting-deadline') ||
|
|
1814
|
+
((terminalClass !== 'request-expired' && terminalClass !== 'posting-deadline') ||
|
|
1552
1815
|
!isRetryTerminalRecovery(record.recovery)))) {
|
|
1553
1816
|
throw new Error('Corrupt retry-request record');
|
|
1554
1817
|
}
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
}
|
|
1560
|
-
return cloneReliabilityValue(record);
|
|
1818
|
+
const parsed = cloneReliabilityValue(record);
|
|
1819
|
+
if (parsed.terminal)
|
|
1820
|
+
parsed.terminal.class = terminalClass;
|
|
1821
|
+
return parsed;
|
|
1561
1822
|
}
|
|
1562
1823
|
/**
|
|
1563
1824
|
* Mark the stored attempt of a retry request answered when a resend or a
|
|
@@ -1630,6 +1891,8 @@ export async function listRetryRequestIds(store, now = Date.now()) {
|
|
|
1630
1891
|
}
|
|
1631
1892
|
/** The key prefix of the retry request records before the lifecycle fields. */
|
|
1632
1893
|
const RETRY_REQUESTED_V1_PREFIX = 'reliability:retry-requested:v1';
|
|
1894
|
+
/** The key prefix of the retry request records of SDK 13.x, with the class `expired-content`. */
|
|
1895
|
+
const RETRY_REQUESTED_V2_PREFIX = 'reliability:retry-requested:v2';
|
|
1633
1896
|
/** A v1 retry request record, or null for a value that does not parse. */
|
|
1634
1897
|
function parseV1RetryRequest(value) {
|
|
1635
1898
|
try {
|
|
@@ -1640,20 +1903,57 @@ function parseV1RetryRequest(value) {
|
|
|
1640
1903
|
return null;
|
|
1641
1904
|
}
|
|
1642
1905
|
}
|
|
1906
|
+
/** A v2 retry request record through the dual reader, or null for a value that does not parse. */
|
|
1907
|
+
function parseV2RetryRequest(value, failedEnvelopeId) {
|
|
1908
|
+
try {
|
|
1909
|
+
return parseRetryRequest(value, failedEnvelopeId);
|
|
1910
|
+
}
|
|
1911
|
+
catch {
|
|
1912
|
+
return null;
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1643
1915
|
/**
|
|
1644
|
-
* Move each retry request record of
|
|
1645
|
-
* and delete the
|
|
1646
|
-
*
|
|
1647
|
-
* v1 record
|
|
1648
|
-
*
|
|
1649
|
-
*
|
|
1650
|
-
* record
|
|
1651
|
-
*
|
|
1916
|
+
* Move each retry request record of an earlier prefix to the current prefix,
|
|
1917
|
+
* and delete the earlier keys.
|
|
1918
|
+
*
|
|
1919
|
+
* The v1 step: a v1 record has no deadline, so the moved record gets the
|
|
1920
|
+
* time of its stored attempt plus `RETRY_REQUEST_DEADLINE_MS`. A v1 record
|
|
1921
|
+
* does not tell whether the post of its attempt reached the Relay, so an
|
|
1922
|
+
* unanswered record is due at the time of its stored attempt, and the
|
|
1923
|
+
* scheduler posts its stored bytes again.
|
|
1924
|
+
*
|
|
1925
|
+
* The v2 step: the dual reader reads each v2 record, so a stored
|
|
1926
|
+
* `expired-content` terminal moves as `request-expired`, with its `at`, its
|
|
1927
|
+
* `emitted` and the `recovery` of the record. The step deletes the v2
|
|
1928
|
+
* records, then the v2 buckets, and the v2 manifest last, so a stop leaves
|
|
1929
|
+
* each record that did not move in the v2 index, and the next start moves it.
|
|
1930
|
+
*
|
|
1931
|
+
* In each step, a record older than `RETRY_REQUESTER_RECORD_TTL_MS`, and a
|
|
1932
|
+
* record that does not parse, is deleted. With no v1 or v2 keys, it writes
|
|
1933
|
+
* nothing.
|
|
1934
|
+
*
|
|
1935
|
+
* The whole migration, the reads of the two manifests included, runs under
|
|
1936
|
+
* one lock. When `navigator.locks.request` exists and the store sets
|
|
1937
|
+
* `storeIdentity`, the lock is the Web Lock
|
|
1938
|
+
* `open-e2ee:retry-request-migration:<store identity>`, so two tabs, workers
|
|
1939
|
+
* or threads of one origin that start on one store do not move the same
|
|
1940
|
+
* records at the same time. Otherwise the in-process mutation order of the
|
|
1941
|
+
* store is the only lock.
|
|
1652
1942
|
*/
|
|
1653
1943
|
export async function migrateRetryRequestRecords(store, now = Date.now()) {
|
|
1944
|
+
const scope = originLockScope(store);
|
|
1945
|
+
if (scope !== null) {
|
|
1946
|
+
return withOriginLock(`open-e2ee:retry-request-migration:${scope}`, () => moveRetryRequestRecords(store, now));
|
|
1947
|
+
}
|
|
1948
|
+
return moveRetryRequestRecords(store, now);
|
|
1949
|
+
}
|
|
1950
|
+
/** The body of `migrateRetryRequestRecords`, which its caller runs under the migration lock. */
|
|
1951
|
+
async function moveRetryRequestRecords(store, now) {
|
|
1654
1952
|
const v1ManifestKey = `${RETRY_REQUESTED_V1_PREFIX}:manifest`;
|
|
1655
1953
|
const days = parseJson(await store.getMetadata(v1ManifestKey), []);
|
|
1656
|
-
|
|
1954
|
+
const v2ManifestKey = `${RETRY_REQUESTED_V2_PREFIX}:manifest`;
|
|
1955
|
+
const v2Days = parseJson(await store.getMetadata(v2ManifestKey), []);
|
|
1956
|
+
if (days.length === 0 && v2Days.length === 0)
|
|
1657
1957
|
return;
|
|
1658
1958
|
await serializeMutation(store, async () => {
|
|
1659
1959
|
for (const day of days) {
|
|
@@ -1677,7 +1977,187 @@ export async function migrateRetryRequestRecords(store, now = Date.now()) {
|
|
|
1677
1977
|
}
|
|
1678
1978
|
await store.deleteMetadata(v1BucketKey);
|
|
1679
1979
|
}
|
|
1980
|
+
if (days.length > 0)
|
|
1981
|
+
await store.deleteMetadata(v1ManifestKey);
|
|
1982
|
+
for (const day of v2Days) {
|
|
1983
|
+
const v2BucketKey = `${RETRY_REQUESTED_V2_PREFIX}:bucket:${day}`;
|
|
1984
|
+
for (const id of parseJson(await store.getMetadata(v2BucketKey), [])) {
|
|
1985
|
+
const v2RecordKey = `${RETRY_REQUESTED_V2_PREFIX}:record:${encodeURIComponent(id)}`;
|
|
1986
|
+
const record = parseV2RetryRequest(await store.getMetadata(v2RecordKey), id);
|
|
1987
|
+
if (record && record.requestedAt >= now - RETRY_REQUESTER_RECORD_TTL_MS) {
|
|
1988
|
+
const retainedDays = await pruneExpiredBuckets(store, 'retry-requested', now);
|
|
1989
|
+
await store.setMetadata(recordKey('retry-requested', id), JSON.stringify(record));
|
|
1990
|
+
await indexRecord(store, 'retry-requested', id, record.requestedAt, retainedDays);
|
|
1991
|
+
}
|
|
1992
|
+
await store.deleteMetadata(v2RecordKey);
|
|
1993
|
+
}
|
|
1994
|
+
await store.deleteMetadata(v2BucketKey);
|
|
1995
|
+
}
|
|
1996
|
+
if (v2Days.length > 0)
|
|
1997
|
+
await store.deleteMetadata(v2ManifestKey);
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
/**
|
|
2001
|
+
* Move each SDK 13.x received-message record to the current prefix, under
|
|
2002
|
+
* the key `[senderUserId, timestamp]`, and delete the v1 keys.
|
|
2003
|
+
*
|
|
2004
|
+
* A v1 record ID is `[senderUserId, senderDeviceId, timestamp, digest]`.
|
|
2005
|
+
* The step writes one record for each sender user and timestamp, with the
|
|
2006
|
+
* earliest `receivedAt` of the v1 records and of a record that the current
|
|
2007
|
+
* prefix already holds for the key. It indexes each record under the day of
|
|
2008
|
+
* that time, so the current ledger prunes it after
|
|
2009
|
+
* `RETRY_REQUESTER_RECORD_TTL_MS`. A v1 record whose ID or value does not
|
|
2010
|
+
* parse is deleted and counted, and does not stop the step.
|
|
2011
|
+
*
|
|
2012
|
+
* The v1 manifest is the marker: with no v1 manifest, the step writes
|
|
2013
|
+
* nothing. The step deletes the v1 records of a bucket, then that bucket,
|
|
2014
|
+
* and the v1 manifest after every bucket. A stop before the manifest delete
|
|
2015
|
+
* runs the whole step again at the next start. That run merges into the
|
|
2016
|
+
* records that the stopped run wrote, so each key keeps one record with the
|
|
2017
|
+
* earliest `receivedAt`.
|
|
2018
|
+
*
|
|
2019
|
+
* The whole step, the read of the v1 manifest included, runs under one lock.
|
|
2020
|
+
* When `navigator.locks.request` exists and the store sets `storeIdentity`,
|
|
2021
|
+
* the lock is the Web Lock
|
|
2022
|
+
* `open-e2ee:received-timestamp-migration:<store identity>`. Otherwise the
|
|
2023
|
+
* in-process mutation order of the store is the only lock. The step takes no
|
|
2024
|
+
* other lock while it holds this one. A step that rewrote at least one record
|
|
2025
|
+
* logs one line with its counts.
|
|
2026
|
+
*/
|
|
2027
|
+
export async function migrateReceivedTimestampRecords(store, logger = defaultSignalProtocolLogger) {
|
|
2028
|
+
const scope = originLockScope(store);
|
|
2029
|
+
const counts = scope !== null
|
|
2030
|
+
? await withOriginLock(`open-e2ee:received-timestamp-migration:${scope}`, () => moveReceivedTimestampRecords(store))
|
|
2031
|
+
: await moveReceivedTimestampRecords(store);
|
|
2032
|
+
if (counts !== null && counts.rewritten > 0) {
|
|
2033
|
+
logger.info('Rewrote received-timestamp records', { category: 'KeyStorage', data: counts });
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
/**
|
|
2037
|
+
* The v1 record ID `[senderUserId, senderDeviceId, timestamp, digest]` as
|
|
2038
|
+
* the current key, or null for an ID that does not parse into four fields.
|
|
2039
|
+
*/
|
|
2040
|
+
function parseV1ReceivedMessageId(id) {
|
|
2041
|
+
let fields;
|
|
2042
|
+
try {
|
|
2043
|
+
fields = JSON.parse(id);
|
|
2044
|
+
}
|
|
2045
|
+
catch {
|
|
2046
|
+
return null;
|
|
2047
|
+
}
|
|
2048
|
+
if (!Array.isArray(fields) || fields.length !== 4)
|
|
2049
|
+
return null;
|
|
2050
|
+
const [senderUserId, senderDeviceId, timestamp, digest] = fields;
|
|
2051
|
+
if (typeof senderUserId !== 'string' ||
|
|
2052
|
+
senderUserId === '' ||
|
|
2053
|
+
!Number.isSafeInteger(senderDeviceId) ||
|
|
2054
|
+
typeof timestamp !== 'number' ||
|
|
2055
|
+
!Number.isFinite(timestamp) ||
|
|
2056
|
+
typeof digest !== 'string') {
|
|
2057
|
+
return null;
|
|
2058
|
+
}
|
|
2059
|
+
return { senderUserId, timestamp };
|
|
2060
|
+
}
|
|
2061
|
+
/** The `receivedAt` of one stored received-message record, or null for a value that does not parse. */
|
|
2062
|
+
function parseReceivedAt(value) {
|
|
2063
|
+
if (value === null)
|
|
2064
|
+
return null;
|
|
2065
|
+
try {
|
|
2066
|
+
const { receivedAt } = JSON.parse(value);
|
|
2067
|
+
return typeof receivedAt === 'number' && Number.isFinite(receivedAt) ? receivedAt : null;
|
|
2068
|
+
}
|
|
2069
|
+
catch {
|
|
2070
|
+
return null;
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
/**
|
|
2074
|
+
* The body of `migrateReceivedTimestampRecords`, which its caller runs under
|
|
2075
|
+
* the migration lock. Resolves to null when the store has no v1 manifest.
|
|
2076
|
+
*
|
|
2077
|
+
* The step works one v1 day bucket at a time, so its store operations grow
|
|
2078
|
+
* linearly with the v1 records. For each bucket it reads each v1 record
|
|
2079
|
+
* once and merges the records of one key to the earliest `receivedAt`. It
|
|
2080
|
+
* then reads and writes the current record of each key once, reads and
|
|
2081
|
+
* writes the current bucket of each affected day once, and writes the
|
|
2082
|
+
* current manifest once. Only then does it delete the v1 records and the v1
|
|
2083
|
+
* bucket. A stop before those deletes leaves the v1 records, and the next run
|
|
2084
|
+
* merges them again into the same current records.
|
|
2085
|
+
*/
|
|
2086
|
+
async function moveReceivedTimestampRecords(store) {
|
|
2087
|
+
const v1ManifestKey = `${RECEIVED_TIMESTAMP_V1_PREFIX}:manifest`;
|
|
2088
|
+
const manifest = await store.getMetadata(v1ManifestKey);
|
|
2089
|
+
if (manifest === null)
|
|
2090
|
+
return null;
|
|
2091
|
+
return serializeMutation(store, async () => {
|
|
2092
|
+
const counts = { rewritten: 0, merged: 0, malformed: 0 };
|
|
2093
|
+
const retainedDays = parseJson(await store.getMetadata(manifestKey('received-timestamp')), []);
|
|
2094
|
+
for (const day of parseJson(manifest, [])) {
|
|
2095
|
+
const v1BucketKey = `${RECEIVED_TIMESTAMP_V1_PREFIX}:bucket:${day}`;
|
|
2096
|
+
const v1Ids = parseJson(await store.getMetadata(v1BucketKey), []);
|
|
2097
|
+
const v1RecordKeys = v1Ids.map((v1Id) => ({
|
|
2098
|
+
v1Id,
|
|
2099
|
+
key: `${RECEIVED_TIMESTAMP_V1_PREFIX}:record:${encodeURIComponent(v1Id)}`,
|
|
2100
|
+
}));
|
|
2101
|
+
// The earliest v1 receivedAt of each current ID, and its number of v1 records.
|
|
2102
|
+
const merges = new Map();
|
|
2103
|
+
for (const { v1Id, key: v1RecordKey } of v1RecordKeys) {
|
|
2104
|
+
const value = await store.getMetadata(v1RecordKey);
|
|
2105
|
+
const key = parseV1ReceivedMessageId(v1Id);
|
|
2106
|
+
const v1ReceivedAt = parseReceivedAt(value);
|
|
2107
|
+
if (key === null || (value !== null && v1ReceivedAt === null)) {
|
|
2108
|
+
counts.malformed += 1;
|
|
2109
|
+
continue;
|
|
2110
|
+
}
|
|
2111
|
+
// A null value is a record that a stopped run already deleted.
|
|
2112
|
+
if (v1ReceivedAt === null)
|
|
2113
|
+
continue;
|
|
2114
|
+
const id = receivedMessageId(key);
|
|
2115
|
+
const merge = merges.get(id);
|
|
2116
|
+
merges.set(id, {
|
|
2117
|
+
receivedAt: merge === undefined ? v1ReceivedAt : Math.min(merge.receivedAt, v1ReceivedAt),
|
|
2118
|
+
records: (merge?.records ?? 0) + 1,
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
const idsByDay = new Map();
|
|
2122
|
+
for (const [id, merge] of merges) {
|
|
2123
|
+
const key = recordKey('received-timestamp', id);
|
|
2124
|
+
const current = parseReceivedAt(await store.getMetadata(key));
|
|
2125
|
+
const receivedAt = current === null ? merge.receivedAt : Math.min(current, merge.receivedAt);
|
|
2126
|
+
await store.setMetadata(key, JSON.stringify({ receivedAt }));
|
|
2127
|
+
const ids = idsByDay.get(dayFor(receivedAt));
|
|
2128
|
+
if (ids === undefined)
|
|
2129
|
+
idsByDay.set(dayFor(receivedAt), [id]);
|
|
2130
|
+
else
|
|
2131
|
+
ids.push(id);
|
|
2132
|
+
counts.rewritten += merge.records;
|
|
2133
|
+
// Each v1 record merges into a current record, except the first v1
|
|
2134
|
+
// record of a key that had no current record.
|
|
2135
|
+
counts.merged += current === null ? merge.records - 1 : merge.records;
|
|
2136
|
+
}
|
|
2137
|
+
let manifestChanged = false;
|
|
2138
|
+
for (const [v2Day, ids] of idsByDay) {
|
|
2139
|
+
const dayKey = bucketKey('received-timestamp', v2Day);
|
|
2140
|
+
const indexed = parseJson(await store.getMetadata(dayKey), []);
|
|
2141
|
+
const present = new Set(indexed);
|
|
2142
|
+
const added = ids.filter((id) => !present.has(id));
|
|
2143
|
+
if (added.length > 0) {
|
|
2144
|
+
await store.setMetadata(dayKey, JSON.stringify([...indexed, ...added]));
|
|
2145
|
+
}
|
|
2146
|
+
if (!retainedDays.includes(v2Day)) {
|
|
2147
|
+
retainedDays.push(v2Day);
|
|
2148
|
+
manifestChanged = true;
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
if (manifestChanged) {
|
|
2152
|
+
retainedDays.sort((a, b) => a - b);
|
|
2153
|
+
await store.setMetadata(manifestKey('received-timestamp'), JSON.stringify(retainedDays));
|
|
2154
|
+
}
|
|
2155
|
+
for (const { key: v1RecordKey } of v1RecordKeys)
|
|
2156
|
+
await store.deleteMetadata(v1RecordKey);
|
|
2157
|
+
await store.deleteMetadata(v1BucketKey);
|
|
2158
|
+
}
|
|
1680
2159
|
await store.deleteMetadata(v1ManifestKey);
|
|
2160
|
+
return counts;
|
|
1681
2161
|
});
|
|
1682
2162
|
}
|
|
1683
2163
|
/**
|