@aztec/pxe 3.0.0-nightly.20251216 → 3.0.0-nightly.20251217

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dest/block_synchronizer/block_synchronizer.d.ts +39 -0
  2. package/dest/block_synchronizer/block_synchronizer.d.ts.map +1 -0
  3. package/dest/{synchronizer/synchronizer.js → block_synchronizer/block_synchronizer.js} +22 -17
  4. package/dest/block_synchronizer/index.d.ts +2 -0
  5. package/dest/block_synchronizer/index.d.ts.map +1 -0
  6. package/dest/block_synchronizer/index.js +1 -0
  7. package/dest/config/index.d.ts +4 -4
  8. package/dest/config/index.d.ts.map +1 -1
  9. package/dest/contract_function_simulator/contract_function_simulator.d.ts +6 -4
  10. package/dest/contract_function_simulator/contract_function_simulator.d.ts.map +1 -1
  11. package/dest/contract_function_simulator/contract_function_simulator.js +7 -6
  12. package/dest/contract_function_simulator/execution_data_provider.d.ts +6 -26
  13. package/dest/contract_function_simulator/execution_data_provider.d.ts.map +1 -1
  14. package/dest/contract_function_simulator/execution_tagging_index_cache.d.ts +1 -1
  15. package/dest/contract_function_simulator/execution_tagging_index_cache.js +1 -1
  16. package/dest/contract_function_simulator/oracle/interfaces.d.ts +2 -2
  17. package/dest/contract_function_simulator/oracle/interfaces.d.ts.map +1 -1
  18. package/dest/contract_function_simulator/oracle/oracle.d.ts +1 -1
  19. package/dest/contract_function_simulator/oracle/oracle.d.ts.map +1 -1
  20. package/dest/contract_function_simulator/oracle/oracle.js +3 -3
  21. package/dest/contract_function_simulator/oracle/private_execution.d.ts +2 -2
  22. package/dest/contract_function_simulator/oracle/private_execution.d.ts.map +1 -1
  23. package/dest/contract_function_simulator/oracle/private_execution.js +0 -1
  24. package/dest/contract_function_simulator/oracle/private_execution_oracle.d.ts +2 -3
  25. package/dest/contract_function_simulator/oracle/private_execution_oracle.d.ts.map +1 -1
  26. package/dest/contract_function_simulator/oracle/private_execution_oracle.js +8 -4
  27. package/dest/contract_function_simulator/oracle/utility_execution_oracle.d.ts +4 -3
  28. package/dest/contract_function_simulator/oracle/utility_execution_oracle.d.ts.map +1 -1
  29. package/dest/contract_function_simulator/oracle/utility_execution_oracle.js +8 -7
  30. package/dest/contract_function_simulator/pxe_oracle_interface.d.ts +9 -20
  31. package/dest/contract_function_simulator/pxe_oracle_interface.d.ts.map +1 -1
  32. package/dest/contract_function_simulator/pxe_oracle_interface.js +41 -106
  33. package/dest/debug/pxe_debug_utils.d.ts +30 -0
  34. package/dest/debug/pxe_debug_utils.d.ts.map +1 -0
  35. package/dest/debug/pxe_debug_utils.js +37 -0
  36. package/dest/events/private_event_filter_validator.d.ts +4 -4
  37. package/dest/events/private_event_filter_validator.d.ts.map +1 -1
  38. package/dest/events/private_event_filter_validator.js +4 -4
  39. package/dest/pxe.d.ts +19 -28
  40. package/dest/pxe.d.ts.map +1 -1
  41. package/dest/pxe.js +70 -86
  42. package/dest/storage/anchor_block_data_provider/anchor_block_data_provider.d.ts +9 -0
  43. package/dest/storage/anchor_block_data_provider/anchor_block_data_provider.d.ts.map +1 -0
  44. package/dest/storage/{sync_data_provider/sync_data_provider.js → anchor_block_data_provider/anchor_block_data_provider.js} +2 -9
  45. package/dest/storage/anchor_block_data_provider/index.d.ts +2 -0
  46. package/dest/storage/anchor_block_data_provider/index.d.ts.map +1 -0
  47. package/dest/storage/anchor_block_data_provider/index.js +1 -0
  48. package/dest/storage/contract_data_provider/contract_data_provider.d.ts +3 -2
  49. package/dest/storage/contract_data_provider/contract_data_provider.d.ts.map +1 -1
  50. package/dest/storage/contract_data_provider/contract_data_provider.js +21 -1
  51. package/dest/storage/index.d.ts +2 -2
  52. package/dest/storage/index.d.ts.map +1 -1
  53. package/dest/storage/index.js +1 -1
  54. package/dest/storage/tagging_data_provider/index.d.ts +3 -2
  55. package/dest/storage/tagging_data_provider/index.d.ts.map +1 -1
  56. package/dest/storage/tagging_data_provider/index.js +2 -1
  57. package/dest/storage/tagging_data_provider/recipient_tagging_data_provider.d.ts +31 -0
  58. package/dest/storage/tagging_data_provider/recipient_tagging_data_provider.d.ts.map +1 -0
  59. package/dest/storage/tagging_data_provider/recipient_tagging_data_provider.js +65 -0
  60. package/dest/storage/tagging_data_provider/sender_tagging_data_provider.d.ts +67 -0
  61. package/dest/storage/tagging_data_provider/sender_tagging_data_provider.d.ts.map +1 -0
  62. package/dest/storage/tagging_data_provider/sender_tagging_data_provider.js +196 -0
  63. package/dest/tagging/constants.d.ts +1 -1
  64. package/dest/tagging/constants.d.ts.map +1 -1
  65. package/dest/tagging/constants.js +1 -0
  66. package/dest/tagging/sync/sync_sender_tagging_indexes.d.ts +21 -0
  67. package/dest/tagging/sync/sync_sender_tagging_indexes.d.ts.map +1 -0
  68. package/dest/tagging/sync/sync_sender_tagging_indexes.js +89 -0
  69. package/dest/tagging/sync/utils/get_status_change_of_pending.d.ts +11 -0
  70. package/dest/tagging/sync/utils/get_status_change_of_pending.d.ts.map +1 -0
  71. package/dest/tagging/sync/utils/get_status_change_of_pending.js +32 -0
  72. package/dest/tagging/sync/utils/load_and_store_new_tagging_indexes.d.ts +18 -0
  73. package/dest/tagging/sync/utils/load_and_store_new_tagging_indexes.d.ts.map +1 -0
  74. package/dest/tagging/sync/utils/load_and_store_new_tagging_indexes.js +57 -0
  75. package/dest/tagging/utils.d.ts +2 -2
  76. package/dest/tagging/utils.d.ts.map +1 -1
  77. package/dest/tagging/utils.js +3 -2
  78. package/package.json +16 -16
  79. package/src/{synchronizer/synchronizer.ts → block_synchronizer/block_synchronizer.ts} +21 -17
  80. package/src/block_synchronizer/index.ts +1 -0
  81. package/src/config/index.ts +3 -3
  82. package/src/contract_function_simulator/contract_function_simulator.ts +21 -5
  83. package/src/contract_function_simulator/execution_data_provider.ts +7 -28
  84. package/src/contract_function_simulator/execution_tagging_index_cache.ts +1 -1
  85. package/src/contract_function_simulator/oracle/interfaces.ts +1 -1
  86. package/src/contract_function_simulator/oracle/oracle.ts +3 -3
  87. package/src/contract_function_simulator/oracle/private_execution.ts +1 -3
  88. package/src/contract_function_simulator/oracle/private_execution_oracle.ts +11 -5
  89. package/src/contract_function_simulator/oracle/utility_execution_oracle.ts +6 -6
  90. package/src/contract_function_simulator/pxe_oracle_interface.ts +41 -114
  91. package/src/debug/pxe_debug_utils.ts +47 -0
  92. package/src/events/private_event_filter_validator.ts +3 -3
  93. package/src/pxe.ts +74 -96
  94. package/src/storage/{sync_data_provider/sync_data_provider.ts → anchor_block_data_provider/anchor_block_data_provider.ts} +2 -12
  95. package/src/storage/anchor_block_data_provider/index.ts +1 -0
  96. package/src/storage/contract_data_provider/contract_data_provider.ts +27 -0
  97. package/src/storage/index.ts +1 -1
  98. package/src/storage/tagging_data_provider/index.ts +2 -1
  99. package/src/storage/tagging_data_provider/recipient_tagging_data_provider.ts +86 -0
  100. package/src/storage/tagging_data_provider/sender_tagging_data_provider.ts +244 -0
  101. package/src/tagging/constants.ts +1 -0
  102. package/src/tagging/sync/sync_sender_tagging_indexes.ts +112 -0
  103. package/src/tagging/sync/utils/get_status_change_of_pending.ts +44 -0
  104. package/src/tagging/sync/utils/load_and_store_new_tagging_indexes.ts +74 -0
  105. package/src/tagging/utils.ts +3 -2
  106. package/dest/storage/sync_data_provider/index.d.ts +0 -2
  107. package/dest/storage/sync_data_provider/index.d.ts.map +0 -1
  108. package/dest/storage/sync_data_provider/index.js +0 -1
  109. package/dest/storage/sync_data_provider/sync_data_provider.d.ts +0 -11
  110. package/dest/storage/sync_data_provider/sync_data_provider.d.ts.map +0 -1
  111. package/dest/storage/tagging_data_provider/tagging_data_provider.d.ts +0 -40
  112. package/dest/storage/tagging_data_provider/tagging_data_provider.d.ts.map +0 -1
  113. package/dest/storage/tagging_data_provider/tagging_data_provider.js +0 -89
  114. package/dest/synchronizer/index.d.ts +0 -2
  115. package/dest/synchronizer/index.d.ts.map +0 -1
  116. package/dest/synchronizer/index.js +0 -1
  117. package/dest/synchronizer/synchronizer.d.ts +0 -36
  118. package/dest/synchronizer/synchronizer.d.ts.map +0 -1
  119. package/src/storage/sync_data_provider/index.ts +0 -1
  120. package/src/storage/tagging_data_provider/tagging_data_provider.ts +0 -120
  121. package/src/synchronizer/index.ts +0 -1
@@ -0,0 +1,244 @@
1
+ import { toArray } from '@aztec/foundation/iterable';
2
+ import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
3
+ import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs';
4
+ import { TxHash } from '@aztec/stdlib/tx';
5
+
6
+ import { WINDOW_LEN as SENDER_TAGGING_INDEXES_SYNC_WINDOW_LEN } from '../../tagging/sync/sync_sender_tagging_indexes.js';
7
+
8
+ /**
9
+ * Data provider of tagging data used when syncing the sender tagging indexes. The recipient counterpart of this class
10
+ * is called RecipientTaggingDataProvider. We have the providers separate for the sender and recipient because
11
+ * the algorithms are completely disjoint and there is not data reuse between the two.
12
+ */
13
+ export class SenderTaggingDataProvider {
14
+ #store: AztecAsyncKVStore;
15
+
16
+ // Stores the pending indexes for each directional app tagging secret. Pending here means that the tx that contained
17
+ // the private logs with tags corresponding to these indexes has not been finalized yet.
18
+ //
19
+ // We don't store just the highest index because if their transaction is dropped we'd then need the information about
20
+ // the lower pending indexes. For each secret-tx pair however we only store the largest index used in that tx, since
21
+ // the smaller ones are irrelevant due to tx atomicity.
22
+ //
23
+ // TODO(#17615): This assumes no logs are used in the non-revertible phase.
24
+ #pendingIndexes: AztecAsyncMap<string, { index: number; txHash: string }[]>;
25
+
26
+ // Stores the last (highest) finalized index for each directional app tagging secret. We care only about the last
27
+ // index because unlike the pending indexes, it will never happen that a finalized index would be removed and hence
28
+ // we don't need to store the history.
29
+ #lastFinalizedIndexes: AztecAsyncMap<string, number>;
30
+
31
+ constructor(store: AztecAsyncKVStore) {
32
+ this.#store = store;
33
+
34
+ this.#pendingIndexes = this.#store.openMap('pending_indexes');
35
+ this.#lastFinalizedIndexes = this.#store.openMap('last_finalized_indexes');
36
+ }
37
+
38
+ /**
39
+ * Stores pending indexes.
40
+ * @remarks Ignores the index if the same preTag + txHash combination already exists in the db with the same index.
41
+ * This is expected to happen because whenever we start sync we start from the last finalized index and we can have
42
+ * pending indexes already stored from previous syncs.
43
+ * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be
44
+ * stored in the db.
45
+ * @param txHash - The tx in which the pretags were used in private logs.
46
+ * @throws If any two pre-tags contain the same directional app tagging secret. This is enforced because we care
47
+ * only about the highest index for a given secret that was used in the tx. Hence this check is a good way to catch
48
+ * bugs.
49
+ * @throws If the newly stored pending index is further than window length from the highest finalized index for the
50
+ * same secret. This is enforced in order to give a guarantee to a recipient that he doesn't need to look further than
51
+ * window length ahead of the highest finalized index.
52
+ * @throws If a secret + txHash pair already exists in the db with a different index value. It should never happen
53
+ * that we would attempt to store a different index for a given secret-txHash pair because we always store just the
54
+ * highest index for a given secret-txHash pair. Hence this is a good way to catch bugs.
55
+ * @throws If the newly stored pending index is lower than or equal to the last finalized index for the same secret.
56
+ * This is enforced because this should never happen if the syncing is done correctly as we look for logs from higher
57
+ * indexes than finalized ones.
58
+ */
59
+ async storePendingIndexes(preTags: PreTag[], txHash: TxHash) {
60
+ // The secrets in pre-tags should be unique because we always store just the highest index per given secret-txHash
61
+ // pair. Below we check that this is the case.
62
+ const secretsSet = new Set(preTags.map(preTag => preTag.secret.toString()));
63
+ if (secretsSet.size !== preTags.length) {
64
+ throw new Error(`Duplicate secrets found when storing pending indexes`);
65
+ }
66
+
67
+ for (const { secret, index } of preTags) {
68
+ // First we check that for any secret the highest used index in tx is not further than window length from
69
+ // the highest finalized index.
70
+ const finalizedIndex = (await this.getLastFinalizedIndex(secret)) ?? 0;
71
+ if (index > finalizedIndex + SENDER_TAGGING_INDEXES_SYNC_WINDOW_LEN) {
72
+ throw new Error(
73
+ `Highest used index ${index} is further than window length from the highest finalized index ${finalizedIndex}.
74
+ Tagging window length ${SENDER_TAGGING_INDEXES_SYNC_WINDOW_LEN} is configured too low. Contact the Aztec team
75
+ to increase it!`,
76
+ );
77
+ }
78
+
79
+ // Throw if the new pending index is lower than or equal to the last finalized index
80
+ const secretStr = secret.toString();
81
+ const lastFinalizedIndex = await this.#lastFinalizedIndexes.getAsync(secretStr);
82
+ if (lastFinalizedIndex !== undefined && index <= lastFinalizedIndex) {
83
+ throw new Error(
84
+ `Cannot store pending index ${index} for secret ${secretStr}: ` +
85
+ `it is lower than or equal to the last finalized index ${lastFinalizedIndex}`,
86
+ );
87
+ }
88
+
89
+ // Check if this secret + txHash combination already exists
90
+ const txHashStr = txHash.toString();
91
+ const existingForSecret = (await this.#pendingIndexes.getAsync(secretStr)) ?? [];
92
+ const existingForSecretAndTx = existingForSecret.find(entry => entry.txHash === txHashStr);
93
+
94
+ if (existingForSecretAndTx) {
95
+ // If it exists with a different index, throw an error
96
+ if (existingForSecretAndTx.index !== index) {
97
+ throw new Error(
98
+ `Cannot store index ${index} for secret ${secretStr} and txHash ${txHashStr}: ` +
99
+ `a different index ${existingForSecretAndTx.index} already exists for this secret-txHash pair`,
100
+ );
101
+ }
102
+ // If it exists with the same index, ignore the update (no-op)
103
+ } else {
104
+ // If it doesn't exist, add it
105
+ await this.#pendingIndexes.set(secretStr, [...existingForSecret, { index, txHash: txHashStr }]);
106
+ }
107
+ }
108
+ }
109
+
110
+ /**
111
+ * Returns the transaction hashes of all pending transactions that contain indexes within a specified range
112
+ * for a given directional app tagging secret.
113
+ * @param secret - The directional app tagging secret to query pending indexes for.
114
+ * @param startIndex - The lower bound of the index range (inclusive).
115
+ * @param endIndex - The upper bound of the index range (exclusive).
116
+ * @returns An array of unique transaction hashes for pending transactions that contain indexes in the range
117
+ * [startIndex, endIndex). Returns an empty array if no pending indexes exist in the range.
118
+ */
119
+ async getTxHashesOfPendingIndexes(
120
+ secret: DirectionalAppTaggingSecret,
121
+ startIndex: number,
122
+ endIndex: number,
123
+ ): Promise<TxHash[]> {
124
+ const existing = (await this.#pendingIndexes.getAsync(secret.toString())) ?? [];
125
+ const txHashes = existing
126
+ .filter(entry => entry.index >= startIndex && entry.index < endIndex)
127
+ .map(entry => entry.txHash);
128
+ return Array.from(new Set(txHashes)).map(TxHash.fromString);
129
+ }
130
+
131
+ /**
132
+ * Returns the last (highest) finalized index for a given secret.
133
+ * @param secret - The secret to get the last finalized index for.
134
+ * @returns The last (highest) finalized index for the given secret.
135
+ */
136
+ getLastFinalizedIndex(secret: DirectionalAppTaggingSecret): Promise<number | undefined> {
137
+ return this.#lastFinalizedIndexes.getAsync(secret.toString());
138
+ }
139
+
140
+ /**
141
+ * Returns the last used index for a given directional app tagging secret, considering both finalized and pending
142
+ * indexes.
143
+ * @param secret - The directional app tagging secret to query the last used index for.
144
+ * @returns The last used index.
145
+ */
146
+ async getLastUsedIndex(secret: DirectionalAppTaggingSecret): Promise<number | undefined> {
147
+ const secretStr = secret.toString();
148
+ const pendingTxScopedIndexes = (await this.#pendingIndexes.getAsync(secretStr)) ?? [];
149
+ const pendingIndexes = pendingTxScopedIndexes.map(entry => entry.index);
150
+
151
+ if (pendingTxScopedIndexes.length === 0) {
152
+ return this.#lastFinalizedIndexes.getAsync(secretStr);
153
+ }
154
+
155
+ // As the last used index we return the highest one from the pending indexes. Note that this value will be always
156
+ // higher than the last finalized index because we prune lower pending indexes when a tx is finalized.
157
+ return Math.max(...pendingIndexes);
158
+ }
159
+
160
+ /**
161
+ * Drops all pending indexes corresponding to the given transaction hashes.
162
+ */
163
+ async dropPendingIndexes(txHashes: TxHash[]) {
164
+ if (txHashes.length === 0) {
165
+ return;
166
+ }
167
+
168
+ const txHashStrs = new Set<string>(txHashes.map(txHash => txHash.toString()));
169
+ const allSecrets = await toArray(this.#pendingIndexes.keysAsync());
170
+
171
+ for (const secret of allSecrets) {
172
+ const pendingData = await this.#pendingIndexes.getAsync(secret);
173
+ if (pendingData) {
174
+ const filtered = pendingData.filter(item => !txHashStrs.has(item.txHash));
175
+ if (filtered.length === 0) {
176
+ await this.#pendingIndexes.delete(secret);
177
+ } else if (filtered.length !== pendingData.length) {
178
+ // Some items were filtered out, so update the pending data
179
+ await this.#pendingIndexes.set(secret, filtered);
180
+ }
181
+ // else: No items were filtered out (txHashes not found for this secret) --> no-op
182
+ }
183
+ }
184
+ }
185
+
186
+ /**
187
+ * Updates pending indexes corresponding to the given transaction hashes to be finalized and prunes any lower pending
188
+ * indexes.
189
+ */
190
+ async finalizePendingIndexes(txHashes: TxHash[]) {
191
+ if (txHashes.length === 0) {
192
+ return;
193
+ }
194
+
195
+ for (const txHash of txHashes) {
196
+ const txHashStr = txHash.toString();
197
+
198
+ const allSecrets = await toArray(this.#pendingIndexes.keysAsync());
199
+
200
+ for (const secret of allSecrets) {
201
+ const pendingData = await this.#pendingIndexes.getAsync(secret);
202
+ if (!pendingData) {
203
+ continue;
204
+ }
205
+
206
+ const matchingIndexes = pendingData.filter(item => item.txHash === txHashStr).map(item => item.index);
207
+ if (matchingIndexes.length === 0) {
208
+ continue;
209
+ }
210
+
211
+ if (matchingIndexes.length > 1) {
212
+ // We should always just store the highest pending index for a given tx hash and secret because the lower
213
+ // values are irrelevant.
214
+ throw new Error(`Multiple pending indexes found for tx hash ${txHashStr} and secret ${secret}`);
215
+ }
216
+
217
+ let lastFinalized = await this.#lastFinalizedIndexes.getAsync(secret);
218
+ const newFinalized = matchingIndexes[0];
219
+
220
+ if (newFinalized < (lastFinalized ?? 0)) {
221
+ // This should never happen because when last finalized index was finalized we should have pruned the lower
222
+ // pending indexes.
223
+ throw new Error(
224
+ `New finalized index ${newFinalized} is smaller than the current last finalized index ${lastFinalized}`,
225
+ );
226
+ }
227
+
228
+ await this.#lastFinalizedIndexes.set(secret, newFinalized);
229
+ lastFinalized = newFinalized;
230
+
231
+ // When we add pending indexes, we ensure they are higher than the last finalized index. However, because we
232
+ // cannot control the order in which transactions are finalized, there may be pending indexes that are now
233
+ // obsolete because they are lower than the most recently finalized index. For this reason, we prune these
234
+ // outdated pending indexes.
235
+ const remainingItemsOfHigherIndex = pendingData.filter(item => item.index > (lastFinalized ?? 0));
236
+ if (remainingItemsOfHigherIndex.length === 0) {
237
+ await this.#pendingIndexes.delete(secret);
238
+ } else {
239
+ await this.#pendingIndexes.set(secret, remainingItemsOfHigherIndex);
240
+ }
241
+ }
242
+ }
243
+ }
244
+ }
@@ -1,2 +1,3 @@
1
1
  // Half the size of the window we slide over the tagging indexes.
2
+ // TODO(#17775): Move this to the recipient log sync function once it's implemented in this directory.
2
3
  export const WINDOW_HALF_SIZE = 10;
@@ -0,0 +1,112 @@
1
+ import type { AztecAddress } from '@aztec/stdlib/aztec-address';
2
+ import type { AztecNode } from '@aztec/stdlib/interfaces/server';
3
+ import type { DirectionalAppTaggingSecret } from '@aztec/stdlib/logs';
4
+
5
+ import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js';
6
+ import { getStatusChangeOfPending } from './utils/get_status_change_of_pending.js';
7
+ import { loadAndStoreNewTaggingIndexes } from './utils/load_and_store_new_tagging_indexes.js';
8
+
9
+ // This window has to be as large as the largest expected number of logs emitted in a tx for a given directional app
10
+ // tagging secret. If we get more tag indexes consumed than this window, an error is thrown in `PXE::proveTx` function.
11
+ // This is set to a larger value than MAX_PRIVATE_LOGS_PER_TX (currently 64) because there could be more than
12
+ // MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note
13
+ // and the note is nullified in the same tx.
14
+ //
15
+ // Rationale for value 95:
16
+ // - The `e2e_pending_note_hashes_contract` test's "Should handle overflowing the kernel data structures in nested
17
+ // calls" test case hits 95 tagging indexes emitted in a single transaction. This test creates and nullifies many
18
+ // notes recursively to test kernel reset circuit behavior, which causes logs to be squashed but still consume
19
+ // tagging indexes during the sync process. Since this is testing MAX_PRIVATE_LOGS_PER_TX overflow we can be
20
+ // reasonably certain that this value is large enough for standard use cases.
21
+ // - This value is below MAX_RPC_LEN (100) which is the limit for array parameters in the JSON RPC schema for
22
+ // `getLogsByTags`. Any test that would perform sync over JSON RPC (not by having access to the Aztec node instance
23
+ // directly) would error out if that maximum was hit (docs_examples.test.ts is an example of this).
24
+ export const WINDOW_LEN = 95;
25
+
26
+ /**
27
+ * Syncs tagging indexes. This function needs to be called whenever a private log is being sent.
28
+ *
29
+ * @param secret - The secret that's unique for (sender, recipient, contract) tuple while the direction of
30
+ * sender -> recipient matters.
31
+ * @param app - The address of the contract that the logs are tagged for. Needs to be provided because we perform
32
+ * second round of siloing in this function which is necessary because kernels do it as well (they silo first field
33
+ * of the private log which corresponds to the tag).
34
+ * @remarks When syncing the indexes as sender we don't care about the log contents - we only care about the highest
35
+ * pending and highest finalized indexes as that guides the next index choice when sending a log. The next index choice
36
+ * is simply the highest pending index plus one (or finalized if pending is undefined).
37
+ * @dev This function looks for new indexes, adds them to pending, then it checks status of each pending index and
38
+ * updates its status accordingly.
39
+ */
40
+ export async function syncSenderTaggingIndexes(
41
+ secret: DirectionalAppTaggingSecret,
42
+ app: AztecAddress,
43
+ aztecNode: AztecNode,
44
+ taggingDataProvider: SenderTaggingDataProvider,
45
+ ): Promise<void> {
46
+ // # Explanation of how syncing works
47
+ //
48
+ // When choosing an index, we select: highest pending index + 1 (or highest finalized index + 1 if no pending).
49
+ // If the chosen index is more than WINDOW_LEN from the highest finalized index, we throw an error. By having this
50
+ // hard limit we give a guarantee to a recipient that he doesn't need to look further than WINDOW_LEN ahead of the
51
+ // highest finalized index.
52
+ //
53
+ // This function synchronizes the finalized and pending indexes by iteratively querying the node for a window of
54
+ // indexes at a time, storing all those indexes as pending, and then checking the status of each pending index to
55
+ // update its finalization status accordingly. If we stumble upon a window with no indexes, we stop the loop.
56
+ //
57
+ // Stopping at that point is safe because of the limit described above - there can never be an index that is more
58
+ // than WINDOW_LEN from the highest finalized index.
59
+ //
60
+ // # Note on performance
61
+ // Each window advance requires two queries (logs + tx status). For example, syncing indexes 0–500 with a window of
62
+ // 100 takes at least 10 round trips (5 windows × 2 queries).
63
+
64
+ const finalizedIndex = await taggingDataProvider.getLastFinalizedIndex(secret);
65
+
66
+ let start = finalizedIndex === undefined ? 0 : finalizedIndex + 1;
67
+ let end = start + WINDOW_LEN;
68
+
69
+ let previousFinalizedIndex = finalizedIndex;
70
+ let newFinalizedIndex = undefined;
71
+
72
+ while (true) {
73
+ // Load and store indexes for the current window. These indexes may already exist in the database if txs using
74
+ // them were previously sent from this PXE. Any duplicates are handled by the tagging data provider.
75
+ await loadAndStoreNewTaggingIndexes(secret, app, start, end, aztecNode, taggingDataProvider);
76
+
77
+ // Retrieve all indexes within the current window from storage and update their status accordingly.
78
+ const pendingTxHashes = await taggingDataProvider.getTxHashesOfPendingIndexes(secret, start, end);
79
+ if (pendingTxHashes.length === 0) {
80
+ break;
81
+ }
82
+
83
+ const { txHashesToFinalize, txHashesToDrop } = await getStatusChangeOfPending(pendingTxHashes, aztecNode);
84
+
85
+ await taggingDataProvider.dropPendingIndexes(txHashesToDrop);
86
+ await taggingDataProvider.finalizePendingIndexes(txHashesToFinalize);
87
+
88
+ // We check if the finalized index has been updated.
89
+ newFinalizedIndex = await taggingDataProvider.getLastFinalizedIndex(secret);
90
+ if (previousFinalizedIndex !== newFinalizedIndex) {
91
+ // A new finalized index was found, so we'll run the loop again. For example:
92
+ // - Previous finalized index: 10
93
+ // - New finalized index: 13
94
+ // - Window length: 10
95
+ //
96
+ // In the last iteration, we processed indexes 11-20. To avoid reprocessing the same logs,
97
+ // we'll only look at the new indexes 21-23:
98
+ //
99
+ // Previous window: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
100
+ // New window: [21, 22, 23]
101
+
102
+ const previousEnd = end;
103
+ // Add 1 because `end` is exclusive and the known finalized index is not included in the window.
104
+ end = newFinalizedIndex! + WINDOW_LEN + 1;
105
+ start = previousEnd;
106
+ previousFinalizedIndex = newFinalizedIndex;
107
+ } else {
108
+ // No new finalized index was found, so we don't need to process the next window.
109
+ break;
110
+ }
111
+ }
112
+ }
@@ -0,0 +1,44 @@
1
+ import type { AztecNode } from '@aztec/stdlib/interfaces/server';
2
+ import { TxHash, TxStatus } from '@aztec/stdlib/tx';
3
+
4
+ /**
5
+ * Based on receipts obtained from `aztecNode` returns which pending transactions changed their status to finalized or
6
+ * dropped.
7
+ */
8
+ export async function getStatusChangeOfPending(
9
+ pending: TxHash[],
10
+ aztecNode: AztecNode,
11
+ ): Promise<{ txHashesToFinalize: TxHash[]; txHashesToDrop: TxHash[] }> {
12
+ // Get receipts for all pending tx hashes and the finalized block number.
13
+ const [receipts, { finalized }] = await Promise.all([
14
+ Promise.all(pending.map(pendingTxHash => aztecNode.getTxReceipt(pendingTxHash))),
15
+ aztecNode.getL2Tips(),
16
+ ]);
17
+
18
+ const txHashesToFinalize: TxHash[] = [];
19
+ const txHashesToDrop: TxHash[] = [];
20
+
21
+ for (let i = 0; i < receipts.length; i++) {
22
+ const receipt = receipts[i];
23
+ const txHash = pending[i];
24
+
25
+ if (receipt.status === TxStatus.SUCCESS && receipt.blockNumber && receipt.blockNumber <= finalized.number) {
26
+ // Tx has been included in a block and the corresponding block is finalized --> we mark the indexes as
27
+ // finalized.
28
+ txHashesToFinalize.push(txHash);
29
+ } else if (
30
+ receipt.status === TxStatus.DROPPED ||
31
+ receipt.status === TxStatus.APP_LOGIC_REVERTED ||
32
+ receipt.status === TxStatus.TEARDOWN_REVERTED ||
33
+ receipt.status === TxStatus.BOTH_REVERTED
34
+ ) {
35
+ // Tx was dropped or reverted --> we drop the corresponding pending indexes.
36
+ // TODO(#17615): Don't drop pending indexes corresponding to non-revertible phases.
37
+ txHashesToDrop.push(txHash);
38
+ } else {
39
+ // Tx is still pending or the corresponding block is not yet finalized --> we don't do anything.
40
+ }
41
+ }
42
+
43
+ return { txHashesToFinalize, txHashesToDrop };
44
+ }
@@ -0,0 +1,74 @@
1
+ import type { AztecAddress } from '@aztec/stdlib/aztec-address';
2
+ import type { AztecNode } from '@aztec/stdlib/interfaces/server';
3
+ import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs';
4
+ import { TxHash } from '@aztec/stdlib/tx';
5
+
6
+ import type { SenderTaggingDataProvider } from '../../../storage/tagging_data_provider/sender_tagging_data_provider.js';
7
+ import { SiloedTag } from '../../siloed_tag.js';
8
+ import { Tag } from '../../tag.js';
9
+
10
+ /**
11
+ * Loads tagging indexes from the Aztec node and stores them in the tagging data provider.
12
+ * @remarks This function is one of two places by which a pending index can get to the tagging data provider. The other
13
+ * place is when a tx is being sent from this PXE.
14
+ * @param secret - The directional app tagging secret that's unique for (sender, recipient, contract) tuple.
15
+ * @param app - The address of the contract that the logs are tagged for. Used for siloing tags to match
16
+ * kernel circuit behavior.
17
+ * @param start - The starting index (inclusive) of the window to process.
18
+ * @param end - The ending index (exclusive) of the window to process.
19
+ * @param aztecNode - The Aztec node instance to query for logs.
20
+ * @param taggingDataProvider - The data provider to store pending indexes.
21
+ */
22
+ export async function loadAndStoreNewTaggingIndexes(
23
+ secret: DirectionalAppTaggingSecret,
24
+ app: AztecAddress,
25
+ start: number,
26
+ end: number,
27
+ aztecNode: AztecNode,
28
+ taggingDataProvider: SenderTaggingDataProvider,
29
+ ) {
30
+ // We compute the tags for the current window of indexes
31
+ const preTagsForWindow: PreTag[] = Array(end - start)
32
+ .fill(0)
33
+ .map((_, i) => ({ secret, index: start + i }));
34
+ const siloedTagsForWindow = await Promise.all(
35
+ preTagsForWindow.map(async preTag => SiloedTag.compute(await Tag.compute(preTag), app)),
36
+ );
37
+
38
+ const txsForTags = await getTxsContainingTags(siloedTagsForWindow, aztecNode);
39
+ const highestIndexMap = getTxHighestIndexMap(txsForTags, preTagsForWindow);
40
+
41
+ // Now we iterate over the map, reconstruct the preTags and tx hash and store them in the db.
42
+ for (const [txHashStr, highestIndex] of highestIndexMap.entries()) {
43
+ const txHash = TxHash.fromString(txHashStr);
44
+ await taggingDataProvider.storePendingIndexes([{ secret, index: highestIndex }], txHash);
45
+ }
46
+ }
47
+
48
+ // Returns txs that used the given tags. A tag might have been used in multiple txs and for this reason we return
49
+ // an array for each tag.
50
+ async function getTxsContainingTags(tags: SiloedTag[], aztecNode: AztecNode): Promise<TxHash[][]> {
51
+ const tagsAsFr = tags.map(tag => tag.value);
52
+ const allLogs = await aztecNode.getLogsByTags(tagsAsFr);
53
+ return allLogs.map(logs => logs.filter(log => !log.isFromPublic).map(log => log.txHash));
54
+ }
55
+
56
+ // Returns a map of txHash to the highest index for that txHash.
57
+ function getTxHighestIndexMap(txHashesForTags: TxHash[][], preTagsForWindow: PreTag[]): Map<string, number> {
58
+ if (txHashesForTags.length !== preTagsForWindow.length) {
59
+ throw new Error(
60
+ `Number of tx hashes arrays does not match number of pre-tags. ${txHashesForTags.length} !== ${preTagsForWindow.length}`,
61
+ );
62
+ }
63
+
64
+ const highestIndexMap = new Map<string, number>();
65
+ for (let i = 0; i < txHashesForTags.length; i++) {
66
+ const taggingIndex = preTagsForWindow[i].index;
67
+ const txHashesForTag = txHashesForTags[i];
68
+ for (const txHash of txHashesForTag) {
69
+ const key = txHash.toString();
70
+ highestIndexMap.set(key, Math.max(highestIndexMap.get(key) ?? 0, taggingIndex));
71
+ }
72
+ }
73
+ return highestIndexMap;
74
+ }
@@ -1,6 +1,7 @@
1
1
  import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs';
2
2
 
3
- // TODO(benesjan): Make this return tags instead - this will moves some complexity from syncTaggedLogs
3
+ // TODO(#17775): If this does not get dropped when implementing the linked issue make this return tags instead. This
4
+ // will move some complexity from syncTaggedLogs to here.
4
5
  export function getPreTagsForTheWindow(
5
6
  secretsAndWindows: { secret: DirectionalAppTaggingSecret; leftMostIndex: number; rightMostIndex: number }[],
6
7
  ): PreTag[] {
@@ -15,7 +16,7 @@ export function getPreTagsForTheWindow(
15
16
 
16
17
  /**
17
18
  * Creates a map from directional app tagging secret to initial index.
18
- * @param preTags - The pre tags to get the initial indexes map from.
19
+ * @param preTags - The pre-tags to get the initial indexes map from.
19
20
  * @returns The map from directional app tagging secret to initial index.
20
21
  */
21
22
  export function getInitialIndexesMap(preTags: { secret: DirectionalAppTaggingSecret; index: number | undefined }[]): {
@@ -1,2 +0,0 @@
1
- export { SyncDataProvider } from './sync_data_provider.js';
2
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zdG9yYWdlL3N5bmNfZGF0YV9wcm92aWRlci9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQyJ9
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/storage/sync_data_provider/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC"}
@@ -1 +0,0 @@
1
- export { SyncDataProvider } from './sync_data_provider.js';
@@ -1,11 +0,0 @@
1
- import type { BlockNumber } from '@aztec/foundation/branded-types';
2
- import type { AztecAsyncKVStore } from '@aztec/kv-store';
3
- import { BlockHeader } from '@aztec/stdlib/tx';
4
- export declare class SyncDataProvider {
5
- #private;
6
- constructor(store: AztecAsyncKVStore);
7
- setHeader(header: BlockHeader): Promise<void>;
8
- getBlockNumber(): Promise<BlockNumber>;
9
- getBlockHeader(): Promise<BlockHeader>;
10
- }
11
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3luY19kYXRhX3Byb3ZpZGVyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3RvcmFnZS9zeW5jX2RhdGFfcHJvdmlkZXIvc3luY19kYXRhX3Byb3ZpZGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBQ25FLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUF1QixNQUFNLGlCQUFpQixDQUFDO0FBQzlFLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUUvQyxxQkFBYSxnQkFBZ0I7O0lBSTNCLFlBQVksS0FBSyxFQUFFLGlCQUFpQixFQUduQztJQUVLLFNBQVMsQ0FBQyxNQUFNLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFbEQ7SUFFSyxjQUFjLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxDQU8zQztJQUVLLGNBQWMsSUFBSSxPQUFPLENBQUMsV0FBVyxDQUFDLENBTzNDO0NBQ0YifQ==
@@ -1 +0,0 @@
1
- {"version":3,"file":"sync_data_provider.d.ts","sourceRoot":"","sources":["../../../src/storage/sync_data_provider/sync_data_provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAuB,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,qBAAa,gBAAgB;;IAI3B,YAAY,KAAK,EAAE,iBAAiB,EAGnC;IAEK,SAAS,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAElD;IAEK,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,CAO3C;IAEK,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,CAO3C;CACF"}
@@ -1,40 +0,0 @@
1
- import type { AztecAsyncKVStore } from '@aztec/kv-store';
2
- import { AztecAddress } from '@aztec/stdlib/aztec-address';
3
- import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs';
4
- export declare class TaggingDataProvider {
5
- #private;
6
- constructor(store: AztecAsyncKVStore);
7
- /**
8
- * Sets the last used indexes when sending a log.
9
- * @param preTags - The pre tags containing the directional app tagging secrets and the indexes that are to be
10
- * updated in the db.
11
- * @throws If any two pre tags contain the same directional app tagging secret
12
- */
13
- setLastUsedIndexesAsSender(preTags: PreTag[]): Promise<void[]>;
14
- /**
15
- * Sets the last used indexes when looking for logs.
16
- * @param preTags - The pre tags containing the directional app tagging secrets and the indexes that are to be
17
- * updated in the db.
18
- * @throws If any two pre tags contain the same directional app tagging secret
19
- */
20
- setLastUsedIndexesAsRecipient(preTags: PreTag[]): Promise<void[]>;
21
- /**
22
- * Returns the last used index when sending a log with a given secret.
23
- * @param secret - The directional app tagging secret.
24
- * @returns The last used index for the given directional app tagging secret, or undefined if not found.
25
- */
26
- getLastUsedIndexesAsSender(secret: DirectionalAppTaggingSecret): Promise<number | undefined>;
27
- /**
28
- * Returns the last used indexes when looking for logs as a recipient.
29
- * @param secrets - The directional app tagging secrets to obtain the indexes for.
30
- * @returns The last used indexes for the given directional app tagging secrets, or undefined if have never yet found
31
- * a log for a given secret.
32
- */
33
- getLastUsedIndexesAsRecipient(secrets: DirectionalAppTaggingSecret[]): Promise<(number | undefined)[]>;
34
- resetNoteSyncData(): Promise<void>;
35
- addSenderAddress(address: AztecAddress): Promise<boolean>;
36
- getSenderAddresses(): Promise<AztecAddress[]>;
37
- removeSenderAddress(address: AztecAddress): Promise<boolean>;
38
- getSize(): Promise<number>;
39
- }
40
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGFnZ2luZ19kYXRhX3Byb3ZpZGVyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc3RvcmFnZS90YWdnaW5nX2RhdGFfcHJvdmlkZXIvdGFnZ2luZ19kYXRhX3Byb3ZpZGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFpQixNQUFNLGlCQUFpQixDQUFDO0FBQ3hFLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUMzRCxPQUFPLEtBQUssRUFBRSwyQkFBMkIsRUFBRSxNQUFNLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUU5RSxxQkFBYSxtQkFBbUI7O0lBUzlCLFlBQVksS0FBSyxFQUFFLGlCQUFpQixFQU9uQztJQUVEOzs7OztPQUtHO0lBQ0gsMEJBQTBCLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxtQkFNM0M7SUFFRDs7Ozs7T0FLRztJQUNILDZCQUE2QixDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsbUJBTTlDO0lBYUQ7Ozs7T0FJRztJQUNHLDBCQUEwQixDQUFDLE1BQU0sRUFBRSwyQkFBMkIsR0FBRyxPQUFPLENBQUMsTUFBTSxHQUFHLFNBQVMsQ0FBQyxDQUVqRztJQUVEOzs7OztPQUtHO0lBQ0gsNkJBQTZCLENBQUMsT0FBTyxFQUFFLDJCQUEyQixFQUFFLEdBQUcsT0FBTyxDQUFDLENBQUMsTUFBTSxHQUFHLFNBQVMsQ0FBQyxFQUFFLENBQUMsQ0FFckc7SUFFRCxpQkFBaUIsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBT2pDO0lBRUssZ0JBQWdCLENBQUMsT0FBTyxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBUTlEO0lBRUssa0JBQWtCLElBQUksT0FBTyxDQUFDLFlBQVksRUFBRSxDQUFDLENBRWxEO0lBRUssbUJBQW1CLENBQUMsT0FBTyxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBUWpFO0lBRUssT0FBTyxvQkFJWjtDQUNGIn0=
@@ -1 +0,0 @@
1
- {"version":3,"file":"tagging_data_provider.d.ts","sourceRoot":"","sources":["../../../src/storage/tagging_data_provider/tagging_data_provider.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAiB,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE9E,qBAAa,mBAAmB;;IAS9B,YAAY,KAAK,EAAE,iBAAiB,EAOnC;IAED;;;;;OAKG;IACH,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,mBAM3C;IAED;;;;;OAKG;IACH,6BAA6B,CAAC,OAAO,EAAE,MAAM,EAAE,mBAM9C;IAaD;;;;OAIG;IACG,0BAA0B,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAEjG;IAED;;;;;OAKG;IACH,6BAA6B,CAAC,OAAO,EAAE,2BAA2B,EAAE,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC,CAErG;IAED,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAOjC;IAEK,gBAAgB,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ9D;IAEK,kBAAkB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,CAElD;IAEK,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAQjE;IAEK,OAAO,oBAIZ;CACF"}