@bsv/overlay 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/cjs/package.json +3 -3
  2. package/dist/cjs/src/Engine.js +9 -7
  3. package/dist/cjs/src/Engine.js.map +1 -1
  4. package/dist/cjs/src/GASP/OverlayGASPStorage.js +89 -44
  5. package/dist/cjs/src/GASP/OverlayGASPStorage.js.map +1 -1
  6. package/dist/cjs/src/storage/knex/KnexStorage.js +23 -18
  7. package/dist/cjs/src/storage/knex/KnexStorage.js.map +1 -1
  8. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
  9. package/dist/esm/src/Engine.js +9 -7
  10. package/dist/esm/src/Engine.js.map +1 -1
  11. package/dist/esm/src/GASP/OverlayGASPStorage.js +89 -44
  12. package/dist/esm/src/GASP/OverlayGASPStorage.js.map +1 -1
  13. package/dist/esm/src/storage/knex/KnexStorage.js +23 -18
  14. package/dist/esm/src/storage/knex/KnexStorage.js.map +1 -1
  15. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  16. package/dist/types/src/Engine.d.ts +1 -1
  17. package/dist/types/src/Engine.d.ts.map +1 -1
  18. package/dist/types/src/GASP/OverlayGASPStorage.d.ts +10 -0
  19. package/dist/types/src/GASP/OverlayGASPStorage.d.ts.map +1 -1
  20. package/dist/types/src/TopicManager.d.ts +1 -1
  21. package/dist/types/src/TopicManager.d.ts.map +1 -1
  22. package/dist/types/src/storage/knex/KnexStorage.d.ts.map +1 -1
  23. package/dist/types/tsconfig.types.tsbuildinfo +1 -1
  24. package/docs/API.md +16 -503
  25. package/docs/README.md +9 -3
  26. package/docs/Synchronization.md +203 -0
  27. package/docs/concepts/00-overview.md +85 -0
  28. package/docs/concepts/01-best-practices.md +202 -0
  29. package/docs/concepts/02-query-performance.md +345 -0
  30. package/docs/concepts/03-database-monitoring.md +211 -0
  31. package/docs/concepts/04-pagination-example.md +186 -0
  32. package/docs/concepts/05-recommendations-summary.md +158 -0
  33. package/docs/concepts/README.md +16 -0
  34. package/docs/examples/README.md +11 -1
  35. package/docs/examples/gs-wip.md +7 -1
  36. package/docs/internal/README.md +8 -1
  37. package/package.json +3 -3
  38. package/src/Engine.ts +10 -7
  39. package/src/GASP/OverlayGASPStorage.ts +105 -49
  40. package/src/TopicManager.ts +6 -1
  41. package/src/__tests/Engine.test.ts +6 -2
  42. package/src/storage/knex/KnexStorage.ts +25 -20
@@ -0,0 +1,186 @@
1
+ # Pagination in Overlay Queries: Convo Messenger Example
2
+
3
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](./README.md) | [📖 Examples](../examples/README.md) | [⚙️ Internal](../internal/README.md)
4
+
5
+ **Navigation:** [Overview](./00-overview.md) | [Best Practices](./01-best-practices.md) | [Query Performance](./02-query-performance.md) | [Database Monitoring](./03-database-monitoring.md) | [Pagination Example](./04-pagination-example.md) | [Recommendations Summary](./05-recommendations-summary.md)
6
+
7
+ ---
8
+
9
+ ## Overview
10
+
11
+ Pagination is one of the simplest and most effective techniques for ensuring overlay lookup performance remains stable as datasets grow.
12
+ Without pagination, queries that return lists—messages, reactions, or other records—can become increasingly slow and resource-intensive.
13
+
14
+ This document illustrates why pagination matters and uses Convo Messenger as a real example of how it improves performance.
15
+ It does not prescribe a required pattern or mandate changes; it simply shows how one overlay benefits from predictable, bounded queries.
16
+
17
+ ---
18
+
19
+ ## 1. Why Pagination Matters
20
+
21
+ Overlay datasets grow continuously. Without pagination:
22
+
23
+ * A lookup endpoint may return thousands of documents at once
24
+ * Memory usage spikes on both the server and the client
25
+ * Query latency increases as more documents accumulate
26
+ * Sorting happens in memory instead of using indexes
27
+
28
+ Pagination ensures each lookup returns only a small, fixed-size window of results.
29
+
30
+ **Benefits**
31
+
32
+ * Consistent performance regardless of dataset size
33
+ * Lower memory pressure
34
+ * Index-friendly access patterns
35
+ * Improved UX in applications that display messages or lists
36
+
37
+ ---
38
+
39
+ ## 2. How Convo Messenger Uses Pagination (Example)
40
+
41
+ Convo Messenger added pagination to all Lookup queries that return lists, including:
42
+
43
+ * listThreadMessages
44
+ * listLatestMessages
45
+ * listReplies
46
+ * listThreadReactions
47
+
48
+ Below is one example from Convo’s Lookup Service showing how `skip` and `limit` values are interpreted when provided:
49
+
50
+ ```typescript
51
+ if (query.type === 'listThreadMessages') {
52
+ const threadId = query.threadId ?? query.value?.threadId;
53
+ if (!threadId) throw new Error("threadId required");
54
+
55
+ const skip = query.skip ?? query.value?.skip ?? 0;
56
+ const limit = query.limit ?? query.value?.limit ?? 50;
57
+
58
+ const messages = await this.storage.listThreadMessages(threadId, skip, limit);
59
+ return this.formatAsLookupAnswers(messages);
60
+ }
61
+ ```
62
+
63
+ **Storage Layer Example (Convo)**
64
+
65
+ Convo’s storage layer performs an indexed and paginated fetch:
66
+
67
+ ```typescript
68
+ async listThreadMessages(threadId: string, skip = 0, limit = 50) {
69
+ const results = await this.messages
70
+ .aggregate([
71
+ { $match: { threadId } },
72
+ { $sort: { createdAt: -1 } }, // newest → oldest (indexed)
73
+ { $skip: skip },
74
+ { $limit: limit }
75
+ ])
76
+ .toArray();
77
+
78
+ return results.reverse(); // oldest → newest for UI ordering
79
+ }
80
+ ```
81
+
82
+ This matches Convo’s existing index:
83
+
84
+ ```typescript
85
+ db.convoMessages.createIndex({ threadId: 1, createdAt: -1 });
86
+ ```
87
+
88
+ **Client-Side Integration**
89
+
90
+ In the frontend (React), pagination can be managed by maintaining a `page` or `offset` variable:
91
+
92
+ ```typescript
93
+ const [messages, setMessages] = useState([]);
94
+ const [page, setPage] = useState(0);
95
+ const pageSize = 50;
96
+
97
+ async function loadNextPage() {
98
+ const result = await lookup.query({
99
+ service: 'convo_lookup',
100
+ query: {
101
+ type: 'listThreadMessages',
102
+ threadId,
103
+ skip: page * pageSize,
104
+ limit: pageSize,
105
+ }
106
+ });
107
+
108
+ setMessages([...messages, ...result]);
109
+ setPage(page + 1);
110
+ }
111
+ ```
112
+
113
+ This simple approach allows seamless infinite scrolling or “Load More” functionality.
114
+
115
+ Again, this is not a prescription—it simply demonstrates how Convo uses indexed pagination to maintain predictable performance.
116
+
117
+ ---
118
+
119
+ ## 3. Performance Impact (Convo Example)
120
+
121
+ After adding pagination to its lookup queries, Convo observed substantial performance improvements.
122
+
123
+ | Metric | Before Pagination | After Pagination |
124
+ | -------------------------- | -----------------: | ---------------------: |
125
+ | Average Query Time | ~420 ms | **< 70 ms** |
126
+ | Memory Usage (per request) | ~75 MB | **< 8 MB** |
127
+ | CPU Load (peak) | Very High | **Much Lower** |
128
+ | UI Latency | Noticeable stutter | **Instantaneous load** |
129
+
130
+ **Takeaways**
131
+ * Avoiding unbounded queries dramatically stabilizes performance
132
+ * Server resource usage becomes predictable
133
+ * Larger threads no longer degrade lookup time
134
+ * User experience improves without changing overlay protocols
135
+
136
+ ---
137
+
138
+ ## 4. Developer Notes (General Guidance)
139
+
140
+ ### 4.1 Recommended Defaults
141
+
142
+ | Parameter | Recommended Value | Notes |
143
+ | --------- | ------------------- | ---------------------------------------------------------- |
144
+ | `limit` | 50 | Reasonable for message lists; tweak based on dataset size. |
145
+ | `skip` | 0 | Always provide, even if zero. |
146
+ | sort | `{ createdAt: -1 }` | Ensures chronological consistency. |
147
+
148
+ ### 4.2 Index Alignment
149
+
150
+ For overlays that paginate ordered data, an index on the sort field is essential:
151
+
152
+ ```js
153
+ { threadId: 1, createdAt: -1 }
154
+ ```
155
+ This avoids expensive in-memory sorts.
156
+
157
+ ### 4.3 Cursor-Based Pagination (Optional)
158
+
159
+ Some overlays may use cursor-based pagination (e.g., createdAt > X) for large-scale workloads.
160
+ ---
161
+
162
+ ## 5. Visual Results (Convo Example)
163
+
164
+ Pagination allowed Convo to:
165
+ * Avoid returning entire threads at once
166
+ * Use MongoDB indexes effectively
167
+ * Keep lookup latency stable as message volume grew
168
+ * Implement infinite scroll UX without heavy data loads
169
+
170
+ This demonstrates how pagination helps overlays scale smoothly without requiring complicated architecture.
171
+
172
+ ---
173
+
174
+ ## 6. Recommendations Summary
175
+
176
+ | Area | Recommendation |
177
+ | ------------------ | ---------------------------------------------------- |
178
+ | Implementation | Always bound list queries with pagination parameters |
179
+ | Client Integration | Incremental loading improves UX and stability |
180
+ | Indexing | Align indexes with sort and filter fields |
181
+ | Optimization | Cursor-based pagination can help for very large sets |
182
+ | Benchmarking | Measure latency and memory before/after deployment |
183
+
184
+ ---
185
+
186
+ Next file: [`05-recommendations-summary.md`](./05-recommendations-summary.md) — Summarizes all proposed overlay improvements for BSVA documentation and Overlay Express feature suggestions.
@@ -0,0 +1,158 @@
1
+ # Overlay Recommendations Summary
2
+
3
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](./README.md) | [📖 Examples](../examples/README.md) | [⚙️ Internal](../internal/README.md)
4
+
5
+ **Navigation:** [Overview](./00-overview.md) | [Best Practices](./01-best-practices.md) | [Query Performance](./02-query-performance.md) | [Database Monitoring](./03-database-monitoring.md) | [Pagination Example](./04-pagination-example.md) | [Recommendations Summary](./05-recommendations-summary.md)
6
+
7
+ ---
8
+
9
+ ## Overview
10
+
11
+ This document summarizes the key recommendations from the overlay optimization series.
12
+ These best practices help developers build overlays that remain efficient, stable, and easy to debug as datasets grow.
13
+ The recommendations also highlight future opportunities for improvements to Overlay Express and related tooling.
14
+
15
+ This summary is written for overlay authors and maintainers. It does not prescribe protocol-level changes.
16
+
17
+ ---
18
+
19
+ ## 1. General Design Recommendations
20
+
21
+ | Area | Recommendation | Description |
22
+ | ---------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ |
23
+ | **Lookup Data Structure** | Store only indexable metadata | Keep Lookup Services lightweight by storing only the fields needed for indexing and filtering. Store large data (media, payloads) externally. |
24
+ | **Topic Manager Validation** | Enforce schema and idempotency | Ensure each admitted transaction matches the expected structure and avoid duplicate writes. |
25
+ | **Retention Policy** | Implement periodic cleanup | Schedule jobs to prune stale data and rebuild indexes to maintain predictable performance. |
26
+ | **Security** | Restrict admin/debug access | Provide safe read-only views only to trusted admins (e.g., CARS admins or overlay maintainers). |
27
+
28
+ ---
29
+
30
+ ## 2. Query and Indexing Recommendations
31
+
32
+ ### 2.1 Query Design
33
+
34
+ * Always filter on indexed fields (e.g., `threadId`, `parentMessageId`, `createdAt`).
35
+ * Avoid unbounded queries (`find({})`, no limit, no filter).
36
+ * Do not use regex or `$text` queries—they prevent index usage.
37
+ * Prefer range queries `($gte, $lte)` when filtering by time.
38
+
39
+ ### 2.2 Indexing Strategy
40
+
41
+ * Add indexes that match your overlay’s most common query patterns:
42
+ * Example from Convo: `{ threadId: 1, createdAt: -1 }`
43
+ * Example for replies: `{ parentMessageId: 1, createdAt: -1 }`
44
+ * Use MongoDB’s explain("executionStats") to confirm index usage.
45
+ * Rebuild indexes periodically for overlays that experience frequent writes/deletes.
46
+
47
+ ### 2.3 Pagination
48
+
49
+ * Implement skip/limit pagination on all list endpoints.
50
+ * Align pagination with indexed sorting to avoid in-memory sorts.
51
+ * For large-scale overlays, consider cursor-based pagination (e.g., createdAt > X).
52
+
53
+ ---
54
+
55
+ ## 3. Monitoring and Debugging
56
+
57
+ ### 3.1 Read-Only Admin Access
58
+
59
+ Provide safe ways for overlay and CARS admins to inspect stored data:
60
+ * Read-only database credentials
61
+ * Or a protected REST endpoint for paginated inspection
62
+
63
+ These allow debugging without risking data mutation.
64
+
65
+ ### 3.2 Health Metrics
66
+
67
+ Track operational metrics such as:
68
+ * Query latency (per lookup type)
69
+ * Failed or rejected transactions
70
+ * Topic Manager ingestion delay
71
+ * Database and index size
72
+ * Slow query counts
73
+
74
+ Visualize these with Prometheus, Grafana, or MongoDB tools.
75
+
76
+ ### 3.3 Logging Standards
77
+
78
+ * Log Topic Manager admissions and Lookup Service query timings.
79
+ * Use consistent log tags (e.g., `[tm_convo]`, `[ls_market]`, `[overlay]`).
80
+ * Avoid logging sensitive or encrypted data.
81
+
82
+ ---
83
+
84
+ ## 4. Performance Enhancements Verified in Convo Messenger
85
+
86
+ | Optimization | Result |
87
+ | --------------------------------- | --------------------------------------------- |
88
+ | Added pagination (`skip`/`limit`) | Reduced avg query time from 420ms → 70ms |
89
+ | Indexed `(threadId, createdAt)` | Reduced CPU load and consistent scaling |
90
+ | Range Based queries | More predictable performance |
91
+ | MongoDB profiler + timing logs | Identified slow or unindexed queries |
92
+
93
+ These tests confirm that even small backend changes yield **major end-user improvements** in message loading and overlay stability.
94
+
95
+ ---
96
+
97
+ ## 5. Proposed Overlay Express Enhancements
98
+
99
+ ### 5.1 Built-In Query Monitor (Future Proposal)
100
+
101
+ A lightweight module that tracks and displays:
102
+ * Average execution time per query type
103
+ * Query counts
104
+ * Slow query alerts
105
+ * Index usage indicators
106
+
107
+ ### 5.2 Health Endpoint Standardization
108
+
109
+ A recommended addition:
110
+
111
+ ```bash
112
+ /health → basic liveness
113
+ /metrics → Prometheus-compatible performance stats
114
+ ```
115
+
116
+ ### 5.3 Admin Console Integration
117
+
118
+ A browser-based console for authorized users to:
119
+ * Inspect Lookup data (read-only)
120
+ * View recent admissions
121
+ * Inspect slow queries
122
+ * Monitor index statistics
123
+
124
+ ### 5.4 SDK Convenience Utilities
125
+
126
+ Potential optional helpers:
127
+ * `lookup.paginate()` wrapper
128
+ * Standardized pagination response format
129
+ * Typed client helpers for common query shapes
130
+
131
+ These features would reduce boilerplate for overlay developers.
132
+ ---
133
+
134
+ ## 6. Next Steps for BSVA Integration
135
+
136
+ 1. Add these best practices to BSVA documentation and onboarding materials.
137
+ 2. Update example overlays (e.g., in Metanet Academy) to demonstrate pagination and indexing.
138
+ 3. Create an example overlay dashboard (Prometheus/Grafana) for developers.
139
+ 4. Review potential Overlay Express enhancements with maintainers.
140
+
141
+ ---
142
+
143
+ ## 7. Summary Table
144
+
145
+ | Category | Key Action | Impact |
146
+ | --------------- | ------------------------------- | --------------------------------------- |
147
+ | Lookup Design | Store metadata only | Lower storage load and faster queries |
148
+ | Indexing | Add indexes based on query usage| Lower latency and predictable behavior |
149
+ | Pagination | Apply universally | Stable performance as data grows |
150
+ | Monitoring | Add metrics + profiling | Early detection of issues |
151
+ | Admin Tools | Provide safe, read-only access | Easier debugging and transparency |
152
+ | Overlay Express | Consider Query Monitor tooling | Unified visibility across overlays |
153
+
154
+ ---
155
+
156
+ **Conclusion:**
157
+ These recommendations form a solid baseline for building fast, scalable, and maintainable overlays.
158
+ They are intentionally lightweight and compatible with existing designs—no protocol changes required.
@@ -1,4 +1,20 @@
1
1
  # Conceptual Topics
2
2
 
3
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](./README.md) | [📖 Examples](../examples/README.md) | [⚙️ Internal](../internal/README.md)
4
+
5
+ ---
6
+
3
7
  These documents cover high-level conceptual information that will augment developers' understanding of the code-level Overlay Services Engine documentation:
4
8
 
9
+ ## Available Topics
10
+
11
+ - [Overview](./00-overview.md) — Introduction to Overlay Use and Optimization in the BSV Ecosystem
12
+ - [Best Practices](./01-best-practices.md) — Best practices for overlay design and implementation
13
+ - [Query Performance](./02-query-performance.md) — Query optimization and performance considerations
14
+ - [Database Monitoring](./03-database-monitoring.md) — Monitoring and debugging overlays
15
+ - [Pagination Example](./04-pagination-example.md) — Pagination and practical examples
16
+ - [Recommendations Summary](./05-recommendations-summary.md) — Quick reference of best practices
17
+
18
+ ---
19
+
20
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](./README.md) | [📖 Examples](../examples/README.md) | [⚙️ Internal](../internal/README.md)
@@ -1,5 +1,15 @@
1
1
  # Examples
2
2
 
3
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](../concepts/README.md) | [📖 Examples](./README.md) | [⚙️ Internal](../internal/README.md)
4
+
5
+ ---
6
+
3
7
  Here, you will find documentation for common example usages of the Overlay Services Engine.
4
8
 
5
- - [Getting Started (WIP)](./gs-wip.md)
9
+ ## Available Examples
10
+
11
+ - [Getting Started (WIP)](./gs-wip.md) — Introduction and getting started guide
12
+
13
+ ---
14
+
15
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](../concepts/README.md) | [📖 Examples](./README.md) | [⚙️ Internal](../internal/README.md)
@@ -1,4 +1,10 @@
1
- ### Introduction to BSV Overlay Services Engine
1
+ # Getting Started with BSV Overlay Services Engine
2
+
3
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](../concepts/README.md) | [📖 Examples](./README.md) | [⚙️ Internal](../internal/README.md)
4
+
5
+ ---
6
+
7
+ ## Introduction to BSV Overlay Services Engine
2
8
 
3
9
  The BSV Overlay Services Engine is designed to process transactions and manage data within a blockchain-based system, specifically targeting the Bitcoin SV (BSV) blockchain. It integrates various components such as Topic Managers, Lookup Services, Storage, and Chain Tracker to provide a robust environment for managing transaction data and overlay services.
4
10
 
@@ -1,4 +1,11 @@
1
1
  # Internals
2
2
 
3
- These documents cover the internal components of the Overlay Services engine. Generally, these are only useful for creating customized deployments:
3
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](../concepts/README.md) | [📖 Examples](../examples/README.md) | [⚙️ Internal](./README.md)
4
4
 
5
+ ---
6
+
7
+ These documents cover the internal components of the Overlay Services engine. Generally, these are only useful for creating customized deployments.
8
+
9
+ ---
10
+
11
+ [🏠 Home](../README.md) | [📚 API](../API.md) | [💡 Concepts](../concepts/README.md) | [📖 Examples](../examples/README.md) | [⚙️ Internal](./README.md)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bsv/overlay",
3
- "version": "0.5.4",
3
+ "version": "0.6.1",
4
4
  "type": "module",
5
5
  "description": "BSV Blockchain Overlay Services Engine",
6
6
  "main": "dist/cjs/mod.js",
@@ -73,8 +73,8 @@
73
73
  "typescript": "^5.2.2"
74
74
  },
75
75
  "dependencies": {
76
- "@bsv/gasp": "^1.2.0",
77
- "@bsv/sdk": "^1.9.1",
76
+ "@bsv/gasp": "^1.2.2",
77
+ "@bsv/sdk": "^2.0.0",
78
78
  "knex": "^3.1.0"
79
79
  }
80
80
  }
package/src/Engine.ts CHANGED
@@ -134,7 +134,7 @@ export class Engine {
134
134
  *
135
135
  * @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
136
136
  */
137
- async submit(taggedBEEF: TaggedBEEF, onSteakReady?: (steak: STEAK) => void, mode: 'historical-tx' | 'current-tx' = 'current-tx', offChainValues?: number[]): Promise<STEAK> {
137
+ async submit(taggedBEEF: TaggedBEEF, onSteakReady?: (steak: STEAK) => void, mode: 'historical-tx' | 'current-tx' | 'historical-tx-no-spv' = 'current-tx', offChainValues?: number[]): Promise<STEAK> {
138
138
  for (const t of taggedBEEF.topics) {
139
139
  if (this.managers[t] === undefined || this.managers[t] === null) {
140
140
  throw new Error(`This server does not support this topic: ${t}`)
@@ -146,10 +146,12 @@ export class Engine {
146
146
  const txid = tx.id('hex')
147
147
 
148
148
  this.startTime(`submit_${txid}`)
149
- this.startTime(`chainTracker_${txid.substring(0, 10)}`)
150
- const txValid = await tx.verify(this.chainTracker)
151
- if (!txValid) throw new Error('Unable to verify SPV information.')
152
- this.endTime(`chainTracker_${txid.substring(0, 10)}`)
149
+ if (mode !== 'historical-tx-no-spv') {
150
+ this.startTime(`chainTracker_${txid.substring(0, 10)}`)
151
+ const txValid = await tx.verify(this.chainTracker)
152
+ if (!txValid) throw new Error('Unable to verify SPV information.')
153
+ this.endTime(`chainTracker_${txid.substring(0, 10)}`)
154
+ }
153
155
 
154
156
  const steak: STEAK = {}
155
157
  const dupeTopics = new Set<string>()
@@ -212,7 +214,8 @@ export class Engine {
212
214
  const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(
213
215
  taggedBEEF.beef,
214
216
  previousCoins,
215
- offChainValues
217
+ offChainValues,
218
+ mode
216
219
  )
217
220
  this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
218
221
 
@@ -499,7 +502,7 @@ export class Engine {
499
502
  }
500
503
 
501
504
  // If we don't have an advertiser or we are dealing with historical transactions, just return the steak
502
- if (this.advertiser === undefined || mode === 'historical-tx') {
505
+ if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
503
506
  return steak
504
507
  }
505
508
 
@@ -21,9 +21,45 @@ export interface GraphNode {
21
21
 
22
22
  export class OverlayGASPStorage implements GASPStorage {
23
23
  readonly temporaryGraphNodeRefs: Record<string, GraphNode> = {}
24
+ private static activeAnchorValidations = 0
25
+ private static readonly anchorValidationQueue: Array<() => void> = []
26
+ private static activeFinalizations = 0
27
+ private static readonly finalizationQueue: Array<() => void> = []
28
+ private static readonly MAX_CONCURRENT_ANCHOR_VALIDATIONS = 4
29
+ private static readonly MAX_CONCURRENT_FINALIZATIONS = 2
24
30
 
25
31
  constructor (public topic: string, public engine: Engine, public maxNodesInGraph?: number) { }
26
32
 
33
+ private static async acquireAnchorValidationSlot (): Promise<void> {
34
+ if (OverlayGASPStorage.activeAnchorValidations >= OverlayGASPStorage.MAX_CONCURRENT_ANCHOR_VALIDATIONS) {
35
+ await new Promise<void>(resolve => { OverlayGASPStorage.anchorValidationQueue.push(resolve) })
36
+ }
37
+ OverlayGASPStorage.activeAnchorValidations++
38
+ }
39
+
40
+ private static releaseAnchorValidationSlot (): void {
41
+ OverlayGASPStorage.activeAnchorValidations--
42
+ const next = OverlayGASPStorage.anchorValidationQueue.shift()
43
+ if (next !== undefined) {
44
+ next()
45
+ }
46
+ }
47
+
48
+ private static async acquireFinalizationSlot (): Promise<void> {
49
+ if (OverlayGASPStorage.activeFinalizations >= OverlayGASPStorage.MAX_CONCURRENT_FINALIZATIONS) {
50
+ await new Promise<void>(resolve => { OverlayGASPStorage.finalizationQueue.push(resolve) })
51
+ }
52
+ OverlayGASPStorage.activeFinalizations++
53
+ }
54
+
55
+ private static releaseFinalizationSlot (): void {
56
+ OverlayGASPStorage.activeFinalizations--
57
+ const next = OverlayGASPStorage.finalizationQueue.shift()
58
+ if (next !== undefined) {
59
+ next()
60
+ }
61
+ }
62
+
27
63
  /**
28
64
  *
29
65
  * @param since
@@ -91,8 +127,12 @@ export class OverlayGASPStorage implements GASPStorage {
91
127
 
92
128
  // Attempt to check if the current transaction is admissible
93
129
  parsedTx.merklePath = MerklePath.fromHex(tx.proof)
94
- const admittanceResult = await this.engine.managers[this.topic].identifyAdmissibleOutputs(parsedTx.toBEEF(), [], typeof tx.txMetadata === 'string' ? Utils.toArray(tx.txMetadata) : undefined)
95
-
130
+ const admittanceResult = await this.engine.managers[this.topic].identifyAdmissibleOutputs(
131
+ parsedTx.toBEEF(),
132
+ [],
133
+ typeof tx.txMetadata === 'string' ? Utils.toArray(tx.txMetadata) : undefined,
134
+ 'historical-tx'
135
+ )
96
136
  if (admittanceResult.outputsToAdmit.includes(tx.outputIndex)) {
97
137
  // The transaction is admissible, no further inputs are needed
98
138
  } else {
@@ -196,51 +236,61 @@ export class OverlayGASPStorage implements GASPStorage {
196
236
  * @throws If the graph is not well-anchored, according to the rules of Bitcoin or the rules of the Overlay Topic Manager.
197
237
  */
198
238
  async validateGraphAnchor (graphID: string): Promise<void> {
199
- const rootNode = this.temporaryGraphNodeRefs[graphID]
200
- if (rootNode === undefined) {
201
- throw new Error(`Graph node with ID ${graphID} not found`)
202
- }
203
-
204
- // Check that the root node is Bitcoin-valid.
205
- const beef = this.getBEEFForNode(rootNode)
206
- const spvTx = Transaction.fromBEEF(beef)
207
- const isBitcoinValid = await spvTx.verify(this.engine.chainTracker)
208
- if (!isBitcoinValid) {
209
- throw new Error('The graph is not well-anchored according to the rules of Bitcoin.')
210
- }
211
-
212
- // Then, ensure the node is Overlay-valid.
213
- const beefs = this.computeOrderedBEEFsForGraph(graphID)
239
+ await OverlayGASPStorage.acquireAnchorValidationSlot()
240
+ try {
241
+ const rootNode = this.temporaryGraphNodeRefs[graphID]
242
+ if (rootNode === undefined) {
243
+ throw new Error(`Graph node with ID ${graphID} not found`)
244
+ }
214
245
 
215
- // coins: a Set of all historical coins to retain (no need to remove them), used to emulate topical admittance of previous inputs over time.
216
- const coins = new Set<string>()
246
+ // Check that the root node is Bitcoin-valid.
247
+ const beef = this.getBEEFForNode(rootNode)
248
+ const spvTx = Transaction.fromBEEF(beef)
249
+ const isBitcoinValid = await spvTx.verify(this.engine.chainTracker)
250
+ if (!isBitcoinValid) {
251
+ throw new Error('The graph is not well-anchored according to the rules of Bitcoin.')
252
+ }
217
253
 
218
- // Submit all historical BEEFs in order through the topic manager, tracking what would be retained until we submit the root node last.
219
- // If, at the end, the root node is admitted, we have a valid overlay-specific graph.
220
- for (const beef of beefs) {
221
- // For any input to this transaction, see if it's a valid coin that's admitted. If so, it's a previous coin.
222
- const previousCoins: number[] = []
223
- const tx = Transaction.fromBEEF(beef)
224
- for (const [inputIndex, input] of tx.inputs.entries()) {
225
- const sourceTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex')
226
- if (sourceTXID != null && sourceTXID !== '') {
227
- const coin = `${sourceTXID}.${input.sourceOutputIndex}`
228
- if (coins.has(coin)) {
229
- previousCoins.push(Number(inputIndex))
254
+ // Then, ensure the node is Overlay-valid.
255
+ const beefs = this.computeOrderedBEEFsForGraph(graphID)
256
+
257
+ // coins: a Set of all historical coins to retain (no need to remove them), used to emulate topical admittance of previous inputs over time.
258
+ const coins = new Set<string>()
259
+
260
+ // Submit all historical BEEFs in order through the topic manager, tracking what would be retained until we submit the root node last.
261
+ // If, at the end, the root node is admitted, we have a valid overlay-specific graph.
262
+ for (const beef of beefs) {
263
+ // For any input to this transaction, see if it's a valid coin that's admitted. If so, it's a previous coin.
264
+ const previousCoins: number[] = []
265
+ const tx = Transaction.fromBEEF(beef)
266
+ for (const [inputIndex, input] of tx.inputs.entries()) {
267
+ const sourceTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex')
268
+ if (sourceTXID != null && sourceTXID !== '') {
269
+ const coin = `${sourceTXID}.${input.sourceOutputIndex}`
270
+ if (coins.has(coin)) {
271
+ previousCoins.push(Number(inputIndex))
272
+ }
230
273
  }
231
274
  }
275
+ const admittanceInstructions = await this.engine.managers[this.topic].identifyAdmissibleOutputs(
276
+ beef,
277
+ previousCoins,
278
+ undefined,
279
+ 'historical-tx'
280
+ )
281
+ // Every admitted output is now a coin.
282
+ for (const outputIndex of admittanceInstructions.outputsToAdmit) {
283
+ coins.add(`${tx.id('hex')}.${outputIndex}`)
284
+ }
232
285
  }
233
- const admittanceInstructions = await this.engine.managers[this.topic].identifyAdmissibleOutputs(beef, previousCoins)
234
- // Every admitted output is now a coin.
235
- for (const outputIndex of admittanceInstructions.outputsToAdmit) {
236
- coins.add(`${tx.id('hex')}.${outputIndex}`)
286
+ // After sending through all the graph's BEEFs...
287
+ // If the root node is now a coin, we have acceptance by the overlay.
288
+ // Otherwise, throw.
289
+ if (!coins.has(graphID)) {
290
+ throw new Error('This graph did not result in topical admittance of the root node. Rejecting.')
237
291
  }
238
- }
239
- // After sending through all the graph's BEEFs...
240
- // If the root node is now a coin, we have acceptance by the overlay.
241
- // Otherwise, throw.
242
- if (!coins.has(graphID)) {
243
- throw new Error('This graph did not result in topical admittance of the root node. Rejecting.')
292
+ } finally {
293
+ OverlayGASPStorage.releaseAnchorValidationSlot()
244
294
  }
245
295
  }
246
296
 
@@ -263,14 +313,20 @@ export class OverlayGASPStorage implements GASPStorage {
263
313
  * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the root of this graph.
264
314
  */
265
315
  async finalizeGraph (graphID: string): Promise<void> {
266
- const beefs = this.computeOrderedBEEFsForGraph(graphID)
267
-
268
- // Submit all historical BEEFs in order, finalizing the graph for the current UTXO
269
- for (const beef of beefs) {
270
- await this.engine.submit({
271
- beef,
272
- topics: [this.topic]
273
- }, () => { }, 'historical-tx')
316
+ await OverlayGASPStorage.acquireFinalizationSlot()
317
+ try {
318
+ const beefs = this.computeOrderedBEEFsForGraph(graphID)
319
+
320
+ // Submit all historical BEEFs in order, finalizing the graph for the current UTXO.
321
+ // We skip SPV verification here because validateGraphAnchor has already done it.
322
+ for (const beef of beefs) {
323
+ await this.engine.submit({
324
+ beef,
325
+ topics: [this.topic]
326
+ }, () => { }, 'historical-tx-no-spv')
327
+ }
328
+ } finally {
329
+ OverlayGASPStorage.releaseFinalizationSlot()
274
330
  }
275
331
  }
276
332