@optimystic/db-core 1.4.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +336 -336
  2. package/dist/src/collection/collection.d.ts +44 -4
  3. package/dist/src/collection/collection.d.ts.map +1 -1
  4. package/dist/src/collection/collection.js +95 -14
  5. package/dist/src/collection/collection.js.map +1 -1
  6. package/dist/src/collections/tree/struct.d.ts +9 -4
  7. package/dist/src/collections/tree/struct.d.ts.map +1 -1
  8. package/dist/src/collections/tree/struct.js +9 -4
  9. package/dist/src/collections/tree/struct.js.map +1 -1
  10. package/dist/src/log/log.d.ts +41 -8
  11. package/dist/src/log/log.d.ts.map +1 -1
  12. package/dist/src/log/log.js +15 -10
  13. package/dist/src/log/log.js.map +1 -1
  14. package/dist/src/testing/test-transactor.d.ts +36 -0
  15. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  16. package/dist/src/testing/test-transactor.js +49 -0
  17. package/dist/src/testing/test-transactor.js.map +1 -1
  18. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  19. package/dist/src/transaction/coordinator.js +26 -9
  20. package/dist/src/transaction/coordinator.js.map +1 -1
  21. package/package.json +1 -1
  22. package/src/blocks/block-store.ts +31 -31
  23. package/src/blocks/block-types.ts +11 -11
  24. package/src/blocks/index.ts +5 -5
  25. package/src/blocks/structs.ts +17 -17
  26. package/src/btree/btree.ts +872 -872
  27. package/src/btree/independent-trunk.ts +53 -53
  28. package/src/btree/index.ts +5 -5
  29. package/src/btree/key-range.ts +15 -15
  30. package/src/btree/keyset.ts +6 -6
  31. package/src/btree/nodes.ts +25 -25
  32. package/src/btree/path.ts +37 -37
  33. package/src/btree/tree-block.ts +11 -11
  34. package/src/btree/trunk.ts +14 -14
  35. package/src/chain/chain-nodes.ts +26 -26
  36. package/src/chain/chain.ts +349 -349
  37. package/src/chain/index.ts +3 -3
  38. package/src/cluster/i-cluster.ts +6 -6
  39. package/src/cluster/index.ts +3 -3
  40. package/src/cohort-topic/addressing.ts +120 -120
  41. package/src/cohort-topic/antidos/bootstrap-evidence.ts +106 -106
  42. package/src/cohort-topic/antidos/index.ts +5 -5
  43. package/src/cohort-topic/antidos/rate-limiter.ts +210 -210
  44. package/src/cohort-topic/antidos/replay-guard.ts +146 -146
  45. package/src/cohort-topic/antidos/topic-budget.ts +160 -160
  46. package/src/cohort-topic/antiflood/index.ts +2 -2
  47. package/src/cohort-topic/antiflood/invariants.ts +108 -108
  48. package/src/cohort-topic/antiflood/jitter.ts +117 -117
  49. package/src/cohort-topic/coldstart.ts +237 -237
  50. package/src/cohort-topic/dmax.ts +88 -88
  51. package/src/cohort-topic/gossip/bus.ts +254 -254
  52. package/src/cohort-topic/gossip/index.ts +3 -3
  53. package/src/cohort-topic/gossip/records.ts +45 -45
  54. package/src/cohort-topic/gossip/view.ts +91 -91
  55. package/src/cohort-topic/index.ts +20 -20
  56. package/src/cohort-topic/load/barometer.ts +134 -134
  57. package/src/cohort-topic/load/index.ts +1 -1
  58. package/src/cohort-topic/member-engine.ts +430 -430
  59. package/src/cohort-topic/membership/index.ts +3 -3
  60. package/src/cohort-topic/membership/publisher.ts +163 -163
  61. package/src/cohort-topic/membership/source.ts +41 -41
  62. package/src/cohort-topic/ports.ts +157 -157
  63. package/src/cohort-topic/promotion.ts +477 -477
  64. package/src/cohort-topic/registration/bytes.ts +37 -37
  65. package/src/cohort-topic/registration/handoff.ts +154 -154
  66. package/src/cohort-topic/registration/index.ts +6 -6
  67. package/src/cohort-topic/registration/renewal.ts +495 -495
  68. package/src/cohort-topic/registration/sharding.ts +61 -61
  69. package/src/cohort-topic/registration/store.ts +81 -81
  70. package/src/cohort-topic/registration/types.ts +91 -91
  71. package/src/cohort-topic/ring-hash.ts +50 -50
  72. package/src/cohort-topic/service.ts +416 -416
  73. package/src/cohort-topic/sig/index.ts +2 -2
  74. package/src/cohort-topic/sig/payloads.ts +59 -59
  75. package/src/cohort-topic/sig/threshold.ts +64 -64
  76. package/src/cohort-topic/tiers.ts +74 -74
  77. package/src/cohort-topic/traffic.ts +233 -233
  78. package/src/cohort-topic/walk.ts +326 -326
  79. package/src/cohort-topic/willingness.ts +237 -237
  80. package/src/cohort-topic/wire/index.ts +18 -18
  81. package/src/cohort-topic/wire/payloads.ts +126 -126
  82. package/src/collection/action.ts +48 -48
  83. package/src/collection/collection-type-registry.ts +37 -37
  84. package/src/collection/collection.ts +2203 -2118
  85. package/src/collection/index.ts +4 -4
  86. package/src/collections/diary/index.ts +2 -2
  87. package/src/collections/diary/struct.ts +3 -3
  88. package/src/collections/index.ts +2 -2
  89. package/src/collections/tree/collection-trunk.ts +25 -25
  90. package/src/collections/tree/index.ts +2 -2
  91. package/src/collections/tree/readme.md +55 -55
  92. package/src/collections/tree/struct.ts +9 -4
  93. package/src/log/index.ts +2 -2
  94. package/src/log/log.ts +304 -268
  95. package/src/log/struct.ts +165 -165
  96. package/src/matchmaking/capability-filter.ts +45 -45
  97. package/src/matchmaking/config.ts +98 -98
  98. package/src/matchmaking/index.ts +21 -21
  99. package/src/matchmaking/multi-cohort-seeker.ts +234 -234
  100. package/src/matchmaking/provider.ts +123 -123
  101. package/src/matchmaking/query-eval.ts +105 -105
  102. package/src/matchmaking/seeker-walk.ts +127 -127
  103. package/src/matchmaking/seeker.ts +86 -86
  104. package/src/matchmaking/topic-anchor.ts +90 -90
  105. package/src/matchmaking/voting-quorum.ts +394 -394
  106. package/src/network/i-peer-network.ts +25 -25
  107. package/src/network/repo-protocol.ts +13 -13
  108. package/src/network/types.ts +37 -37
  109. package/src/reactivity/backfill.ts +220 -220
  110. package/src/reactivity/backpressure.ts +191 -191
  111. package/src/reactivity/checkpoint.ts +308 -308
  112. package/src/reactivity/config.ts +172 -172
  113. package/src/reactivity/dedupe.ts +132 -132
  114. package/src/reactivity/forwarder.ts +87 -87
  115. package/src/reactivity/index.ts +34 -34
  116. package/src/reactivity/notification.ts +123 -123
  117. package/src/reactivity/policy.ts +79 -79
  118. package/src/reactivity/push-state.ts +310 -310
  119. package/src/reactivity/recover.ts +153 -153
  120. package/src/reactivity/replay-buffer.ts +141 -141
  121. package/src/reactivity/resume.ts +549 -549
  122. package/src/reactivity/rotation.ts +415 -415
  123. package/src/reactivity/subscriber.ts +132 -132
  124. package/src/reactivity/subscription.ts +66 -66
  125. package/src/reactivity/topic-anchor.ts +71 -71
  126. package/src/reactivity/verify.ts +73 -73
  127. package/src/reactivity/wire-validate.ts +13 -13
  128. package/src/testing/async-wait.ts +65 -65
  129. package/src/testing/index.ts +2 -2
  130. package/src/testing/test-transactor.ts +62 -0
  131. package/src/transaction/actions-engine.ts +68 -68
  132. package/src/transaction/coordinator.ts +28 -11
  133. package/src/transaction/index.ts +52 -52
  134. package/src/transaction/operations-hash.ts +196 -196
  135. package/src/transaction/read-dependency-collector.ts +78 -78
  136. package/src/transaction/session.ts +238 -238
  137. package/src/transaction/transaction.ts +442 -442
  138. package/src/transaction/validator.ts +197 -197
  139. package/src/transactor/block-floors.ts +126 -126
  140. package/src/transactor/index.ts +6 -6
  141. package/src/transactor/network-transactor.ts +1545 -1545
  142. package/src/transform/atomic-proxy.ts +92 -92
  143. package/src/transform/cache-source.ts +370 -370
  144. package/src/transform/helpers.ts +164 -164
  145. package/src/transform/index.ts +7 -7
  146. package/src/transform/struct.ts +21 -21
  147. package/src/utility/actor.ts +62 -62
  148. package/src/utility/backoff.ts +95 -95
  149. package/src/utility/batch-coordinator.ts +191 -191
  150. package/src/utility/ensured.ts +32 -32
  151. package/src/utility/groupby.ts +18 -18
  152. package/src/utility/hash-string.ts +14 -14
  153. package/src/utility/is-record-empty.ts +5 -5
  154. package/src/utility/latches.ts +42 -42
  155. package/src/utility/lru-map.ts +60 -60
  156. package/src/utility/nameof.ts +7 -7
  157. package/src/utility/pending.ts +43 -43
package/README.md CHANGED
@@ -1,336 +1,336 @@
1
- # Optimystic DB Core
2
-
3
- A distributed database system designed for peer-to-peer networks, providing ACID transactions across content-addressed storage with strong consistency guarantees and conflict resolution.
4
-
5
- ## Overview
6
-
7
- Optimystic DB Core solves the fundamental challenge of maintaining **atomic transactions across content-addressed distributed storage**. Unlike traditional databases where related data resides on a single server, Optimystic distributes blocks across multiple independent peers.
8
-
9
- ### The Challenge
10
-
11
- In distributed content-addressed systems:
12
- - **Logically related blocks** are scattered across different network peers
13
- - **Atomic transactions** must coordinate across multiple independent clusters
14
- - **Consensus ordering** must be established without centralized coordination
15
- - **Network failures** can occur independently at any peer or cluster
16
- - **Conflict resolution** must handle concurrent modifications gracefully
17
-
18
- ### The Solution
19
-
20
- Optimystic provides a **layered architecture** that maintains ACID properties while achieving horizontal scalability and fault tolerance:
21
-
22
- 1. **Randomly distributed block storage** with immutable, versioned units
23
- 2. **Block-based data structures** (B-trees, chains) built on block primitives
24
- 3. **Transaction logging** with integrity guarantees and checkpointing
25
- 4. **Collection abstractions** that combine data structures with distributed transactions
26
- 5. **Distributed coordination** through log-first transaction ordering
27
- 6. **Peer-to-peer networking** with cluster-aware consensus and failure recovery
28
-
29
- A key innovation of this system is that the transaction payload is transmitted prior to the commital, as part of the "pend" phase. This minimizes the overhead of commitment since the propagation and validation of the transaction are complete prior to commit.
30
-
31
- ## Architecture
32
-
33
- ```
34
- ┌─────────────────────────────────────────────────────────────┐
35
- │ Collections Layer │
36
- │ High-level abstractions: Tree, Diary, Custom Collections │
37
- │ • Action-based mutations with conflict resolution │
38
- │ • Local snapshots with explicit synchronization │
39
- │ • Integration with specialized data structures │
40
- └─────────────────┬───────────────────────────────────────────┘
41
- │
42
- ┌─────────────────▼───────────────────────────────────────────┐
43
- │ Transaction Layer │
44
- │ Logs (transaction ordering) + Transactors │
45
- │ • Log-first transaction ordering across collections │
46
- │ • Distributed consensus with conflict detection │
47
- │ • Missing/pending transaction resolution │
48
- └─────────────────┬───────────────────────────────────────────┘
49
- │
50
- ┌─────────────────▼───────────────────────────────────────────┐
51
- │ Data Structures Layer │
52
- │ Chains (ordered) + BTrees (indexed) │
53
- │ • Chains: stacks, queues, logs with linked blocks │
54
- │ • BTrees: sorted access with range queries │
55
- │ • Both built on immutable block operations │
56
- └─────────────────┬───────────────────────────────────────────┘
57
- │
58
- ┌─────────────────▼───────────────────────────────────────────┐
59
- │ Block Layer │
60
- │ Immutable, versioned storage units │
61
- │ • Content-addressed with base32 IDs │
62
- │ • Atomic operations with transform tracking │
63
- │ • Network-ready serialization and distribution │
64
- └─────────────────────────────────────────────────────────────┘
65
- ```
66
-
67
- ## Core Components
68
-
69
- ### [Blocks](docs/blocks.md) - Foundation Storage Units
70
-
71
- Immutable, versioned data storage units that form the foundation of the system:
72
-
73
- - **Content-addressed**: Uniquely identified by base32-encoded block IDs
74
- - **Atomic operations**: Precise modifications through block operations
75
- - **Transform tracking**: Collect changes without immediate application
76
- - **Network-ready**: Designed for distributed storage and retrieval
77
-
78
- ```typescript
79
- // Basic block with header
80
- type IBlock = {
81
- header: {
82
- id: BlockId;
83
- type: BlockType;
84
- collectionId: BlockId;
85
- }
86
- }
87
-
88
- // Atomic operations on block properties
89
- apply(store, block, ['items', 0, 1, ['newValue']]);
90
- ```
91
-
92
- ### [Chains](docs/chains.md) - Ordered Data Structures
93
-
94
- Linked block primitives that can function as stacks, queues, or logs:
95
-
96
- - **Flexible access patterns**: LIFO (stack), FIFO (queue), or sequential (log)
97
- - **Efficient traversal**: Forward and backward iteration with precise navigation
98
- - **Block efficiency**: 32 entries per block optimized for network transfer
99
- - **Distributed ready**: Lazy loading and efficient network operations
100
-
101
- ```typescript
102
- // Stack operations (LIFO)
103
- await chain.add('item1', 'item2');
104
- const items = await chain.pop(1); // ['item2']
105
-
106
- // Queue operations (FIFO)
107
- await chain.add('first', 'second');
108
- const items = await chain.dequeue(1); // ['first']
109
- ```
110
-
111
- ### [BTrees](docs/btree.md) - Indexed Data Structures
112
-
113
- B+tree implementation providing efficient sorted access to data:
114
-
115
- - **Sorted access**: Keys maintained in order for efficient range queries
116
- - **Path-based navigation**: Cursor-like traversal through the tree
117
- - **Automatic rebalancing**: Splits, merges, and borrowing maintain optimal structure
118
- - **Type-safe**: Generic implementation supporting custom key/entry types
119
-
120
- ```typescript
121
- // Range queries with efficient iteration
122
- const range = new KeyRange(startKey, endKey, true);
123
- for await (const path of tree.range(range)) {
124
- const entry = tree.at(path);
125
- console.log(entry);
126
- }
127
- ```
128
-
129
- ### [Logs](docs/logs.md) - Transaction Logging
130
-
131
- Transaction logging built on chains with integrity guarantees:
132
-
133
- - **SHA256 integrity**: Block hashing ensures tamper detection
134
- - **Action tracking**: Transaction IDs, revision numbers, affected blocks
135
- - **Checkpoint management**: Efficient transaction state queries
136
- - **Multi-collection coordination**: Cross-collection transaction support
137
-
138
- ```typescript
139
- // Record transaction with affected blocks
140
- await log.addActions(
141
- [action1, action2],
142
- transactionId,
143
- revisionNumber,
144
- () => affectedBlockIds,
145
- involvedCollectionIds
146
- );
147
- ```
148
-
149
- ### [Collections](docs/collections.md) - High-Level Abstractions
150
-
151
- Logical groupings that combine data structures with distributed transactions:
152
-
153
- - **Local snapshots**: Immediate local changes with explicit synchronization
154
- - **Action abstraction**: Replay actions from local, remote, or conflict resolution
155
- - **Conflict resolution**: Customizable strategies for concurrent modifications
156
- - **Data structure integration**: Tree collections (indexed), Diary collections (append-only)
157
-
158
- ```typescript
159
- // Local changes applied immediately to snapshot
160
- await collection.act(action1, action2, action3);
161
-
162
- // Explicit synchronization with distributed state
163
- await collection.updateAndSync();
164
- ```
165
-
166
- ### [Transactors](docs/transactor.md) - Distributed Coordination
167
-
168
- Abstract coordination layer managing distributed transactions:
169
-
170
- - **Log-first ordering**: Transaction order established by log tail commits
171
- - **Two-phase coordination**: Pend → commit pattern for atomic operations
172
- - **Conflict detection**: Missing vs pending transaction differentiation
173
- - **Network abstraction**: Clean interface hiding consensus complexity
174
-
175
- ```typescript
176
- // Two-phase distributed transaction
177
- const pendResult = await transactor.pend(request);
178
- if (pendResult.success) {
179
- const commitResult = await transactor.commit(request);
180
- }
181
- ```
182
-
183
- ### [Network Layer](docs/network.md) - Peer-to-Peer Implementation
184
-
185
- Concrete implementation for peer-to-peer networks:
186
-
187
- - **Content-to-cluster mapping**: Deterministic routing via consistent hashing
188
- - **Parallel cluster coordination**: Execute operations across multiple clusters
189
- - **Failure-aware retry**: Handle peer failures and network partitions
190
- - **Cluster-aware batching**: Group operations by responsible peers
191
-
192
- ## Key Features
193
-
194
- ### Distributed ACID Transactions
195
-
196
- - **Atomicity**: All changes in a transaction succeed or fail together
197
- - **Consistency**: Strong consistency across distributed participants
198
- - **Isolation**: Conflict detection and resolution for concurrent operations
199
- - **Durability**: Committed transactions survive network partitions
200
-
201
- ### Content-Addressed Storage
202
-
203
- - **Deterministic distribution**: Blocks distributed based on content hash
204
- - **Location independence**: Access blocks without knowing physical location
205
- - **Efficient routing**: Consistent hashing maps blocks to responsible peers
206
- - **Fault tolerance**: Multiple peers can coordinate same keyspace regions
207
-
208
- ### Conflict Resolution
209
-
210
- - **Action filtering**: Customizable strategies for resolving conflicts
211
- - **State replay**: Re-apply local actions on updated remote state
212
- - **Missing transactions**: Automatic rebasing on newer committed changes
213
- - **Pending awareness**: Handle concurrent modifications gracefully
214
-
215
- ### Horizontal Scalability
216
-
217
- - **Peer-to-peer architecture**: No single points of failure
218
- - **Parallel execution**: Operations on different blocks execute concurrently
219
- - **Load distribution**: Coordination responsibility spread across network
220
- - **Dynamic membership**: Peers can join and leave without disruption
221
-
222
- ## Usage Patterns
223
-
224
- ### Simple Append-Only Storage
225
-
226
- ```typescript
227
- // Create diary for event logging
228
- const eventLog = await Diary.createOrOpen<Event>(transactor, 'events');
229
-
230
- // Add events
231
- await eventLog.append({ type: 'user_login', userId: '123' });
232
- await eventLog.append({ type: 'user_logout', userId: '123' });
233
-
234
- // Read events in order
235
- for await (const event of eventLog.select()) {
236
- processEvent(event);
237
- }
238
- ```
239
-
240
- ### Indexed Data with Range Queries
241
-
242
- ```typescript
243
- // Create tree collection for user data
244
- const userTree = await Tree.createOrOpen<string, User>(
245
- transactor,
246
- 'users',
247
- user => user.id, // Key extractor
248
- (a, b) => a.localeCompare(b) // Comparator
249
- );
250
-
251
- // Batch updates
252
- await userTree.replace([
253
- ['user1', { id: 'user1', name: 'Alice' }],
254
- ['user2', { id: 'user2', name: 'Bob' }]
255
- ]);
256
-
257
- // Range queries
258
- for await (const path of userTree.range({ from: 'user1', to: 'user9' })) {
259
- console.log(userTree.at(path));
260
- }
261
- ```
262
-
263
- ### Custom Collections with Conflict Resolution
264
-
265
- ```typescript
266
- class CounterCollection {
267
- static async create(transactor: ITransactor, id: CollectionId) {
268
- const init: CollectionInitOptions<IncrementAction> = {
269
- modules: {
270
- "increment": async (action, store) => {
271
- // Custom increment logic
272
- }
273
- },
274
- filterConflict: (local, remote) => {
275
- // Merge conflicting increments
276
- const remoteSum = remote.reduce((sum, r) => sum + r.data.value, 0);
277
- return { ...local, data: { ...local.data, value: local.data.value + remoteSum }};
278
- }
279
- };
280
-
281
- return await Collection.createOrOpen(transactor, id, init);
282
- }
283
- }
284
- ```
285
-
286
- ## Getting Started
287
-
288
- ### Installation
289
-
290
- ```bash
291
- npm install @optimystic/db-core
292
- ```
293
-
294
- ### Basic Setup
295
-
296
- ```typescript
297
- import {
298
- NetworkTransactor,
299
- Tree,
300
- Diary,
301
- Collection
302
- } from '@optimystic/db-core';
303
-
304
- // Set up distributed transactor
305
- const transactor = new NetworkTransactor({
306
- keyNetwork, // Peer discovery implementation
307
- peerNetwork, // Communication layer
308
- getRepo // Repository factory
309
- });
310
-
311
- // Create collections
312
- const userTree = await Tree.createOrOpen(transactor, 'users', /* ... */);
313
- const eventLog = await Diary.createOrOpen(transactor, 'events');
314
-
315
- // Use collections
316
- await userTree.replace([['user1', userData]]);
317
- await eventLog.append(eventData);
318
- ```
319
-
320
- ## Documentation
321
-
322
- - **[Blocks](docs/blocks.md)** - Immutable storage units and operations
323
- - **[BTrees](docs/btree.md)** - Sorted data access and range queries
324
- - **[Chains](docs/chains.md)** - Ordered data structures (stacks, queues, logs)
325
- - **[Logs](docs/logs.md)** - Transaction logging with integrity guarantees
326
- - **[Collections](docs/collections.md)** - High-level abstractions with conflict resolution
327
- - **[Transactors](docs/transactor.md)** - Distributed transaction coordination
328
- - **[Network](docs/network.md)** - Peer-to-peer implementation architecture
329
-
330
- ### Internal Architecture
331
-
332
- For developers and AI agents working on internals, see [Internals Guide](../../docs/internals.md) for:
333
- - Data flow diagrams (read/write/commit paths)
334
- - Mutation contracts (which functions mutate vs clone)
335
- - Key invariants and common pitfalls
336
- - Type glossary and debugging tips
1
+ # Optimystic DB Core
2
+
3
+ A distributed database system designed for peer-to-peer networks, providing ACID transactions across content-addressed storage with strong consistency guarantees and conflict resolution.
4
+
5
+ ## Overview
6
+
7
+ Optimystic DB Core solves the fundamental challenge of maintaining **atomic transactions across content-addressed distributed storage**. Unlike traditional databases where related data resides on a single server, Optimystic distributes blocks across multiple independent peers.
8
+
9
+ ### The Challenge
10
+
11
+ In distributed content-addressed systems:
12
+ - **Logically related blocks** are scattered across different network peers
13
+ - **Atomic transactions** must coordinate across multiple independent clusters
14
+ - **Consensus ordering** must be established without centralized coordination
15
+ - **Network failures** can occur independently at any peer or cluster
16
+ - **Conflict resolution** must handle concurrent modifications gracefully
17
+
18
+ ### The Solution
19
+
20
+ Optimystic provides a **layered architecture** that maintains ACID properties while achieving horizontal scalability and fault tolerance:
21
+
22
+ 1. **Randomly distributed block storage** with immutable, versioned units
23
+ 2. **Block-based data structures** (B-trees, chains) built on block primitives
24
+ 3. **Transaction logging** with integrity guarantees and checkpointing
25
+ 4. **Collection abstractions** that combine data structures with distributed transactions
26
+ 5. **Distributed coordination** through log-first transaction ordering
27
+ 6. **Peer-to-peer networking** with cluster-aware consensus and failure recovery
28
+
29
+ A key innovation of this system is that the transaction payload is transmitted prior to the commital, as part of the "pend" phase. This minimizes the overhead of commitment since the propagation and validation of the transaction are complete prior to commit.
30
+
31
+ ## Architecture
32
+
33
+ ```
34
+ ┌─────────────────────────────────────────────────────────────┐
35
+ │ Collections Layer │
36
+ │ High-level abstractions: Tree, Diary, Custom Collections │
37
+ │ • Action-based mutations with conflict resolution │
38
+ │ • Local snapshots with explicit synchronization │
39
+ │ • Integration with specialized data structures │
40
+ └─────────────────┬───────────────────────────────────────────┘
41
+ │
42
+ ┌─────────────────▼───────────────────────────────────────────┐
43
+ │ Transaction Layer │
44
+ │ Logs (transaction ordering) + Transactors │
45
+ │ • Log-first transaction ordering across collections │
46
+ │ • Distributed consensus with conflict detection │
47
+ │ • Missing/pending transaction resolution │
48
+ └─────────────────┬───────────────────────────────────────────┘
49
+ │
50
+ ┌─────────────────▼───────────────────────────────────────────┐
51
+ │ Data Structures Layer │
52
+ │ Chains (ordered) + BTrees (indexed) │
53
+ │ • Chains: stacks, queues, logs with linked blocks │
54
+ │ • BTrees: sorted access with range queries │
55
+ │ • Both built on immutable block operations │
56
+ └─────────────────┬───────────────────────────────────────────┘
57
+ │
58
+ ┌─────────────────▼───────────────────────────────────────────┐
59
+ │ Block Layer │
60
+ │ Immutable, versioned storage units │
61
+ │ • Content-addressed with base32 IDs │
62
+ │ • Atomic operations with transform tracking │
63
+ │ • Network-ready serialization and distribution │
64
+ └─────────────────────────────────────────────────────────────┘
65
+ ```
66
+
67
+ ## Core Components
68
+
69
+ ### [Blocks](docs/blocks.md) - Foundation Storage Units
70
+
71
+ Immutable, versioned data storage units that form the foundation of the system:
72
+
73
+ - **Content-addressed**: Uniquely identified by base32-encoded block IDs
74
+ - **Atomic operations**: Precise modifications through block operations
75
+ - **Transform tracking**: Collect changes without immediate application
76
+ - **Network-ready**: Designed for distributed storage and retrieval
77
+
78
+ ```typescript
79
+ // Basic block with header
80
+ type IBlock = {
81
+ header: {
82
+ id: BlockId;
83
+ type: BlockType;
84
+ collectionId: BlockId;
85
+ }
86
+ }
87
+
88
+ // Atomic operations on block properties
89
+ apply(store, block, ['items', 0, 1, ['newValue']]);
90
+ ```
91
+
92
+ ### [Chains](docs/chains.md) - Ordered Data Structures
93
+
94
+ Linked block primitives that can function as stacks, queues, or logs:
95
+
96
+ - **Flexible access patterns**: LIFO (stack), FIFO (queue), or sequential (log)
97
+ - **Efficient traversal**: Forward and backward iteration with precise navigation
98
+ - **Block efficiency**: 32 entries per block optimized for network transfer
99
+ - **Distributed ready**: Lazy loading and efficient network operations
100
+
101
+ ```typescript
102
+ // Stack operations (LIFO)
103
+ await chain.add('item1', 'item2');
104
+ const items = await chain.pop(1); // ['item2']
105
+
106
+ // Queue operations (FIFO)
107
+ await chain.add('first', 'second');
108
+ const items = await chain.dequeue(1); // ['first']
109
+ ```
110
+
111
+ ### [BTrees](docs/btree.md) - Indexed Data Structures
112
+
113
+ B+tree implementation providing efficient sorted access to data:
114
+
115
+ - **Sorted access**: Keys maintained in order for efficient range queries
116
+ - **Path-based navigation**: Cursor-like traversal through the tree
117
+ - **Automatic rebalancing**: Splits, merges, and borrowing maintain optimal structure
118
+ - **Type-safe**: Generic implementation supporting custom key/entry types
119
+
120
+ ```typescript
121
+ // Range queries with efficient iteration
122
+ const range = new KeyRange(startKey, endKey, true);
123
+ for await (const path of tree.range(range)) {
124
+ const entry = tree.at(path);
125
+ console.log(entry);
126
+ }
127
+ ```
128
+
129
+ ### [Logs](docs/logs.md) - Transaction Logging
130
+
131
+ Transaction logging built on chains with integrity guarantees:
132
+
133
+ - **SHA256 integrity**: Block hashing ensures tamper detection
134
+ - **Action tracking**: Transaction IDs, revision numbers, affected blocks
135
+ - **Checkpoint management**: Efficient transaction state queries
136
+ - **Multi-collection coordination**: Cross-collection transaction support
137
+
138
+ ```typescript
139
+ // Record transaction with affected blocks
140
+ await log.addActions(
141
+ [action1, action2],
142
+ transactionId,
143
+ revisionNumber,
144
+ () => affectedBlockIds,
145
+ involvedCollectionIds
146
+ );
147
+ ```
148
+
149
+ ### [Collections](docs/collections.md) - High-Level Abstractions
150
+
151
+ Logical groupings that combine data structures with distributed transactions:
152
+
153
+ - **Local snapshots**: Immediate local changes with explicit synchronization
154
+ - **Action abstraction**: Replay actions from local, remote, or conflict resolution
155
+ - **Conflict resolution**: Customizable strategies for concurrent modifications
156
+ - **Data structure integration**: Tree collections (indexed), Diary collections (append-only)
157
+
158
+ ```typescript
159
+ // Local changes applied immediately to snapshot
160
+ await collection.act(action1, action2, action3);
161
+
162
+ // Explicit synchronization with distributed state
163
+ await collection.updateAndSync();
164
+ ```
165
+
166
+ ### [Transactors](docs/transactor.md) - Distributed Coordination
167
+
168
+ Abstract coordination layer managing distributed transactions:
169
+
170
+ - **Log-first ordering**: Transaction order established by log tail commits
171
+ - **Two-phase coordination**: Pend → commit pattern for atomic operations
172
+ - **Conflict detection**: Missing vs pending transaction differentiation
173
+ - **Network abstraction**: Clean interface hiding consensus complexity
174
+
175
+ ```typescript
176
+ // Two-phase distributed transaction
177
+ const pendResult = await transactor.pend(request);
178
+ if (pendResult.success) {
179
+ const commitResult = await transactor.commit(request);
180
+ }
181
+ ```
182
+
183
+ ### [Network Layer](docs/network.md) - Peer-to-Peer Implementation
184
+
185
+ Concrete implementation for peer-to-peer networks:
186
+
187
+ - **Content-to-cluster mapping**: Deterministic routing via consistent hashing
188
+ - **Parallel cluster coordination**: Execute operations across multiple clusters
189
+ - **Failure-aware retry**: Handle peer failures and network partitions
190
+ - **Cluster-aware batching**: Group operations by responsible peers
191
+
192
+ ## Key Features
193
+
194
+ ### Distributed ACID Transactions
195
+
196
+ - **Atomicity**: All changes in a transaction succeed or fail together
197
+ - **Consistency**: Strong consistency across distributed participants
198
+ - **Isolation**: Conflict detection and resolution for concurrent operations
199
+ - **Durability**: Committed transactions survive network partitions
200
+
201
+ ### Content-Addressed Storage
202
+
203
+ - **Deterministic distribution**: Blocks distributed based on content hash
204
+ - **Location independence**: Access blocks without knowing physical location
205
+ - **Efficient routing**: Consistent hashing maps blocks to responsible peers
206
+ - **Fault tolerance**: Multiple peers can coordinate same keyspace regions
207
+
208
+ ### Conflict Resolution
209
+
210
+ - **Action filtering**: Customizable strategies for resolving conflicts
211
+ - **State replay**: Re-apply local actions on updated remote state
212
+ - **Missing transactions**: Automatic rebasing on newer committed changes
213
+ - **Pending awareness**: Handle concurrent modifications gracefully
214
+
215
+ ### Horizontal Scalability
216
+
217
+ - **Peer-to-peer architecture**: No single points of failure
218
+ - **Parallel execution**: Operations on different blocks execute concurrently
219
+ - **Load distribution**: Coordination responsibility spread across network
220
+ - **Dynamic membership**: Peers can join and leave without disruption
221
+
222
+ ## Usage Patterns
223
+
224
+ ### Simple Append-Only Storage
225
+
226
+ ```typescript
227
+ // Create diary for event logging
228
+ const eventLog = await Diary.createOrOpen<Event>(transactor, 'events');
229
+
230
+ // Add events
231
+ await eventLog.append({ type: 'user_login', userId: '123' });
232
+ await eventLog.append({ type: 'user_logout', userId: '123' });
233
+
234
+ // Read events in order
235
+ for await (const event of eventLog.select()) {
236
+ processEvent(event);
237
+ }
238
+ ```
239
+
240
+ ### Indexed Data with Range Queries
241
+
242
+ ```typescript
243
+ // Create tree collection for user data
244
+ const userTree = await Tree.createOrOpen<string, User>(
245
+ transactor,
246
+ 'users',
247
+ user => user.id, // Key extractor
248
+ (a, b) => a.localeCompare(b) // Comparator
249
+ );
250
+
251
+ // Batch updates
252
+ await userTree.replace([
253
+ ['user1', { id: 'user1', name: 'Alice' }],
254
+ ['user2', { id: 'user2', name: 'Bob' }]
255
+ ]);
256
+
257
+ // Range queries
258
+ for await (const path of userTree.range({ from: 'user1', to: 'user9' })) {
259
+ console.log(userTree.at(path));
260
+ }
261
+ ```
262
+
263
+ ### Custom Collections with Conflict Resolution
264
+
265
+ ```typescript
266
+ class CounterCollection {
267
+ static async create(transactor: ITransactor, id: CollectionId) {
268
+ const init: CollectionInitOptions<IncrementAction> = {
269
+ modules: {
270
+ "increment": async (action, store) => {
271
+ // Custom increment logic
272
+ }
273
+ },
274
+ filterConflict: (local, remote) => {
275
+ // Merge conflicting increments
276
+ const remoteSum = remote.reduce((sum, r) => sum + r.data.value, 0);
277
+ return { ...local, data: { ...local.data, value: local.data.value + remoteSum }};
278
+ }
279
+ };
280
+
281
+ return await Collection.createOrOpen(transactor, id, init);
282
+ }
283
+ }
284
+ ```
285
+
286
+ ## Getting Started
287
+
288
+ ### Installation
289
+
290
+ ```bash
291
+ npm install @optimystic/db-core
292
+ ```
293
+
294
+ ### Basic Setup
295
+
296
+ ```typescript
297
+ import {
298
+ NetworkTransactor,
299
+ Tree,
300
+ Diary,
301
+ Collection
302
+ } from '@optimystic/db-core';
303
+
304
+ // Set up distributed transactor
305
+ const transactor = new NetworkTransactor({
306
+ keyNetwork, // Peer discovery implementation
307
+ peerNetwork, // Communication layer
308
+ getRepo // Repository factory
309
+ });
310
+
311
+ // Create collections
312
+ const userTree = await Tree.createOrOpen(transactor, 'users', /* ... */);
313
+ const eventLog = await Diary.createOrOpen(transactor, 'events');
314
+
315
+ // Use collections
316
+ await userTree.replace([['user1', userData]]);
317
+ await eventLog.append(eventData);
318
+ ```
319
+
320
+ ## Documentation
321
+
322
+ - **[Blocks](docs/blocks.md)** - Immutable storage units and operations
323
+ - **[BTrees](docs/btree.md)** - Sorted data access and range queries
324
+ - **[Chains](docs/chains.md)** - Ordered data structures (stacks, queues, logs)
325
+ - **[Logs](docs/logs.md)** - Transaction logging with integrity guarantees
326
+ - **[Collections](docs/collections.md)** - High-level abstractions with conflict resolution
327
+ - **[Transactors](docs/transactor.md)** - Distributed transaction coordination
328
+ - **[Network](docs/network.md)** - Peer-to-peer implementation architecture
329
+
330
+ ### Internal Architecture
331
+
332
+ For developers and AI agents working on internals, see [Internals Guide](../../docs/internals.md) for:
333
+ - Data flow diagrams (read/write/commit paths)
334
+ - Mutation contracts (which functions mutate vs clone)
335
+ - Key invariants and common pitfalls
336
+ - Type glossary and debugging tips