@harperfast/harper 5.3.0 → 5.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/bin/copyDb.ts +5 -1
  2. package/bin/deployRenderer.ts +3 -0
  3. package/components/Application.ts +66 -22
  4. package/components/DESIGN.md +89 -36
  5. package/components/componentLoader.ts +2 -0
  6. package/components/deploymentRecorder.ts +18 -1
  7. package/components/installFingerprint.ts +123 -0
  8. package/components/mcp/DESIGN.md +4 -4
  9. package/components/operations.js +29 -0
  10. package/config/DESIGN.md +5 -5
  11. package/dataLayer/DESIGN.md +37 -22
  12. package/dataLayer/backupRepository.ts +248 -0
  13. package/dataLayer/restoreMarker.ts +30 -75
  14. package/dataLayer/rocksdbBackup.ts +253 -110
  15. package/dist/bin/copyDb.js +3 -0
  16. package/dist/bin/copyDb.js.map +1 -1
  17. package/dist/bin/deployRenderer.js +3 -0
  18. package/dist/bin/deployRenderer.js.map +1 -1
  19. package/dist/components/Application.d.ts +5 -0
  20. package/dist/components/Application.js +51 -19
  21. package/dist/components/Application.js.map +1 -1
  22. package/dist/components/componentLoader.js +2 -0
  23. package/dist/components/componentLoader.js.map +1 -1
  24. package/dist/components/deploymentRecorder.js +18 -1
  25. package/dist/components/deploymentRecorder.js.map +1 -1
  26. package/dist/components/installFingerprint.d.ts +39 -0
  27. package/dist/components/installFingerprint.js +113 -0
  28. package/dist/components/installFingerprint.js.map +1 -0
  29. package/dist/components/operations.d.ts +2 -0
  30. package/dist/components/operations.js +31 -0
  31. package/dist/components/operations.js.map +1 -1
  32. package/dist/dataLayer/backupRepository.d.ts +33 -0
  33. package/dist/dataLayer/backupRepository.js +232 -0
  34. package/dist/dataLayer/backupRepository.js.map +1 -0
  35. package/dist/dataLayer/restoreMarker.d.ts +4 -1
  36. package/dist/dataLayer/restoreMarker.js +29 -68
  37. package/dist/dataLayer/restoreMarker.js.map +1 -1
  38. package/dist/dataLayer/rocksdbBackup.js +221 -112
  39. package/dist/dataLayer/rocksdbBackup.js.map +1 -1
  40. package/dist/index.d.ts +1 -0
  41. package/dist/index.js.map +1 -1
  42. package/dist/json/systemSchema.json +14 -0
  43. package/dist/resources/DatabaseTransaction.d.ts +140 -11
  44. package/dist/resources/DatabaseTransaction.js +1055 -464
  45. package/dist/resources/DatabaseTransaction.js.map +1 -1
  46. package/dist/resources/IterableEventQueue.d.ts +2 -1
  47. package/dist/resources/IterableEventQueue.js +16 -8
  48. package/dist/resources/IterableEventQueue.js.map +1 -1
  49. package/dist/resources/LMDBTransaction.d.ts +13 -2
  50. package/dist/resources/LMDBTransaction.js +219 -93
  51. package/dist/resources/LMDBTransaction.js.map +1 -1
  52. package/dist/resources/RecordEncoder.js +2 -3
  53. package/dist/resources/RecordEncoder.js.map +1 -1
  54. package/dist/resources/Resource.js +5 -6
  55. package/dist/resources/Resource.js.map +1 -1
  56. package/dist/resources/ResourceInterface.d.ts +3 -1
  57. package/dist/resources/ResourceInterface.js.map +1 -1
  58. package/dist/resources/RocksIndexStore.d.ts +2 -1
  59. package/dist/resources/RocksIndexStore.js +3 -0
  60. package/dist/resources/RocksIndexStore.js.map +1 -1
  61. package/dist/resources/Table.js +116 -12
  62. package/dist/resources/Table.js.map +1 -1
  63. package/dist/resources/analytics/write.js +6 -2
  64. package/dist/resources/analytics/write.js.map +1 -1
  65. package/dist/resources/auditStore.d.ts +71 -17
  66. package/dist/resources/auditStore.js +248 -64
  67. package/dist/resources/auditStore.js.map +1 -1
  68. package/dist/resources/branchDatabase.js +2 -0
  69. package/dist/resources/branchDatabase.js.map +1 -1
  70. package/dist/resources/databases.js +8 -2
  71. package/dist/resources/databases.js.map +1 -1
  72. package/dist/resources/indexes/fullTextNativeBinding.d.ts +6 -2
  73. package/dist/resources/indexes/fullTextNativeBinding.js +12 -2
  74. package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -1
  75. package/dist/resources/indexes/fullTextQueryIndex.d.ts +4 -0
  76. package/dist/resources/indexes/fullTextQueryIndex.js +207 -10
  77. package/dist/resources/indexes/fullTextQueryIndex.js.map +1 -1
  78. package/dist/resources/models/Models.d.ts +11 -0
  79. package/dist/resources/models/Models.js +95 -4
  80. package/dist/resources/models/Models.js.map +1 -1
  81. package/dist/resources/models/backendRegistry.d.ts +8 -0
  82. package/dist/resources/models/backendRegistry.js +15 -1
  83. package/dist/resources/models/backendRegistry.js.map +1 -1
  84. package/dist/resources/models/bootstrap.d.ts +2 -0
  85. package/dist/resources/models/bootstrap.js +6 -0
  86. package/dist/resources/models/bootstrap.js.map +1 -1
  87. package/dist/resources/models/calibration.d.ts +88 -0
  88. package/dist/resources/models/calibration.js +244 -0
  89. package/dist/resources/models/calibration.js.map +1 -0
  90. package/dist/resources/models/calibrationStore.d.ts +163 -0
  91. package/dist/resources/models/calibrationStore.js +986 -0
  92. package/dist/resources/models/calibrationStore.js.map +1 -0
  93. package/dist/resources/models/decisionStore.d.ts +24 -0
  94. package/dist/resources/models/decisionStore.js +45 -0
  95. package/dist/resources/models/decisionStore.js.map +1 -1
  96. package/dist/resources/models/generativeDecision.js +44 -15
  97. package/dist/resources/models/generativeDecision.js.map +1 -1
  98. package/dist/resources/models/types.d.ts +7 -0
  99. package/dist/resources/recordLock.d.ts +8 -5
  100. package/dist/resources/recordLock.js +6 -4
  101. package/dist/resources/recordLock.js.map +1 -1
  102. package/dist/resources/replayLogs.js +8 -1
  103. package/dist/resources/replayLogs.js.map +1 -1
  104. package/dist/resources/scheduler/engine.d.ts +4 -0
  105. package/dist/resources/scheduler/engine.js +30 -2
  106. package/dist/resources/scheduler/engine.js.map +1 -1
  107. package/dist/resources/search.d.ts +16 -4
  108. package/dist/resources/search.js +117 -13
  109. package/dist/resources/search.js.map +1 -1
  110. package/dist/resources/transaction.js +85 -5
  111. package/dist/resources/transaction.js.map +1 -1
  112. package/dist/resources/transactionBroadcast.d.ts +8 -0
  113. package/dist/resources/transactionBroadcast.js +65 -0
  114. package/dist/resources/transactionBroadcast.js.map +1 -1
  115. package/dist/security/keys.d.ts +1 -1
  116. package/dist/security/keys.js +170 -120
  117. package/dist/security/keys.js.map +1 -1
  118. package/dist/server/http.js +10 -0
  119. package/dist/server/http.js.map +1 -1
  120. package/dist/server/serverHelpers/Request.js +11 -2
  121. package/dist/server/serverHelpers/Request.js.map +1 -1
  122. package/dist/server/serverHelpers/contentTypes.js +8 -3
  123. package/dist/server/serverHelpers/contentTypes.js.map +1 -1
  124. package/dist/server/serverHelpers/uwsServer.js +38 -7
  125. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  126. package/dist/server/threads/manageThreads.d.ts +3 -3
  127. package/dist/server/threads/manageThreads.js +15 -13
  128. package/dist/server/threads/manageThreads.js.map +1 -1
  129. package/dist/server/threads/threadServer.d.ts +3 -0
  130. package/dist/server/threads/threadServer.js +24 -6
  131. package/dist/server/threads/threadServer.js.map +1 -1
  132. package/dist/upgrade/directives/5-3-0.js +9 -3
  133. package/dist/upgrade/directives/5-3-0.js.map +1 -1
  134. package/dist/utility/durableFile.d.ts +36 -0
  135. package/dist/utility/durableFile.js +106 -0
  136. package/dist/utility/durableFile.js.map +1 -0
  137. package/dist/utility/errors/hdbError.d.ts +4 -0
  138. package/dist/utility/errors/hdbError.js +10 -1
  139. package/dist/utility/errors/hdbError.js.map +1 -1
  140. package/dist/utility/globalSchema.d.ts +9 -0
  141. package/dist/utility/hdbTerms.d.ts +1 -0
  142. package/dist/utility/hdbTerms.js +1 -0
  143. package/dist/utility/hdbTerms.js.map +1 -1
  144. package/dist/validation/configValidator.js +29 -0
  145. package/dist/validation/configValidator.js.map +1 -1
  146. package/index.ts +7 -0
  147. package/json/systemSchema.json +14 -0
  148. package/npm-shrinkwrap.json +22 -22
  149. package/package.json +2 -2
  150. package/resources/DESIGN.md +57 -272
  151. package/resources/DatabaseTransaction.ts +708 -87
  152. package/resources/FULL_TEXT.md +6 -2
  153. package/resources/IterableEventQueue.ts +14 -7
  154. package/resources/LMDBTransaction.ts +212 -84
  155. package/resources/RecordEncoder.ts +2 -3
  156. package/resources/Resource.ts +5 -6
  157. package/resources/ResourceInterface.ts +3 -1
  158. package/resources/RocksIndexStore.ts +5 -0
  159. package/resources/Table.ts +118 -15
  160. package/resources/analytics/write.ts +6 -2
  161. package/resources/audit-retention.md +203 -0
  162. package/resources/auditStore.ts +306 -84
  163. package/resources/branchDatabase.ts +2 -0
  164. package/resources/databases.ts +6 -2
  165. package/resources/indexes/DESIGN.md +120 -177
  166. package/resources/indexes/fullTextNativeBinding.ts +22 -3
  167. package/resources/indexes/fullTextQueryIndex.ts +274 -11
  168. package/resources/models/DESIGN.md +11 -3
  169. package/resources/models/Models.ts +127 -4
  170. package/resources/models/backendRegistry.ts +17 -0
  171. package/resources/models/bootstrap.ts +9 -1
  172. package/resources/models/calibration.ts +298 -0
  173. package/resources/models/calibrationStore.ts +1167 -0
  174. package/resources/models/decisionStore.ts +57 -0
  175. package/resources/models/generativeDecision.ts +54 -16
  176. package/resources/models/types.ts +6 -0
  177. package/resources/record-locks.md +155 -141
  178. package/resources/recordLock.ts +12 -6
  179. package/resources/replayLogs.ts +9 -2
  180. package/resources/scheduler/DESIGN.md +4 -3
  181. package/resources/scheduler/engine.ts +35 -2
  182. package/resources/search.ts +154 -16
  183. package/resources/transaction.ts +50 -5
  184. package/resources/transactionBroadcast.ts +69 -0
  185. package/security/DESIGN.md +9 -30
  186. package/security/keys.ts +175 -127
  187. package/server/DESIGN.md +70 -36
  188. package/server/http.ts +10 -0
  189. package/server/serverHelpers/Request.ts +10 -2
  190. package/server/serverHelpers/contentTypes.ts +5 -3
  191. package/server/serverHelpers/uwsServer.ts +34 -7
  192. package/server/threads/manageThreads.js +16 -14
  193. package/server/threads/threadServer.js +24 -6
  194. package/studio/web/assets/{Chat-CNEKMllJ.js → Chat-4LMxI0yB.js} +1 -1
  195. package/studio/web/assets/{FloatingChat-CEnWhHeR.js → FloatingChat-kWHApEtF.js} +3 -3
  196. package/studio/web/assets/{SearchDialog-CwhJdEdY.js → SearchDialog-B7ymna97.js} +1 -1
  197. package/studio/web/assets/{apiToken-DC5vsaaO.js → apiToken-CE3AAt9f.js} +1 -1
  198. package/studio/web/assets/{applications-DQZxDy0x.js → applications-DdN7CqsB.js} +1 -1
  199. package/studio/web/assets/{cssMode-Dz2DxF4e.js → cssMode-BpySZUSt.js} +1 -1
  200. package/studio/web/assets/{editor-B3mjd3tz.js → editor-myLCVv9C.js} +1 -1
  201. package/studio/web/assets/{html-BsRGM7tp.js → html-DGNYO55O.js} +1 -1
  202. package/studio/web/assets/{htmlMode-tUKJaA3o.js → htmlMode-BUgpQr41.js} +1 -1
  203. package/studio/web/assets/{index-BuqFjN5K.js → index-FN4KC9va.js} +39 -39
  204. package/studio/web/assets/index-QLZw6xrU.css +1 -0
  205. package/studio/web/assets/{index.lazy-CcpfQFY_.js → index.lazy-DZFuaumG.js} +1 -1
  206. package/studio/web/assets/{javascript-BZ_uRGHl.js → javascript-DiBzh4pj.js} +1 -1
  207. package/studio/web/assets/{jsonMode-DgB-aFMj.js → jsonMode-Bm312nmu.js} +1 -1
  208. package/studio/web/assets/{languageServices-BleuBw17.js → languageServices-Dz2K9rZN.js} +1 -1
  209. package/studio/web/assets/{lspLanguageFeatures-ByVYLSsQ.js → lspLanguageFeatures-CNu1L5xn.js} +1 -1
  210. package/studio/web/assets/{notifications-TuzN0IMi.js → notifications--nwFLGmM.js} +1 -1
  211. package/studio/web/assets/{notifications-D1Nf-Edn.js → notifications-BelKw5bR.js} +1 -1
  212. package/studio/web/assets/{profile-D7C7Lupz.js → profile-AU5FjOBx.js} +1 -1
  213. package/studio/web/assets/{regions-nGQCm-v8.js → regions-C0sTSEHH.js} +1 -1
  214. package/studio/web/assets/{register-C3x9Ze6P.js → register-DyNR2rGk.js} +2 -2
  215. package/studio/web/assets/{setComponentFile-DpjrRqeG.js → setComponentFile-BkWx8AN3.js} +1 -1
  216. package/studio/web/assets/{setup-CGiKLZ9e.js → setup-D6LZS78d.js} +2 -2
  217. package/studio/web/assets/{status-CGJwrNPH.js → status-BCWnn70o.js} +1 -1
  218. package/studio/web/assets/{toggleHighContrast-C-Ne6P7T.js → toggleHighContrast-pyeGJ6W1.js} +1 -1
  219. package/studio/web/assets/{tsMode-CSreYTys.js → tsMode-CGV3iaWC.js} +1 -1
  220. package/studio/web/assets/{typescript-MnxBk89W.js → typescript-Ci7Khvwo.js} +1 -1
  221. package/studio/web/assets/{useEntityRestURL-TNrh77-s.js → useEntityRestURL-DNiz3aZ0.js} +1 -1
  222. package/studio/web/assets/{workers-BfZlUNVI.js → workers-byzxjbqy.js} +1 -1
  223. package/studio/web/assets/{xml-BHd6qGH7.js → xml-DGnHQ3CX.js} +1 -1
  224. package/studio/web/assets/{yaml-C53R8OQq.js → yaml-BUoTnPWb.js} +1 -1
  225. package/studio/web/index.html +2 -2
  226. package/upgrade/directives/5-3-0.ts +10 -3
  227. package/utility/durableFile.ts +96 -0
  228. package/utility/errors/hdbError.ts +9 -0
  229. package/utility/hdbTerms.ts +1 -0
  230. package/validation/configValidator.ts +31 -0
  231. package/studio/web/assets/index-72cEH474.css +0 -1
@@ -88,10 +88,14 @@ Weight and highlighting changes preserve the physical index and refresh query re
88
88
 
89
89
  ## Native package and limits
90
90
 
91
- Harper pins `@harperfast/fulltext` to **0.3.0** and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation.
91
+ Harper pins `@harperfast/fulltext` to **0.5.0** and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation.
92
+
93
+ A format-incompatible native package upgrade causes each node to rebuild its local derived index from authoritative records. Full-text queries on that node return 503 until the index is ready. Use a rolling upgrade to keep full-text search available on other replicas while each local index rebuilds.
92
94
 
93
95
  Earlier beta declarations and indexes are unsupported. This field-only API does not provide a compatibility or upgrade path for them.
94
96
 
95
- Native queries have finite result windows and execution budgets. Reduce the requested offset/limit or narrow the query when a window is exceeded; filtering can also exhaust the window. Prefix modes use the native autocomplete window. Highlight tracing has separate record and source-byte bounds. These are functional constraints, not catalog-scale performance guarantees.
97
+ Native queries have finite result windows and execution budgets. Reduce the requested offset/limit or narrow the query when a window is exceeded; filtering can also exhaust the window. Prefix modes use the native autocomplete window, clamped when necessary so its maximum result set fits in one native response. Highlight tracing has separate record and source-byte bounds. These are functional constraints, not catalog-scale performance guarantees.
98
+
99
+ Harper does not configure Fulltext's native filter metadata or submit native filter or candidate-ID requests. Ordinary conditions continue through Harper's secondary-index gates and authoritative record predicates.
96
100
 
97
101
  For implementation details, see the [full-text invariants in the Resource design guide](DESIGN.md#full-text-declarations-and-reader-snapshots). The schema contract lives in [schema.graphql](../schema.graphql); the compiler and generation rules are in [fullTextSchema.ts](fullTextSchema.ts).
@@ -30,19 +30,26 @@ export class IterableEventQueue<Event extends object = any> extends EventEmitter
30
30
  return true;
31
31
  }
32
32
  /**
33
- * Permanently close the queue. A final message is delivered before iteration ends when supplied.
33
+ * Permanently close the queue. A final message is delivered before iteration ends when supplied. The queue
34
+ * closes and emits 'close' once even if a listener throws on the final message; that error is then rethrown.
34
35
  */
35
36
  close(finalMessage?: Event) {
36
37
  if (this.closed) return;
37
38
  // Closing is authoritative: buffered events must not leak after revocation or policy failure.
38
39
  if (this.queue) this.queue.length = 0;
39
- if (finalMessage !== undefined) this.send(finalMessage);
40
- this.closed = true;
41
- if (this.resolveNext) {
42
- this.resolveNext({ value: undefined, done: true });
43
- this.resolveNext = null;
40
+ try {
41
+ if (finalMessage !== undefined) this.send(finalMessage);
42
+ } finally {
43
+ // a listener handling the final message may already have closed the queue
44
+ if (!this.closed) {
45
+ this.closed = true;
46
+ if (this.resolveNext) {
47
+ this.resolveNext({ value: undefined, done: true });
48
+ this.resolveNext = null;
49
+ }
50
+ this.emit('close');
51
+ }
44
52
  }
45
- this.emit('close');
46
53
  }
47
54
  getNextMessage() {
48
55
  const message = this.queue?.shift();
@@ -1,8 +1,10 @@
1
1
  import {
2
2
  DatabaseTransaction,
3
3
  closeWriteInstance,
4
+ deferForCommitInFlight,
4
5
  validateWrite,
5
6
  shouldSpareCommitPhase,
7
+ requestAbortedError,
6
8
  transactionOpenTooLongError,
7
9
  type CommitOptions,
8
10
  type TransactionWrite,
@@ -43,6 +45,16 @@ export class LMDBTransaction extends DatabaseTransaction {
43
45
  this.timeout = Math.max(txnExpiration, this.timeoutBudget ?? 0);
44
46
  }
45
47
 
48
+ protected markNativeCommitSubmitted(): void {
49
+ super.markNativeCommitSubmitted();
50
+ this.timeout = Math.max(this.timeout || 0, txnExpiration, this.timeoutBudget ?? 0);
51
+ trackedTxns.add(this);
52
+ }
53
+
54
+ protected nativeCommitAttemptEnded(): void {
55
+ if (!this.readTxn) trackedTxns.delete(this);
56
+ }
57
+
46
58
  getReadTxn(): any {
47
59
  // used optimistically
48
60
  this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
@@ -58,6 +70,7 @@ export class LMDBTransaction extends DatabaseTransaction {
58
70
  this.readTxn = (this.db as any).useReadTransaction();
59
71
 
60
72
  this.readTxnsUsed = 1;
73
+ this.baseReadRefConsumed = false;
61
74
  if ((this.readTxn as any).openTimer) (this.readTxn as any).openTimer = 0;
62
75
  trackedTxns.add(this as any);
63
76
  return this.readTxn;
@@ -84,8 +97,11 @@ export class LMDBTransaction extends DatabaseTransaction {
84
97
  (this.readTxn as any).done();
85
98
  }
86
99
  if (--this.readTxnsUsed === 0) {
87
- trackedTxns.delete(this as any);
100
+ // A submitted write remains monitor-visible after its separate read snapshot is released.
101
+ // nativeCommitAttemptEnded() removes this enrollment once the outcome is known.
102
+ if (!this.nativeCommitSubmitted) trackedTxns.delete(this as any);
88
103
  this.readTxn = null;
104
+ this.completeDeferredContextRelease();
89
105
  }
90
106
  }
91
107
 
@@ -96,16 +112,16 @@ export class LMDBTransaction extends DatabaseTransaction {
96
112
  }
97
113
 
98
114
  addWrite(operation: TransactionWrite): any {
99
- if (this.timedOut) throw transactionOpenTooLongError();
100
- if (this.open === TRANSACTION_STATE.CLOSED) {
101
- throw new Error('Can not use a transaction that is no longer open');
102
- }
103
-
115
+ if (this.timedOut || this.postSubmitPoisoned) throw transactionOpenTooLongError();
116
+ if (this.disconnected) throw requestAbortedError();
104
117
  if (this.open === TRANSACTION_STATE.LINGERING) {
105
118
  // if the transaction is lingering, it is already committed, so we need to commit the write immediately
106
119
  const immediateTxn = new ImmediateTransaction(this.db);
120
+ // The throwaway owns this write, so it owns the request's cancellation too: this settled chain
121
+ // would neither refuse on its behalf nor ever release a subscription taken for it.
122
+ immediateTxn.requestSignal = (this.root ?? this).requestSignal;
107
123
  immediateTxn.addWrite(operation);
108
- const result = immediateTxn.commit({});
124
+ const result = immediateTxn.commit({}) as any;
109
125
  // Nothing may be sent back to this throwaway: the write is already committed, and its
110
126
  // durability is the promise below.
111
127
  operation.stagedIn = undefined;
@@ -116,6 +132,10 @@ export class LMDBTransaction extends DatabaseTransaction {
116
132
  }
117
133
  return result;
118
134
  }
135
+ this.admitRequestWrite();
136
+ if (this.open === TRANSACTION_STATE.CLOSED) {
137
+ throw new Error('Can not use a transaction that is no longer open');
138
+ }
119
139
 
120
140
  this.linkWrite(operation);
121
141
  this.writes.push(operation); // standard path, add to current transaction
@@ -132,8 +152,20 @@ export class LMDBTransaction extends DatabaseTransaction {
132
152
  /**
133
153
  * Resolves with information on the timestamp and success of the commit
134
154
  */
135
- commit(options: CommitOptions = {}): any {
136
- if (this.timedOut) throw transactionOpenTooLongError();
155
+ protected performCommit(options: CommitOptions = {}): any {
156
+ const next = this.next;
157
+ let result;
158
+ try {
159
+ result = this.performCommitAttempt(options);
160
+ } catch (error) {
161
+ this.abortCommitFailure(error, next);
162
+ }
163
+ if (typeof result?.then === 'function') return result.catch((error) => this.abortCommitFailure(error, next));
164
+ return result;
165
+ }
166
+
167
+ /** Preserve the synchronous empty-commit path while the outer method funnels every failure once. */
168
+ private performCommitAttempt(options: CommitOptions = {}): any {
137
169
  options = options || {};
138
170
  let txnTime = this.timestamp;
139
171
  if (!txnTime) txnTime = this.timestamp = options.timestamp || getNextMonotonicTime();
@@ -141,74 +173,73 @@ export class LMDBTransaction extends DatabaseTransaction {
141
173
  const retries = options.retries || 0;
142
174
  // now validate
143
175
  if (this.validated < this.writes.length) {
144
- try {
145
- const start = this.validated;
146
- // record the number of writes that have been validated so if we re-execute
147
- // and the number is increased we can validate the new entries
148
- this.validated = this.writes.length;
149
- for (let i = start; i < this.validated; i++) {
150
- const write = this.writes[i];
151
- try {
152
- if (write) validateWrite(write, this.timestamp, this);
153
- } finally {
154
- closeWriteInstance(write);
155
- }
176
+ const start = this.validated;
177
+ // record the number of writes that have been validated so if we re-execute
178
+ // and the number is increased we can validate the new entries
179
+ this.validated = this.writes.length;
180
+ for (let i = start; i < this.validated; i++) {
181
+ const write = this.writes[i];
182
+ try {
183
+ if (write) validateWrite(write, this.timestamp, this);
184
+ } finally {
185
+ closeWriteInstance(write);
156
186
  }
157
- let hasBefore;
158
- for (let i = start; i < this.validated; i++) {
159
- const write = this.writes[i];
160
- if (!write) continue;
161
- if (write.before || write.beforeIntermediate) {
162
- hasBefore = true;
163
- }
187
+ }
188
+ let hasBefore;
189
+ for (let i = start; i < this.validated; i++) {
190
+ const write = this.writes[i];
191
+ if (!write) continue;
192
+ if (write.before || write.beforeIntermediate) {
193
+ hasBefore = true;
164
194
  }
165
- // Now we need to let any "before" actions execute. These are calls to the sources,
166
- // and we want to follow the order of the source sequence so that later, more canonical
167
- // source writes will finish (with right to refuse/abort) before proceeeding to less
168
- // canonical sources.
169
- if (hasBefore) {
170
- // see `committing` in DatabaseTransaction (issue #2062)
171
- this.setCommitPhase(true);
172
- const stagedWrites = this.writes.length;
173
- return (async () => {
174
- try {
175
- for (let phase = 0; phase < 2; phase++) {
176
- let completion;
177
- for (let i = start; i < this.validated; i++) {
178
- const write = this.writes[i];
179
- if (!write) continue;
180
- const before = write[phase === 0 ? 'before' : 'beforeIntermediate'];
181
- if (before) {
182
- const nextCompletion = before();
183
- if (completion) {
184
- if (completion.push) completion.push(nextCompletion);
185
- else completion = [completion, nextCompletion];
186
- } else completion = nextCompletion;
187
- }
195
+ }
196
+ // Now we need to let any "before" actions execute. These are calls to the sources,
197
+ // and we want to follow the order of the source sequence so that later, more canonical
198
+ // source writes will finish (with right to refuse/abort) before proceeeding to less
199
+ // canonical sources.
200
+ if (hasBefore) {
201
+ // see `committing` in DatabaseTransaction (issue #2062)
202
+ this.setCommitPhase(true);
203
+ const stagedWrites = this.writes.length;
204
+ return (async () => {
205
+ try {
206
+ for (let phase = 0; phase < 2; phase++) {
207
+ let completion;
208
+ for (let i = start; i < this.validated; i++) {
209
+ const write = this.writes[i];
210
+ if (!write) continue;
211
+ const before = write[phase === 0 ? 'before' : 'beforeIntermediate'];
212
+ if (before) {
213
+ const nextCompletion = before();
214
+ if (completion) {
215
+ if (completion.push) completion.push(nextCompletion);
216
+ else completion = [completion, nextCompletion];
217
+ } else completion = nextCompletion;
188
218
  }
189
- if (completion) await (completion.push ? Promise.all(completion) : completion);
190
219
  }
191
- } catch (error) {
192
- this.setCommitPhase(false);
193
- this.abort();
194
- throw error;
220
+ if (completion) await (completion.push ? Promise.all(completion) : completion);
195
221
  }
222
+ } finally {
196
223
  this.setCommitPhase(false);
197
- // aborted underneath us while parked above — see DatabaseTransaction's twin guard
224
+ }
225
+ // aborted underneath us while parked above — see DatabaseTransaction's twin guard. A chain
226
+ // poisoned after one of its stores submitted rides the cascade instead (abortAndPoison).
227
+ if (!this.poisonedMidCommit) {
198
228
  if (this.timedOut) throw transactionOpenTooLongError();
199
- if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
200
- throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
201
- return this.commit(options);
202
- })();
203
- }
204
- } catch (error) {
205
- this.abort();
206
- throw error;
229
+ if (this.disconnected) throw requestAbortedError();
230
+ }
231
+ if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
232
+ throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
233
+ return this.commit({ ...options, continuation: true });
234
+ })();
207
235
  }
208
236
  }
209
237
  if (!retries && this.writes.some((write) => write)) this.renewChainForNativeCommit();
210
238
  // release the read snapshot so we don't keep it open longer than necessary
211
- if (!retries) this.doneReadTxn();
239
+ if (!retries && !this.baseReadRefConsumed) {
240
+ this.baseReadRefConsumed = true;
241
+ this.doneReadTxn();
242
+ }
212
243
  this.open = options?.doneWriting ? TRANSACTION_STATE.LINGERING : TRANSACTION_STATE.OPEN;
213
244
  let resolution;
214
245
  const completions = [];
@@ -264,6 +295,7 @@ export class LMDBTransaction extends DatabaseTransaction {
264
295
  // use a real (async) transaction to get exclusive access to the data
265
296
  if (db?.retryRisk) db.retryRisk *= 0.99; // gradually decay the retry risk
266
297
  if (this.writes.length + (db?.retryRisk || 0) < MAX_OPTIMISTIC_SIZE >> retries) {
298
+ this.markNativeCommitSubmitted();
267
299
  nextCondition();
268
300
  if (commitCompletions) {
269
301
  if (resolution) {
@@ -278,6 +310,7 @@ export class LMDBTransaction extends DatabaseTransaction {
278
310
  } else {
279
311
  // if it is too big to expect optimistic writes to work, or we have done too many retries we use
280
312
  // a real LMDB transaction to get exclusive access to reading and writing
313
+ this.markNativeCommitSubmitted();
281
314
  const transactionResolution = this.writes[0].store.transaction(() => {
282
315
  for (const write of this.writes) {
283
316
  // we load latest data while in the transaction
@@ -296,17 +329,18 @@ export class LMDBTransaction extends DatabaseTransaction {
296
329
  return resolution.then((resolution) => {
297
330
  if (resolution) {
298
331
  if (this.next) {
299
- completions.push(this.next.commit(options));
332
+ completions.push(this.next.commit({ ...options, continuation: true }));
300
333
  }
301
334
  if (options?.flush) {
302
- completions.push(this.writes[0].store.flushed);
335
+ const store = this.writes[0]?.store;
336
+ if (store) completions.push(store.flushed);
303
337
  }
304
338
  if (this.replicatedConfirmation) {
305
339
  // if we want to wait for replication confirmation, we need to track the transaction times
306
340
  // and when replication notifications come in, we count the number of confirms until we reach the desired number
307
- const databaseName = this.writes[0].store.rootStore.databaseName;
341
+ const databaseName = this.writes[0]?.store.rootStore.databaseName;
308
342
  const lastEntry = this.lastConfirmableEntry();
309
- if (confirmReplication && lastEntry)
343
+ if (confirmReplication && databaseName && lastEntry)
310
344
  completions.push(
311
345
  confirmReplication(databaseName, (lastEntry as any).localTime, this.replicatedConfirmation)
312
346
  );
@@ -337,7 +371,8 @@ export class LMDBTransaction extends DatabaseTransaction {
337
371
  }
338
372
  if (options) options.retries = retries + 1;
339
373
  else options = { retries: 1 };
340
- return this.commit(options); // try again
374
+ // A continuation: a poison that landed mid-commit must not abandon the retry ladder.
375
+ return this.commit({ ...options, continuation: true }); // try again
341
376
  }
342
377
  });
343
378
  }
@@ -346,7 +381,7 @@ export class LMDBTransaction extends DatabaseTransaction {
346
381
  };
347
382
  if (this.next) {
348
383
  // now run any other transactions
349
- const nextResolution = this.next?.commit(options);
384
+ const nextResolution = this.next?.commit({ ...options, continuation: true });
350
385
  if ((nextResolution as any)?.then)
351
386
  return (nextResolution as any)?.then((nextResolution) => ({
352
387
  txnTime,
@@ -356,19 +391,79 @@ export class LMDBTransaction extends DatabaseTransaction {
356
391
  }
357
392
  return txnResolution;
358
393
  }
359
- abort(): void {
360
- while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
361
- this.open = TRANSACTION_STATE.CLOSED;
362
- this.drainCompletions();
363
- // any blobs that were pre-saved as part of these writes will never be referenced; schedule deletion
364
- // (retaining any fileId the current on-disk record still references — an aborted write may carry an
365
- // already-saved blob shared with the surviving record; see harper-pro#406).
366
- for (const write of this.writes) {
367
- if (write?.savedBlobs)
368
- cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
394
+
395
+ private abortCommitFailure(error, next: DatabaseTransaction | undefined): never {
396
+ const nextWasAttached = Boolean(next && this.next === next);
397
+ try {
398
+ this.abort(true);
399
+ } catch (abortError) {
400
+ harperLogger.debug?.('aborting LMDB transaction after a failed commit', abortError);
401
+ }
402
+ // A detached pre-submit child has no other cleanup path. A submitted child owns an unknown native
403
+ // outcome and will clean itself when its own commit promise settles.
404
+ if (next && !nextWasAttached && !next.nativeCommitSubmitted) {
405
+ try {
406
+ next.abort(true);
407
+ } catch (abortError) {
408
+ harperLogger.debug?.('aborting a detached LMDB transaction after a failed commit', abortError);
409
+ }
410
+ }
411
+ throw error;
412
+ }
413
+ /**
414
+ * Force-release the retained read snapshot without touching staged writes. The base class's
415
+ * version acts on `this.transaction`; LMDB's snapshot lives on `this.readTxn`, so the monitor
416
+ * would otherwise have nothing that can reclaim a handle abort(true) retained.
417
+ */
418
+ releaseReadTxn(): void {
419
+ while (this.readTxn && this.readTxnsUsed > 0) this.doneReadTxn();
420
+ this.completeDeferredContextRelease();
421
+ }
422
+
423
+ abort(retainReadTransaction = false, cascade = true): void {
424
+ const next = cascade ? this.next : undefined;
425
+ if (cascade && !retainReadTransaction) this.next = null;
426
+ try {
427
+ const hasOpenReadIterator =
428
+ retainReadTransaction &&
429
+ this.readTxn &&
430
+ (this.readTxnsUsed > 1 || (this.baseReadRefConsumed && this.readTxnsUsed > 0));
431
+ if (hasOpenReadIterator) {
432
+ if (!this.baseReadRefConsumed) {
433
+ this.doneReadTxn();
434
+ this.baseReadRefConsumed = true;
435
+ }
436
+ } else {
437
+ while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
438
+ }
439
+ this.open = TRANSACTION_STATE.CLOSED;
440
+ this.drainCompletions();
441
+ // Any blobs that were pre-saved as part of these writes will never be referenced; schedule deletion
442
+ // (retaining any fileId the current on-disk record still references — an aborted write may carry an
443
+ // already-saved blob shared with the surviving record; see harper-pro#406).
444
+ for (const write of this.writes) {
445
+ if (write?.savedBlobs)
446
+ cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
447
+ }
448
+ } finally {
449
+ try {
450
+ this.open = TRANSACTION_STATE.CLOSED;
451
+ this.endScopeOwnership();
452
+ this.clearWrites();
453
+ this.releaseContext(!this.timedOut && !this.disconnected);
454
+ if (!this.root) this.releaseRequestAbortListener();
455
+ } finally {
456
+ // Same guard as DatabaseTransaction.abort(): a child whose native commit is outstanding owns
457
+ // its own cleanup, and clearing its writes here would unlink blobs that commit still references.
458
+ if (next && !(next.nativeCommitSubmitted && next.commitsInFlight)) {
459
+ try {
460
+ next.abort(retainReadTransaction);
461
+ } catch (error) {
462
+ harperLogger.debug?.('cleaning up a chained LMDB transaction during abort', error);
463
+ }
464
+ }
465
+ }
369
466
  }
370
- // reset the transaction
371
- this.clearWrites();
372
467
  }
373
468
  save(..._args: any[]): any {
374
469
  // noop for LMDB
@@ -382,6 +477,7 @@ export class ImmediateTransaction extends LMDBTransaction {
382
477
  this.db = db;
383
478
  }
384
479
  save(..._args: any[]): any {
480
+ this.admitRequestWrite(); // a deferred save reaches here without passing through addWrite()
385
481
  return this.commit();
386
482
  }
387
483
  // @ts-expect-error accessor overriding property
@@ -406,6 +502,38 @@ function startMonitoringTxns() {
406
502
  const commitChainHead = txn.commitChainHead ?? txn;
407
503
  if (txn.timeout <= 0) {
408
504
  const url = (txn.getContext() as any)?.url;
505
+ if (deferForCommitInFlight(txn, url, txnExpiration)) continue;
506
+ if (
507
+ txn.open === TRANSACTION_STATE.CLOSED &&
508
+ !txn.commitAttemptDoomedByPoison() &&
509
+ shouldSpareCommitPhase(txn, checkedCommitPhaseChains)
510
+ ) {
511
+ harperLogger.warn?.(
512
+ `Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work; letting it complete, from table: ${
513
+ (txn.db as any)?.name + (url ? ' path: ' + url : '')
514
+ }`
515
+ );
516
+ txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
517
+ continue;
518
+ }
519
+ if (txn.open === TRANSACTION_STATE.CLOSED) {
520
+ // Only reachable through abort(true), which retained the snapshot for read iterators
521
+ // that own it. Nothing else here can reclaim it: the branches below would re-enter
522
+ // commit() on a poisoned transaction every tick, throw on its own poison check, and
523
+ // re-arm the timer — pinning an LMDB read snapshot, which blocks free-page reuse and
524
+ // grows the data file, for the life of the process. Close the owning iterators first so
525
+ // the reference comes back through doneReadTxn(), and force the release if it does not.
526
+ const closedIterators = txn.closeOwnedReadIterators();
527
+ if (closedIterators > 0 || txn.readTxn)
528
+ harperLogger.warn?.(
529
+ `Read iterators held a closed transaction's snapshot past the open-transaction limit; ${
530
+ txn.readTxn ? 'releasing it' : `closed ${closedIterators} abandoned iterator(s)`
531
+ }, from table: ${(txn.db as any)?.name + (url ? ' path: ' + url : '')}`
532
+ );
533
+ if (txn.readTxn) txn.releaseReadTxn();
534
+ trackedTxns.delete(txn);
535
+ continue;
536
+ }
409
537
  if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
410
538
  // see DatabaseTransaction's monitor (issue #2062)
411
539
  harperLogger.warn?.(
@@ -967,10 +967,9 @@ export function recordUpdater(store, tableId, auditStore) {
967
967
  extendedType |= HAS_ADDITIONAL_AUDIT_REFS_AUDIT;
968
968
  }
969
969
  if (options?.localOnly) {
970
- // Mark this write as local-only: set the bit in BOTH the persisted record metadata
971
- // (so the full-copy send loop, which reads entry.metadataFlags, skips it) AND the audit
972
- // entry's extendedType (so the audit-forward send path skips it by bitmask without decode).
973
970
  metadataInNextEncoding |= LOCAL_ONLY;
971
+ }
972
+ if (options?.localOnly || options?.auditLocalOnly) {
974
973
  extendedType |= LOCAL_ONLY;
975
974
  }
976
975
  if (previousResidencyId !== residencyId) {
@@ -873,15 +873,14 @@ function transactional(
873
873
  // opposite treatment:
874
874
  // - committed: the prior call ran to completion normally. Safe to silently start a fresh
875
875
  // transaction for this independent call (the case above).
876
- // - aborted due to exceeding storage.maxTransactionOpenTime (#1411, DatabaseTransaction.ts
877
- // abortDueToTimeout(), marked via the `timedOut` poison flag): the whole logical operation
876
+ // - poisoned (`timedOut`, #1411; or `disconnected`, harper#2001): the whole logical operation
878
877
  // must fail atomically, not have this write quietly land on a brand-new transaction while
879
878
  // the earlier write(s) were rolled back. Joining the poisoned transaction here (instead of
880
- // starting fresh) makes the write throw transactionOpenTooLongError via addWrite()/commit()'s
881
- // poison check, correctly propagating the abort to the caller. See
879
+ // starting fresh) makes the write throw transactionOpenTooLongError/requestAbortedError via
880
+ // addWrite()/commit()'s poison check, correctly propagating the abort to the caller. See
882
881
  // integrationTests/resources/txn-overtime-atomicity.test.ts.
883
- if (isJoinableScope(context?.transaction) || context?.transaction?.timedOut) {
884
- // we are already in a transaction (or it was poisoned by a timeout abort and must fail), proceed
882
+ if (isJoinableScope(context?.transaction) || context?.transaction?.timedOut || context?.transaction?.disconnected) {
883
+ // we are already in a transaction (or it was poisoned and must fail), proceed
885
884
  const resource = this.getResource(query, context, resourceOptions);
886
885
  return resource.then
887
886
  ? resource.then(authorizeActionOnResource)
@@ -143,7 +143,9 @@ export interface Context {
143
143
  /**
144
144
  * Abort signal carried through ALS so generator bodies can forward cancellation to
145
145
  * external work (e.g. `scope.models.generateStream({ signal })`). Populated on the
146
- * Request that becomes the ALS-bound Context for HTTP/WS paths via #513.
146
+ * Request that becomes the ALS-bound Context for HTTP/WS paths via #513. Also the request's
147
+ * lifetime for its writes (harper#2001): once it aborts, no write for the request is admitted
148
+ * and a write-bearing transaction is aborted at once (`DatabaseTransaction.admitRequestWrite`).
147
149
  */
148
150
  signal?: AbortSignal;
149
151
  }
@@ -4,6 +4,7 @@ import {
4
4
  type CountEstimateOptions,
5
5
  DBI,
6
6
  type StoreIteratorOptions,
7
+ type StoreGetOptions,
7
8
  type StorePutOptions,
8
9
  type StoreRemoveOptions,
9
10
  RocksDatabase,
@@ -44,6 +45,10 @@ function translateIndexBounds<
44
45
  * RocksDatabase rather than a store because it actually alters the interface
45
46
  */
46
47
  export class RocksIndexStore extends RocksDatabase {
48
+ hasIndexEntry(indexedValue: any, primaryKey: Id, options?: StoreGetOptions): boolean {
49
+ return super.getBinaryFastSync([indexedValue, primaryKey], options) !== undefined;
50
+ }
51
+
47
52
  /**
48
53
  * Get all entries matching the range
49
54
  * @param options