@harperfast/harper 5.3.0 → 5.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/copyDb.ts +5 -1
- package/bin/deployRenderer.ts +3 -0
- package/components/Application.ts +66 -22
- package/components/DESIGN.md +89 -36
- package/components/componentLoader.ts +2 -0
- package/components/deploymentRecorder.ts +18 -1
- package/components/installFingerprint.ts +123 -0
- package/components/mcp/DESIGN.md +4 -4
- package/components/operations.js +29 -0
- package/config/DESIGN.md +5 -5
- package/dataLayer/DESIGN.md +37 -22
- package/dataLayer/backupRepository.ts +248 -0
- package/dataLayer/restoreMarker.ts +30 -75
- package/dataLayer/rocksdbBackup.ts +253 -110
- package/dist/bin/copyDb.js +3 -0
- package/dist/bin/copyDb.js.map +1 -1
- package/dist/bin/deployRenderer.js +3 -0
- package/dist/bin/deployRenderer.js.map +1 -1
- package/dist/components/Application.d.ts +5 -0
- package/dist/components/Application.js +51 -19
- package/dist/components/Application.js.map +1 -1
- package/dist/components/componentLoader.js +2 -0
- package/dist/components/componentLoader.js.map +1 -1
- package/dist/components/deploymentRecorder.js +18 -1
- package/dist/components/deploymentRecorder.js.map +1 -1
- package/dist/components/installFingerprint.d.ts +39 -0
- package/dist/components/installFingerprint.js +113 -0
- package/dist/components/installFingerprint.js.map +1 -0
- package/dist/components/operations.d.ts +2 -0
- package/dist/components/operations.js +31 -0
- package/dist/components/operations.js.map +1 -1
- package/dist/dataLayer/backupRepository.d.ts +33 -0
- package/dist/dataLayer/backupRepository.js +232 -0
- package/dist/dataLayer/backupRepository.js.map +1 -0
- package/dist/dataLayer/restoreMarker.d.ts +4 -1
- package/dist/dataLayer/restoreMarker.js +29 -68
- package/dist/dataLayer/restoreMarker.js.map +1 -1
- package/dist/dataLayer/rocksdbBackup.js +221 -112
- package/dist/dataLayer/rocksdbBackup.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js.map +1 -1
- package/dist/json/systemSchema.json +14 -0
- package/dist/resources/DatabaseTransaction.d.ts +140 -11
- package/dist/resources/DatabaseTransaction.js +1055 -464
- package/dist/resources/DatabaseTransaction.js.map +1 -1
- package/dist/resources/IterableEventQueue.d.ts +2 -1
- package/dist/resources/IterableEventQueue.js +16 -8
- package/dist/resources/IterableEventQueue.js.map +1 -1
- package/dist/resources/LMDBTransaction.d.ts +13 -2
- package/dist/resources/LMDBTransaction.js +219 -93
- package/dist/resources/LMDBTransaction.js.map +1 -1
- package/dist/resources/RecordEncoder.js +2 -3
- package/dist/resources/RecordEncoder.js.map +1 -1
- package/dist/resources/Resource.js +5 -6
- package/dist/resources/Resource.js.map +1 -1
- package/dist/resources/ResourceInterface.d.ts +3 -1
- package/dist/resources/ResourceInterface.js.map +1 -1
- package/dist/resources/RocksIndexStore.d.ts +2 -1
- package/dist/resources/RocksIndexStore.js +3 -0
- package/dist/resources/RocksIndexStore.js.map +1 -1
- package/dist/resources/Table.js +116 -12
- package/dist/resources/Table.js.map +1 -1
- package/dist/resources/analytics/write.js +6 -2
- package/dist/resources/analytics/write.js.map +1 -1
- package/dist/resources/auditStore.d.ts +71 -17
- package/dist/resources/auditStore.js +248 -64
- package/dist/resources/auditStore.js.map +1 -1
- package/dist/resources/branchDatabase.js +2 -0
- package/dist/resources/branchDatabase.js.map +1 -1
- package/dist/resources/databases.js +8 -2
- package/dist/resources/databases.js.map +1 -1
- package/dist/resources/indexes/fullTextNativeBinding.d.ts +6 -2
- package/dist/resources/indexes/fullTextNativeBinding.js +12 -2
- package/dist/resources/indexes/fullTextNativeBinding.js.map +1 -1
- package/dist/resources/indexes/fullTextQueryIndex.d.ts +4 -0
- package/dist/resources/indexes/fullTextQueryIndex.js +207 -10
- package/dist/resources/indexes/fullTextQueryIndex.js.map +1 -1
- package/dist/resources/models/Models.d.ts +11 -0
- package/dist/resources/models/Models.js +95 -4
- package/dist/resources/models/Models.js.map +1 -1
- package/dist/resources/models/backendRegistry.d.ts +8 -0
- package/dist/resources/models/backendRegistry.js +15 -1
- package/dist/resources/models/backendRegistry.js.map +1 -1
- package/dist/resources/models/bootstrap.d.ts +2 -0
- package/dist/resources/models/bootstrap.js +6 -0
- package/dist/resources/models/bootstrap.js.map +1 -1
- package/dist/resources/models/calibration.d.ts +88 -0
- package/dist/resources/models/calibration.js +244 -0
- package/dist/resources/models/calibration.js.map +1 -0
- package/dist/resources/models/calibrationStore.d.ts +163 -0
- package/dist/resources/models/calibrationStore.js +986 -0
- package/dist/resources/models/calibrationStore.js.map +1 -0
- package/dist/resources/models/decisionStore.d.ts +24 -0
- package/dist/resources/models/decisionStore.js +45 -0
- package/dist/resources/models/decisionStore.js.map +1 -1
- package/dist/resources/models/generativeDecision.js +44 -15
- package/dist/resources/models/generativeDecision.js.map +1 -1
- package/dist/resources/models/types.d.ts +7 -0
- package/dist/resources/recordLock.d.ts +8 -5
- package/dist/resources/recordLock.js +6 -4
- package/dist/resources/recordLock.js.map +1 -1
- package/dist/resources/replayLogs.js +8 -1
- package/dist/resources/replayLogs.js.map +1 -1
- package/dist/resources/scheduler/engine.d.ts +4 -0
- package/dist/resources/scheduler/engine.js +30 -2
- package/dist/resources/scheduler/engine.js.map +1 -1
- package/dist/resources/search.d.ts +16 -4
- package/dist/resources/search.js +117 -13
- package/dist/resources/search.js.map +1 -1
- package/dist/resources/transaction.js +85 -5
- package/dist/resources/transaction.js.map +1 -1
- package/dist/resources/transactionBroadcast.d.ts +8 -0
- package/dist/resources/transactionBroadcast.js +65 -0
- package/dist/resources/transactionBroadcast.js.map +1 -1
- package/dist/security/keys.d.ts +1 -1
- package/dist/security/keys.js +170 -120
- package/dist/security/keys.js.map +1 -1
- package/dist/server/http.js +10 -0
- package/dist/server/http.js.map +1 -1
- package/dist/server/serverHelpers/Request.js +11 -2
- package/dist/server/serverHelpers/Request.js.map +1 -1
- package/dist/server/serverHelpers/contentTypes.js +8 -3
- package/dist/server/serverHelpers/contentTypes.js.map +1 -1
- package/dist/server/serverHelpers/uwsServer.js +38 -7
- package/dist/server/serverHelpers/uwsServer.js.map +1 -1
- package/dist/server/threads/manageThreads.d.ts +3 -3
- package/dist/server/threads/manageThreads.js +15 -13
- package/dist/server/threads/manageThreads.js.map +1 -1
- package/dist/server/threads/threadServer.d.ts +3 -0
- package/dist/server/threads/threadServer.js +24 -6
- package/dist/server/threads/threadServer.js.map +1 -1
- package/dist/upgrade/directives/5-3-0.js +9 -3
- package/dist/upgrade/directives/5-3-0.js.map +1 -1
- package/dist/utility/durableFile.d.ts +36 -0
- package/dist/utility/durableFile.js +106 -0
- package/dist/utility/durableFile.js.map +1 -0
- package/dist/utility/errors/hdbError.d.ts +4 -0
- package/dist/utility/errors/hdbError.js +10 -1
- package/dist/utility/errors/hdbError.js.map +1 -1
- package/dist/utility/globalSchema.d.ts +9 -0
- package/dist/utility/hdbTerms.d.ts +1 -0
- package/dist/utility/hdbTerms.js +1 -0
- package/dist/utility/hdbTerms.js.map +1 -1
- package/dist/validation/configValidator.js +29 -0
- package/dist/validation/configValidator.js.map +1 -1
- package/index.ts +7 -0
- package/json/systemSchema.json +14 -0
- package/npm-shrinkwrap.json +22 -22
- package/package.json +2 -2
- package/resources/DESIGN.md +57 -272
- package/resources/DatabaseTransaction.ts +708 -87
- package/resources/FULL_TEXT.md +6 -2
- package/resources/IterableEventQueue.ts +14 -7
- package/resources/LMDBTransaction.ts +212 -84
- package/resources/RecordEncoder.ts +2 -3
- package/resources/Resource.ts +5 -6
- package/resources/ResourceInterface.ts +3 -1
- package/resources/RocksIndexStore.ts +5 -0
- package/resources/Table.ts +118 -15
- package/resources/analytics/write.ts +6 -2
- package/resources/audit-retention.md +203 -0
- package/resources/auditStore.ts +306 -84
- package/resources/branchDatabase.ts +2 -0
- package/resources/databases.ts +6 -2
- package/resources/indexes/DESIGN.md +120 -177
- package/resources/indexes/fullTextNativeBinding.ts +22 -3
- package/resources/indexes/fullTextQueryIndex.ts +274 -11
- package/resources/models/DESIGN.md +11 -3
- package/resources/models/Models.ts +127 -4
- package/resources/models/backendRegistry.ts +17 -0
- package/resources/models/bootstrap.ts +9 -1
- package/resources/models/calibration.ts +298 -0
- package/resources/models/calibrationStore.ts +1167 -0
- package/resources/models/decisionStore.ts +57 -0
- package/resources/models/generativeDecision.ts +54 -16
- package/resources/models/types.ts +6 -0
- package/resources/record-locks.md +155 -141
- package/resources/recordLock.ts +12 -6
- package/resources/replayLogs.ts +9 -2
- package/resources/scheduler/DESIGN.md +4 -3
- package/resources/scheduler/engine.ts +35 -2
- package/resources/search.ts +154 -16
- package/resources/transaction.ts +50 -5
- package/resources/transactionBroadcast.ts +69 -0
- package/security/DESIGN.md +9 -30
- package/security/keys.ts +175 -127
- package/server/DESIGN.md +70 -36
- package/server/http.ts +10 -0
- package/server/serverHelpers/Request.ts +10 -2
- package/server/serverHelpers/contentTypes.ts +5 -3
- package/server/serverHelpers/uwsServer.ts +34 -7
- package/server/threads/manageThreads.js +16 -14
- package/server/threads/threadServer.js +24 -6
- package/studio/web/assets/{Chat-CNEKMllJ.js → Chat-4LMxI0yB.js} +1 -1
- package/studio/web/assets/{FloatingChat-CEnWhHeR.js → FloatingChat-kWHApEtF.js} +3 -3
- package/studio/web/assets/{SearchDialog-CwhJdEdY.js → SearchDialog-B7ymna97.js} +1 -1
- package/studio/web/assets/{apiToken-DC5vsaaO.js → apiToken-CE3AAt9f.js} +1 -1
- package/studio/web/assets/{applications-DQZxDy0x.js → applications-DdN7CqsB.js} +1 -1
- package/studio/web/assets/{cssMode-Dz2DxF4e.js → cssMode-BpySZUSt.js} +1 -1
- package/studio/web/assets/{editor-B3mjd3tz.js → editor-myLCVv9C.js} +1 -1
- package/studio/web/assets/{html-BsRGM7tp.js → html-DGNYO55O.js} +1 -1
- package/studio/web/assets/{htmlMode-tUKJaA3o.js → htmlMode-BUgpQr41.js} +1 -1
- package/studio/web/assets/{index-BuqFjN5K.js → index-FN4KC9va.js} +39 -39
- package/studio/web/assets/index-QLZw6xrU.css +1 -0
- package/studio/web/assets/{index.lazy-CcpfQFY_.js → index.lazy-DZFuaumG.js} +1 -1
- package/studio/web/assets/{javascript-BZ_uRGHl.js → javascript-DiBzh4pj.js} +1 -1
- package/studio/web/assets/{jsonMode-DgB-aFMj.js → jsonMode-Bm312nmu.js} +1 -1
- package/studio/web/assets/{languageServices-BleuBw17.js → languageServices-Dz2K9rZN.js} +1 -1
- package/studio/web/assets/{lspLanguageFeatures-ByVYLSsQ.js → lspLanguageFeatures-CNu1L5xn.js} +1 -1
- package/studio/web/assets/{notifications-TuzN0IMi.js → notifications--nwFLGmM.js} +1 -1
- package/studio/web/assets/{notifications-D1Nf-Edn.js → notifications-BelKw5bR.js} +1 -1
- package/studio/web/assets/{profile-D7C7Lupz.js → profile-AU5FjOBx.js} +1 -1
- package/studio/web/assets/{regions-nGQCm-v8.js → regions-C0sTSEHH.js} +1 -1
- package/studio/web/assets/{register-C3x9Ze6P.js → register-DyNR2rGk.js} +2 -2
- package/studio/web/assets/{setComponentFile-DpjrRqeG.js → setComponentFile-BkWx8AN3.js} +1 -1
- package/studio/web/assets/{setup-CGiKLZ9e.js → setup-D6LZS78d.js} +2 -2
- package/studio/web/assets/{status-CGJwrNPH.js → status-BCWnn70o.js} +1 -1
- package/studio/web/assets/{toggleHighContrast-C-Ne6P7T.js → toggleHighContrast-pyeGJ6W1.js} +1 -1
- package/studio/web/assets/{tsMode-CSreYTys.js → tsMode-CGV3iaWC.js} +1 -1
- package/studio/web/assets/{typescript-MnxBk89W.js → typescript-Ci7Khvwo.js} +1 -1
- package/studio/web/assets/{useEntityRestURL-TNrh77-s.js → useEntityRestURL-DNiz3aZ0.js} +1 -1
- package/studio/web/assets/{workers-BfZlUNVI.js → workers-byzxjbqy.js} +1 -1
- package/studio/web/assets/{xml-BHd6qGH7.js → xml-DGnHQ3CX.js} +1 -1
- package/studio/web/assets/{yaml-C53R8OQq.js → yaml-BUoTnPWb.js} +1 -1
- package/studio/web/index.html +2 -2
- package/upgrade/directives/5-3-0.ts +10 -3
- package/utility/durableFile.ts +96 -0
- package/utility/errors/hdbError.ts +9 -0
- package/utility/hdbTerms.ts +1 -0
- package/validation/configValidator.ts +31 -0
- package/studio/web/assets/index-72cEH474.css +0 -1
package/resources/FULL_TEXT.md
CHANGED
|
@@ -88,10 +88,14 @@ Weight and highlighting changes preserve the physical index and refresh query re
|
|
|
88
88
|
|
|
89
89
|
## Native package and limits
|
|
90
90
|
|
|
91
|
-
Harper pins `@harperfast/fulltext` to **0.
|
|
91
|
+
Harper pins `@harperfast/fulltext` to **0.5.0** and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation.
|
|
92
|
+
|
|
93
|
+
A format-incompatible native package upgrade causes each node to rebuild its local derived index from authoritative records. Full-text queries on that node return 503 until the index is ready. Use a rolling upgrade to keep full-text search available on other replicas while each local index rebuilds.
|
|
92
94
|
|
|
93
95
|
Earlier beta declarations and indexes are unsupported. This field-only API does not provide a compatibility or upgrade path for them.
|
|
94
96
|
|
|
95
|
-
Native queries have finite result windows and execution budgets. Reduce the requested offset/limit or narrow the query when a window is exceeded; filtering can also exhaust the window. Prefix modes use the native autocomplete window. Highlight tracing has separate record and source-byte bounds. These are functional constraints, not catalog-scale performance guarantees.
|
|
97
|
+
Native queries have finite result windows and execution budgets. Reduce the requested offset/limit or narrow the query when a window is exceeded; filtering can also exhaust the window. Prefix modes use the native autocomplete window, clamped when necessary so its maximum result set fits in one native response. Highlight tracing has separate record and source-byte bounds. These are functional constraints, not catalog-scale performance guarantees.
|
|
98
|
+
|
|
99
|
+
Harper does not configure Fulltext's native filter metadata or submit native filter or candidate-ID requests. Ordinary conditions continue through Harper's secondary-index gates and authoritative record predicates.
|
|
96
100
|
|
|
97
101
|
For implementation details, see the [full-text invariants in the Resource design guide](DESIGN.md#full-text-declarations-and-reader-snapshots). The schema contract lives in [schema.graphql](../schema.graphql); the compiler and generation rules are in [fullTextSchema.ts](fullTextSchema.ts).
|
|
@@ -30,19 +30,26 @@ export class IterableEventQueue<Event extends object = any> extends EventEmitter
|
|
|
30
30
|
return true;
|
|
31
31
|
}
|
|
32
32
|
/**
|
|
33
|
-
* Permanently close the queue. A final message is delivered before iteration ends when supplied.
|
|
33
|
+
* Permanently close the queue. A final message is delivered before iteration ends when supplied. The queue
|
|
34
|
+
* closes and emits 'close' once even if a listener throws on the final message; that error is then rethrown.
|
|
34
35
|
*/
|
|
35
36
|
close(finalMessage?: Event) {
|
|
36
37
|
if (this.closed) return;
|
|
37
38
|
// Closing is authoritative: buffered events must not leak after revocation or policy failure.
|
|
38
39
|
if (this.queue) this.queue.length = 0;
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
this.
|
|
40
|
+
try {
|
|
41
|
+
if (finalMessage !== undefined) this.send(finalMessage);
|
|
42
|
+
} finally {
|
|
43
|
+
// a listener handling the final message may already have closed the queue
|
|
44
|
+
if (!this.closed) {
|
|
45
|
+
this.closed = true;
|
|
46
|
+
if (this.resolveNext) {
|
|
47
|
+
this.resolveNext({ value: undefined, done: true });
|
|
48
|
+
this.resolveNext = null;
|
|
49
|
+
}
|
|
50
|
+
this.emit('close');
|
|
51
|
+
}
|
|
44
52
|
}
|
|
45
|
-
this.emit('close');
|
|
46
53
|
}
|
|
47
54
|
getNextMessage() {
|
|
48
55
|
const message = this.queue?.shift();
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DatabaseTransaction,
|
|
3
3
|
closeWriteInstance,
|
|
4
|
+
deferForCommitInFlight,
|
|
4
5
|
validateWrite,
|
|
5
6
|
shouldSpareCommitPhase,
|
|
7
|
+
requestAbortedError,
|
|
6
8
|
transactionOpenTooLongError,
|
|
7
9
|
type CommitOptions,
|
|
8
10
|
type TransactionWrite,
|
|
@@ -43,6 +45,16 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
43
45
|
this.timeout = Math.max(txnExpiration, this.timeoutBudget ?? 0);
|
|
44
46
|
}
|
|
45
47
|
|
|
48
|
+
protected markNativeCommitSubmitted(): void {
|
|
49
|
+
super.markNativeCommitSubmitted();
|
|
50
|
+
this.timeout = Math.max(this.timeout || 0, txnExpiration, this.timeoutBudget ?? 0);
|
|
51
|
+
trackedTxns.add(this);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
protected nativeCommitAttemptEnded(): void {
|
|
55
|
+
if (!this.readTxn) trackedTxns.delete(this);
|
|
56
|
+
}
|
|
57
|
+
|
|
46
58
|
getReadTxn(): any {
|
|
47
59
|
// used optimistically
|
|
48
60
|
this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
|
|
@@ -58,6 +70,7 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
58
70
|
this.readTxn = (this.db as any).useReadTransaction();
|
|
59
71
|
|
|
60
72
|
this.readTxnsUsed = 1;
|
|
73
|
+
this.baseReadRefConsumed = false;
|
|
61
74
|
if ((this.readTxn as any).openTimer) (this.readTxn as any).openTimer = 0;
|
|
62
75
|
trackedTxns.add(this as any);
|
|
63
76
|
return this.readTxn;
|
|
@@ -84,8 +97,11 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
84
97
|
(this.readTxn as any).done();
|
|
85
98
|
}
|
|
86
99
|
if (--this.readTxnsUsed === 0) {
|
|
87
|
-
|
|
100
|
+
// A submitted write remains monitor-visible after its separate read snapshot is released.
|
|
101
|
+
// nativeCommitAttemptEnded() removes this enrollment once the outcome is known.
|
|
102
|
+
if (!this.nativeCommitSubmitted) trackedTxns.delete(this as any);
|
|
88
103
|
this.readTxn = null;
|
|
104
|
+
this.completeDeferredContextRelease();
|
|
89
105
|
}
|
|
90
106
|
}
|
|
91
107
|
|
|
@@ -96,16 +112,16 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
96
112
|
}
|
|
97
113
|
|
|
98
114
|
addWrite(operation: TransactionWrite): any {
|
|
99
|
-
if (this.timedOut) throw transactionOpenTooLongError();
|
|
100
|
-
if (this.
|
|
101
|
-
throw new Error('Can not use a transaction that is no longer open');
|
|
102
|
-
}
|
|
103
|
-
|
|
115
|
+
if (this.timedOut || this.postSubmitPoisoned) throw transactionOpenTooLongError();
|
|
116
|
+
if (this.disconnected) throw requestAbortedError();
|
|
104
117
|
if (this.open === TRANSACTION_STATE.LINGERING) {
|
|
105
118
|
// if the transaction is lingering, it is already committed, so we need to commit the write immediately
|
|
106
119
|
const immediateTxn = new ImmediateTransaction(this.db);
|
|
120
|
+
// The throwaway owns this write, so it owns the request's cancellation too: this settled chain
|
|
121
|
+
// would neither refuse on its behalf nor ever release a subscription taken for it.
|
|
122
|
+
immediateTxn.requestSignal = (this.root ?? this).requestSignal;
|
|
107
123
|
immediateTxn.addWrite(operation);
|
|
108
|
-
const result = immediateTxn.commit({});
|
|
124
|
+
const result = immediateTxn.commit({}) as any;
|
|
109
125
|
// Nothing may be sent back to this throwaway: the write is already committed, and its
|
|
110
126
|
// durability is the promise below.
|
|
111
127
|
operation.stagedIn = undefined;
|
|
@@ -116,6 +132,10 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
116
132
|
}
|
|
117
133
|
return result;
|
|
118
134
|
}
|
|
135
|
+
this.admitRequestWrite();
|
|
136
|
+
if (this.open === TRANSACTION_STATE.CLOSED) {
|
|
137
|
+
throw new Error('Can not use a transaction that is no longer open');
|
|
138
|
+
}
|
|
119
139
|
|
|
120
140
|
this.linkWrite(operation);
|
|
121
141
|
this.writes.push(operation); // standard path, add to current transaction
|
|
@@ -132,8 +152,20 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
132
152
|
/**
|
|
133
153
|
* Resolves with information on the timestamp and success of the commit
|
|
134
154
|
*/
|
|
135
|
-
|
|
136
|
-
|
|
155
|
+
protected performCommit(options: CommitOptions = {}): any {
|
|
156
|
+
const next = this.next;
|
|
157
|
+
let result;
|
|
158
|
+
try {
|
|
159
|
+
result = this.performCommitAttempt(options);
|
|
160
|
+
} catch (error) {
|
|
161
|
+
this.abortCommitFailure(error, next);
|
|
162
|
+
}
|
|
163
|
+
if (typeof result?.then === 'function') return result.catch((error) => this.abortCommitFailure(error, next));
|
|
164
|
+
return result;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** Preserve the synchronous empty-commit path while the outer method funnels every failure once. */
|
|
168
|
+
private performCommitAttempt(options: CommitOptions = {}): any {
|
|
137
169
|
options = options || {};
|
|
138
170
|
let txnTime = this.timestamp;
|
|
139
171
|
if (!txnTime) txnTime = this.timestamp = options.timestamp || getNextMonotonicTime();
|
|
@@ -141,74 +173,73 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
141
173
|
const retries = options.retries || 0;
|
|
142
174
|
// now validate
|
|
143
175
|
if (this.validated < this.writes.length) {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
closeWriteInstance(write);
|
|
155
|
-
}
|
|
176
|
+
const start = this.validated;
|
|
177
|
+
// record the number of writes that have been validated so if we re-execute
|
|
178
|
+
// and the number is increased we can validate the new entries
|
|
179
|
+
this.validated = this.writes.length;
|
|
180
|
+
for (let i = start; i < this.validated; i++) {
|
|
181
|
+
const write = this.writes[i];
|
|
182
|
+
try {
|
|
183
|
+
if (write) validateWrite(write, this.timestamp, this);
|
|
184
|
+
} finally {
|
|
185
|
+
closeWriteInstance(write);
|
|
156
186
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
187
|
+
}
|
|
188
|
+
let hasBefore;
|
|
189
|
+
for (let i = start; i < this.validated; i++) {
|
|
190
|
+
const write = this.writes[i];
|
|
191
|
+
if (!write) continue;
|
|
192
|
+
if (write.before || write.beforeIntermediate) {
|
|
193
|
+
hasBefore = true;
|
|
164
194
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
}
|
|
195
|
+
}
|
|
196
|
+
// Now we need to let any "before" actions execute. These are calls to the sources,
|
|
197
|
+
// and we want to follow the order of the source sequence so that later, more canonical
|
|
198
|
+
// source writes will finish (with right to refuse/abort) before proceeeding to less
|
|
199
|
+
// canonical sources.
|
|
200
|
+
if (hasBefore) {
|
|
201
|
+
// see `committing` in DatabaseTransaction (issue #2062)
|
|
202
|
+
this.setCommitPhase(true);
|
|
203
|
+
const stagedWrites = this.writes.length;
|
|
204
|
+
return (async () => {
|
|
205
|
+
try {
|
|
206
|
+
for (let phase = 0; phase < 2; phase++) {
|
|
207
|
+
let completion;
|
|
208
|
+
for (let i = start; i < this.validated; i++) {
|
|
209
|
+
const write = this.writes[i];
|
|
210
|
+
if (!write) continue;
|
|
211
|
+
const before = write[phase === 0 ? 'before' : 'beforeIntermediate'];
|
|
212
|
+
if (before) {
|
|
213
|
+
const nextCompletion = before();
|
|
214
|
+
if (completion) {
|
|
215
|
+
if (completion.push) completion.push(nextCompletion);
|
|
216
|
+
else completion = [completion, nextCompletion];
|
|
217
|
+
} else completion = nextCompletion;
|
|
188
218
|
}
|
|
189
|
-
if (completion) await (completion.push ? Promise.all(completion) : completion);
|
|
190
219
|
}
|
|
191
|
-
|
|
192
|
-
this.setCommitPhase(false);
|
|
193
|
-
this.abort();
|
|
194
|
-
throw error;
|
|
220
|
+
if (completion) await (completion.push ? Promise.all(completion) : completion);
|
|
195
221
|
}
|
|
222
|
+
} finally {
|
|
196
223
|
this.setCommitPhase(false);
|
|
197
|
-
|
|
224
|
+
}
|
|
225
|
+
// aborted underneath us while parked above — see DatabaseTransaction's twin guard. A chain
|
|
226
|
+
// poisoned after one of its stores submitted rides the cascade instead (abortAndPoison).
|
|
227
|
+
if (!this.poisonedMidCommit) {
|
|
198
228
|
if (this.timedOut) throw transactionOpenTooLongError();
|
|
199
|
-
if (
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
this.abort();
|
|
206
|
-
throw error;
|
|
229
|
+
if (this.disconnected) throw requestAbortedError();
|
|
230
|
+
}
|
|
231
|
+
if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
|
|
232
|
+
throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
|
|
233
|
+
return this.commit({ ...options, continuation: true });
|
|
234
|
+
})();
|
|
207
235
|
}
|
|
208
236
|
}
|
|
209
237
|
if (!retries && this.writes.some((write) => write)) this.renewChainForNativeCommit();
|
|
210
238
|
// release the read snapshot so we don't keep it open longer than necessary
|
|
211
|
-
if (!retries
|
|
239
|
+
if (!retries && !this.baseReadRefConsumed) {
|
|
240
|
+
this.baseReadRefConsumed = true;
|
|
241
|
+
this.doneReadTxn();
|
|
242
|
+
}
|
|
212
243
|
this.open = options?.doneWriting ? TRANSACTION_STATE.LINGERING : TRANSACTION_STATE.OPEN;
|
|
213
244
|
let resolution;
|
|
214
245
|
const completions = [];
|
|
@@ -264,6 +295,7 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
264
295
|
// use a real (async) transaction to get exclusive access to the data
|
|
265
296
|
if (db?.retryRisk) db.retryRisk *= 0.99; // gradually decay the retry risk
|
|
266
297
|
if (this.writes.length + (db?.retryRisk || 0) < MAX_OPTIMISTIC_SIZE >> retries) {
|
|
298
|
+
this.markNativeCommitSubmitted();
|
|
267
299
|
nextCondition();
|
|
268
300
|
if (commitCompletions) {
|
|
269
301
|
if (resolution) {
|
|
@@ -278,6 +310,7 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
278
310
|
} else {
|
|
279
311
|
// if it is too big to expect optimistic writes to work, or we have done too many retries we use
|
|
280
312
|
// a real LMDB transaction to get exclusive access to reading and writing
|
|
313
|
+
this.markNativeCommitSubmitted();
|
|
281
314
|
const transactionResolution = this.writes[0].store.transaction(() => {
|
|
282
315
|
for (const write of this.writes) {
|
|
283
316
|
// we load latest data while in the transaction
|
|
@@ -296,17 +329,18 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
296
329
|
return resolution.then((resolution) => {
|
|
297
330
|
if (resolution) {
|
|
298
331
|
if (this.next) {
|
|
299
|
-
completions.push(this.next.commit(options));
|
|
332
|
+
completions.push(this.next.commit({ ...options, continuation: true }));
|
|
300
333
|
}
|
|
301
334
|
if (options?.flush) {
|
|
302
|
-
|
|
335
|
+
const store = this.writes[0]?.store;
|
|
336
|
+
if (store) completions.push(store.flushed);
|
|
303
337
|
}
|
|
304
338
|
if (this.replicatedConfirmation) {
|
|
305
339
|
// if we want to wait for replication confirmation, we need to track the transaction times
|
|
306
340
|
// and when replication notifications come in, we count the number of confirms until we reach the desired number
|
|
307
|
-
const databaseName = this.writes[0]
|
|
341
|
+
const databaseName = this.writes[0]?.store.rootStore.databaseName;
|
|
308
342
|
const lastEntry = this.lastConfirmableEntry();
|
|
309
|
-
if (confirmReplication && lastEntry)
|
|
343
|
+
if (confirmReplication && databaseName && lastEntry)
|
|
310
344
|
completions.push(
|
|
311
345
|
confirmReplication(databaseName, (lastEntry as any).localTime, this.replicatedConfirmation)
|
|
312
346
|
);
|
|
@@ -337,7 +371,8 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
337
371
|
}
|
|
338
372
|
if (options) options.retries = retries + 1;
|
|
339
373
|
else options = { retries: 1 };
|
|
340
|
-
|
|
374
|
+
// A continuation: a poison that landed mid-commit must not abandon the retry ladder.
|
|
375
|
+
return this.commit({ ...options, continuation: true }); // try again
|
|
341
376
|
}
|
|
342
377
|
});
|
|
343
378
|
}
|
|
@@ -346,7 +381,7 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
346
381
|
};
|
|
347
382
|
if (this.next) {
|
|
348
383
|
// now run any other transactions
|
|
349
|
-
const nextResolution = this.next?.commit(options);
|
|
384
|
+
const nextResolution = this.next?.commit({ ...options, continuation: true });
|
|
350
385
|
if ((nextResolution as any)?.then)
|
|
351
386
|
return (nextResolution as any)?.then((nextResolution) => ({
|
|
352
387
|
txnTime,
|
|
@@ -356,19 +391,79 @@ export class LMDBTransaction extends DatabaseTransaction {
|
|
|
356
391
|
}
|
|
357
392
|
return txnResolution;
|
|
358
393
|
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
this.
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
394
|
+
|
|
395
|
+
private abortCommitFailure(error, next: DatabaseTransaction | undefined): never {
|
|
396
|
+
const nextWasAttached = Boolean(next && this.next === next);
|
|
397
|
+
try {
|
|
398
|
+
this.abort(true);
|
|
399
|
+
} catch (abortError) {
|
|
400
|
+
harperLogger.debug?.('aborting LMDB transaction after a failed commit', abortError);
|
|
401
|
+
}
|
|
402
|
+
// A detached pre-submit child has no other cleanup path. A submitted child owns an unknown native
|
|
403
|
+
// outcome and will clean itself when its own commit promise settles.
|
|
404
|
+
if (next && !nextWasAttached && !next.nativeCommitSubmitted) {
|
|
405
|
+
try {
|
|
406
|
+
next.abort(true);
|
|
407
|
+
} catch (abortError) {
|
|
408
|
+
harperLogger.debug?.('aborting a detached LMDB transaction after a failed commit', abortError);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
throw error;
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Force-release the retained read snapshot without touching staged writes. The base class's
|
|
415
|
+
* version acts on `this.transaction`; LMDB's snapshot lives on `this.readTxn`, so the monitor
|
|
416
|
+
* would otherwise have nothing that can reclaim a handle abort(true) retained.
|
|
417
|
+
*/
|
|
418
|
+
releaseReadTxn(): void {
|
|
419
|
+
while (this.readTxn && this.readTxnsUsed > 0) this.doneReadTxn();
|
|
420
|
+
this.completeDeferredContextRelease();
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
abort(retainReadTransaction = false, cascade = true): void {
|
|
424
|
+
const next = cascade ? this.next : undefined;
|
|
425
|
+
if (cascade && !retainReadTransaction) this.next = null;
|
|
426
|
+
try {
|
|
427
|
+
const hasOpenReadIterator =
|
|
428
|
+
retainReadTransaction &&
|
|
429
|
+
this.readTxn &&
|
|
430
|
+
(this.readTxnsUsed > 1 || (this.baseReadRefConsumed && this.readTxnsUsed > 0));
|
|
431
|
+
if (hasOpenReadIterator) {
|
|
432
|
+
if (!this.baseReadRefConsumed) {
|
|
433
|
+
this.doneReadTxn();
|
|
434
|
+
this.baseReadRefConsumed = true;
|
|
435
|
+
}
|
|
436
|
+
} else {
|
|
437
|
+
while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
|
|
438
|
+
}
|
|
439
|
+
this.open = TRANSACTION_STATE.CLOSED;
|
|
440
|
+
this.drainCompletions();
|
|
441
|
+
// Any blobs that were pre-saved as part of these writes will never be referenced; schedule deletion
|
|
442
|
+
// (retaining any fileId the current on-disk record still references — an aborted write may carry an
|
|
443
|
+
// already-saved blob shared with the surviving record; see harper-pro#406).
|
|
444
|
+
for (const write of this.writes) {
|
|
445
|
+
if (write?.savedBlobs)
|
|
446
|
+
cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
|
|
447
|
+
}
|
|
448
|
+
} finally {
|
|
449
|
+
try {
|
|
450
|
+
this.open = TRANSACTION_STATE.CLOSED;
|
|
451
|
+
this.endScopeOwnership();
|
|
452
|
+
this.clearWrites();
|
|
453
|
+
this.releaseContext(!this.timedOut && !this.disconnected);
|
|
454
|
+
if (!this.root) this.releaseRequestAbortListener();
|
|
455
|
+
} finally {
|
|
456
|
+
// Same guard as DatabaseTransaction.abort(): a child whose native commit is outstanding owns
|
|
457
|
+
// its own cleanup, and clearing its writes here would unlink blobs that commit still references.
|
|
458
|
+
if (next && !(next.nativeCommitSubmitted && next.commitsInFlight)) {
|
|
459
|
+
try {
|
|
460
|
+
next.abort(retainReadTransaction);
|
|
461
|
+
} catch (error) {
|
|
462
|
+
harperLogger.debug?.('cleaning up a chained LMDB transaction during abort', error);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
369
466
|
}
|
|
370
|
-
// reset the transaction
|
|
371
|
-
this.clearWrites();
|
|
372
467
|
}
|
|
373
468
|
save(..._args: any[]): any {
|
|
374
469
|
// noop for LMDB
|
|
@@ -382,6 +477,7 @@ export class ImmediateTransaction extends LMDBTransaction {
|
|
|
382
477
|
this.db = db;
|
|
383
478
|
}
|
|
384
479
|
save(..._args: any[]): any {
|
|
480
|
+
this.admitRequestWrite(); // a deferred save reaches here without passing through addWrite()
|
|
385
481
|
return this.commit();
|
|
386
482
|
}
|
|
387
483
|
// @ts-expect-error accessor overriding property
|
|
@@ -406,6 +502,38 @@ function startMonitoringTxns() {
|
|
|
406
502
|
const commitChainHead = txn.commitChainHead ?? txn;
|
|
407
503
|
if (txn.timeout <= 0) {
|
|
408
504
|
const url = (txn.getContext() as any)?.url;
|
|
505
|
+
if (deferForCommitInFlight(txn, url, txnExpiration)) continue;
|
|
506
|
+
if (
|
|
507
|
+
txn.open === TRANSACTION_STATE.CLOSED &&
|
|
508
|
+
!txn.commitAttemptDoomedByPoison() &&
|
|
509
|
+
shouldSpareCommitPhase(txn, checkedCommitPhaseChains)
|
|
510
|
+
) {
|
|
511
|
+
harperLogger.warn?.(
|
|
512
|
+
`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work; letting it complete, from table: ${
|
|
513
|
+
(txn.db as any)?.name + (url ? ' path: ' + url : '')
|
|
514
|
+
}`
|
|
515
|
+
);
|
|
516
|
+
txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
if (txn.open === TRANSACTION_STATE.CLOSED) {
|
|
520
|
+
// Only reachable through abort(true), which retained the snapshot for read iterators
|
|
521
|
+
// that own it. Nothing else here can reclaim it: the branches below would re-enter
|
|
522
|
+
// commit() on a poisoned transaction every tick, throw on its own poison check, and
|
|
523
|
+
// re-arm the timer — pinning an LMDB read snapshot, which blocks free-page reuse and
|
|
524
|
+
// grows the data file, for the life of the process. Close the owning iterators first so
|
|
525
|
+
// the reference comes back through doneReadTxn(), and force the release if it does not.
|
|
526
|
+
const closedIterators = txn.closeOwnedReadIterators();
|
|
527
|
+
if (closedIterators > 0 || txn.readTxn)
|
|
528
|
+
harperLogger.warn?.(
|
|
529
|
+
`Read iterators held a closed transaction's snapshot past the open-transaction limit; ${
|
|
530
|
+
txn.readTxn ? 'releasing it' : `closed ${closedIterators} abandoned iterator(s)`
|
|
531
|
+
}, from table: ${(txn.db as any)?.name + (url ? ' path: ' + url : '')}`
|
|
532
|
+
);
|
|
533
|
+
if (txn.readTxn) txn.releaseReadTxn();
|
|
534
|
+
trackedTxns.delete(txn);
|
|
535
|
+
continue;
|
|
536
|
+
}
|
|
409
537
|
if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
|
|
410
538
|
// see DatabaseTransaction's monitor (issue #2062)
|
|
411
539
|
harperLogger.warn?.(
|
|
@@ -967,10 +967,9 @@ export function recordUpdater(store, tableId, auditStore) {
|
|
|
967
967
|
extendedType |= HAS_ADDITIONAL_AUDIT_REFS_AUDIT;
|
|
968
968
|
}
|
|
969
969
|
if (options?.localOnly) {
|
|
970
|
-
// Mark this write as local-only: set the bit in BOTH the persisted record metadata
|
|
971
|
-
// (so the full-copy send loop, which reads entry.metadataFlags, skips it) AND the audit
|
|
972
|
-
// entry's extendedType (so the audit-forward send path skips it by bitmask without decode).
|
|
973
970
|
metadataInNextEncoding |= LOCAL_ONLY;
|
|
971
|
+
}
|
|
972
|
+
if (options?.localOnly || options?.auditLocalOnly) {
|
|
974
973
|
extendedType |= LOCAL_ONLY;
|
|
975
974
|
}
|
|
976
975
|
if (previousResidencyId !== residencyId) {
|
package/resources/Resource.ts
CHANGED
|
@@ -873,15 +873,14 @@ function transactional(
|
|
|
873
873
|
// opposite treatment:
|
|
874
874
|
// - committed: the prior call ran to completion normally. Safe to silently start a fresh
|
|
875
875
|
// transaction for this independent call (the case above).
|
|
876
|
-
// -
|
|
877
|
-
// abortDueToTimeout(), marked via the `timedOut` poison flag): the whole logical operation
|
|
876
|
+
// - poisoned (`timedOut`, #1411; or `disconnected`, harper#2001): the whole logical operation
|
|
878
877
|
// must fail atomically, not have this write quietly land on a brand-new transaction while
|
|
879
878
|
// the earlier write(s) were rolled back. Joining the poisoned transaction here (instead of
|
|
880
|
-
// starting fresh) makes the write throw transactionOpenTooLongError via
|
|
881
|
-
// poison check, correctly propagating the abort to the caller. See
|
|
879
|
+
// starting fresh) makes the write throw transactionOpenTooLongError/requestAbortedError via
|
|
880
|
+
// addWrite()/commit()'s poison check, correctly propagating the abort to the caller. See
|
|
882
881
|
// integrationTests/resources/txn-overtime-atomicity.test.ts.
|
|
883
|
-
if (isJoinableScope(context?.transaction) || context?.transaction?.timedOut) {
|
|
884
|
-
// we are already in a transaction (or it was poisoned
|
|
882
|
+
if (isJoinableScope(context?.transaction) || context?.transaction?.timedOut || context?.transaction?.disconnected) {
|
|
883
|
+
// we are already in a transaction (or it was poisoned and must fail), proceed
|
|
885
884
|
const resource = this.getResource(query, context, resourceOptions);
|
|
886
885
|
return resource.then
|
|
887
886
|
? resource.then(authorizeActionOnResource)
|
|
@@ -143,7 +143,9 @@ export interface Context {
|
|
|
143
143
|
/**
|
|
144
144
|
* Abort signal carried through ALS so generator bodies can forward cancellation to
|
|
145
145
|
* external work (e.g. `scope.models.generateStream({ signal })`). Populated on the
|
|
146
|
-
* Request that becomes the ALS-bound Context for HTTP/WS paths via #513.
|
|
146
|
+
* Request that becomes the ALS-bound Context for HTTP/WS paths via #513. Also the request's
|
|
147
|
+
* lifetime for its writes (harper#2001): once it aborts, no write for the request is admitted
|
|
148
|
+
* and a write-bearing transaction is aborted at once (`DatabaseTransaction.admitRequestWrite`).
|
|
147
149
|
*/
|
|
148
150
|
signal?: AbortSignal;
|
|
149
151
|
}
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
type CountEstimateOptions,
|
|
5
5
|
DBI,
|
|
6
6
|
type StoreIteratorOptions,
|
|
7
|
+
type StoreGetOptions,
|
|
7
8
|
type StorePutOptions,
|
|
8
9
|
type StoreRemoveOptions,
|
|
9
10
|
RocksDatabase,
|
|
@@ -44,6 +45,10 @@ function translateIndexBounds<
|
|
|
44
45
|
* RocksDatabase rather than a store because it actually alters the interface
|
|
45
46
|
*/
|
|
46
47
|
export class RocksIndexStore extends RocksDatabase {
|
|
48
|
+
hasIndexEntry(indexedValue: any, primaryKey: Id, options?: StoreGetOptions): boolean {
|
|
49
|
+
return super.getBinaryFastSync([indexedValue, primaryKey], options) !== undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
47
52
|
/**
|
|
48
53
|
* Get all entries matching the range
|
|
49
54
|
* @param options
|