@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +4 -5
- package/dist/engine-wasm/wasm/lix_engine.js +1 -1
- package/dist/engine-wasm/wasm/lix_engine.wasm +0 -0
- package/dist/generated/builtin-schemas.d.ts +87 -162
- package/dist/generated/builtin-schemas.js +139 -236
- package/dist/open-lix.d.ts +1 -1
- package/dist-engine-src/src/binary_cas/types.rs +0 -6
- package/dist-engine-src/src/catalog/context.rs +412 -0
- package/dist-engine-src/src/catalog/mod.rs +10 -0
- package/dist-engine-src/src/catalog/schema.rs +4 -0
- package/dist-engine-src/src/catalog/snapshot.rs +1114 -0
- package/dist-engine-src/src/cel/mod.rs +1 -1
- package/dist-engine-src/src/cel/provider.rs +1 -1
- package/dist-engine-src/src/commit_graph/context.rs +328 -1015
- package/dist-engine-src/src/commit_graph/mod.rs +2 -3
- package/dist-engine-src/src/commit_graph/types.rs +7 -43
- package/dist-engine-src/src/commit_graph/walker.rs +57 -81
- package/dist-engine-src/src/commit_store/codec.rs +887 -0
- package/dist-engine-src/src/commit_store/context.rs +944 -0
- package/dist-engine-src/src/commit_store/materialization.rs +84 -0
- package/dist-engine-src/src/commit_store/mod.rs +16 -0
- package/dist-engine-src/src/commit_store/storage.rs +600 -0
- package/dist-engine-src/src/commit_store/types.rs +215 -0
- package/dist-engine-src/src/common/identity.rs +15 -5
- package/dist-engine-src/src/common/json_pointer.rs +67 -0
- package/dist-engine-src/src/common/metadata.rs +17 -12
- package/dist-engine-src/src/common/mod.rs +5 -5
- package/dist-engine-src/src/domain.rs +324 -0
- package/dist-engine-src/src/engine.rs +29 -43
- package/dist-engine-src/src/entity_identity.rs +238 -118
- package/dist-engine-src/src/functions/context.rs +17 -52
- package/dist-engine-src/src/functions/deterministic.rs +1 -1
- package/dist-engine-src/src/functions/mod.rs +1 -1
- package/dist-engine-src/src/functions/provider.rs +4 -4
- package/dist-engine-src/src/functions/state.rs +39 -66
- package/dist-engine-src/src/functions/types.rs +1 -1
- package/dist-engine-src/src/init.rs +204 -151
- package/dist-engine-src/src/json_store/context.rs +354 -60
- package/dist-engine-src/src/json_store/encoded.rs +6 -6
- package/dist-engine-src/src/json_store/mod.rs +4 -1
- package/dist-engine-src/src/json_store/store.rs +884 -11
- package/dist-engine-src/src/json_store/types.rs +166 -1
- package/dist-engine-src/src/lib.rs +10 -9
- package/dist-engine-src/src/live_state/context.rs +608 -830
- package/dist-engine-src/src/live_state/mod.rs +3 -3
- package/dist-engine-src/src/live_state/overlay.rs +7 -7
- package/dist-engine-src/src/live_state/reader.rs +5 -5
- package/dist-engine-src/src/live_state/types.rs +19 -36
- package/dist-engine-src/src/live_state/visibility.rs +19 -14
- package/dist-engine-src/src/plugin/archive.rs +3 -6
- package/dist-engine-src/src/plugin/install.rs +0 -18
- package/dist-engine-src/src/plugin/plugin_manifest.json +0 -1
- package/dist-engine-src/src/schema/annotations/defaults.rs +2 -7
- package/dist-engine-src/src/schema/builtin/lix_account.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_active_account.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_binary_blob_ref.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_change.json +11 -10
- package/dist-engine-src/src/schema/builtin/lix_change_author.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_commit.json +8 -46
- package/dist-engine-src/src/schema/builtin/lix_commit_edge.json +29 -22
- package/dist-engine-src/src/schema/builtin/lix_directory_descriptor.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_file_descriptor.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_key_value.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_label.json +10 -3
- package/dist-engine-src/src/schema/builtin/lix_label_assignment.json +74 -0
- package/dist-engine-src/src/schema/builtin/lix_registered_schema.json +2 -8
- package/dist-engine-src/src/schema/builtin/lix_version_descriptor.json +0 -1
- package/dist-engine-src/src/schema/builtin/lix_version_ref.json +0 -1
- package/dist-engine-src/src/schema/builtin/mod.rs +10 -59
- package/dist-engine-src/src/schema/compatibility.rs +787 -0
- package/dist-engine-src/src/schema/definition.json +47 -17
- package/dist-engine-src/src/schema/definition.rs +202 -96
- package/dist-engine-src/src/schema/key.rs +9 -77
- package/dist-engine-src/src/schema/mod.rs +4 -4
- package/dist-engine-src/src/schema/tests.rs +133 -92
- package/dist-engine-src/src/session/context.rs +40 -42
- package/dist-engine-src/src/session/create_version.rs +22 -14
- package/dist-engine-src/src/session/execute.rs +45 -14
- package/dist-engine-src/src/session/merge/apply.rs +4 -4
- package/dist-engine-src/src/session/merge/conflicts.rs +3 -2
- package/dist-engine-src/src/session/merge/stats.rs +1 -1
- package/dist-engine-src/src/session/merge/version.rs +35 -45
- package/dist-engine-src/src/session/mod.rs +4 -2
- package/dist-engine-src/src/session/optimization9_sql2_bench.rs +100 -0
- package/dist-engine-src/src/session/switch_version.rs +16 -28
- package/dist-engine-src/src/sql2/change_provider.rs +14 -20
- package/dist-engine-src/src/sql2/classify.rs +61 -26
- package/dist-engine-src/src/sql2/context.rs +22 -18
- package/dist-engine-src/src/sql2/directory_history_provider.rs +28 -20
- package/dist-engine-src/src/sql2/directory_provider.rs +131 -83
- package/dist-engine-src/src/sql2/entity_history_provider.rs +10 -14
- package/dist-engine-src/src/sql2/entity_provider.rs +680 -169
- package/dist-engine-src/src/sql2/error.rs +21 -1
- package/dist-engine-src/src/sql2/execute.rs +325 -264
- package/dist-engine-src/src/sql2/file_history_provider.rs +29 -21
- package/dist-engine-src/src/sql2/file_provider.rs +533 -108
- package/dist-engine-src/src/sql2/filesystem_planner.rs +58 -94
- package/dist-engine-src/src/sql2/filesystem_visibility.rs +37 -23
- package/dist-engine-src/src/sql2/history_projection.rs +3 -27
- package/dist-engine-src/src/sql2/history_provider.rs +11 -17
- package/dist-engine-src/src/sql2/history_route.rs +22 -8
- package/dist-engine-src/src/sql2/lix_state_provider.rs +178 -96
- package/dist-engine-src/src/sql2/mod.rs +6 -3
- package/dist-engine-src/src/sql2/predicate_typecheck.rs +246 -0
- package/dist-engine-src/src/sql2/public_bind/assignment.rs +46 -0
- package/dist-engine-src/src/sql2/public_bind/capability.rs +41 -0
- package/dist-engine-src/src/sql2/public_bind/dml.rs +166 -0
- package/dist-engine-src/src/sql2/public_bind/mod.rs +25 -0
- package/dist-engine-src/src/sql2/public_bind/table.rs +168 -0
- package/dist-engine-src/src/sql2/read_only.rs +10 -12
- package/dist-engine-src/src/sql2/session.rs +7 -10
- package/dist-engine-src/src/sql2/udfs/lix_timestamp.rs +76 -0
- package/dist-engine-src/src/sql2/udfs/mod.rs +8 -1
- package/dist-engine-src/src/sql2/udfs/public_call.rs +211 -0
- package/dist-engine-src/src/sql2/version_provider.rs +46 -31
- package/dist-engine-src/src/sql2/version_scope.rs +4 -4
- package/dist-engine-src/src/storage_bench.rs +1782 -325
- package/dist-engine-src/src/test_support.rs +183 -36
- package/dist-engine-src/src/tracked_state/by_file_index.rs +20 -24
- package/dist-engine-src/src/tracked_state/codec.rs +1519 -181
- package/dist-engine-src/src/tracked_state/context.rs +1155 -271
- package/dist-engine-src/src/tracked_state/diff.rs +249 -57
- package/dist-engine-src/src/tracked_state/materialization.rs +365 -103
- package/dist-engine-src/src/tracked_state/materializer.rs +488 -0
- package/dist-engine-src/src/tracked_state/merge.rs +37 -19
- package/dist-engine-src/src/tracked_state/mod.rs +8 -7
- package/dist-engine-src/src/tracked_state/storage.rs +138 -6
- package/dist-engine-src/src/tracked_state/tree.rs +695 -252
- package/dist-engine-src/src/tracked_state/types.rs +176 -6
- package/dist-engine-src/src/transaction/commit.rs +695 -435
- package/dist-engine-src/src/transaction/context.rs +551 -310
- package/dist-engine-src/src/transaction/live_state_overlay.rs +9 -8
- package/dist-engine-src/src/transaction/mod.rs +2 -0
- package/dist-engine-src/src/transaction/normalization.rs +311 -447
- package/dist-engine-src/src/transaction/prep.rs +37 -0
- package/dist-engine-src/src/transaction/schema_resolver.rs +93 -71
- package/dist-engine-src/src/transaction/staging.rs +701 -406
- package/dist-engine-src/src/transaction/types.rs +231 -122
- package/dist-engine-src/src/transaction/validation.rs +2717 -1698
- package/dist-engine-src/src/untracked_state/codec.rs +40 -96
- package/dist-engine-src/src/untracked_state/context.rs +21 -5
- package/dist-engine-src/src/untracked_state/materialization.rs +10 -104
- package/dist-engine-src/src/untracked_state/mod.rs +3 -5
- package/dist-engine-src/src/untracked_state/storage.rs +105 -57
- package/dist-engine-src/src/untracked_state/types.rs +63 -13
- package/dist-engine-src/src/version/context.rs +1 -13
- package/dist-engine-src/src/version/lifecycle.rs +221 -0
- package/dist-engine-src/src/version/mod.rs +3 -2
- package/dist-engine-src/src/version/refs.rs +12 -103
- package/dist-engine-src/src/version/stage_rows.rs +15 -19
- package/package.json +1 -1
- package/dist-engine-src/src/changelog/codec.rs +0 -321
- package/dist-engine-src/src/changelog/context.rs +0 -92
- package/dist-engine-src/src/changelog/materialization.rs +0 -121
- package/dist-engine-src/src/changelog/mod.rs +0 -13
- package/dist-engine-src/src/changelog/reader.rs +0 -20
- package/dist-engine-src/src/changelog/storage.rs +0 -220
- package/dist-engine-src/src/changelog/types.rs +0 -38
- package/dist-engine-src/src/schema/builtin/lix_change_set.json +0 -18
- package/dist-engine-src/src/schema/builtin/lix_change_set_element.json +0 -75
- package/dist-engine-src/src/schema/builtin/lix_entity_label.json +0 -63
- package/dist-engine-src/src/schema_registry.rs +0 -294
- package/dist-engine-src/src/sql2/commit_derived_provider.rs +0 -591
- package/dist-engine-src/src/tracked_state/rebuild.rs +0 -771
- package/dist-engine-src/src/tracked_state/tree_types.rs +0 -176
|
@@ -1,29 +1,53 @@
|
|
|
1
1
|
use crate::binary_cas::{BinaryCasContext, BlobHash, BlobWrite};
|
|
2
|
-
use crate::
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
use crate::catalog::CatalogContext;
|
|
3
|
+
use crate::commit_graph::CommitGraphChangeHistoryRequest;
|
|
4
|
+
use crate::commit_store::{
|
|
5
|
+
Change, ChangeScanRequest, CommitDraftRef, CommitStoreContext, MaterializedChange,
|
|
5
6
|
};
|
|
6
7
|
use crate::entity_identity::EntityIdentity;
|
|
7
|
-
use crate::entity_identity::EntityIdentityPart;
|
|
8
8
|
use crate::json_store::context::JsonStoreContext;
|
|
9
|
-
use crate::json_store::types::{
|
|
9
|
+
use crate::json_store::types::{
|
|
10
|
+
JsonLoadRequestRef, JsonProjectionLoadRequestRef, JsonProjectionPath, JsonReadScopeRef,
|
|
11
|
+
JsonRef, JsonWritePlacementRef, NormalizedJsonRef,
|
|
12
|
+
};
|
|
13
|
+
use crate::live_state::LiveStateContext;
|
|
14
|
+
use crate::session::SessionMode;
|
|
10
15
|
use crate::storage::{
|
|
11
16
|
KvGetGroup, KvGetRequest, KvScanRange, KvScanRequest, KvWriteBatch, StorageContext,
|
|
12
17
|
StorageWriteSet,
|
|
13
18
|
};
|
|
14
19
|
use crate::tracked_state::{
|
|
15
|
-
|
|
16
|
-
|
|
20
|
+
MaterializedTrackedStateRow, TrackedStateContext, TrackedStateDeltaRef,
|
|
21
|
+
TrackedStateDiffRequest, TrackedStateFilter, TrackedStateProjection, TrackedStateRowRequest,
|
|
22
|
+
TrackedStateScanRequest,
|
|
23
|
+
};
|
|
24
|
+
use crate::transaction::open_transaction;
|
|
25
|
+
use crate::transaction::types::{
|
|
26
|
+
TransactionJson, TransactionWrite, TransactionWriteMode, TransactionWriteRow,
|
|
17
27
|
};
|
|
18
28
|
use crate::untracked_state::{
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
UntrackedStateScanRequest,
|
|
29
|
+
MaterializedUntrackedStateRow, UntrackedStateContext, UntrackedStateFilter,
|
|
30
|
+
UntrackedStateProjection, UntrackedStateRowRequest, UntrackedStateScanRequest,
|
|
22
31
|
};
|
|
32
|
+
use crate::version::VersionContext;
|
|
23
33
|
use crate::{Backend, LixError, NullableKeyFilter};
|
|
34
|
+
use std::collections::{BTreeMap, HashSet};
|
|
35
|
+
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
24
36
|
use std::sync::Arc;
|
|
37
|
+
use std::sync::Mutex;
|
|
38
|
+
use std::sync::OnceLock;
|
|
25
39
|
use std::time::{Duration, Instant};
|
|
26
40
|
|
|
41
|
+
fn prepare_json_ref(document: &[u8]) -> Result<JsonRef, LixError> {
|
|
42
|
+
let text = std::str::from_utf8(document).map_err(|error| {
|
|
43
|
+
LixError::new(
|
|
44
|
+
LixError::CODE_UNKNOWN,
|
|
45
|
+
format!("benchmark JSON document is invalid UTF-8: {error}"),
|
|
46
|
+
)
|
|
47
|
+
})?;
|
|
48
|
+
Ok(JsonRef::for_content(text.as_bytes()))
|
|
49
|
+
}
|
|
50
|
+
|
|
27
51
|
#[derive(Debug, Clone, Copy)]
|
|
28
52
|
pub struct StorageBenchConfig {
|
|
29
53
|
pub rows: usize,
|
|
@@ -115,13 +139,647 @@ pub struct StorageBenchReport {
|
|
|
115
139
|
pub elapsed: Duration,
|
|
116
140
|
}
|
|
117
141
|
|
|
142
|
+
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
143
|
+
pub struct TransactionBenchCounters {
|
|
144
|
+
pub rows_staged: usize,
|
|
145
|
+
pub untracked_rows: usize,
|
|
146
|
+
pub validation_version_count: usize,
|
|
147
|
+
pub schema_catalog_loads: usize,
|
|
148
|
+
pub json_store_stage_bytes_calls: usize,
|
|
149
|
+
pub unique_json_refs: usize,
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
153
|
+
pub struct TransactionAccountingReport {
|
|
154
|
+
pub counters: TransactionBenchCounters,
|
|
155
|
+
pub storage_write_batches: usize,
|
|
156
|
+
pub kv_puts_by_namespace: BTreeMap<String, usize>,
|
|
157
|
+
pub bytes_by_namespace: BTreeMap<String, usize>,
|
|
158
|
+
}
|
|
159
|
+
|
|
118
160
|
pub struct StorageApiFixture {
|
|
119
161
|
storage: StorageContext,
|
|
120
162
|
rows: usize,
|
|
121
163
|
}
|
|
122
164
|
|
|
165
|
+
pub struct TransactionBenchFixture {
|
|
166
|
+
storage: StorageContext,
|
|
167
|
+
live_state: Arc<LiveStateContext>,
|
|
168
|
+
tracked_state: Arc<TrackedStateContext>,
|
|
169
|
+
binary_cas: Arc<BinaryCasContext>,
|
|
170
|
+
commit_store: Arc<CommitStoreContext>,
|
|
171
|
+
version_ctx: Arc<VersionContext>,
|
|
172
|
+
catalog_context: Arc<CatalogContext>,
|
|
173
|
+
rows: Vec<TransactionWriteRow>,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
pub struct TransactionCommitOnlyFixture {
|
|
177
|
+
runtime_functions: crate::functions::FunctionContext,
|
|
178
|
+
transaction: crate::transaction::Transaction,
|
|
179
|
+
rows: usize,
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
static TRANSACTION_ROWS_STAGED: AtomicUsize = AtomicUsize::new(0);
|
|
183
|
+
static TRANSACTION_UNTRACKED_ROWS: AtomicUsize = AtomicUsize::new(0);
|
|
184
|
+
static TRANSACTION_VALIDATION_VERSION_COUNT: AtomicUsize = AtomicUsize::new(0);
|
|
185
|
+
static TRANSACTION_SCHEMA_CATALOG_LOADS: AtomicUsize = AtomicUsize::new(0);
|
|
186
|
+
static JSON_STORE_STAGE_BYTES_CALLS: AtomicUsize = AtomicUsize::new(0);
|
|
187
|
+
static JSON_STORE_UNIQUE_REFS: OnceLock<Mutex<HashSet<[u8; 32]>>> = OnceLock::new();
|
|
188
|
+
|
|
123
189
|
const STORAGE_API_NAMESPACE: &str = "bench.storage_api";
|
|
124
190
|
const STORAGE_API_ALT_NAMESPACE: &str = "bench.storage_api.alt";
|
|
191
|
+
const TRANSACTION_BENCH_SCHEMA_KEY: &str = "bench_transaction_entity";
|
|
192
|
+
|
|
193
|
+
pub fn reset_transaction_bench_counters() {
|
|
194
|
+
TRANSACTION_ROWS_STAGED.store(0, Ordering::Relaxed);
|
|
195
|
+
TRANSACTION_UNTRACKED_ROWS.store(0, Ordering::Relaxed);
|
|
196
|
+
TRANSACTION_VALIDATION_VERSION_COUNT.store(0, Ordering::Relaxed);
|
|
197
|
+
TRANSACTION_SCHEMA_CATALOG_LOADS.store(0, Ordering::Relaxed);
|
|
198
|
+
JSON_STORE_STAGE_BYTES_CALLS.store(0, Ordering::Relaxed);
|
|
199
|
+
json_store_unique_refs()
|
|
200
|
+
.lock()
|
|
201
|
+
.expect("json store unique ref counter mutex should lock")
|
|
202
|
+
.clear();
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
pub fn transaction_bench_counters() -> TransactionBenchCounters {
|
|
206
|
+
TransactionBenchCounters {
|
|
207
|
+
rows_staged: TRANSACTION_ROWS_STAGED.load(Ordering::Relaxed),
|
|
208
|
+
untracked_rows: TRANSACTION_UNTRACKED_ROWS.load(Ordering::Relaxed),
|
|
209
|
+
validation_version_count: TRANSACTION_VALIDATION_VERSION_COUNT.load(Ordering::Relaxed),
|
|
210
|
+
schema_catalog_loads: TRANSACTION_SCHEMA_CATALOG_LOADS.load(Ordering::Relaxed),
|
|
211
|
+
json_store_stage_bytes_calls: JSON_STORE_STAGE_BYTES_CALLS.load(Ordering::Relaxed),
|
|
212
|
+
unique_json_refs: json_store_unique_refs()
|
|
213
|
+
.lock()
|
|
214
|
+
.expect("json store unique ref counter mutex should lock")
|
|
215
|
+
.len(),
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
pub(crate) fn record_transaction_rows_staged(rows: usize) {
|
|
220
|
+
TRANSACTION_ROWS_STAGED.fetch_add(rows, Ordering::Relaxed);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
pub(crate) fn record_transaction_untracked_rows(rows: usize) {
|
|
224
|
+
TRANSACTION_UNTRACKED_ROWS.fetch_add(rows, Ordering::Relaxed);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
pub(crate) fn record_transaction_validation_version() {
|
|
228
|
+
TRANSACTION_VALIDATION_VERSION_COUNT.fetch_add(1, Ordering::Relaxed);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
pub(crate) fn record_transaction_schema_catalog_load() {
|
|
232
|
+
TRANSACTION_SCHEMA_CATALOG_LOADS.fetch_add(1, Ordering::Relaxed);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
pub(crate) fn record_json_store_stage_bytes(hash: [u8; 32]) {
|
|
236
|
+
JSON_STORE_STAGE_BYTES_CALLS.fetch_add(1, Ordering::Relaxed);
|
|
237
|
+
json_store_unique_refs()
|
|
238
|
+
.lock()
|
|
239
|
+
.expect("json store unique ref counter mutex should lock")
|
|
240
|
+
.insert(hash);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
fn json_store_unique_refs() -> &'static Mutex<HashSet<[u8; 32]>> {
|
|
244
|
+
JSON_STORE_UNIQUE_REFS.get_or_init(|| Mutex::new(HashSet::new()))
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
pub async fn prepare_transaction_commit_empty(
|
|
248
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
249
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
250
|
+
prepare_transaction_fixture(backend, Vec::new()).await
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
pub async fn prepare_transaction_commit_schema_only(
|
|
254
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
255
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
256
|
+
prepare_transaction_fixture(backend, vec![transaction_registered_schema_row()]).await
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
pub async fn prepare_transaction_commit_entities_no_payload(
|
|
260
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
261
|
+
rows: usize,
|
|
262
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
263
|
+
prepare_transaction_fixture(
|
|
264
|
+
backend,
|
|
265
|
+
transaction_entity_rows(TransactionEntityRows {
|
|
266
|
+
rows,
|
|
267
|
+
payload_bytes: 0,
|
|
268
|
+
payload_pattern: TransactionPayloadPattern::Unique,
|
|
269
|
+
metadata_pattern: TransactionPayloadPattern::None,
|
|
270
|
+
untracked: false,
|
|
271
|
+
key_prefix: "entity-no-payload",
|
|
272
|
+
}),
|
|
273
|
+
)
|
|
274
|
+
.await
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
pub async fn prepare_transaction_commit_entities_payload_1k_unique(
|
|
278
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
279
|
+
rows: usize,
|
|
280
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
281
|
+
prepare_transaction_payload_fixture(
|
|
282
|
+
backend,
|
|
283
|
+
rows,
|
|
284
|
+
1024,
|
|
285
|
+
TransactionPayloadPattern::Unique,
|
|
286
|
+
false,
|
|
287
|
+
"entity-payload-1k-unique",
|
|
288
|
+
)
|
|
289
|
+
.await
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
pub async fn prepare_transaction_commit_entities_payload_1k_same(
|
|
293
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
294
|
+
rows: usize,
|
|
295
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
296
|
+
prepare_transaction_payload_fixture(
|
|
297
|
+
backend,
|
|
298
|
+
rows,
|
|
299
|
+
1024,
|
|
300
|
+
TransactionPayloadPattern::Same,
|
|
301
|
+
false,
|
|
302
|
+
"entity-payload-1k-same",
|
|
303
|
+
)
|
|
304
|
+
.await
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
pub async fn prepare_transaction_commit_entities_payload_1k_half_duplicate(
|
|
308
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
309
|
+
rows: usize,
|
|
310
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
311
|
+
prepare_transaction_payload_fixture(
|
|
312
|
+
backend,
|
|
313
|
+
rows,
|
|
314
|
+
1024,
|
|
315
|
+
TransactionPayloadPattern::HalfDuplicate,
|
|
316
|
+
false,
|
|
317
|
+
"entity-payload-1k-half-duplicate",
|
|
318
|
+
)
|
|
319
|
+
.await
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
pub async fn prepare_transaction_commit_entities_metadata_1k_same(
|
|
323
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
324
|
+
rows: usize,
|
|
325
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
326
|
+
prepare_transaction_fixture(
|
|
327
|
+
backend,
|
|
328
|
+
transaction_entity_rows(TransactionEntityRows {
|
|
329
|
+
rows,
|
|
330
|
+
payload_bytes: 0,
|
|
331
|
+
payload_pattern: TransactionPayloadPattern::Unique,
|
|
332
|
+
metadata_pattern: TransactionPayloadPattern::Same,
|
|
333
|
+
untracked: false,
|
|
334
|
+
key_prefix: "entity-metadata-1k-same",
|
|
335
|
+
}),
|
|
336
|
+
)
|
|
337
|
+
.await
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
pub async fn prepare_transaction_commit_entities_payload_16k_unique(
|
|
341
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
342
|
+
rows: usize,
|
|
343
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
344
|
+
prepare_transaction_payload_fixture(
|
|
345
|
+
backend,
|
|
346
|
+
rows,
|
|
347
|
+
16 * 1024,
|
|
348
|
+
TransactionPayloadPattern::Unique,
|
|
349
|
+
false,
|
|
350
|
+
"entity-payload-16k-unique",
|
|
351
|
+
)
|
|
352
|
+
.await
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
pub async fn prepare_transaction_commit_untracked_payload_1k_same(
|
|
356
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
357
|
+
rows: usize,
|
|
358
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
359
|
+
prepare_transaction_payload_fixture(
|
|
360
|
+
backend,
|
|
361
|
+
rows,
|
|
362
|
+
1024,
|
|
363
|
+
TransactionPayloadPattern::Same,
|
|
364
|
+
true,
|
|
365
|
+
"untracked-payload-1k-same",
|
|
366
|
+
)
|
|
367
|
+
.await
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
pub async fn prepare_transaction_update_existing_payload_1k(
|
|
371
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
372
|
+
root_rows: usize,
|
|
373
|
+
update_rows: usize,
|
|
374
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
375
|
+
let fixture = prepare_transaction_payload_fixture(
|
|
376
|
+
backend,
|
|
377
|
+
root_rows,
|
|
378
|
+
1024,
|
|
379
|
+
TransactionPayloadPattern::Unique,
|
|
380
|
+
false,
|
|
381
|
+
"update-existing-root",
|
|
382
|
+
)
|
|
383
|
+
.await?;
|
|
384
|
+
transaction_commit_prepared(&fixture).await?;
|
|
385
|
+
let rows = transaction_entity_rows(TransactionEntityRows {
|
|
386
|
+
rows: update_rows,
|
|
387
|
+
payload_bytes: 1024,
|
|
388
|
+
payload_pattern: TransactionPayloadPattern::Unique,
|
|
389
|
+
metadata_pattern: TransactionPayloadPattern::None,
|
|
390
|
+
untracked: false,
|
|
391
|
+
key_prefix: "update-existing-root",
|
|
392
|
+
});
|
|
393
|
+
Ok(TransactionBenchFixture { rows, ..fixture })
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
pub async fn transaction_commit_prepared(
|
|
397
|
+
fixture: &TransactionBenchFixture,
|
|
398
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
399
|
+
let opened = open_transaction(
|
|
400
|
+
&SessionMode::Pinned {
|
|
401
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
402
|
+
},
|
|
403
|
+
fixture.storage.clone(),
|
|
404
|
+
Arc::clone(&fixture.live_state),
|
|
405
|
+
Arc::clone(&fixture.tracked_state),
|
|
406
|
+
Arc::clone(&fixture.binary_cas),
|
|
407
|
+
Arc::clone(&fixture.commit_store),
|
|
408
|
+
Arc::clone(&fixture.version_ctx),
|
|
409
|
+
Arc::clone(&fixture.catalog_context),
|
|
410
|
+
)
|
|
411
|
+
.await?;
|
|
412
|
+
let mut transaction = opened.transaction;
|
|
413
|
+
let runtime_functions = opened.runtime_functions;
|
|
414
|
+
let started_at = Instant::now();
|
|
415
|
+
if !fixture.rows.is_empty() {
|
|
416
|
+
transaction
|
|
417
|
+
.stage_write(TransactionWrite::Rows {
|
|
418
|
+
mode: TransactionWriteMode::Replace,
|
|
419
|
+
rows: fixture.rows.clone(),
|
|
420
|
+
})
|
|
421
|
+
.await?;
|
|
422
|
+
}
|
|
423
|
+
transaction.commit(&runtime_functions).await?;
|
|
424
|
+
Ok(StorageBenchReport {
|
|
425
|
+
measured_rows: fixture.rows.len(),
|
|
426
|
+
verified_rows: fixture.rows.len(),
|
|
427
|
+
elapsed: started_at.elapsed(),
|
|
428
|
+
})
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
pub async fn transaction_open_empty_prepared(
|
|
432
|
+
fixture: &TransactionBenchFixture,
|
|
433
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
434
|
+
let started_at = Instant::now();
|
|
435
|
+
let opened = open_transaction(
|
|
436
|
+
&SessionMode::Pinned {
|
|
437
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
438
|
+
},
|
|
439
|
+
fixture.storage.clone(),
|
|
440
|
+
Arc::clone(&fixture.live_state),
|
|
441
|
+
Arc::clone(&fixture.tracked_state),
|
|
442
|
+
Arc::clone(&fixture.binary_cas),
|
|
443
|
+
Arc::clone(&fixture.commit_store),
|
|
444
|
+
Arc::clone(&fixture.version_ctx),
|
|
445
|
+
Arc::clone(&fixture.catalog_context),
|
|
446
|
+
)
|
|
447
|
+
.await?;
|
|
448
|
+
let elapsed = started_at.elapsed();
|
|
449
|
+
opened.transaction.rollback().await?;
|
|
450
|
+
Ok(StorageBenchReport {
|
|
451
|
+
measured_rows: 0,
|
|
452
|
+
verified_rows: 0,
|
|
453
|
+
elapsed,
|
|
454
|
+
})
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
pub async fn transaction_stage_only_prepared(
|
|
458
|
+
fixture: &TransactionBenchFixture,
|
|
459
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
460
|
+
let opened = open_transaction(
|
|
461
|
+
&SessionMode::Pinned {
|
|
462
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
463
|
+
},
|
|
464
|
+
fixture.storage.clone(),
|
|
465
|
+
Arc::clone(&fixture.live_state),
|
|
466
|
+
Arc::clone(&fixture.tracked_state),
|
|
467
|
+
Arc::clone(&fixture.binary_cas),
|
|
468
|
+
Arc::clone(&fixture.commit_store),
|
|
469
|
+
Arc::clone(&fixture.version_ctx),
|
|
470
|
+
Arc::clone(&fixture.catalog_context),
|
|
471
|
+
)
|
|
472
|
+
.await?;
|
|
473
|
+
let mut transaction = opened.transaction;
|
|
474
|
+
let started_at = Instant::now();
|
|
475
|
+
if !fixture.rows.is_empty() {
|
|
476
|
+
transaction
|
|
477
|
+
.stage_write(TransactionWrite::Rows {
|
|
478
|
+
mode: TransactionWriteMode::Replace,
|
|
479
|
+
rows: fixture.rows.clone(),
|
|
480
|
+
})
|
|
481
|
+
.await?;
|
|
482
|
+
}
|
|
483
|
+
let elapsed = started_at.elapsed();
|
|
484
|
+
transaction.rollback().await?;
|
|
485
|
+
Ok(StorageBenchReport {
|
|
486
|
+
measured_rows: fixture.rows.len(),
|
|
487
|
+
verified_rows: fixture.rows.len(),
|
|
488
|
+
elapsed,
|
|
489
|
+
})
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
pub async fn prepare_transaction_commit_only(
|
|
493
|
+
fixture: TransactionBenchFixture,
|
|
494
|
+
) -> Result<TransactionCommitOnlyFixture, LixError> {
|
|
495
|
+
let opened = open_transaction(
|
|
496
|
+
&SessionMode::Pinned {
|
|
497
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
498
|
+
},
|
|
499
|
+
fixture.storage.clone(),
|
|
500
|
+
Arc::clone(&fixture.live_state),
|
|
501
|
+
Arc::clone(&fixture.tracked_state),
|
|
502
|
+
Arc::clone(&fixture.binary_cas),
|
|
503
|
+
Arc::clone(&fixture.commit_store),
|
|
504
|
+
Arc::clone(&fixture.version_ctx),
|
|
505
|
+
Arc::clone(&fixture.catalog_context),
|
|
506
|
+
)
|
|
507
|
+
.await?;
|
|
508
|
+
let mut transaction = opened.transaction;
|
|
509
|
+
let runtime_functions = opened.runtime_functions;
|
|
510
|
+
let rows = fixture.rows.len();
|
|
511
|
+
if !fixture.rows.is_empty() {
|
|
512
|
+
transaction
|
|
513
|
+
.stage_write(TransactionWrite::Rows {
|
|
514
|
+
mode: TransactionWriteMode::Replace,
|
|
515
|
+
rows: fixture.rows,
|
|
516
|
+
})
|
|
517
|
+
.await?;
|
|
518
|
+
}
|
|
519
|
+
Ok(TransactionCommitOnlyFixture {
|
|
520
|
+
runtime_functions,
|
|
521
|
+
transaction,
|
|
522
|
+
rows,
|
|
523
|
+
})
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
pub async fn transaction_commit_only_prepared(
|
|
527
|
+
fixture: TransactionCommitOnlyFixture,
|
|
528
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
529
|
+
let rows = fixture.rows;
|
|
530
|
+
let started_at = Instant::now();
|
|
531
|
+
fixture
|
|
532
|
+
.transaction
|
|
533
|
+
.commit(&fixture.runtime_functions)
|
|
534
|
+
.await?;
|
|
535
|
+
Ok(StorageBenchReport {
|
|
536
|
+
measured_rows: rows,
|
|
537
|
+
verified_rows: rows,
|
|
538
|
+
elapsed: started_at.elapsed(),
|
|
539
|
+
})
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
async fn prepare_transaction_payload_fixture(
|
|
543
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
544
|
+
rows: usize,
|
|
545
|
+
payload_bytes: usize,
|
|
546
|
+
payload_pattern: TransactionPayloadPattern,
|
|
547
|
+
untracked: bool,
|
|
548
|
+
key_prefix: &'static str,
|
|
549
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
550
|
+
prepare_transaction_fixture(
|
|
551
|
+
backend,
|
|
552
|
+
transaction_entity_rows(TransactionEntityRows {
|
|
553
|
+
rows,
|
|
554
|
+
payload_bytes,
|
|
555
|
+
payload_pattern,
|
|
556
|
+
metadata_pattern: TransactionPayloadPattern::None,
|
|
557
|
+
untracked,
|
|
558
|
+
key_prefix,
|
|
559
|
+
}),
|
|
560
|
+
)
|
|
561
|
+
.await
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
async fn prepare_transaction_fixture(
|
|
565
|
+
backend: Arc<dyn Backend + Send + Sync>,
|
|
566
|
+
rows: Vec<TransactionWriteRow>,
|
|
567
|
+
) -> Result<TransactionBenchFixture, LixError> {
|
|
568
|
+
let storage = StorageContext::new(backend);
|
|
569
|
+
let tracked_state = Arc::new(TrackedStateContext::new());
|
|
570
|
+
let untracked_state = Arc::new(UntrackedStateContext::new());
|
|
571
|
+
let commit_store = Arc::new(CommitStoreContext::new());
|
|
572
|
+
let live_state = Arc::new(LiveStateContext::new(
|
|
573
|
+
tracked_state.as_ref().clone(),
|
|
574
|
+
untracked_state.as_ref().clone(),
|
|
575
|
+
crate::commit_graph::CommitGraphContext::new(),
|
|
576
|
+
));
|
|
577
|
+
let binary_cas = Arc::new(BinaryCasContext::new());
|
|
578
|
+
let version_ctx = Arc::new(VersionContext::new(untracked_state));
|
|
579
|
+
let catalog_context = Arc::new(CatalogContext::new());
|
|
580
|
+
seed_transaction_visible_schema_rows(storage.clone()).await?;
|
|
581
|
+
Ok(TransactionBenchFixture {
|
|
582
|
+
storage,
|
|
583
|
+
live_state,
|
|
584
|
+
tracked_state,
|
|
585
|
+
binary_cas,
|
|
586
|
+
commit_store,
|
|
587
|
+
version_ctx,
|
|
588
|
+
catalog_context,
|
|
589
|
+
rows,
|
|
590
|
+
})
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
async fn seed_transaction_visible_schema_rows(storage: StorageContext) -> Result<(), LixError> {
|
|
594
|
+
let mut writes = StorageWriteSet::new();
|
|
595
|
+
let rows = crate::schema::seed_schema_definitions()
|
|
596
|
+
.into_iter()
|
|
597
|
+
.cloned()
|
|
598
|
+
.chain(std::iter::once(transaction_entity_schema_definition()))
|
|
599
|
+
.map(|schema| {
|
|
600
|
+
let key = crate::schema::schema_key_from_definition(&schema)
|
|
601
|
+
.expect("seed schema key should derive");
|
|
602
|
+
let snapshot_content = serde_json::json!({ "value": schema }).to_string();
|
|
603
|
+
Ok(crate::untracked_state::UntrackedStateRow {
|
|
604
|
+
entity_id: crate::schema::registered_schema_entity_id(&key.schema_key)
|
|
605
|
+
.expect("registered schema identity should derive"),
|
|
606
|
+
schema_key: "lix_registered_schema".to_string(),
|
|
607
|
+
file_id: None,
|
|
608
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
609
|
+
snapshot_content: Some(snapshot_content),
|
|
610
|
+
metadata: None,
|
|
611
|
+
created_at: "1970-01-01T00:00:00.000Z".to_string(),
|
|
612
|
+
updated_at: "1970-01-01T00:00:00.000Z".to_string(),
|
|
613
|
+
global: true,
|
|
614
|
+
})
|
|
615
|
+
})
|
|
616
|
+
.collect::<Result<Vec<_>, LixError>>()?;
|
|
617
|
+
let mut transaction = storage.begin_write_transaction().await?;
|
|
618
|
+
UntrackedStateContext::new()
|
|
619
|
+
.writer(&mut writes)
|
|
620
|
+
.stage_rows(rows.iter().map(|row| row.as_ref()))?;
|
|
621
|
+
writes.apply(&mut transaction.as_mut()).await?;
|
|
622
|
+
transaction.commit().await
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
fn transaction_entity_schema_definition() -> serde_json::Value {
|
|
626
|
+
serde_json::json!({
|
|
627
|
+
"x-lix-key": TRANSACTION_BENCH_SCHEMA_KEY,
|
|
628
|
+
"type": "object",
|
|
629
|
+
"properties": {
|
|
630
|
+
"value": {
|
|
631
|
+
"anyOf": [
|
|
632
|
+
{ "type": "string" },
|
|
633
|
+
{ "type": "object" },
|
|
634
|
+
{ "type": "array" },
|
|
635
|
+
{ "type": "number" },
|
|
636
|
+
{ "type": "boolean" },
|
|
637
|
+
{ "type": "null" }
|
|
638
|
+
]
|
|
639
|
+
}
|
|
640
|
+
},
|
|
641
|
+
"required": ["value"],
|
|
642
|
+
"additionalProperties": false
|
|
643
|
+
})
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
#[derive(Debug, Clone, Copy)]
|
|
647
|
+
enum TransactionPayloadPattern {
|
|
648
|
+
None,
|
|
649
|
+
Unique,
|
|
650
|
+
Same,
|
|
651
|
+
HalfDuplicate,
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
struct TransactionEntityRows {
|
|
655
|
+
rows: usize,
|
|
656
|
+
payload_bytes: usize,
|
|
657
|
+
payload_pattern: TransactionPayloadPattern,
|
|
658
|
+
metadata_pattern: TransactionPayloadPattern,
|
|
659
|
+
untracked: bool,
|
|
660
|
+
key_prefix: &'static str,
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
fn transaction_entity_rows(config: TransactionEntityRows) -> Vec<TransactionWriteRow> {
|
|
664
|
+
(0..config.rows)
|
|
665
|
+
.map(|index| {
|
|
666
|
+
let key = format!("{}-{index:06}", config.key_prefix);
|
|
667
|
+
let value_index = payload_pattern_index(config.payload_pattern, index);
|
|
668
|
+
let metadata_index = payload_pattern_index(config.metadata_pattern, index);
|
|
669
|
+
TransactionWriteRow {
|
|
670
|
+
entity_id: Some(EntityIdentity::single(key.clone())),
|
|
671
|
+
schema_key: TRANSACTION_BENCH_SCHEMA_KEY.to_string(),
|
|
672
|
+
file_id: None,
|
|
673
|
+
snapshot: Some(transaction_snapshot_json(
|
|
674
|
+
&key,
|
|
675
|
+
value_index,
|
|
676
|
+
config.payload_bytes,
|
|
677
|
+
)),
|
|
678
|
+
metadata: transaction_metadata(config.metadata_pattern, metadata_index),
|
|
679
|
+
origin: None,
|
|
680
|
+
created_at: None,
|
|
681
|
+
updated_at: None,
|
|
682
|
+
global: true,
|
|
683
|
+
change_id: None,
|
|
684
|
+
commit_id: None,
|
|
685
|
+
untracked: config.untracked,
|
|
686
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
687
|
+
}
|
|
688
|
+
})
|
|
689
|
+
.collect()
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
fn transaction_registered_schema_row() -> TransactionWriteRow {
|
|
693
|
+
let schema = serde_json::json!({
|
|
694
|
+
"x-lix-key": "bench_transaction_schema",
|
|
695
|
+
"x-lix-primary-key": ["/id"],
|
|
696
|
+
"type": "object",
|
|
697
|
+
"properties": {
|
|
698
|
+
"id": { "type": "string" },
|
|
699
|
+
"value": { "type": "string" }
|
|
700
|
+
},
|
|
701
|
+
"required": ["id", "value"],
|
|
702
|
+
"additionalProperties": false
|
|
703
|
+
});
|
|
704
|
+
let key =
|
|
705
|
+
crate::schema::schema_key_from_definition(&schema).expect("seed schema key should derive");
|
|
706
|
+
TransactionWriteRow {
|
|
707
|
+
entity_id: Some(
|
|
708
|
+
crate::schema::registered_schema_entity_id(&key.schema_key)
|
|
709
|
+
.expect("registered schema identity should derive"),
|
|
710
|
+
),
|
|
711
|
+
schema_key: "lix_registered_schema".to_string(),
|
|
712
|
+
file_id: None,
|
|
713
|
+
snapshot: Some(TransactionJson::from_value_unchecked(
|
|
714
|
+
serde_json::json!({ "value": schema }),
|
|
715
|
+
)),
|
|
716
|
+
metadata: None,
|
|
717
|
+
origin: None,
|
|
718
|
+
created_at: None,
|
|
719
|
+
updated_at: None,
|
|
720
|
+
global: true,
|
|
721
|
+
change_id: None,
|
|
722
|
+
commit_id: None,
|
|
723
|
+
untracked: false,
|
|
724
|
+
version_id: crate::GLOBAL_VERSION_ID.to_string(),
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
fn transaction_snapshot_json(
|
|
729
|
+
_key: &str,
|
|
730
|
+
payload_index: usize,
|
|
731
|
+
target_bytes: usize,
|
|
732
|
+
) -> TransactionJson {
|
|
733
|
+
let base_value = format!("/entities/{payload_index}/value");
|
|
734
|
+
let value = if target_bytes == 0 {
|
|
735
|
+
base_value
|
|
736
|
+
} else {
|
|
737
|
+
let current = serde_json::json!({
|
|
738
|
+
"value": base_value,
|
|
739
|
+
})
|
|
740
|
+
.to_string()
|
|
741
|
+
.len();
|
|
742
|
+
let padding = target_bytes.saturating_sub(current);
|
|
743
|
+
format!("{base_value}:{}", "x".repeat(padding))
|
|
744
|
+
};
|
|
745
|
+
let mut object = serde_json::Map::new();
|
|
746
|
+
object.insert("value".to_string(), serde_json::Value::String(value));
|
|
747
|
+
TransactionJson::from_value_unchecked(serde_json::Value::Object(object))
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
fn transaction_metadata(
|
|
751
|
+
pattern: TransactionPayloadPattern,
|
|
752
|
+
metadata_index: usize,
|
|
753
|
+
) -> Option<TransactionJson> {
|
|
754
|
+
match pattern {
|
|
755
|
+
TransactionPayloadPattern::None => None,
|
|
756
|
+
TransactionPayloadPattern::Unique
|
|
757
|
+
| TransactionPayloadPattern::Same
|
|
758
|
+
| TransactionPayloadPattern::HalfDuplicate => {
|
|
759
|
+
let mut object = serde_json::Map::new();
|
|
760
|
+
object.insert(
|
|
761
|
+
"source".to_string(),
|
|
762
|
+
serde_json::Value::String("transaction-bench".to_string()),
|
|
763
|
+
);
|
|
764
|
+
object.insert(
|
|
765
|
+
"metadata_index".to_string(),
|
|
766
|
+
serde_json::Value::String(metadata_index.to_string()),
|
|
767
|
+
);
|
|
768
|
+
pad_json_object(&mut object, 1024);
|
|
769
|
+
Some(TransactionJson::from_value_unchecked(
|
|
770
|
+
serde_json::Value::Object(object),
|
|
771
|
+
))
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
fn payload_pattern_index(pattern: TransactionPayloadPattern, index: usize) -> usize {
|
|
777
|
+
match pattern {
|
|
778
|
+
TransactionPayloadPattern::None | TransactionPayloadPattern::Unique => index,
|
|
779
|
+
TransactionPayloadPattern::Same => 0,
|
|
780
|
+
TransactionPayloadPattern::HalfDuplicate => index % 2,
|
|
781
|
+
}
|
|
782
|
+
}
|
|
125
783
|
|
|
126
784
|
pub async fn storage_api_write_kv_batch_puts(
|
|
127
785
|
backend: Arc<dyn Backend + Send + Sync>,
|
|
@@ -689,7 +1347,7 @@ fn storage_api_updated_value(index: usize) -> Vec<u8> {
|
|
|
689
1347
|
|
|
690
1348
|
pub struct TrackedStateWriteRootFixture {
|
|
691
1349
|
context: TrackedStateContext,
|
|
692
|
-
rows: Vec<
|
|
1350
|
+
rows: Vec<MaterializedTrackedStateRow>,
|
|
693
1351
|
}
|
|
694
1352
|
|
|
695
1353
|
pub struct TrackedStateReadFixture {
|
|
@@ -702,7 +1360,7 @@ pub struct TrackedStateReadFixture {
|
|
|
702
1360
|
|
|
703
1361
|
pub struct TrackedStateUpdateFixture {
|
|
704
1362
|
context: TrackedStateContext,
|
|
705
|
-
rows: Vec<
|
|
1363
|
+
rows: Vec<MaterializedTrackedStateRow>,
|
|
706
1364
|
}
|
|
707
1365
|
|
|
708
1366
|
pub struct TrackedStateDiffFixture {
|
|
@@ -712,6 +1370,32 @@ pub struct TrackedStateDiffFixture {
|
|
|
712
1370
|
expected_entries: usize,
|
|
713
1371
|
}
|
|
714
1372
|
|
|
1373
|
+
pub struct TrackedStateMaterializeFixture {
|
|
1374
|
+
context: TrackedStateContext,
|
|
1375
|
+
commit_id: String,
|
|
1376
|
+
expected_rows: usize,
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
#[derive(Clone)]
|
|
1380
|
+
pub struct JsonPointerStorageRow {
|
|
1381
|
+
pub path: String,
|
|
1382
|
+
pub value_json: String,
|
|
1383
|
+
pub updated_value_json: String,
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
pub struct JsonPointerTrackedStateReadFixture {
|
|
1387
|
+
context: TrackedStateContext,
|
|
1388
|
+
rows: Vec<JsonPointerStorageRow>,
|
|
1389
|
+
commit_id: String,
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
pub struct JsonPointerTrackedStateDiffFixture {
|
|
1393
|
+
context: TrackedStateContext,
|
|
1394
|
+
left_commit_id: String,
|
|
1395
|
+
right_commit_id: String,
|
|
1396
|
+
expected_entries: usize,
|
|
1397
|
+
}
|
|
1398
|
+
|
|
715
1399
|
pub struct UntrackedStateWriteFixture {
|
|
716
1400
|
context: UntrackedStateContext,
|
|
717
1401
|
rows: Vec<MaterializedUntrackedStateRow>,
|
|
@@ -725,20 +1409,25 @@ pub struct UntrackedStateReadFixture {
|
|
|
725
1409
|
}
|
|
726
1410
|
|
|
727
1411
|
pub struct ChangelogAppendFixture {
|
|
728
|
-
context:
|
|
729
|
-
changes: Vec<
|
|
1412
|
+
context: CommitStoreContext,
|
|
1413
|
+
changes: Vec<MaterializedChange>,
|
|
730
1414
|
}
|
|
731
1415
|
|
|
732
1416
|
pub struct ChangelogReadFixture {
|
|
733
|
-
context:
|
|
1417
|
+
context: CommitStoreContext,
|
|
734
1418
|
rows: usize,
|
|
735
1419
|
}
|
|
736
1420
|
|
|
737
1421
|
pub struct ChangelogCodecFixture {
|
|
738
|
-
changes: Vec<
|
|
1422
|
+
changes: Vec<Change>,
|
|
739
1423
|
encoded_changes: Vec<Vec<u8>>,
|
|
740
1424
|
}
|
|
741
1425
|
|
|
1426
|
+
pub struct CommitGraphReadFixture {
|
|
1427
|
+
head_commit_id: String,
|
|
1428
|
+
rows: usize,
|
|
1429
|
+
}
|
|
1430
|
+
|
|
742
1431
|
pub struct BinaryCasWriteFixture {
|
|
743
1432
|
context: BinaryCasContext,
|
|
744
1433
|
file_ids: Vec<String>,
|
|
@@ -839,97 +1528,161 @@ pub async fn prepare_tracked_state_read_file_selective(
|
|
|
839
1528
|
})
|
|
840
1529
|
}
|
|
841
1530
|
|
|
842
|
-
pub async fn
|
|
843
|
-
backend: &Arc<dyn Backend + Send + Sync>,
|
|
844
|
-
fixture: &TrackedStateReadFixture,
|
|
845
|
-
) -> Result<StorageBenchReport, LixError> {
|
|
846
|
-
let mut verified_rows = 0;
|
|
847
|
-
let mut reader = fixture
|
|
848
|
-
.context
|
|
849
|
-
.reader(StorageContext::new(Arc::clone(backend)));
|
|
850
|
-
for index in 0..fixture.rows {
|
|
851
|
-
if reader
|
|
852
|
-
.load_row_at_commit(
|
|
853
|
-
&fixture.commit_id,
|
|
854
|
-
&TrackedStateRowRequest {
|
|
855
|
-
schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
|
|
856
|
-
entity_id: EntityIdentity::single(entity_id(
|
|
857
|
-
"tracked",
|
|
858
|
-
index,
|
|
859
|
-
fixture.key_pattern,
|
|
860
|
-
)),
|
|
861
|
-
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
862
|
-
},
|
|
863
|
-
)
|
|
864
|
-
.await?
|
|
865
|
-
.is_some()
|
|
866
|
-
{
|
|
867
|
-
verified_rows += 1;
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
Ok(report(fixture.rows, verified_rows, Duration::ZERO))
|
|
871
|
-
}
|
|
872
|
-
|
|
873
|
-
pub async fn tracked_state_read_point_hit_constant_prepared(
|
|
1531
|
+
pub async fn prepare_tracked_state_read_after_update_rows(
|
|
874
1532
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
875
|
-
|
|
876
|
-
|
|
1533
|
+
config: StorageBenchConfig,
|
|
1534
|
+
updated_rows: usize,
|
|
1535
|
+
) -> Result<TrackedStateReadFixture, LixError> {
|
|
1536
|
+
let fixture = prepare_tracked_state_update_rows(backend, config, updated_rows).await?;
|
|
1537
|
+
tracked_state_update_existing_prepared(backend, &fixture).await?;
|
|
1538
|
+
Ok(TrackedStateReadFixture {
|
|
1539
|
+
context: fixture.context,
|
|
1540
|
+
rows: config.rows,
|
|
1541
|
+
commit_id: "bench-tracked-child".to_string(),
|
|
1542
|
+
key_pattern: config.key_pattern,
|
|
1543
|
+
selectivity: config.selectivity,
|
|
1544
|
+
})
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
pub async fn prepare_tracked_state_read_delta_chain(
|
|
1548
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1549
|
+
config: StorageBenchConfig,
|
|
1550
|
+
delta_commits: usize,
|
|
1551
|
+
updated_rows_per_commit: usize,
|
|
1552
|
+
) -> Result<TrackedStateReadFixture, LixError> {
|
|
1553
|
+
let (context, final_commit_id) =
|
|
1554
|
+
write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
|
|
1555
|
+
Ok(TrackedStateReadFixture {
|
|
1556
|
+
context,
|
|
1557
|
+
rows: config.rows,
|
|
1558
|
+
commit_id: final_commit_id,
|
|
1559
|
+
key_pattern: config.key_pattern,
|
|
1560
|
+
selectivity: config.selectivity,
|
|
1561
|
+
})
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
pub async fn prepare_tracked_state_read_materialized_delta_chain(
|
|
1565
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1566
|
+
config: StorageBenchConfig,
|
|
1567
|
+
delta_commits: usize,
|
|
1568
|
+
updated_rows_per_commit: usize,
|
|
1569
|
+
) -> Result<TrackedStateReadFixture, LixError> {
|
|
1570
|
+
let (context, final_commit_id) =
|
|
1571
|
+
write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
|
|
1572
|
+
materialize_tracked_root(backend, &context, &final_commit_id).await?;
|
|
1573
|
+
Ok(TrackedStateReadFixture {
|
|
1574
|
+
context,
|
|
1575
|
+
rows: config.rows,
|
|
1576
|
+
commit_id: final_commit_id,
|
|
1577
|
+
key_pattern: config.key_pattern,
|
|
1578
|
+
selectivity: config.selectivity,
|
|
1579
|
+
})
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
pub async fn prepare_tracked_state_materialize_delta_chain(
|
|
1583
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1584
|
+
config: StorageBenchConfig,
|
|
1585
|
+
delta_commits: usize,
|
|
1586
|
+
updated_rows_per_commit: usize,
|
|
1587
|
+
) -> Result<TrackedStateMaterializeFixture, LixError> {
|
|
1588
|
+
let (context, final_commit_id) =
|
|
1589
|
+
write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
|
|
1590
|
+
Ok(TrackedStateMaterializeFixture {
|
|
1591
|
+
context,
|
|
1592
|
+
commit_id: final_commit_id,
|
|
1593
|
+
expected_rows: config.rows,
|
|
1594
|
+
})
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
fn tracked_point_hit_requests(
|
|
1598
|
+
rows: usize,
|
|
1599
|
+
key_pattern: StorageBenchKeyPattern,
|
|
1600
|
+
) -> Vec<TrackedStateRowRequest> {
|
|
1601
|
+
(0..rows)
|
|
1602
|
+
.map(|index| TrackedStateRowRequest {
|
|
1603
|
+
schema_key: tracked_schema_key(index, StorageBenchSelectivity::Percent100),
|
|
1604
|
+
entity_id: EntityIdentity::single(entity_id("tracked", index, key_pattern)),
|
|
1605
|
+
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
1606
|
+
})
|
|
1607
|
+
.collect()
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
fn tracked_point_miss_requests(
|
|
1611
|
+
rows: usize,
|
|
1612
|
+
selectivity: StorageBenchSelectivity,
|
|
1613
|
+
) -> Vec<TrackedStateRowRequest> {
|
|
1614
|
+
(0..rows)
|
|
1615
|
+
.map(|index| TrackedStateRowRequest {
|
|
1616
|
+
schema_key: tracked_schema_key(index, selectivity),
|
|
1617
|
+
entity_id: EntityIdentity::single(format!("missing-{index}")),
|
|
1618
|
+
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
1619
|
+
})
|
|
1620
|
+
.collect()
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
fn tracked_point_miss_requests_for_schema(
|
|
1624
|
+
rows: usize,
|
|
1625
|
+
schema_key: &str,
|
|
1626
|
+
) -> Vec<TrackedStateRowRequest> {
|
|
1627
|
+
(0..rows)
|
|
1628
|
+
.map(|index| TrackedStateRowRequest {
|
|
1629
|
+
schema_key: schema_key.to_string(),
|
|
1630
|
+
entity_id: EntityIdentity::single(format!("missing-{index}")),
|
|
1631
|
+
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
1632
|
+
})
|
|
1633
|
+
.collect()
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
pub async fn tracked_state_read_point_hit_prepared(
|
|
1637
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1638
|
+
fixture: &TrackedStateReadFixture,
|
|
877
1639
|
) -> Result<StorageBenchReport, LixError> {
|
|
878
|
-
let mut verified_rows = 0;
|
|
879
1640
|
let mut reader = fixture
|
|
880
1641
|
.context
|
|
881
1642
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
1643
|
+
let requests = tracked_point_hit_requests(fixture.rows, fixture.key_pattern);
|
|
1644
|
+
let verified_rows = reader
|
|
1645
|
+
.load_rows_at_commit(&fixture.commit_id, &requests)
|
|
1646
|
+
.await?
|
|
1647
|
+
.into_iter()
|
|
1648
|
+
.filter(Option::is_some)
|
|
1649
|
+
.count();
|
|
1650
|
+
Ok(report(fixture.rows, verified_rows, Duration::ZERO))
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
pub async fn tracked_state_read_point_hit_constant_prepared(
|
|
1654
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1655
|
+
fixture: &TrackedStateReadFixture,
|
|
1656
|
+
measured_reads: usize,
|
|
1657
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
1658
|
+
let measured_rows = measured_reads.min(fixture.rows);
|
|
1659
|
+
let mut reader = fixture
|
|
1660
|
+
.context
|
|
1661
|
+
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1662
|
+
let requests = tracked_point_hit_requests(measured_rows, fixture.key_pattern);
|
|
1663
|
+
let verified_rows = reader
|
|
1664
|
+
.load_rows_at_commit(&fixture.commit_id, &requests)
|
|
1665
|
+
.await?
|
|
1666
|
+
.into_iter()
|
|
1667
|
+
.filter(Option::is_some)
|
|
1668
|
+
.count();
|
|
1669
|
+
Ok(report(measured_rows, verified_rows, Duration::ZERO))
|
|
907
1670
|
}
|
|
908
1671
|
|
|
909
1672
|
pub async fn tracked_state_read_point_miss_prepared(
|
|
910
1673
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
911
1674
|
fixture: &TrackedStateReadFixture,
|
|
912
1675
|
) -> Result<StorageBenchReport, LixError> {
|
|
913
|
-
let mut misses = 0;
|
|
914
1676
|
let mut reader = fixture
|
|
915
1677
|
.context
|
|
916
1678
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
925
|
-
},
|
|
926
|
-
)
|
|
927
|
-
.await?
|
|
928
|
-
.is_none()
|
|
929
|
-
{
|
|
930
|
-
misses += 1;
|
|
931
|
-
}
|
|
932
|
-
}
|
|
1679
|
+
let requests = tracked_point_miss_requests_for_schema(fixture.rows, TRACKED_MATCH_SCHEMA_KEY);
|
|
1680
|
+
let misses = reader
|
|
1681
|
+
.load_rows_at_commit(&fixture.commit_id, &requests)
|
|
1682
|
+
.await?
|
|
1683
|
+
.into_iter()
|
|
1684
|
+
.filter(Option::is_none)
|
|
1685
|
+
.count();
|
|
933
1686
|
Ok(report(fixture.rows, misses, Duration::ZERO))
|
|
934
1687
|
}
|
|
935
1688
|
|
|
@@ -1261,6 +2014,22 @@ pub async fn prepare_tracked_state_diff_update_rows(
|
|
|
1261
2014
|
})
|
|
1262
2015
|
}
|
|
1263
2016
|
|
|
2017
|
+
pub async fn prepare_tracked_state_diff_delta_chain(
|
|
2018
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2019
|
+
config: StorageBenchConfig,
|
|
2020
|
+
delta_commits: usize,
|
|
2021
|
+
updated_rows_per_commit: usize,
|
|
2022
|
+
) -> Result<TrackedStateDiffFixture, LixError> {
|
|
2023
|
+
let (context, final_commit_id) =
|
|
2024
|
+
write_tracked_delta_chain(backend, config, delta_commits, updated_rows_per_commit).await?;
|
|
2025
|
+
Ok(TrackedStateDiffFixture {
|
|
2026
|
+
context,
|
|
2027
|
+
left_commit_id: "bench-tracked-base".to_string(),
|
|
2028
|
+
right_commit_id: final_commit_id,
|
|
2029
|
+
expected_entries: updated_rows_per_commit.min(config.rows),
|
|
2030
|
+
})
|
|
2031
|
+
}
|
|
2032
|
+
|
|
1264
2033
|
pub async fn prepare_tracked_state_diff_tombstone_rows(
|
|
1265
2034
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1266
2035
|
config: StorageBenchConfig,
|
|
@@ -1276,24 +2045,315 @@ pub async fn prepare_tracked_state_diff_tombstone_rows(
|
|
|
1276
2045
|
})
|
|
1277
2046
|
}
|
|
1278
2047
|
|
|
1279
|
-
pub async fn prepare_tracked_state_diff_equal(
|
|
2048
|
+
pub async fn prepare_tracked_state_diff_equal(
|
|
2049
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2050
|
+
config: StorageBenchConfig,
|
|
2051
|
+
) -> Result<TrackedStateDiffFixture, LixError> {
|
|
2052
|
+
let context = TrackedStateContext::new();
|
|
2053
|
+
let rows = tracked_rows(config, "bench-tracked-parent");
|
|
2054
|
+
write_tracked_root(backend, &context, "bench-tracked-parent", None, &rows).await?;
|
|
2055
|
+
Ok(TrackedStateDiffFixture {
|
|
2056
|
+
context,
|
|
2057
|
+
left_commit_id: "bench-tracked-parent".to_string(),
|
|
2058
|
+
right_commit_id: "bench-tracked-parent".to_string(),
|
|
2059
|
+
expected_entries: 0,
|
|
2060
|
+
})
|
|
2061
|
+
}
|
|
2062
|
+
|
|
2063
|
+
pub async fn tracked_state_diff_commits_prepared(
|
|
2064
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2065
|
+
fixture: &TrackedStateDiffFixture,
|
|
2066
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2067
|
+
let mut reader = fixture
|
|
2068
|
+
.context
|
|
2069
|
+
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2070
|
+
let diff = reader
|
|
2071
|
+
.diff_commits(
|
|
2072
|
+
&fixture.left_commit_id,
|
|
2073
|
+
&fixture.right_commit_id,
|
|
2074
|
+
&TrackedStateDiffRequest::default(),
|
|
2075
|
+
)
|
|
2076
|
+
.await?;
|
|
2077
|
+
Ok(report(
|
|
2078
|
+
fixture.expected_entries,
|
|
2079
|
+
diff.entries.len(),
|
|
2080
|
+
Duration::ZERO,
|
|
2081
|
+
))
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
pub async fn tracked_state_materialize_root_prepared(
|
|
2085
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2086
|
+
fixture: &TrackedStateMaterializeFixture,
|
|
2087
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2088
|
+
materialize_tracked_root(backend, &fixture.context, &fixture.commit_id).await?;
|
|
2089
|
+
Ok(report(
|
|
2090
|
+
fixture.expected_rows,
|
|
2091
|
+
fixture.expected_rows,
|
|
2092
|
+
Duration::ZERO,
|
|
2093
|
+
))
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
pub async fn prepare_json_pointer_tracked_state_write_root(
|
|
2097
|
+
rows: &[JsonPointerStorageRow],
|
|
2098
|
+
) -> Result<TrackedStateWriteRootFixture, LixError> {
|
|
2099
|
+
Ok(TrackedStateWriteRootFixture {
|
|
2100
|
+
context: TrackedStateContext::new(),
|
|
2101
|
+
rows: json_pointer_tracked_rows(rows, "json-pointer-base", false),
|
|
2102
|
+
})
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
pub async fn prepare_json_pointer_tracked_state_read(
|
|
2106
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2107
|
+
rows: &[JsonPointerStorageRow],
|
|
2108
|
+
) -> Result<JsonPointerTrackedStateReadFixture, LixError> {
|
|
2109
|
+
let context = TrackedStateContext::new();
|
|
2110
|
+
let materialized_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
|
|
2111
|
+
write_tracked_root(
|
|
2112
|
+
backend,
|
|
2113
|
+
&context,
|
|
2114
|
+
"json-pointer-base",
|
|
2115
|
+
None,
|
|
2116
|
+
&materialized_rows,
|
|
2117
|
+
)
|
|
2118
|
+
.await?;
|
|
2119
|
+
Ok(JsonPointerTrackedStateReadFixture {
|
|
2120
|
+
context,
|
|
2121
|
+
rows: rows.to_vec(),
|
|
2122
|
+
commit_id: "json-pointer-base".to_string(),
|
|
2123
|
+
})
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
pub async fn prepare_json_pointer_tracked_state_diff_update_rows(
|
|
2127
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2128
|
+
rows: &[JsonPointerStorageRow],
|
|
2129
|
+
updated_rows: usize,
|
|
2130
|
+
) -> Result<JsonPointerTrackedStateDiffFixture, LixError> {
|
|
2131
|
+
let context = TrackedStateContext::new();
|
|
2132
|
+
let base_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
|
|
2133
|
+
write_tracked_root(backend, &context, "json-pointer-base", None, &base_rows).await?;
|
|
2134
|
+
let child_rows = json_pointer_tracked_rows(
|
|
2135
|
+
&rows[..updated_rows.min(rows.len())],
|
|
2136
|
+
"json-pointer-child",
|
|
2137
|
+
true,
|
|
2138
|
+
);
|
|
2139
|
+
write_tracked_root(
|
|
2140
|
+
backend,
|
|
2141
|
+
&context,
|
|
2142
|
+
"json-pointer-child",
|
|
2143
|
+
Some("json-pointer-base"),
|
|
2144
|
+
&child_rows,
|
|
2145
|
+
)
|
|
2146
|
+
.await?;
|
|
2147
|
+
Ok(JsonPointerTrackedStateDiffFixture {
|
|
2148
|
+
context,
|
|
2149
|
+
left_commit_id: "json-pointer-base".to_string(),
|
|
2150
|
+
right_commit_id: "json-pointer-child".to_string(),
|
|
2151
|
+
expected_entries: child_rows.len(),
|
|
2152
|
+
})
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
pub async fn json_pointer_tracked_state_get_many_prepared(
|
|
2156
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2157
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2158
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2159
|
+
let mut reader = fixture
|
|
2160
|
+
.context
|
|
2161
|
+
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2162
|
+
let requests = fixture
|
|
2163
|
+
.rows
|
|
2164
|
+
.iter()
|
|
2165
|
+
.map(|row| TrackedStateRowRequest {
|
|
2166
|
+
schema_key: "json_pointer".to_string(),
|
|
2167
|
+
entity_id: EntityIdentity::single(row.path.as_str()),
|
|
2168
|
+
file_id: NullableKeyFilter::Null,
|
|
2169
|
+
})
|
|
2170
|
+
.collect::<Vec<_>>();
|
|
2171
|
+
let verified_rows = reader
|
|
2172
|
+
.load_rows_at_commit(&fixture.commit_id, &requests)
|
|
2173
|
+
.await?
|
|
2174
|
+
.into_iter()
|
|
2175
|
+
.filter(Option::is_some)
|
|
2176
|
+
.count();
|
|
2177
|
+
Ok(report(fixture.rows.len(), verified_rows, Duration::ZERO))
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
pub async fn json_pointer_tracked_state_get_many_missing_prepared(
|
|
2181
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2182
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2183
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2184
|
+
let mut reader = fixture
|
|
2185
|
+
.context
|
|
2186
|
+
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2187
|
+
let requests = fixture
|
|
2188
|
+
.rows
|
|
2189
|
+
.iter()
|
|
2190
|
+
.map(|row| TrackedStateRowRequest {
|
|
2191
|
+
schema_key: "json_pointer".to_string(),
|
|
2192
|
+
entity_id: EntityIdentity::single(format!("missing{}", row.path)),
|
|
2193
|
+
file_id: NullableKeyFilter::Null,
|
|
2194
|
+
})
|
|
2195
|
+
.collect::<Vec<_>>();
|
|
2196
|
+
let verified_rows = reader
|
|
2197
|
+
.load_rows_at_commit(&fixture.commit_id, &requests)
|
|
2198
|
+
.await?
|
|
2199
|
+
.into_iter()
|
|
2200
|
+
.filter(Option::is_none)
|
|
2201
|
+
.count();
|
|
2202
|
+
Ok(report(fixture.rows.len(), verified_rows, Duration::ZERO))
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
pub async fn json_pointer_tracked_state_scan_keys_only_prepared(
|
|
2206
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2207
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2208
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2209
|
+
json_pointer_scan_with_projection(
|
|
2210
|
+
backend,
|
|
2211
|
+
fixture,
|
|
2212
|
+
TrackedStateProjection {
|
|
2213
|
+
columns: vec!["entity_id".to_string()],
|
|
2214
|
+
},
|
|
2215
|
+
)
|
|
2216
|
+
.await
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
pub async fn json_pointer_tracked_state_scan_headers_only_prepared(
|
|
2220
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2221
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2222
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2223
|
+
json_pointer_scan_with_projection(
|
|
2224
|
+
backend,
|
|
2225
|
+
fixture,
|
|
2226
|
+
TrackedStateProjection {
|
|
2227
|
+
columns: tracked_state_header_columns(),
|
|
2228
|
+
},
|
|
2229
|
+
)
|
|
2230
|
+
.await
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
pub async fn json_pointer_tracked_state_scan_full_rows_prepared(
|
|
2234
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2235
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2236
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2237
|
+
json_pointer_scan_with_projection(backend, fixture, TrackedStateProjection::default()).await
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
pub async fn json_pointer_tracked_state_prefix_scan_schema_prepared(
|
|
2241
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2242
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2243
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2244
|
+
json_pointer_scan_with_projection(backend, fixture, TrackedStateProjection::default()).await
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
pub async fn json_pointer_tracked_state_prefix_scan_schema_file_null_prepared(
|
|
2248
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2249
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2250
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2251
|
+
json_pointer_scan_with_projection(backend, fixture, TrackedStateProjection::default()).await
|
|
2252
|
+
}
|
|
2253
|
+
|
|
2254
|
+
async fn json_pointer_scan_with_projection(
|
|
2255
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2256
|
+
fixture: &JsonPointerTrackedStateReadFixture,
|
|
2257
|
+
projection: TrackedStateProjection,
|
|
2258
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2259
|
+
let mut reader = fixture
|
|
2260
|
+
.context
|
|
2261
|
+
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2262
|
+
let verified_rows = reader
|
|
2263
|
+
.scan_rows_at_commit(
|
|
2264
|
+
&fixture.commit_id,
|
|
2265
|
+
&TrackedStateScanRequest {
|
|
2266
|
+
filter: TrackedStateFilter {
|
|
2267
|
+
schema_keys: vec!["json_pointer".to_string()],
|
|
2268
|
+
file_ids: vec![NullableKeyFilter::Null],
|
|
2269
|
+
..Default::default()
|
|
2270
|
+
},
|
|
2271
|
+
projection,
|
|
2272
|
+
..Default::default()
|
|
2273
|
+
},
|
|
2274
|
+
)
|
|
2275
|
+
.await?
|
|
2276
|
+
.len();
|
|
2277
|
+
Ok(report(fixture.rows.len(), verified_rows, Duration::ZERO))
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
pub async fn prepare_json_pointer_tracked_state_update_rows(
|
|
2281
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2282
|
+
rows: &[JsonPointerStorageRow],
|
|
2283
|
+
updated_rows: usize,
|
|
2284
|
+
) -> Result<TrackedStateUpdateFixture, LixError> {
|
|
2285
|
+
let context = TrackedStateContext::new();
|
|
2286
|
+
let base_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
|
|
2287
|
+
write_tracked_root(backend, &context, "json-pointer-base", None, &base_rows).await?;
|
|
2288
|
+
let child_rows = json_pointer_tracked_rows(
|
|
2289
|
+
&rows[..updated_rows.min(rows.len())],
|
|
2290
|
+
"json-pointer-child",
|
|
2291
|
+
true,
|
|
2292
|
+
);
|
|
2293
|
+
Ok(TrackedStateUpdateFixture {
|
|
2294
|
+
context,
|
|
2295
|
+
rows: child_rows,
|
|
2296
|
+
})
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
pub async fn prepare_json_pointer_tracked_state_tombstone_rows(
|
|
2300
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2301
|
+
rows: &[JsonPointerStorageRow],
|
|
2302
|
+
tombstone_rows: usize,
|
|
2303
|
+
) -> Result<TrackedStateUpdateFixture, LixError> {
|
|
2304
|
+
let context = TrackedStateContext::new();
|
|
2305
|
+
let base_rows = json_pointer_tracked_rows(rows, "json-pointer-base", false);
|
|
2306
|
+
write_tracked_root(backend, &context, "json-pointer-base", None, &base_rows).await?;
|
|
2307
|
+
let mut child_rows = json_pointer_tracked_rows(
|
|
2308
|
+
&rows[..tombstone_rows.min(rows.len())],
|
|
2309
|
+
"json-pointer-child",
|
|
2310
|
+
true,
|
|
2311
|
+
);
|
|
2312
|
+
for row in &mut child_rows {
|
|
2313
|
+
row.snapshot_content = None;
|
|
2314
|
+
}
|
|
2315
|
+
Ok(TrackedStateUpdateFixture {
|
|
2316
|
+
context,
|
|
2317
|
+
rows: child_rows,
|
|
2318
|
+
})
|
|
2319
|
+
}
|
|
2320
|
+
|
|
2321
|
+
pub async fn prepare_json_pointer_tracked_state_diff_delta_chain(
|
|
2322
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2323
|
+
rows: &[JsonPointerStorageRow],
|
|
2324
|
+
delta_commits: usize,
|
|
2325
|
+
updated_rows_per_commit: usize,
|
|
2326
|
+
) -> Result<JsonPointerTrackedStateDiffFixture, LixError> {
|
|
2327
|
+
let (context, final_commit_id) =
|
|
2328
|
+
write_json_pointer_delta_chain(backend, rows, delta_commits, updated_rows_per_commit)
|
|
2329
|
+
.await?;
|
|
2330
|
+
Ok(JsonPointerTrackedStateDiffFixture {
|
|
2331
|
+
context,
|
|
2332
|
+
left_commit_id: "json-pointer-base".to_string(),
|
|
2333
|
+
right_commit_id: final_commit_id,
|
|
2334
|
+
expected_entries: updated_rows_per_commit.min(rows.len()),
|
|
2335
|
+
})
|
|
2336
|
+
}
|
|
2337
|
+
|
|
2338
|
+
pub async fn prepare_json_pointer_tracked_state_materialize_delta_chain(
|
|
1280
2339
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
2340
|
+
rows: &[JsonPointerStorageRow],
|
|
2341
|
+
delta_commits: usize,
|
|
2342
|
+
updated_rows_per_commit: usize,
|
|
2343
|
+
) -> Result<TrackedStateMaterializeFixture, LixError> {
|
|
2344
|
+
let (context, final_commit_id) =
|
|
2345
|
+
write_json_pointer_delta_chain(backend, rows, delta_commits, updated_rows_per_commit)
|
|
2346
|
+
.await?;
|
|
2347
|
+
Ok(TrackedStateMaterializeFixture {
|
|
1287
2348
|
context,
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
expected_entries: 0,
|
|
2349
|
+
commit_id: final_commit_id,
|
|
2350
|
+
expected_rows: rows.len(),
|
|
1291
2351
|
})
|
|
1292
2352
|
}
|
|
1293
2353
|
|
|
1294
|
-
pub async fn
|
|
2354
|
+
pub async fn json_pointer_tracked_state_changed_keys_prepared(
|
|
1295
2355
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1296
|
-
fixture: &
|
|
2356
|
+
fixture: &JsonPointerTrackedStateDiffFixture,
|
|
1297
2357
|
) -> Result<StorageBenchReport, LixError> {
|
|
1298
2358
|
let mut reader = fixture
|
|
1299
2359
|
.context
|
|
@@ -1617,7 +2677,7 @@ pub async fn prepare_changelog_append_changes(
|
|
|
1617
2677
|
config: StorageBenchConfig,
|
|
1618
2678
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1619
2679
|
Ok(ChangelogAppendFixture {
|
|
1620
|
-
context:
|
|
2680
|
+
context: CommitStoreContext::new(),
|
|
1621
2681
|
changes: changelog_materialized_changes(config),
|
|
1622
2682
|
})
|
|
1623
2683
|
}
|
|
@@ -1626,7 +2686,7 @@ pub async fn prepare_changelog_append_tombstones(
|
|
|
1626
2686
|
config: StorageBenchConfig,
|
|
1627
2687
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1628
2688
|
Ok(ChangelogAppendFixture {
|
|
1629
|
-
context:
|
|
2689
|
+
context: CommitStoreContext::new(),
|
|
1630
2690
|
changes: changelog_tombstone_changes(config),
|
|
1631
2691
|
})
|
|
1632
2692
|
}
|
|
@@ -1635,7 +2695,7 @@ pub async fn prepare_changelog_append_metadata(
|
|
|
1635
2695
|
config: StorageBenchConfig,
|
|
1636
2696
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1637
2697
|
Ok(ChangelogAppendFixture {
|
|
1638
|
-
context:
|
|
2698
|
+
context: CommitStoreContext::new(),
|
|
1639
2699
|
changes: changelog_metadata_changes(config),
|
|
1640
2700
|
})
|
|
1641
2701
|
}
|
|
@@ -1644,7 +2704,7 @@ pub async fn prepare_changelog_append_shared_payload(
|
|
|
1644
2704
|
config: StorageBenchConfig,
|
|
1645
2705
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1646
2706
|
Ok(ChangelogAppendFixture {
|
|
1647
|
-
context:
|
|
2707
|
+
context: CommitStoreContext::new(),
|
|
1648
2708
|
changes: changelog_shared_payload_changes(config),
|
|
1649
2709
|
})
|
|
1650
2710
|
}
|
|
@@ -1653,7 +2713,7 @@ pub async fn prepare_changelog_append_shared_metadata(
|
|
|
1653
2713
|
config: StorageBenchConfig,
|
|
1654
2714
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1655
2715
|
Ok(ChangelogAppendFixture {
|
|
1656
|
-
context:
|
|
2716
|
+
context: CommitStoreContext::new(),
|
|
1657
2717
|
changes: changelog_shared_metadata_changes(config),
|
|
1658
2718
|
})
|
|
1659
2719
|
}
|
|
@@ -1662,7 +2722,7 @@ pub async fn prepare_changelog_append_shared_payload_and_metadata(
|
|
|
1662
2722
|
config: StorageBenchConfig,
|
|
1663
2723
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1664
2724
|
Ok(ChangelogAppendFixture {
|
|
1665
|
-
context:
|
|
2725
|
+
context: CommitStoreContext::new(),
|
|
1666
2726
|
changes: changelog_shared_payload_and_metadata_changes(config),
|
|
1667
2727
|
})
|
|
1668
2728
|
}
|
|
@@ -1671,7 +2731,7 @@ pub async fn prepare_changelog_append_composite_entity_ids(
|
|
|
1671
2731
|
config: StorageBenchConfig,
|
|
1672
2732
|
) -> Result<ChangelogAppendFixture, LixError> {
|
|
1673
2733
|
Ok(ChangelogAppendFixture {
|
|
1674
|
-
context:
|
|
2734
|
+
context: CommitStoreContext::new(),
|
|
1675
2735
|
changes: changelog_composite_entity_id_changes(config),
|
|
1676
2736
|
})
|
|
1677
2737
|
}
|
|
@@ -1682,7 +2742,7 @@ pub async fn prepare_changelog_codec(
|
|
|
1682
2742
|
let changes = changelog_changes(config);
|
|
1683
2743
|
let encoded_changes = changes
|
|
1684
2744
|
.iter()
|
|
1685
|
-
.map(crate::
|
|
2745
|
+
.map(|change| crate::commit_store::codec::encode_change_ref(change.as_ref()))
|
|
1686
2746
|
.collect::<Result<Vec<_>, _>>()?;
|
|
1687
2747
|
Ok(ChangelogCodecFixture {
|
|
1688
2748
|
changes,
|
|
@@ -1699,7 +2759,7 @@ pub async fn changelog_append_changes_prepared(
|
|
|
1699
2759
|
.context
|
|
1700
2760
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1701
2761
|
let verified_rows = reader
|
|
1702
|
-
.scan_changes(&
|
|
2762
|
+
.scan_changes(&ChangeScanRequest::default())
|
|
1703
2763
|
.await?
|
|
1704
2764
|
.len();
|
|
1705
2765
|
Ok(report(fixture.changes.len(), verified_rows, Duration::ZERO))
|
|
@@ -1709,7 +2769,7 @@ pub async fn prepare_changelog_read(
|
|
|
1709
2769
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1710
2770
|
config: StorageBenchConfig,
|
|
1711
2771
|
) -> Result<ChangelogReadFixture, LixError> {
|
|
1712
|
-
let context =
|
|
2772
|
+
let context = CommitStoreContext::new();
|
|
1713
2773
|
let changes = changelog_materialized_changes(config);
|
|
1714
2774
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
1715
2775
|
Ok(ChangelogReadFixture {
|
|
@@ -1722,7 +2782,7 @@ pub async fn prepare_changelog_read_with_selectivity(
|
|
|
1722
2782
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1723
2783
|
config: StorageBenchConfig,
|
|
1724
2784
|
) -> Result<ChangelogReadFixture, LixError> {
|
|
1725
|
-
let context =
|
|
2785
|
+
let context = CommitStoreContext::new();
|
|
1726
2786
|
let changes = changelog_selective_changes(config);
|
|
1727
2787
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
1728
2788
|
Ok(ChangelogReadFixture {
|
|
@@ -1735,7 +2795,7 @@ pub async fn prepare_changelog_read_entity_history(
|
|
|
1735
2795
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1736
2796
|
config: StorageBenchConfig,
|
|
1737
2797
|
) -> Result<ChangelogReadFixture, LixError> {
|
|
1738
|
-
let context =
|
|
2798
|
+
let context = CommitStoreContext::new();
|
|
1739
2799
|
let changes = changelog_entity_history_changes(config);
|
|
1740
2800
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
1741
2801
|
Ok(ChangelogReadFixture {
|
|
@@ -1750,7 +2810,7 @@ pub async fn changelog_encode_only_prepared(
|
|
|
1750
2810
|
let mut verified_rows = 0;
|
|
1751
2811
|
let mut encoded_bytes = 0;
|
|
1752
2812
|
for change in &fixture.changes {
|
|
1753
|
-
encoded_bytes += crate::
|
|
2813
|
+
encoded_bytes += crate::commit_store::codec::encode_change_ref(change.as_ref())?.len();
|
|
1754
2814
|
verified_rows += 1;
|
|
1755
2815
|
}
|
|
1756
2816
|
Ok(report(
|
|
@@ -1766,7 +2826,7 @@ pub async fn changelog_decode_only_prepared(
|
|
|
1766
2826
|
let mut verified_rows = 0;
|
|
1767
2827
|
let mut decoded_bytes = 0;
|
|
1768
2828
|
for bytes in &fixture.encoded_changes {
|
|
1769
|
-
let change = crate::
|
|
2829
|
+
let change = crate::commit_store::codec::decode_change(bytes)?;
|
|
1770
2830
|
decoded_bytes += change.schema_key.len();
|
|
1771
2831
|
verified_rows += 1;
|
|
1772
2832
|
}
|
|
@@ -1777,43 +2837,41 @@ pub async fn changelog_decode_only_prepared(
|
|
|
1777
2837
|
))
|
|
1778
2838
|
}
|
|
1779
2839
|
|
|
1780
|
-
pub async fn
|
|
2840
|
+
pub async fn changelog_load_changes_hit_prepared(
|
|
1781
2841
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1782
2842
|
fixture: &ChangelogReadFixture,
|
|
1783
2843
|
) -> Result<StorageBenchReport, LixError> {
|
|
1784
2844
|
let reader = fixture
|
|
1785
2845
|
.context
|
|
1786
2846
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1787
|
-
let
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
}
|
|
2847
|
+
let change_ids = (0..fixture.rows)
|
|
2848
|
+
.map(|index| format!("bench-change-{index}"))
|
|
2849
|
+
.collect::<Vec<_>>();
|
|
2850
|
+
let verified_rows = reader
|
|
2851
|
+
.load_changes(&change_ids)
|
|
2852
|
+
.await?
|
|
2853
|
+
.into_iter()
|
|
2854
|
+
.filter(Option::is_some)
|
|
2855
|
+
.count();
|
|
1797
2856
|
Ok(report(fixture.rows, verified_rows, Duration::ZERO))
|
|
1798
2857
|
}
|
|
1799
2858
|
|
|
1800
|
-
pub async fn
|
|
2859
|
+
pub async fn changelog_load_changes_miss_prepared(
|
|
1801
2860
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1802
2861
|
fixture: &ChangelogReadFixture,
|
|
1803
2862
|
) -> Result<StorageBenchReport, LixError> {
|
|
1804
2863
|
let reader = fixture
|
|
1805
2864
|
.context
|
|
1806
2865
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1807
|
-
let
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
}
|
|
2866
|
+
let change_ids = (0..fixture.rows)
|
|
2867
|
+
.map(|index| format!("missing-change-{index}"))
|
|
2868
|
+
.collect::<Vec<_>>();
|
|
2869
|
+
let misses = reader
|
|
2870
|
+
.load_changes(&change_ids)
|
|
2871
|
+
.await?
|
|
2872
|
+
.into_iter()
|
|
2873
|
+
.filter(Option::is_none)
|
|
2874
|
+
.count();
|
|
1817
2875
|
Ok(report(fixture.rows, misses, Duration::ZERO))
|
|
1818
2876
|
}
|
|
1819
2877
|
|
|
@@ -1825,7 +2883,7 @@ pub async fn changelog_scan_all_prepared(
|
|
|
1825
2883
|
.context
|
|
1826
2884
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1827
2885
|
let verified_rows = reader
|
|
1828
|
-
.scan_changes(&
|
|
2886
|
+
.scan_changes(&ChangeScanRequest::default())
|
|
1829
2887
|
.await?
|
|
1830
2888
|
.len();
|
|
1831
2889
|
Ok(report(fixture.rows, verified_rows, Duration::ZERO))
|
|
@@ -1847,7 +2905,7 @@ pub async fn changelog_scan_limit_100_prepared(
|
|
|
1847
2905
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1848
2906
|
let expected = fixture.rows.min(100);
|
|
1849
2907
|
let verified_rows = reader
|
|
1850
|
-
.scan_changes(&
|
|
2908
|
+
.scan_changes(&ChangeScanRequest {
|
|
1851
2909
|
limit: Some(expected),
|
|
1852
2910
|
})
|
|
1853
2911
|
.await?
|
|
@@ -1855,6 +2913,25 @@ pub async fn changelog_scan_limit_100_prepared(
|
|
|
1855
2913
|
Ok(report(expected, verified_rows, Duration::ZERO))
|
|
1856
2914
|
}
|
|
1857
2915
|
|
|
2916
|
+
pub async fn changelog_scan_change_set_prepared(
|
|
2917
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2918
|
+
fixture: &ChangelogReadFixture,
|
|
2919
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2920
|
+
let reader = fixture
|
|
2921
|
+
.context
|
|
2922
|
+
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2923
|
+
let change_ids = (0..fixture.rows)
|
|
2924
|
+
.map(|index| format!("bench-change-{index}"))
|
|
2925
|
+
.collect::<Vec<_>>();
|
|
2926
|
+
let verified_rows = reader
|
|
2927
|
+
.load_changes(&change_ids)
|
|
2928
|
+
.await?
|
|
2929
|
+
.into_iter()
|
|
2930
|
+
.filter(Option::is_some)
|
|
2931
|
+
.count();
|
|
2932
|
+
Ok(report(fixture.rows, verified_rows, Duration::ZERO))
|
|
2933
|
+
}
|
|
2934
|
+
|
|
1858
2935
|
pub async fn changelog_scan_schema_prepared(
|
|
1859
2936
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
1860
2937
|
fixture: &ChangelogReadFixture,
|
|
@@ -1863,12 +2940,10 @@ pub async fn changelog_scan_schema_prepared(
|
|
|
1863
2940
|
let reader = fixture
|
|
1864
2941
|
.context
|
|
1865
2942
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1866
|
-
let changes = reader
|
|
1867
|
-
.scan_changes(&ChangelogScanRequest::default())
|
|
1868
|
-
.await?;
|
|
2943
|
+
let changes = reader.scan_changes(&ChangeScanRequest::default()).await?;
|
|
1869
2944
|
let verified_rows = changes
|
|
1870
2945
|
.iter()
|
|
1871
|
-
.filter(|change| change.schema_key == CHANGELOG_MATCH_SCHEMA_KEY)
|
|
2946
|
+
.filter(|change| change.record.schema_key == CHANGELOG_MATCH_SCHEMA_KEY)
|
|
1872
2947
|
.count();
|
|
1873
2948
|
Ok(report(
|
|
1874
2949
|
selectivity.expected_rows(fixture.rows),
|
|
@@ -1884,13 +2959,11 @@ pub async fn changelog_scan_entity_history_prepared(
|
|
|
1884
2959
|
let reader = fixture
|
|
1885
2960
|
.context
|
|
1886
2961
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
1887
|
-
let changes = reader
|
|
1888
|
-
.scan_changes(&ChangelogScanRequest::default())
|
|
1889
|
-
.await?;
|
|
2962
|
+
let changes = reader.scan_changes(&ChangeScanRequest::default()).await?;
|
|
1890
2963
|
let target = EntityIdentity::single(CHANGELOG_HISTORY_ENTITY_ID);
|
|
1891
2964
|
let verified_rows = changes
|
|
1892
2965
|
.iter()
|
|
1893
|
-
.filter(|change| change.entity_id == target)
|
|
2966
|
+
.filter(|change| change.record.entity_id == target)
|
|
1894
2967
|
.count();
|
|
1895
2968
|
Ok(report(
|
|
1896
2969
|
fixture.rows.div_ceil(10),
|
|
@@ -1899,6 +2972,41 @@ pub async fn changelog_scan_entity_history_prepared(
|
|
|
1899
2972
|
))
|
|
1900
2973
|
}
|
|
1901
2974
|
|
|
2975
|
+
pub async fn prepare_commit_graph_read(
|
|
2976
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2977
|
+
config: StorageBenchConfig,
|
|
2978
|
+
) -> Result<CommitGraphReadFixture, LixError> {
|
|
2979
|
+
let changelog = CommitStoreContext::new();
|
|
2980
|
+
let mut changes = changelog_materialized_changes(config);
|
|
2981
|
+
let head_commit_id = "bench-commit-head".to_string();
|
|
2982
|
+
changes.push(commit_graph_materialized_commit_change(
|
|
2983
|
+
&head_commit_id,
|
|
2984
|
+
config.rows,
|
|
2985
|
+
));
|
|
2986
|
+
append_changelog_changes(backend, &changelog, &changes).await?;
|
|
2987
|
+
|
|
2988
|
+
Ok(CommitGraphReadFixture {
|
|
2989
|
+
head_commit_id,
|
|
2990
|
+
rows: config.rows,
|
|
2991
|
+
})
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
pub async fn commit_graph_change_history_from_commit_prepared(
|
|
2995
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2996
|
+
fixture: &CommitGraphReadFixture,
|
|
2997
|
+
) -> Result<StorageBenchReport, LixError> {
|
|
2998
|
+
let graph = crate::commit_graph::CommitGraphContext::new();
|
|
2999
|
+
let mut reader = graph.reader(StorageContext::new(Arc::clone(backend)));
|
|
3000
|
+
let verified_rows = reader
|
|
3001
|
+
.change_history_from_commit(
|
|
3002
|
+
&fixture.head_commit_id,
|
|
3003
|
+
&CommitGraphChangeHistoryRequest::default(),
|
|
3004
|
+
)
|
|
3005
|
+
.await?
|
|
3006
|
+
.len();
|
|
3007
|
+
Ok(report(fixture.rows, verified_rows, Duration::ZERO))
|
|
3008
|
+
}
|
|
3009
|
+
|
|
1902
3010
|
pub async fn prepare_binary_cas_write_blobs(
|
|
1903
3011
|
config: StorageBenchConfig,
|
|
1904
3012
|
) -> Result<BinaryCasWriteFixture, LixError> {
|
|
@@ -2029,9 +3137,24 @@ pub async fn json_store_write_prepared(
|
|
|
2029
3137
|
{
|
|
2030
3138
|
let mut writes = StorageWriteSet::new();
|
|
2031
3139
|
let mut writer = fixture.context.writer();
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
3140
|
+
writer.stage_batch(
|
|
3141
|
+
&mut writes,
|
|
3142
|
+
JsonWritePlacementRef::OutOfBand,
|
|
3143
|
+
fixture
|
|
3144
|
+
.documents
|
|
3145
|
+
.iter()
|
|
3146
|
+
.map(|document| {
|
|
3147
|
+
std::str::from_utf8(document)
|
|
3148
|
+
.map(NormalizedJsonRef::new)
|
|
3149
|
+
.map_err(|error| {
|
|
3150
|
+
LixError::new(
|
|
3151
|
+
LixError::CODE_UNKNOWN,
|
|
3152
|
+
format!("benchmark JSON document is invalid UTF-8: {error}"),
|
|
3153
|
+
)
|
|
3154
|
+
})
|
|
3155
|
+
})
|
|
3156
|
+
.collect::<Result<Vec<_>, _>>()?,
|
|
3157
|
+
)?;
|
|
2035
3158
|
writes.apply(&mut transaction.as_mut()).await?;
|
|
2036
3159
|
}
|
|
2037
3160
|
transaction.commit().await?;
|
|
@@ -2070,9 +3193,26 @@ pub async fn prepare_json_store_projection_read(
|
|
|
2070
3193
|
{
|
|
2071
3194
|
let mut writes = StorageWriteSet::new();
|
|
2072
3195
|
let mut writer = context.writer();
|
|
2073
|
-
for document in documents {
|
|
2074
|
-
refs.push(
|
|
3196
|
+
for document in &documents {
|
|
3197
|
+
refs.push(prepare_json_ref(document)?);
|
|
2075
3198
|
}
|
|
3199
|
+
writer.stage_batch(
|
|
3200
|
+
&mut writes,
|
|
3201
|
+
JsonWritePlacementRef::OutOfBand,
|
|
3202
|
+
documents
|
|
3203
|
+
.iter()
|
|
3204
|
+
.map(|document| {
|
|
3205
|
+
std::str::from_utf8(document)
|
|
3206
|
+
.map(NormalizedJsonRef::new)
|
|
3207
|
+
.map_err(|error| {
|
|
3208
|
+
LixError::new(
|
|
3209
|
+
LixError::CODE_UNKNOWN,
|
|
3210
|
+
format!("benchmark JSON document is invalid UTF-8: {error}"),
|
|
3211
|
+
)
|
|
3212
|
+
})
|
|
3213
|
+
})
|
|
3214
|
+
.collect::<Result<Vec<_>, _>>()?,
|
|
3215
|
+
)?;
|
|
2076
3216
|
writes.apply(&mut transaction.as_mut()).await?;
|
|
2077
3217
|
}
|
|
2078
3218
|
transaction.commit().await?;
|
|
@@ -2091,8 +3231,14 @@ pub async fn json_store_read_bytes_prepared(
|
|
|
2091
3231
|
let mut reader = fixture
|
|
2092
3232
|
.context
|
|
2093
3233
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2094
|
-
|
|
2095
|
-
|
|
3234
|
+
let batch = reader
|
|
3235
|
+
.load_bytes_many(JsonLoadRequestRef {
|
|
3236
|
+
refs: &fixture.refs,
|
|
3237
|
+
scope: JsonReadScopeRef::OutOfBand,
|
|
3238
|
+
})
|
|
3239
|
+
.await?;
|
|
3240
|
+
for value in batch.values() {
|
|
3241
|
+
if value.is_some() {
|
|
2096
3242
|
verified_rows += 1;
|
|
2097
3243
|
}
|
|
2098
3244
|
}
|
|
@@ -2107,8 +3253,14 @@ pub async fn json_store_read_value_prepared(
|
|
|
2107
3253
|
let mut reader = fixture
|
|
2108
3254
|
.context
|
|
2109
3255
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2110
|
-
|
|
2111
|
-
|
|
3256
|
+
let batch = reader
|
|
3257
|
+
.load_values_many(JsonLoadRequestRef {
|
|
3258
|
+
refs: &fixture.refs,
|
|
3259
|
+
scope: JsonReadScopeRef::OutOfBand,
|
|
3260
|
+
})
|
|
3261
|
+
.await?;
|
|
3262
|
+
for value in batch.values() {
|
|
3263
|
+
if value.is_some() {
|
|
2112
3264
|
verified_rows += 1;
|
|
2113
3265
|
}
|
|
2114
3266
|
}
|
|
@@ -2123,12 +3275,15 @@ pub async fn json_store_read_projection_prepared(
|
|
|
2123
3275
|
let mut reader = fixture
|
|
2124
3276
|
.context
|
|
2125
3277
|
.reader(StorageContext::new(Arc::clone(backend)));
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
.
|
|
2131
|
-
|
|
3278
|
+
let batch = reader
|
|
3279
|
+
.load_projections_many(JsonProjectionLoadRequestRef {
|
|
3280
|
+
refs: &fixture.refs,
|
|
3281
|
+
scope: JsonReadScopeRef::OutOfBand,
|
|
3282
|
+
paths: &fixture.paths,
|
|
3283
|
+
})
|
|
3284
|
+
.await?;
|
|
3285
|
+
for value in batch.values() {
|
|
3286
|
+
if value.is_some() {
|
|
2132
3287
|
verified_rows += 1;
|
|
2133
3288
|
}
|
|
2134
3289
|
}
|
|
@@ -2162,9 +3317,26 @@ async fn prepare_json_store_base_update(
|
|
|
2162
3317
|
{
|
|
2163
3318
|
let mut writes = StorageWriteSet::new();
|
|
2164
3319
|
let mut writer = context.writer();
|
|
2165
|
-
for document in documents {
|
|
2166
|
-
refs.push(
|
|
3320
|
+
for document in &documents {
|
|
3321
|
+
refs.push(prepare_json_ref(document)?);
|
|
2167
3322
|
}
|
|
3323
|
+
writer.stage_batch(
|
|
3324
|
+
&mut writes,
|
|
3325
|
+
JsonWritePlacementRef::OutOfBand,
|
|
3326
|
+
documents
|
|
3327
|
+
.iter()
|
|
3328
|
+
.map(|document| {
|
|
3329
|
+
std::str::from_utf8(document)
|
|
3330
|
+
.map(NormalizedJsonRef::new)
|
|
3331
|
+
.map_err(|error| {
|
|
3332
|
+
LixError::new(
|
|
3333
|
+
LixError::CODE_UNKNOWN,
|
|
3334
|
+
format!("benchmark JSON document is invalid UTF-8: {error}"),
|
|
3335
|
+
)
|
|
3336
|
+
})
|
|
3337
|
+
})
|
|
3338
|
+
.collect::<Result<Vec<_>, _>>()?,
|
|
3339
|
+
)?;
|
|
2168
3340
|
writes.apply(&mut transaction.as_mut()).await?;
|
|
2169
3341
|
}
|
|
2170
3342
|
transaction.commit().await?;
|
|
@@ -2205,10 +3377,29 @@ async fn json_store_write_against_base_prepared(
|
|
|
2205
3377
|
{
|
|
2206
3378
|
let mut writes = StorageWriteSet::new();
|
|
2207
3379
|
let mut writer = fixture.context.writer();
|
|
3380
|
+
let mut updated_documents = Vec::with_capacity(fixture.refs.len());
|
|
2208
3381
|
for (index, _json_ref) in fixture.refs.iter().enumerate() {
|
|
2209
3382
|
let updated = updated_json_document(shape, index);
|
|
2210
|
-
|
|
3383
|
+
prepare_json_ref(&updated)?;
|
|
3384
|
+
updated_documents.push(updated);
|
|
2211
3385
|
}
|
|
3386
|
+
writer.stage_batch(
|
|
3387
|
+
&mut writes,
|
|
3388
|
+
JsonWritePlacementRef::OutOfBand,
|
|
3389
|
+
updated_documents
|
|
3390
|
+
.iter()
|
|
3391
|
+
.map(|document| {
|
|
3392
|
+
std::str::from_utf8(document)
|
|
3393
|
+
.map(NormalizedJsonRef::new)
|
|
3394
|
+
.map_err(|error| {
|
|
3395
|
+
LixError::new(
|
|
3396
|
+
LixError::CODE_UNKNOWN,
|
|
3397
|
+
format!("benchmark JSON document is invalid UTF-8: {error}"),
|
|
3398
|
+
)
|
|
3399
|
+
})
|
|
3400
|
+
})
|
|
3401
|
+
.collect::<Result<Vec<_>, _>>()?,
|
|
3402
|
+
)?;
|
|
2212
3403
|
writes.apply(&mut transaction.as_mut()).await?;
|
|
2213
3404
|
}
|
|
2214
3405
|
transaction.commit().await?;
|
|
@@ -2243,28 +3434,14 @@ pub async fn tracked_state_read_point_hit(
|
|
|
2243
3434
|
write_tracked_root(backend, &context, "bench-tracked-commit", None, &rows).await?;
|
|
2244
3435
|
|
|
2245
3436
|
let started = Instant::now();
|
|
2246
|
-
let mut verified_rows = 0;
|
|
2247
3437
|
let mut reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
"tracked",
|
|
2256
|
-
index,
|
|
2257
|
-
config.key_pattern,
|
|
2258
|
-
)),
|
|
2259
|
-
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
2260
|
-
},
|
|
2261
|
-
)
|
|
2262
|
-
.await?
|
|
2263
|
-
.is_some()
|
|
2264
|
-
{
|
|
2265
|
-
verified_rows += 1;
|
|
2266
|
-
}
|
|
2267
|
-
}
|
|
3438
|
+
let requests = tracked_point_hit_requests(config.rows, config.key_pattern);
|
|
3439
|
+
let verified_rows = reader
|
|
3440
|
+
.load_rows_at_commit("bench-tracked-commit", &requests)
|
|
3441
|
+
.await?
|
|
3442
|
+
.into_iter()
|
|
3443
|
+
.filter(Option::is_some)
|
|
3444
|
+
.count();
|
|
2268
3445
|
Ok(report(config.rows, verified_rows, started.elapsed()))
|
|
2269
3446
|
}
|
|
2270
3447
|
|
|
@@ -2277,24 +3454,14 @@ pub async fn tracked_state_read_point_miss(
|
|
|
2277
3454
|
write_tracked_root(backend, &context, "bench-tracked-commit", None, &rows).await?;
|
|
2278
3455
|
|
|
2279
3456
|
let started = Instant::now();
|
|
2280
|
-
let mut misses = 0;
|
|
2281
3457
|
let mut reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
file_id: NullableKeyFilter::Value("bench.json".to_string()),
|
|
2290
|
-
},
|
|
2291
|
-
)
|
|
2292
|
-
.await?
|
|
2293
|
-
.is_none()
|
|
2294
|
-
{
|
|
2295
|
-
misses += 1;
|
|
2296
|
-
}
|
|
2297
|
-
}
|
|
3458
|
+
let requests = tracked_point_miss_requests(config.rows, StorageBenchSelectivity::Percent100);
|
|
3459
|
+
let misses = reader
|
|
3460
|
+
.load_rows_at_commit("bench-tracked-commit", &requests)
|
|
3461
|
+
.await?
|
|
3462
|
+
.into_iter()
|
|
3463
|
+
.filter(Option::is_none)
|
|
3464
|
+
.count();
|
|
2298
3465
|
Ok(report(config.rows, misses, started.elapsed()))
|
|
2299
3466
|
}
|
|
2300
3467
|
|
|
@@ -2559,61 +3726,59 @@ pub async fn changelog_append_changes(
|
|
|
2559
3726
|
config: StorageBenchConfig,
|
|
2560
3727
|
) -> Result<StorageBenchReport, LixError> {
|
|
2561
3728
|
let changes = changelog_materialized_changes(config);
|
|
2562
|
-
let context =
|
|
3729
|
+
let context = CommitStoreContext::new();
|
|
2563
3730
|
let started = Instant::now();
|
|
2564
3731
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
2565
3732
|
let elapsed = started.elapsed();
|
|
2566
3733
|
let reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2567
3734
|
let verified_rows = reader
|
|
2568
|
-
.scan_changes(&
|
|
3735
|
+
.scan_changes(&ChangeScanRequest::default())
|
|
2569
3736
|
.await?
|
|
2570
3737
|
.len();
|
|
2571
3738
|
Ok(report(changes.len(), verified_rows, elapsed))
|
|
2572
3739
|
}
|
|
2573
3740
|
|
|
2574
|
-
pub async fn
|
|
3741
|
+
pub async fn changelog_load_changes_hit(
|
|
2575
3742
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2576
3743
|
config: StorageBenchConfig,
|
|
2577
3744
|
) -> Result<StorageBenchReport, LixError> {
|
|
2578
|
-
let context =
|
|
3745
|
+
let context = CommitStoreContext::new();
|
|
2579
3746
|
let changes = changelog_materialized_changes(config);
|
|
2580
3747
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
2581
3748
|
let reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2582
3749
|
|
|
2583
3750
|
let started = Instant::now();
|
|
2584
|
-
let
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
}
|
|
3751
|
+
let change_ids = (0..config.rows)
|
|
3752
|
+
.map(|index| format!("bench-change-{index}"))
|
|
3753
|
+
.collect::<Vec<_>>();
|
|
3754
|
+
let verified_rows = reader
|
|
3755
|
+
.load_changes(&change_ids)
|
|
3756
|
+
.await?
|
|
3757
|
+
.into_iter()
|
|
3758
|
+
.filter(Option::is_some)
|
|
3759
|
+
.count();
|
|
2594
3760
|
Ok(report(config.rows, verified_rows, started.elapsed()))
|
|
2595
3761
|
}
|
|
2596
3762
|
|
|
2597
|
-
pub async fn
|
|
3763
|
+
pub async fn changelog_load_changes_miss(
|
|
2598
3764
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2599
3765
|
config: StorageBenchConfig,
|
|
2600
3766
|
) -> Result<StorageBenchReport, LixError> {
|
|
2601
|
-
let context =
|
|
3767
|
+
let context = CommitStoreContext::new();
|
|
2602
3768
|
let changes = changelog_materialized_changes(config);
|
|
2603
3769
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
2604
3770
|
let reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2605
3771
|
|
|
2606
3772
|
let started = Instant::now();
|
|
2607
|
-
let
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
}
|
|
3773
|
+
let change_ids = (0..config.rows)
|
|
3774
|
+
.map(|index| format!("missing-change-{index}"))
|
|
3775
|
+
.collect::<Vec<_>>();
|
|
3776
|
+
let misses = reader
|
|
3777
|
+
.load_changes(&change_ids)
|
|
3778
|
+
.await?
|
|
3779
|
+
.into_iter()
|
|
3780
|
+
.filter(Option::is_none)
|
|
3781
|
+
.count();
|
|
2617
3782
|
Ok(report(config.rows, misses, started.elapsed()))
|
|
2618
3783
|
}
|
|
2619
3784
|
|
|
@@ -2621,14 +3786,14 @@ pub async fn changelog_scan_all(
|
|
|
2621
3786
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2622
3787
|
config: StorageBenchConfig,
|
|
2623
3788
|
) -> Result<StorageBenchReport, LixError> {
|
|
2624
|
-
let context =
|
|
3789
|
+
let context = CommitStoreContext::new();
|
|
2625
3790
|
let changes = changelog_materialized_changes(config);
|
|
2626
3791
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
2627
3792
|
let reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2628
3793
|
|
|
2629
3794
|
let started = Instant::now();
|
|
2630
3795
|
let verified_rows = reader
|
|
2631
|
-
.scan_changes(&
|
|
3796
|
+
.scan_changes(&ChangeScanRequest::default())
|
|
2632
3797
|
.await?
|
|
2633
3798
|
.len();
|
|
2634
3799
|
Ok(report(config.rows, verified_rows, started.elapsed()))
|
|
@@ -2638,7 +3803,7 @@ pub async fn changelog_scan_limit_100(
|
|
|
2638
3803
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2639
3804
|
config: StorageBenchConfig,
|
|
2640
3805
|
) -> Result<StorageBenchReport, LixError> {
|
|
2641
|
-
let context =
|
|
3806
|
+
let context = CommitStoreContext::new();
|
|
2642
3807
|
let changes = changelog_materialized_changes(config);
|
|
2643
3808
|
append_changelog_changes(backend, &context, &changes).await?;
|
|
2644
3809
|
let reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
@@ -2646,7 +3811,7 @@ pub async fn changelog_scan_limit_100(
|
|
|
2646
3811
|
|
|
2647
3812
|
let started = Instant::now();
|
|
2648
3813
|
let verified_rows = reader
|
|
2649
|
-
.scan_changes(&
|
|
3814
|
+
.scan_changes(&ChangeScanRequest {
|
|
2650
3815
|
limit: Some(expected),
|
|
2651
3816
|
})
|
|
2652
3817
|
.await?
|
|
@@ -2747,36 +3912,181 @@ async fn write_tracked_root(
|
|
|
2747
3912
|
context: &TrackedStateContext,
|
|
2748
3913
|
commit_id: &str,
|
|
2749
3914
|
parent_commit_id: Option<&str>,
|
|
2750
|
-
rows: &[
|
|
3915
|
+
rows: &[MaterializedTrackedStateRow],
|
|
2751
3916
|
) -> Result<(), LixError> {
|
|
2752
3917
|
let storage = StorageContext::new(Arc::clone(backend));
|
|
2753
3918
|
let mut transaction = storage.begin_write_transaction().await?;
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
3919
|
+
let mut writes = StorageWriteSet::new();
|
|
3920
|
+
let changes = rows
|
|
3921
|
+
.iter()
|
|
3922
|
+
.map(tracked_bench_change_from_materialized)
|
|
3923
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
3924
|
+
let payloads = tracked_bench_json_payloads(rows, &changes);
|
|
3925
|
+
let json_report = JsonStoreContext::new().writer().stage_batch_report(
|
|
3926
|
+
&mut writes,
|
|
3927
|
+
JsonWritePlacementRef::CommitPack {
|
|
3928
|
+
commit_id,
|
|
3929
|
+
pack_id: 0,
|
|
3930
|
+
},
|
|
3931
|
+
payloads.iter().map(|(payload, json_ref)| match json_ref {
|
|
3932
|
+
Some(json_ref) => NormalizedJsonRef::trusted_prehashed(payload.as_str(), *json_ref),
|
|
3933
|
+
None => NormalizedJsonRef::new(payload.as_str()),
|
|
3934
|
+
}),
|
|
3935
|
+
)?;
|
|
3936
|
+
|
|
3937
|
+
let parent_ids = parent_commit_id
|
|
3938
|
+
.map(|parent| vec![parent.to_string()])
|
|
3939
|
+
.unwrap_or_default();
|
|
3940
|
+
let commit_change_id = format!("{commit_id}:commit");
|
|
3941
|
+
let commit = CommitDraftRef {
|
|
3942
|
+
id: commit_id,
|
|
3943
|
+
change_id: &commit_change_id,
|
|
3944
|
+
parent_ids: &parent_ids,
|
|
3945
|
+
author_account_ids: &[],
|
|
3946
|
+
created_at: rows
|
|
3947
|
+
.first()
|
|
3948
|
+
.map(|row| row.updated_at.as_str())
|
|
3949
|
+
.unwrap_or("1970-01-01T00:00:00.000Z"),
|
|
3950
|
+
};
|
|
3951
|
+
let commit_store = CommitStoreContext::new();
|
|
3952
|
+
let authored_changes = changes.iter().map(Change::as_ref).collect::<Vec<_>>();
|
|
3953
|
+
let staged = commit_store
|
|
3954
|
+
.writer(&mut transaction.as_mut(), &mut writes)
|
|
3955
|
+
.stage_tracked_commit_draft(commit, authored_changes.clone(), Vec::new())
|
|
3956
|
+
.await?;
|
|
3957
|
+
let mut deltas = Vec::with_capacity(changes.len());
|
|
3958
|
+
deltas.extend(
|
|
3959
|
+
authored_changes
|
|
3960
|
+
.iter()
|
|
3961
|
+
.zip(&staged.authored_locators)
|
|
3962
|
+
.zip(rows)
|
|
3963
|
+
.map(|((change, locator), row)| TrackedStateDeltaRef {
|
|
3964
|
+
change: *change,
|
|
3965
|
+
locator: locator.as_ref(),
|
|
3966
|
+
created_at: row.created_at.as_str(),
|
|
3967
|
+
updated_at: row.updated_at.as_str(),
|
|
3968
|
+
}),
|
|
3969
|
+
);
|
|
3970
|
+
context
|
|
3971
|
+
.writer(&mut transaction.as_mut(), &mut writes)
|
|
3972
|
+
.stage_delta_with_json_pack_indexes(
|
|
3973
|
+
commit_id,
|
|
3974
|
+
parent_commit_id,
|
|
3975
|
+
&deltas,
|
|
3976
|
+
crate::tracked_state::DeltaJsonPackIndexesRef {
|
|
3977
|
+
commit_id,
|
|
3978
|
+
pack_id: 0,
|
|
3979
|
+
indexes: &json_report.pack_indexes,
|
|
3980
|
+
},
|
|
3981
|
+
)
|
|
3982
|
+
.await?;
|
|
3983
|
+
writes.apply(&mut transaction.as_mut()).await?;
|
|
3984
|
+
transaction.commit().await
|
|
3985
|
+
}
|
|
3986
|
+
|
|
3987
|
+
async fn materialize_tracked_root(
|
|
3988
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
3989
|
+
context: &TrackedStateContext,
|
|
3990
|
+
commit_id: &str,
|
|
3991
|
+
) -> Result<(), LixError> {
|
|
3992
|
+
let storage = StorageContext::new(Arc::clone(backend));
|
|
3993
|
+
let mut transaction = storage.begin_write_transaction().await?;
|
|
3994
|
+
let mut writes = StorageWriteSet::new();
|
|
3995
|
+
let commit_store = CommitStoreContext::new();
|
|
3996
|
+
context
|
|
3997
|
+
.materializer(&mut transaction.as_mut(), &mut writes, &commit_store)
|
|
3998
|
+
.materialize_root_at(commit_id)
|
|
3999
|
+
.await?;
|
|
4000
|
+
writes.apply(&mut transaction.as_mut()).await?;
|
|
4001
|
+
transaction.commit().await
|
|
4002
|
+
}
|
|
4003
|
+
|
|
4004
|
+
async fn write_tracked_delta_chain(
|
|
4005
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
4006
|
+
config: StorageBenchConfig,
|
|
4007
|
+
delta_commits: usize,
|
|
4008
|
+
updated_rows_per_commit: usize,
|
|
4009
|
+
) -> Result<(TrackedStateContext, String), LixError> {
|
|
4010
|
+
let context = TrackedStateContext::new();
|
|
4011
|
+
let base_commit_id = "bench-tracked-base";
|
|
4012
|
+
let rows = tracked_rows(config, base_commit_id);
|
|
4013
|
+
write_tracked_root(backend, &context, base_commit_id, None, &rows).await?;
|
|
4014
|
+
|
|
4015
|
+
let mut parent_commit_id = base_commit_id.to_string();
|
|
4016
|
+
for delta_index in 0..delta_commits {
|
|
4017
|
+
let commit_id = format!("bench-tracked-delta-{delta_index}");
|
|
4018
|
+
let mut updated_rows = tracked_rows(
|
|
4019
|
+
config.with_rows(updated_rows_per_commit.min(config.rows)),
|
|
4020
|
+
&commit_id,
|
|
4021
|
+
);
|
|
4022
|
+
for (row_index, row) in updated_rows.iter_mut().enumerate() {
|
|
4023
|
+
row.snapshot_content = Some(delta_chain_snapshot_content(
|
|
4024
|
+
delta_index,
|
|
4025
|
+
row_index,
|
|
4026
|
+
config.state_payload_bytes,
|
|
4027
|
+
));
|
|
4028
|
+
row.updated_at = timestamp(config.rows + delta_index * config.rows + row_index);
|
|
2769
4029
|
}
|
|
2770
|
-
|
|
4030
|
+
write_tracked_root(
|
|
4031
|
+
backend,
|
|
4032
|
+
&context,
|
|
4033
|
+
&commit_id,
|
|
4034
|
+
Some(parent_commit_id.as_str()),
|
|
4035
|
+
&updated_rows,
|
|
4036
|
+
)
|
|
4037
|
+
.await?;
|
|
4038
|
+
parent_commit_id = commit_id;
|
|
2771
4039
|
}
|
|
2772
|
-
|
|
4040
|
+
|
|
4041
|
+
Ok((context, parent_commit_id))
|
|
4042
|
+
}
|
|
4043
|
+
|
|
4044
|
+
fn tracked_bench_change_from_materialized(
|
|
4045
|
+
row: &MaterializedTrackedStateRow,
|
|
4046
|
+
) -> Result<Change, LixError> {
|
|
4047
|
+
Ok(Change {
|
|
4048
|
+
id: row.change_id.clone(),
|
|
4049
|
+
entity_id: row.entity_id.clone(),
|
|
4050
|
+
schema_key: row.schema_key.clone(),
|
|
4051
|
+
file_id: row.file_id.clone(),
|
|
4052
|
+
snapshot_ref: row
|
|
4053
|
+
.snapshot_content
|
|
4054
|
+
.as_deref()
|
|
4055
|
+
.map(|value| prepare_json_ref(value.as_bytes()))
|
|
4056
|
+
.transpose()?,
|
|
4057
|
+
metadata_ref: row
|
|
4058
|
+
.metadata
|
|
4059
|
+
.as_ref()
|
|
4060
|
+
.map(|value| {
|
|
4061
|
+
let serialized = crate::serialize_row_metadata(value);
|
|
4062
|
+
prepare_json_ref(serialized.as_bytes())
|
|
4063
|
+
})
|
|
4064
|
+
.transpose()?,
|
|
4065
|
+
created_at: row.created_at.clone(),
|
|
4066
|
+
})
|
|
4067
|
+
}
|
|
4068
|
+
|
|
4069
|
+
fn tracked_bench_json_payloads(
|
|
4070
|
+
rows: &[MaterializedTrackedStateRow],
|
|
4071
|
+
changes: &[Change],
|
|
4072
|
+
) -> Vec<(String, Option<JsonRef>)> {
|
|
4073
|
+
let mut payloads = Vec::new();
|
|
4074
|
+
for (row, change) in rows.iter().zip(changes) {
|
|
4075
|
+
if let Some(snapshot) = row.snapshot_content.as_deref() {
|
|
4076
|
+
payloads.push((snapshot.to_string(), change.snapshot_ref));
|
|
4077
|
+
}
|
|
4078
|
+
if let Some(metadata) = row.metadata.as_ref() {
|
|
4079
|
+
payloads.push((crate::serialize_row_metadata(metadata), change.metadata_ref));
|
|
4080
|
+
}
|
|
4081
|
+
}
|
|
4082
|
+
payloads
|
|
2773
4083
|
}
|
|
2774
4084
|
|
|
2775
4085
|
async fn scan_tracked(
|
|
2776
4086
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2777
4087
|
context: &TrackedStateContext,
|
|
2778
4088
|
commit_id: &str,
|
|
2779
|
-
) -> Result<Vec<
|
|
4089
|
+
) -> Result<Vec<MaterializedTrackedStateRow>, LixError> {
|
|
2780
4090
|
let mut reader = context.reader(StorageContext::new(Arc::clone(backend)));
|
|
2781
4091
|
reader
|
|
2782
4092
|
.scan_rows_at_commit(commit_id, &TrackedStateScanRequest::default())
|
|
@@ -2792,14 +4102,12 @@ async fn write_untracked_rows(
|
|
|
2792
4102
|
let mut transaction = storage.begin_write_transaction().await?;
|
|
2793
4103
|
{
|
|
2794
4104
|
let mut writes = StorageWriteSet::new();
|
|
2795
|
-
let canonical_rows =
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
.collect::<Result<Vec<_>, _>>()?
|
|
2800
|
-
};
|
|
4105
|
+
let canonical_rows = rows
|
|
4106
|
+
.iter()
|
|
4107
|
+
.map(|row| crate::test_support::untracked_state_row_from_materialized(&mut writes, row))
|
|
4108
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
2801
4109
|
let mut writer = context.writer(&mut writes);
|
|
2802
|
-
writer.stage_rows(
|
|
4110
|
+
writer.stage_rows(canonical_rows.iter().map(|row| row.as_ref()))?;
|
|
2803
4111
|
writes.apply(&mut transaction.as_mut()).await?;
|
|
2804
4112
|
}
|
|
2805
4113
|
transaction.commit().await
|
|
@@ -2816,24 +4124,47 @@ async fn scan_untracked(
|
|
|
2816
4124
|
|
|
2817
4125
|
async fn append_changelog_changes(
|
|
2818
4126
|
backend: &Arc<dyn Backend + Send + Sync>,
|
|
2819
|
-
context: &
|
|
2820
|
-
changes: &[
|
|
4127
|
+
context: &CommitStoreContext,
|
|
4128
|
+
changes: &[MaterializedChange],
|
|
2821
4129
|
) -> Result<(), LixError> {
|
|
2822
4130
|
let storage = StorageContext::new(Arc::clone(backend));
|
|
2823
4131
|
let mut transaction = storage.begin_write_transaction().await?;
|
|
2824
4132
|
{
|
|
2825
4133
|
let mut writes = StorageWriteSet::new();
|
|
2826
|
-
let canonical_changes =
|
|
2827
|
-
|
|
2828
|
-
|
|
4134
|
+
let canonical_changes = changes
|
|
4135
|
+
.iter()
|
|
4136
|
+
.map(canonical_changelog_bench_change)
|
|
4137
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
4138
|
+
let payloads = changelog_bench_json_payloads(changes);
|
|
4139
|
+
JsonStoreContext::new().writer().stage_batch(
|
|
4140
|
+
&mut writes,
|
|
4141
|
+
JsonWritePlacementRef::OutOfBand,
|
|
4142
|
+
payloads
|
|
2829
4143
|
.iter()
|
|
2830
|
-
.map(|
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
4144
|
+
.map(|payload| NormalizedJsonRef::new(payload.as_str())),
|
|
4145
|
+
)?;
|
|
4146
|
+
let parent_ids = Vec::new();
|
|
4147
|
+
let author_account_ids = vec!["bench-author".to_string()];
|
|
4148
|
+
{
|
|
4149
|
+
let mut transaction_ref = transaction.as_mut();
|
|
4150
|
+
let mut writer = context.writer(&mut transaction_ref, &mut writes);
|
|
4151
|
+
writer
|
|
4152
|
+
.stage_commit_draft(
|
|
4153
|
+
CommitDraftRef {
|
|
4154
|
+
id: "bench-changelog-commit-0",
|
|
4155
|
+
change_id: "bench-changelog-header-change-0",
|
|
4156
|
+
parent_ids: &parent_ids,
|
|
4157
|
+
author_account_ids: &author_account_ids,
|
|
4158
|
+
created_at: "2024-01-01T00:00:00.000Z",
|
|
4159
|
+
},
|
|
4160
|
+
canonical_changes
|
|
4161
|
+
.iter()
|
|
4162
|
+
.map(|change| change.as_ref())
|
|
4163
|
+
.collect(),
|
|
4164
|
+
Vec::new(),
|
|
4165
|
+
)
|
|
4166
|
+
.await?;
|
|
4167
|
+
}
|
|
2837
4168
|
writes.apply(&mut transaction.as_mut()).await?;
|
|
2838
4169
|
}
|
|
2839
4170
|
transaction.commit().await
|
|
@@ -2879,29 +4210,101 @@ const CHANGELOG_MATCH_SCHEMA_KEY: &str = "bench_changelog_entity";
|
|
|
2879
4210
|
const CHANGELOG_OTHER_SCHEMA_KEY: &str = "bench_changelog_other_entity";
|
|
2880
4211
|
const CHANGELOG_HISTORY_ENTITY_ID: &str = "change-entity-history-target";
|
|
2881
4212
|
|
|
2882
|
-
fn tracked_rows(config: StorageBenchConfig, commit_id: &str) -> Vec<
|
|
4213
|
+
fn tracked_rows(config: StorageBenchConfig, commit_id: &str) -> Vec<MaterializedTrackedStateRow> {
|
|
2883
4214
|
(0..config.rows)
|
|
2884
|
-
.map(|index|
|
|
4215
|
+
.map(|index| MaterializedTrackedStateRow {
|
|
2885
4216
|
entity_id: EntityIdentity::single(entity_id("tracked", index, config.key_pattern)),
|
|
2886
4217
|
schema_key: tracked_schema_key(index, config.selectivity),
|
|
2887
4218
|
file_id: Some("bench.json".to_string()),
|
|
2888
4219
|
snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
|
|
2889
4220
|
metadata: None,
|
|
2890
|
-
|
|
4221
|
+
deleted: false,
|
|
2891
4222
|
created_at: timestamp(index),
|
|
2892
4223
|
updated_at: timestamp(index),
|
|
2893
|
-
change_id:
|
|
4224
|
+
change_id: tracked_change_id(commit_id, index),
|
|
2894
4225
|
commit_id: commit_id.to_string(),
|
|
2895
4226
|
})
|
|
2896
4227
|
.collect()
|
|
2897
4228
|
}
|
|
2898
4229
|
|
|
4230
|
+
fn json_pointer_tracked_rows(
|
|
4231
|
+
rows: &[JsonPointerStorageRow],
|
|
4232
|
+
commit_id: &str,
|
|
4233
|
+
updated: bool,
|
|
4234
|
+
) -> Vec<MaterializedTrackedStateRow> {
|
|
4235
|
+
rows.iter()
|
|
4236
|
+
.enumerate()
|
|
4237
|
+
.map(|(index, row)| {
|
|
4238
|
+
let value_json = if updated {
|
|
4239
|
+
row.updated_value_json.as_str()
|
|
4240
|
+
} else {
|
|
4241
|
+
row.value_json.as_str()
|
|
4242
|
+
};
|
|
4243
|
+
let value = serde_json::from_str::<serde_json::Value>(value_json)
|
|
4244
|
+
.unwrap_or_else(|_| serde_json::Value::String(value_json.to_string()));
|
|
4245
|
+
let snapshot = serde_json::json!({
|
|
4246
|
+
"path": row.path,
|
|
4247
|
+
"value": value,
|
|
4248
|
+
})
|
|
4249
|
+
.to_string();
|
|
4250
|
+
MaterializedTrackedStateRow {
|
|
4251
|
+
entity_id: EntityIdentity::single(row.path.as_str()),
|
|
4252
|
+
schema_key: "json_pointer".to_string(),
|
|
4253
|
+
file_id: None,
|
|
4254
|
+
snapshot_content: Some(snapshot),
|
|
4255
|
+
metadata: None,
|
|
4256
|
+
deleted: false,
|
|
4257
|
+
created_at: timestamp(index),
|
|
4258
|
+
updated_at: timestamp(index),
|
|
4259
|
+
change_id: tracked_change_id(commit_id, index),
|
|
4260
|
+
commit_id: commit_id.to_string(),
|
|
4261
|
+
}
|
|
4262
|
+
})
|
|
4263
|
+
.collect()
|
|
4264
|
+
}
|
|
4265
|
+
|
|
4266
|
+
async fn write_json_pointer_delta_chain(
|
|
4267
|
+
backend: &Arc<dyn Backend + Send + Sync>,
|
|
4268
|
+
rows: &[JsonPointerStorageRow],
|
|
4269
|
+
delta_commits: usize,
|
|
4270
|
+
updated_rows_per_commit: usize,
|
|
4271
|
+
) -> Result<(TrackedStateContext, String), LixError> {
|
|
4272
|
+
let context = TrackedStateContext::new();
|
|
4273
|
+
let base_commit_id = "json-pointer-base";
|
|
4274
|
+
let base_rows = json_pointer_tracked_rows(rows, base_commit_id, false);
|
|
4275
|
+
write_tracked_root(backend, &context, base_commit_id, None, &base_rows).await?;
|
|
4276
|
+
|
|
4277
|
+
let mut parent_commit_id = base_commit_id.to_string();
|
|
4278
|
+
for delta_index in 0..delta_commits {
|
|
4279
|
+
let commit_id = format!("json-pointer-delta-{delta_index}");
|
|
4280
|
+
let mut child_rows = json_pointer_tracked_rows(
|
|
4281
|
+
&rows[..updated_rows_per_commit.min(rows.len())],
|
|
4282
|
+
&commit_id,
|
|
4283
|
+
true,
|
|
4284
|
+
);
|
|
4285
|
+
for row in &mut child_rows {
|
|
4286
|
+
row.updated_at = timestamp(rows.len() + delta_index);
|
|
4287
|
+
}
|
|
4288
|
+
write_tracked_root(
|
|
4289
|
+
backend,
|
|
4290
|
+
&context,
|
|
4291
|
+
&commit_id,
|
|
4292
|
+
Some(parent_commit_id.as_str()),
|
|
4293
|
+
&child_rows,
|
|
4294
|
+
)
|
|
4295
|
+
.await?;
|
|
4296
|
+
parent_commit_id = commit_id;
|
|
4297
|
+
}
|
|
4298
|
+
|
|
4299
|
+
Ok((context, parent_commit_id))
|
|
4300
|
+
}
|
|
4301
|
+
|
|
2899
4302
|
fn tracked_rows_file_selective(
|
|
2900
4303
|
config: StorageBenchConfig,
|
|
2901
4304
|
commit_id: &str,
|
|
2902
|
-
) -> Vec<
|
|
4305
|
+
) -> Vec<MaterializedTrackedStateRow> {
|
|
2903
4306
|
(0..config.rows)
|
|
2904
|
-
.map(|index|
|
|
4307
|
+
.map(|index| MaterializedTrackedStateRow {
|
|
2905
4308
|
entity_id: EntityIdentity::single(entity_id("tracked", index, config.key_pattern)),
|
|
2906
4309
|
schema_key: TRACKED_MATCH_SCHEMA_KEY.to_string(),
|
|
2907
4310
|
file_id: Some(
|
|
@@ -2914,15 +4317,19 @@ fn tracked_rows_file_selective(
|
|
|
2914
4317
|
),
|
|
2915
4318
|
snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
|
|
2916
4319
|
metadata: None,
|
|
2917
|
-
|
|
4320
|
+
deleted: false,
|
|
2918
4321
|
created_at: timestamp(index),
|
|
2919
4322
|
updated_at: timestamp(index),
|
|
2920
|
-
change_id:
|
|
4323
|
+
change_id: tracked_change_id(commit_id, index),
|
|
2921
4324
|
commit_id: commit_id.to_string(),
|
|
2922
4325
|
})
|
|
2923
4326
|
.collect()
|
|
2924
4327
|
}
|
|
2925
4328
|
|
|
4329
|
+
fn tracked_change_id(commit_id: &str, index: usize) -> String {
|
|
4330
|
+
format!("{commit_id}:tracked-change-{index}")
|
|
4331
|
+
}
|
|
4332
|
+
|
|
2926
4333
|
fn untracked_rows(config: StorageBenchConfig) -> Vec<MaterializedUntrackedStateRow> {
|
|
2927
4334
|
(0..config.rows)
|
|
2928
4335
|
.map(|index| MaterializedUntrackedStateRow {
|
|
@@ -2931,7 +4338,7 @@ fn untracked_rows(config: StorageBenchConfig) -> Vec<MaterializedUntrackedStateR
|
|
|
2931
4338
|
file_id: Some("bench.json".to_string()),
|
|
2932
4339
|
snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
|
|
2933
4340
|
metadata: None,
|
|
2934
|
-
|
|
4341
|
+
deleted: false,
|
|
2935
4342
|
created_at: timestamp(index),
|
|
2936
4343
|
updated_at: timestamp(index),
|
|
2937
4344
|
global: false,
|
|
@@ -2940,16 +4347,16 @@ fn untracked_rows(config: StorageBenchConfig) -> Vec<MaterializedUntrackedStateR
|
|
|
2940
4347
|
.collect()
|
|
2941
4348
|
}
|
|
2942
4349
|
|
|
2943
|
-
fn changelog_changes(config: StorageBenchConfig) -> Vec<
|
|
4350
|
+
fn changelog_changes(config: StorageBenchConfig) -> Vec<Change> {
|
|
2944
4351
|
changelog_materialized_changes(config)
|
|
2945
4352
|
.into_iter()
|
|
2946
|
-
.map(
|
|
4353
|
+
.map(changelog_bench_change_ref_only)
|
|
2947
4354
|
.collect()
|
|
2948
4355
|
}
|
|
2949
4356
|
|
|
2950
|
-
fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<
|
|
4357
|
+
fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
2951
4358
|
(0..config.rows)
|
|
2952
|
-
.map(|index|
|
|
4359
|
+
.map(|index| MaterializedChange {
|
|
2953
4360
|
id: format!("bench-change-{index}"),
|
|
2954
4361
|
entity_id: EntityIdentity::single(entity_id(
|
|
2955
4362
|
"change-entity",
|
|
@@ -2957,7 +4364,6 @@ fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<Materialize
|
|
|
2957
4364
|
config.key_pattern,
|
|
2958
4365
|
)),
|
|
2959
4366
|
schema_key: "bench_changelog_entity".to_string(),
|
|
2960
|
-
schema_version: "1".to_string(),
|
|
2961
4367
|
file_id: Some("bench.json".to_string()),
|
|
2962
4368
|
snapshot_content: Some(snapshot_content(index, config.state_payload_bytes)),
|
|
2963
4369
|
metadata: None,
|
|
@@ -2966,21 +4372,72 @@ fn changelog_materialized_changes(config: StorageBenchConfig) -> Vec<Materialize
|
|
|
2966
4372
|
.collect()
|
|
2967
4373
|
}
|
|
2968
4374
|
|
|
2969
|
-
fn
|
|
4375
|
+
fn commit_graph_materialized_commit_change(commit_id: &str, rows: usize) -> MaterializedChange {
|
|
4376
|
+
let snapshot_content = serde_json::json!({
|
|
4377
|
+
"id": commit_id,
|
|
4378
|
+
})
|
|
4379
|
+
.to_string();
|
|
4380
|
+
|
|
4381
|
+
MaterializedChange {
|
|
4382
|
+
id: format!("bench-commit-change-{commit_id}"),
|
|
4383
|
+
entity_id: EntityIdentity::single(commit_id.to_string()),
|
|
4384
|
+
schema_key: "lix_commit".to_string(),
|
|
4385
|
+
file_id: None,
|
|
4386
|
+
snapshot_content: Some(snapshot_content),
|
|
4387
|
+
metadata: None,
|
|
4388
|
+
created_at: timestamp(rows),
|
|
4389
|
+
}
|
|
4390
|
+
}
|
|
4391
|
+
|
|
4392
|
+
fn canonical_changelog_bench_change(change: &MaterializedChange) -> Result<Change, LixError> {
|
|
4393
|
+
let snapshot_ref = change
|
|
4394
|
+
.snapshot_content
|
|
4395
|
+
.as_ref()
|
|
4396
|
+
.map(|value| prepare_json_ref(value.as_bytes()))
|
|
4397
|
+
.transpose()?;
|
|
4398
|
+
let metadata_ref = change
|
|
4399
|
+
.metadata
|
|
4400
|
+
.as_ref()
|
|
4401
|
+
.map(|value| prepare_json_ref(value.as_bytes()))
|
|
4402
|
+
.transpose()?;
|
|
4403
|
+
Ok(Change {
|
|
4404
|
+
id: change.id.clone(),
|
|
4405
|
+
entity_id: change.entity_id.clone(),
|
|
4406
|
+
schema_key: change.schema_key.clone(),
|
|
4407
|
+
file_id: change.file_id.clone(),
|
|
4408
|
+
snapshot_ref,
|
|
4409
|
+
metadata_ref,
|
|
4410
|
+
created_at: change.created_at.clone(),
|
|
4411
|
+
})
|
|
4412
|
+
}
|
|
4413
|
+
|
|
4414
|
+
fn changelog_bench_json_payloads(changes: &[MaterializedChange]) -> Vec<String> {
|
|
4415
|
+
changes
|
|
4416
|
+
.iter()
|
|
4417
|
+
.flat_map(|change| {
|
|
4418
|
+
change
|
|
4419
|
+
.snapshot_content
|
|
4420
|
+
.iter()
|
|
4421
|
+
.chain(change.metadata.iter())
|
|
4422
|
+
.cloned()
|
|
4423
|
+
.collect::<Vec<_>>()
|
|
4424
|
+
})
|
|
4425
|
+
.collect()
|
|
4426
|
+
}
|
|
4427
|
+
|
|
4428
|
+
fn changelog_bench_change_ref_only(change: MaterializedChange) -> Change {
|
|
2970
4429
|
let snapshot_ref = change
|
|
2971
4430
|
.snapshot_content
|
|
2972
4431
|
.as_ref()
|
|
2973
4432
|
.map(|value| JsonRef::from_hash(blake3::hash(value.as_bytes())));
|
|
2974
|
-
let metadata_ref = change
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
JsonRef::from_hash(blake3::hash(
|
|
2978
|
-
|
|
2979
|
-
CanonicalChange {
|
|
4433
|
+
let metadata_ref = change
|
|
4434
|
+
.metadata
|
|
4435
|
+
.as_ref()
|
|
4436
|
+
.map(|value| JsonRef::from_hash(blake3::hash(value.as_bytes())));
|
|
4437
|
+
Change {
|
|
2980
4438
|
id: change.id,
|
|
2981
4439
|
entity_id: change.entity_id,
|
|
2982
4440
|
schema_key: change.schema_key,
|
|
2983
|
-
schema_version: change.schema_version,
|
|
2984
4441
|
file_id: change.file_id,
|
|
2985
4442
|
snapshot_ref,
|
|
2986
4443
|
metadata_ref,
|
|
@@ -2988,7 +4445,7 @@ fn canonical_changelog_bench_change(change: MaterializedCanonicalChange) -> Cano
|
|
|
2988
4445
|
}
|
|
2989
4446
|
}
|
|
2990
4447
|
|
|
2991
|
-
fn changelog_tombstone_changes(config: StorageBenchConfig) -> Vec<
|
|
4448
|
+
fn changelog_tombstone_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
2992
4449
|
changelog_materialized_changes(config)
|
|
2993
4450
|
.into_iter()
|
|
2994
4451
|
.map(|mut change| {
|
|
@@ -2999,7 +4456,7 @@ fn changelog_tombstone_changes(config: StorageBenchConfig) -> Vec<MaterializedCa
|
|
|
2999
4456
|
.collect()
|
|
3000
4457
|
}
|
|
3001
4458
|
|
|
3002
|
-
fn changelog_metadata_changes(config: StorageBenchConfig) -> Vec<
|
|
4459
|
+
fn changelog_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
3003
4460
|
changelog_materialized_changes(config)
|
|
3004
4461
|
.into_iter()
|
|
3005
4462
|
.enumerate()
|
|
@@ -3010,9 +4467,7 @@ fn changelog_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedCan
|
|
|
3010
4467
|
.collect()
|
|
3011
4468
|
}
|
|
3012
4469
|
|
|
3013
|
-
fn changelog_shared_payload_changes(
|
|
3014
|
-
config: StorageBenchConfig,
|
|
3015
|
-
) -> Vec<MaterializedCanonicalChange> {
|
|
4470
|
+
fn changelog_shared_payload_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
3016
4471
|
let shared_snapshot_content = snapshot_content(0, config.state_payload_bytes);
|
|
3017
4472
|
changelog_materialized_changes(config)
|
|
3018
4473
|
.into_iter()
|
|
@@ -3023,9 +4478,7 @@ fn changelog_shared_payload_changes(
|
|
|
3023
4478
|
.collect()
|
|
3024
4479
|
}
|
|
3025
4480
|
|
|
3026
|
-
fn changelog_shared_metadata_changes(
|
|
3027
|
-
config: StorageBenchConfig,
|
|
3028
|
-
) -> Vec<MaterializedCanonicalChange> {
|
|
4481
|
+
fn changelog_shared_metadata_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
3029
4482
|
let shared_metadata = snapshot_metadata(0, config.state_payload_bytes);
|
|
3030
4483
|
changelog_materialized_changes(config)
|
|
3031
4484
|
.into_iter()
|
|
@@ -3039,7 +4492,7 @@ fn changelog_shared_metadata_changes(
|
|
|
3039
4492
|
|
|
3040
4493
|
fn changelog_shared_payload_and_metadata_changes(
|
|
3041
4494
|
config: StorageBenchConfig,
|
|
3042
|
-
) -> Vec<
|
|
4495
|
+
) -> Vec<MaterializedChange> {
|
|
3043
4496
|
let shared_snapshot_content = snapshot_content(0, config.state_payload_bytes);
|
|
3044
4497
|
let shared_metadata = snapshot_metadata(1, config.state_payload_bytes);
|
|
3045
4498
|
changelog_materialized_changes(config)
|
|
@@ -3052,22 +4505,16 @@ fn changelog_shared_payload_and_metadata_changes(
|
|
|
3052
4505
|
.collect()
|
|
3053
4506
|
}
|
|
3054
4507
|
|
|
3055
|
-
fn changelog_composite_entity_id_changes(
|
|
3056
|
-
config: StorageBenchConfig,
|
|
3057
|
-
) -> Vec<MaterializedCanonicalChange> {
|
|
4508
|
+
fn changelog_composite_entity_id_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
3058
4509
|
changelog_materialized_changes(config)
|
|
3059
4510
|
.into_iter()
|
|
3060
4511
|
.enumerate()
|
|
3061
4512
|
.map(|(index, mut change)| {
|
|
3062
4513
|
change.entity_id = EntityIdentity {
|
|
3063
4514
|
parts: vec![
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
config.key_pattern,
|
|
3068
|
-
)),
|
|
3069
|
-
EntityIdentityPart::Number(index.to_string()),
|
|
3070
|
-
EntityIdentityPart::Bool(index % 2 == 0),
|
|
4515
|
+
entity_id("change-composite", index, config.key_pattern),
|
|
4516
|
+
index.to_string(),
|
|
4517
|
+
(index % 2 == 0).to_string(),
|
|
3071
4518
|
],
|
|
3072
4519
|
};
|
|
3073
4520
|
change
|
|
@@ -3075,7 +4522,7 @@ fn changelog_composite_entity_id_changes(
|
|
|
3075
4522
|
.collect()
|
|
3076
4523
|
}
|
|
3077
4524
|
|
|
3078
|
-
fn changelog_selective_changes(config: StorageBenchConfig) -> Vec<
|
|
4525
|
+
fn changelog_selective_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
3079
4526
|
changelog_materialized_changes(config)
|
|
3080
4527
|
.into_iter()
|
|
3081
4528
|
.enumerate()
|
|
@@ -3086,9 +4533,7 @@ fn changelog_selective_changes(config: StorageBenchConfig) -> Vec<MaterializedCa
|
|
|
3086
4533
|
.collect()
|
|
3087
4534
|
}
|
|
3088
4535
|
|
|
3089
|
-
fn changelog_entity_history_changes(
|
|
3090
|
-
config: StorageBenchConfig,
|
|
3091
|
-
) -> Vec<MaterializedCanonicalChange> {
|
|
4536
|
+
fn changelog_entity_history_changes(config: StorageBenchConfig) -> Vec<MaterializedChange> {
|
|
3092
4537
|
changelog_materialized_changes(config)
|
|
3093
4538
|
.into_iter()
|
|
3094
4539
|
.enumerate()
|
|
@@ -3187,16 +4632,14 @@ fn snapshot_content(index: usize, target_bytes: usize) -> String {
|
|
|
3187
4632
|
value.to_string()
|
|
3188
4633
|
}
|
|
3189
4634
|
|
|
3190
|
-
fn snapshot_metadata(index: usize, target_bytes: usize) ->
|
|
3191
|
-
|
|
3192
|
-
.expect("bench snapshot content should be valid JSON metadata")
|
|
4635
|
+
fn snapshot_metadata(index: usize, target_bytes: usize) -> String {
|
|
4636
|
+
snapshot_content(index, target_bytes)
|
|
3193
4637
|
}
|
|
3194
4638
|
|
|
3195
4639
|
fn tracked_state_header_columns() -> Vec<String> {
|
|
3196
4640
|
[
|
|
3197
4641
|
"entity_id",
|
|
3198
4642
|
"schema_key",
|
|
3199
|
-
"schema_version",
|
|
3200
4643
|
"file_id",
|
|
3201
4644
|
"metadata",
|
|
3202
4645
|
"created_at",
|
|
@@ -3213,7 +4656,6 @@ fn untracked_state_header_columns() -> Vec<String> {
|
|
|
3213
4656
|
[
|
|
3214
4657
|
"entity_id",
|
|
3215
4658
|
"schema_key",
|
|
3216
|
-
"schema_version",
|
|
3217
4659
|
"file_id",
|
|
3218
4660
|
"metadata",
|
|
3219
4661
|
"created_at",
|
|
@@ -3247,6 +4689,21 @@ fn partial_updated_snapshot_content(index: usize, target_bytes: usize) -> String
|
|
|
3247
4689
|
value.to_string()
|
|
3248
4690
|
}
|
|
3249
4691
|
|
|
4692
|
+
fn delta_chain_snapshot_content(
|
|
4693
|
+
delta_index: usize,
|
|
4694
|
+
row_index: usize,
|
|
4695
|
+
target_bytes: usize,
|
|
4696
|
+
) -> String {
|
|
4697
|
+
let mut value = serde_json::json!({
|
|
4698
|
+
"id": format!("entity-{row_index}"),
|
|
4699
|
+
"value": format!("delta-{delta_index}-{row_index}"),
|
|
4700
|
+
"index": row_index,
|
|
4701
|
+
"delta": delta_index
|
|
4702
|
+
});
|
|
4703
|
+
pad_snapshot_content(&mut value, target_bytes);
|
|
4704
|
+
value.to_string()
|
|
4705
|
+
}
|
|
4706
|
+
|
|
3250
4707
|
fn pad_snapshot_content(value: &mut serde_json::Value, target_bytes: usize) {
|
|
3251
4708
|
let current = value.to_string().len();
|
|
3252
4709
|
if target_bytes <= current {
|