@lix-js/sdk 0.6.0-preview.4 → 0.6.0-preview.5

package/dist-engine-src/src/backend/conformance/fixtures.rs

@@ -0,0 +1,26 @@
+use bytes::Bytes;
+
+use crate::backend::{Key, PutBatch, PutEntry, SpaceId, StoredValue};
+
+pub fn space(id: u32) -> SpaceId {
+    SpaceId(id)
+}
+
+pub fn key(bytes: impl AsRef<[u8]>) -> Key {
+    Key(Bytes::copy_from_slice(bytes.as_ref()))
+}
+
+pub fn full_put(key: Key, value: impl Into<Bytes>) -> PutEntry {
+    PutEntry {
+        key,
+        value: StoredValue {
+            bytes: value.into(),
+        },
+    }
+}
+
+pub fn put_batch(entries: impl IntoIterator<Item = PutEntry>) -> PutBatch {
+    PutBatch {
+        entries: entries.into_iter().collect(),
+    }
+}

package/dist-engine-src/src/backend/conformance/mod.rs

@@ -0,0 +1,75 @@
+//! Conformance harness for backend implementations.
+//!
+//! The harness is colocated with the experimental API for now. Once backend
+//! is stable, rs-sdk can re-export this as the public backend author test kit.
+
+mod baseline;
+mod factory;
+#[cfg(test)]
+mod failure_tests;
+#[allow(dead_code)]
+mod fixtures;
+#[allow(dead_code)]
+mod model;
+mod model_based;
+mod persistence;
+mod projection;
+mod pushdown;
+mod runner;
+mod scan;
+mod write;
+
+pub(crate) use factory::open_backend;
+pub use factory::{BackendFactory, BackendFixture, BackendTestConfig};
+pub use runner::{
+    run_backend_conformance, ConformanceReport, ConformanceResult, ConformanceStatus,
+    ConformanceTest,
+};
+
+#[cfg(test)]
+mod tests {
+    use super::{run_backend_conformance, ConformanceStatus};
+    use crate::backend::InMemoryBackendFactory;
+
+    #[test]
+    fn in_memory_backend_passes_baseline_conformance() {
+        let report = run_backend_conformance(&InMemoryBackendFactory);
+
+        report.assert_no_failures();
+
+        let passed = report
+            .tests
+            .iter()
+            .filter(|test| matches!(test.status, ConformanceStatus::Passed))
+            .map(|test| test.name)
+            .collect::<Vec<_>>();
+        assert_eq!(
+            passed,
+            vec![
+                "baseline::get_many_returns_requested_slots",
+                "baseline::get_many_empty_key_list",
+                "baseline::delete_many_missing_keys_is_idempotent",
+                "baseline::delete_many_removes_existing_keys",
+                "baseline::delete_range_removes_exact_range",
+                "baseline::delete_range_applies_after_staged_puts",
+                "baseline::put_many_applies_after_delete_range",
+                "baseline::put_many_overwrites_existing_value",
+                "baseline::scan_range_sees_overwritten_existing_value",
+                "baseline::scan_range_returns_forward_row_bounded_chunks",
+                "baseline::scan_range_honors_bound_variants",
+                "baseline::scan_range_resume_before_lower_does_not_widen_range",
+                "baseline::scan_range_orders_raw_byte_keys",
+                "baseline::scan_range_drains_multi_chunk_limits",
+                "baseline::scan_cursor_drains_multi_chunk_limits",
+                "baseline::scan_range_empty_range_returns_empty_chunk",
+                "baseline::commit_is_atomic",
+                "baseline::rollback_discards_staged_mutations",
+                "baseline::rollback_discards_overwrite_and_delete",
+                "baseline::begin_read_pins_coherent_view",
+                "baseline::full_value_and_key_only_are_core",
+                "baseline::full_value_preserves_opaque_bytes",
+                "model::deterministic_history_matches_reference_model",
+            ]
+        );
+    }
+}

package/dist-engine-src/src/backend/conformance/model.rs

@@ -0,0 +1,28 @@
+use std::collections::BTreeMap;
+
+use bytes::Bytes;
+
+use crate::backend::Key;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ReferenceModel {
+    entries: BTreeMap<Key, Bytes>,
+}
+
+impl ReferenceModel {
+    pub fn get(&self, key: &Key) -> Option<&Bytes> {
+        self.entries.get(key)
+    }
+
+    pub fn put(&mut self, key: Key, value: Bytes) {
+        self.entries.insert(key, value);
+    }
+
+    pub fn delete(&mut self, key: &Key) {
+        self.entries.remove(key);
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&Key, &Bytes)> {
+        self.entries.iter()
+    }
+}

package/dist-engine-src/src/backend/conformance/model_based.rs

@@ -0,0 +1,257 @@
+use std::collections::BTreeMap;
+use std::ops::Bound;
+
+use bytes::Bytes;
+
+use crate::backend::conformance::{
+    fixtures::{full_put, key, put_batch},
+    model::ReferenceModel,
+    open_backend, BackendFactory, ConformanceReport, ConformanceResult,
+};
+use crate::backend::{
+    get_many as backend_get_many, visit_range as backend_visit_range, Backend, BackendRead,
+    BackendWrite, GetOptions, Key, KeyRange, KeyRef, ProjectedValue, ProjectedValueRef, ReadEntry,
+    ReadOptions, ScanChunk, ScanOptions, WriteOptions,
+};
+
+pub(crate) fn register<F>(report: &mut ConformanceReport, factory: &F)
+where
+    F: BackendFactory,
+{
+    report.run(
+        "model::deterministic_history_matches_reference_model",
+        || deterministic_history_matches_reference_model(factory),
+    );
+}
+
+fn deterministic_history_matches_reference_model<F>(factory: &F) -> ConformanceResult
+where
+    F: BackendFactory,
+{
+    let backend = open_backend(factory);
+    let mut model = ReferenceModel::default();
+    let mut rng = TinyRng::new(0x51cedeed);
+    let keys = [
+        key("a"),
+        key("b"),
+        key("c"),
+        key("d"),
+        key("aa"),
+        key("ab"),
+        key("ba"),
+    ];
+
+    for step in 0..48 {
+        let old_read = backend
+            .begin_read(ReadOptions::default())
+            .map_err(|error| format!("step {step}: begin old read failed: {error}"))?;
+        let old_model = model.clone();
+        let mut write = backend
+            .begin_write(WriteOptions::default())
+            .map_err(|error| format!("step {step}: begin write failed: {error}"))?;
+        let mut staged = model.clone();
+
+        let mutation_count = 1 + rng.usize(4);
+        for mutation_index in 0..mutation_count {
+            let target_key = keys[rng.usize(keys.len())].clone();
+            if rng.bool() {
+                let value = Bytes::from(format!("v{step}-{mutation_index}"));
+                write
+                    .put_many(put_batch([full_put(target_key.clone(), value.clone())]))
+                    .map_err(|error| format!("step {step}: put_many failed: {error}"))?;
+                staged.put(target_key, value);
+            } else {
+                write
+                    .delete_many(&[target_key.clone()])
+                    .map_err(|error| format!("step {step}: delete_many failed: {error}"))?;
+                staged.delete(&target_key);
+            }
+        }
+
+        if rng.bool() {
+            write
+                .commit()
+                .map_err(|error| format!("step {step}: commit failed: {error}"))?;
+            model = staged;
+        } else {
+            write
+                .rollback()
+                .map_err(|error| format!("step {step}: rollback failed: {error}"))?;
+        }
+
+        compare_read_to_model(
+            &old_read,
+            &old_model,
+            &keys,
+            &mut rng,
+            &format!("step {step} old snapshot"),
+        )?;
+
+        let new_read = backend
+            .begin_read(ReadOptions::default())
+            .map_err(|error| format!("step {step}: begin new read failed: {error}"))?;
+        compare_read_to_model(
+            &new_read,
+            &model,
+            &keys,
+            &mut rng,
+            &format!("step {step} new snapshot"),
+        )?;
+    }
+
+    Ok(())
+}
+
+fn compare_read_to_model<R>(
+    read: &R,
+    model: &ReferenceModel,
+    keys: &[Key],
+    rng: &mut TinyRng,
+    label: &str,
+) -> ConformanceResult
+where
+    R: BackendRead,
+{
+    let point_keys = [
+        keys[rng.usize(keys.len())].clone(),
+        keys[rng.usize(keys.len())].clone(),
+        key("missing"),
+        keys[rng.usize(keys.len())].clone(),
+    ];
+    let result = backend_get_many(read, &point_keys, GetOptions::default())
+        .map_err(|error| format!("{label}: get_many failed: {error}"))?;
+    let actual = entries_to_map(&result.entries_for_requested_keys(&point_keys));
+    let expected = point_keys
+        .iter()
+        .filter_map(|key| model.get(key).map(|value| (key.clone(), value.clone())))
+        .collect::<BTreeMap<_, _>>();
+    if actual != expected {
+        return Err(format!(
+            "{label}: get_many mismatch: expected {expected:?}, got {actual:?}"
+        ));
+    }
+
+    let lower_key = keys[rng.usize(keys.len())].clone();
+    let upper_key = keys[rng.usize(keys.len())].clone();
+    let (lower, upper) = if lower_key <= upper_key {
+        (lower_key, upper_key)
+    } else {
+        (upper_key, lower_key)
+    };
+    let range = KeyRange {
+        lower: Bound::Included(lower),
+        upper: Bound::Included(upper),
+    };
+    let chunk = scan_range(
+        read,
+        range.clone(),
+        ScanOptions {
+            limit_rows: 3,
+            ..Default::default()
+        },
+    )
+    .map_err(|error| format!("{label}: scan_range failed: {error}"))?;
+    let actual_scan = chunk_entries(&chunk.entries);
+    let expected_scan = model_scan(model, &range, Some(3));
+    if actual_scan != expected_scan {
+        return Err(format!(
+            "{label}: scan_range mismatch: expected {expected_scan:?}, got {actual_scan:?}"
+        ));
+    }
+
+    Ok(())
+}
+
+fn scan_range<R>(
+    read: &R,
+    range: KeyRange,
+    opts: ScanOptions<'_>,
+) -> Result<ScanChunk, crate::backend::BackendError>
+where
+    R: BackendRead,
+{
+    let mut entries = Vec::with_capacity(opts.limit_rows);
+    let result = backend_visit_range(
+        read,
+        range,
+        opts,
+        &mut |key: KeyRef<'_>, value: ProjectedValueRef<'_>| {
+            entries.push(ReadEntry {
+                key: key.to_owned_key(),
+                value: value.to_owned(),
+            });
+            Ok(())
+        },
+    )?;
+    Ok(ScanChunk {
+        entries,
+        has_more: result.has_more,
+    })
+}
+
+fn model_scan(model: &ReferenceModel, range: &KeyRange, limit: Option<usize>) -> Vec<(Key, Bytes)> {
+    model
+        .iter()
+        .filter(|(key, _)| range_contains(range, key))
+        .take(limit.unwrap_or(usize::MAX))
+        .map(|(key, value)| (key.clone(), value.clone()))
+        .collect()
+}
+
+fn range_contains(range: &KeyRange, key: &Key) -> bool {
+    let lower_matches = match &range.lower {
+        Bound::Included(lower) => key >= lower,
+        Bound::Excluded(lower) => key > lower,
+        Bound::Unbounded => true,
+    };
+    let upper_matches = match &range.upper {
+        Bound::Included(upper) => key <= upper,
+        Bound::Excluded(upper) => key < upper,
+        Bound::Unbounded => true,
+    };
+    lower_matches && upper_matches
+}
+
+fn chunk_entries(entries: &[crate::backend::ReadEntry]) -> Vec<(Key, Bytes)> {
+    entries
+        .iter()
+        .map(|entry| (entry.key.clone(), projected_value_bytes(&entry.value)))
+        .collect()
+}
+
+fn entries_to_map(entries: &[crate::backend::ReadEntry]) -> BTreeMap<Key, Bytes> {
+    chunk_entries(entries).into_iter().collect()
+}
+
+fn projected_value_bytes(value: &ProjectedValue) -> Bytes {
+    match value {
+        ProjectedValue::FullValue(bytes) => bytes.clone(),
+        ProjectedValue::KeyOnly => Bytes::new(),
+    }
+}
+
+struct TinyRng {
+    state: u64,
+}
+
+impl TinyRng {
+    fn new(seed: u64) -> Self {
+        Self { state: seed }
+    }
+
+    fn next(&mut self) -> u64 {
+        self.state = self
+            .state
+            .wrapping_mul(6364136223846793005)
+            .wrapping_add(1442695040888963407);
+        self.state
+    }
+
+    fn usize(&mut self, upper: usize) -> usize {
+        (self.next() as usize) % upper
+    }
+
+    fn bool(&mut self) -> bool {
+        self.next() & 1 == 0
+    }
+}

package/dist-engine-src/src/backend/conformance/persistence.rs

@@ -0,0 +1,204 @@
+use crate::backend::conformance::{
+    fixtures::{full_put, key, put_batch, space},
+    BackendFactory, BackendFixture, ConformanceReport, ConformanceResult,
+};
+use crate::backend::{
+    get_many as backend_get_many, Backend, BackendWrite, GetOptions, ProjectedValue, ReadOptions,
+    WriteOptions,
+};
+
+pub(crate) fn register<F>(report: &mut ConformanceReport, factory: &F)
+where
+    F: BackendFactory,
+{
+    report.run(
+        "persistence::durable_write_lock_is_shared_across_reopen",
+        || durable_write_lock_is_shared_across_reopen(factory),
+    );
+    report.run("persistence::committed_data_survives_reopen", || {
+        committed_data_survives_reopen(factory)
+    });
+    report.run(
+        "persistence::rolled_back_data_does_not_survive_reopen",
+        || rolled_back_data_does_not_survive_reopen(factory),
+    );
+    report.run(
+        "persistence::overwrite_and_delete_final_state_survives_reopen",
+        || overwrite_and_delete_final_state_survives_reopen(factory),
+    );
+}
+
+fn durable_write_lock_is_shared_across_reopen<F>(factory: &F) -> ConformanceResult
+where
+    F: BackendFactory,
+{
+    let fixture = factory.create_fixture();
+    let first_lock = {
+        let first = fixture.open();
+        first.durable_write_lock()
+    };
+    let second_lock = {
+        let second = fixture.open();
+        second.durable_write_lock()
+    };
+    if !first_lock.ptr_eq(&second_lock) {
+        return Err(
+            "durable_write_lock must return the same lock for reopened handles to one durable target"
+                .to_string(),
+        );
+    }
+    Ok(())
+}
+
+fn committed_data_survives_reopen<F>(factory: &F) -> ConformanceResult
+where
+    F: BackendFactory,
+{
+    let fixture = factory.create_fixture();
+    let test_space = space(71);
+    let alpha = key("alpha");
+    let beta = key("beta");
+
+    {
+        let backend = fixture.open();
+        let mut write = backend
+            .begin_write(WriteOptions::default())
+            .map_err(|error| format!("begin_write failed: {error}"))?;
+        write
+            .put_many(put_batch([
+                full_put(alpha.clone(), "persisted-alpha"),
+                full_put(beta.clone(), "persisted-beta"),
+            ]))
+            .map_err(|error| format!("put_many failed: {error}"))?;
+        write
+            .commit()
+            .map_err(|error| format!("commit failed: {error}"))?;
+    }
+
+    let reopened = fixture.open();
+    assert_full_values(
+        &reopened,
+        test_space,
+        &[
+            (alpha, Some("persisted-alpha")),
+            (beta, Some("persisted-beta")),
+        ],
+    )
+}
+
+fn rolled_back_data_does_not_survive_reopen<F>(factory: &F) -> ConformanceResult
+where
+    F: BackendFactory,
+{
+    let fixture = factory.create_fixture();
+    let test_space = space(72);
+    let rolled_back = key("rolled-back");
+
+    {
+        let backend = fixture.open();
+        let mut write = backend
+            .begin_write(WriteOptions::default())
+            .map_err(|error| format!("begin_write failed: {error}"))?;
+        write
+            .put_many(put_batch([full_put(
+                rolled_back.clone(),
+                "should-not-persist",
+            )]))
+            .map_err(|error| format!("put_many failed: {error}"))?;
+        write
+            .rollback()
+            .map_err(|error| format!("rollback failed: {error}"))?;
+    }
+
+    let reopened = fixture.open();
+    assert_full_values(&reopened, test_space, &[(rolled_back, None)])
+}
+
+fn overwrite_and_delete_final_state_survives_reopen<F>(factory: &F) -> ConformanceResult
+where
+    F: BackendFactory,
+{
+    let fixture = factory.create_fixture();
+    let test_space = space(73);
+    let overwritten = key("overwritten");
+    let deleted = key("deleted");
+
+    {
+        let backend = fixture.open();
+        let mut write = backend
+            .begin_write(WriteOptions::default())
+            .map_err(|error| format!("begin_write failed: {error}"))?;
+        write
+            .put_many(put_batch([
+                full_put(overwritten.clone(), "old"),
+                full_put(deleted.clone(), "delete-me"),
+            ]))
+            .map_err(|error| format!("initial put_many failed: {error}"))?;
+        write
+            .commit()
+            .map_err(|error| format!("initial commit failed: {error}"))?;
+    }
+
+    {
+        let backend = fixture.open();
+        let mut write = backend
+            .begin_write(WriteOptions::default())
+            .map_err(|error| format!("begin_write failed: {error}"))?;
+        write
+            .put_many(put_batch([full_put(overwritten.clone(), "new")]))
+            .map_err(|error| format!("overwrite put_many failed: {error}"))?;
+        write
+            .delete_many(&[deleted.clone()])
+            .map_err(|error| format!("delete_many failed: {error}"))?;
+        write
+            .commit()
+            .map_err(|error| format!("final commit failed: {error}"))?;
+    }
+
+    let reopened = fixture.open();
+    assert_full_values(
+        &reopened,
+        test_space,
+        &[(overwritten, Some("new")), (deleted, None)],
+    )
+}
+
+fn assert_full_values<B>(
+    backend: &B,
+    _test_space: crate::backend::SpaceId,
+    expected: &[(crate::backend::Key, Option<&str>)],
+) -> ConformanceResult
+where
+    B: Backend,
+{
+    let keys = expected
+        .iter()
+        .map(|(key, _)| key.clone())
+        .collect::<Vec<_>>();
+    let read = backend
+        .begin_read(ReadOptions::default())
+        .map_err(|error| format!("begin_read failed: {error}"))?;
+    let result = backend_get_many(&read, &keys, GetOptions::default())
+        .map_err(|error| format!("get_many failed: {error}"))?;
+
+    for (index, (key, expected_value)) in expected.iter().enumerate() {
+        let actual = match result.values.get(index).and_then(|value| value.as_ref()) {
+            Some(ProjectedValue::FullValue(bytes)) => Some(
+                std::str::from_utf8(bytes.as_ref())
+                    .map_err(|error| format!("slot {index} contained non-utf8 bytes: {error}"))?,
+            ),
+            Some(other) => {
+                return Err(format!("slot {index} returned unexpected value: {other:?}"));
+            }
+            None => None,
+        };
+        if actual != *expected_value {
+            return Err(format!(
+                "key {:?} value mismatch: expected {:?}, got {:?}",
+                key, expected_value, actual
+            ));
+        }
+    }
+
+    Ok(())
+}

package/dist-engine-src/src/backend/conformance/projection.rs

@@ -0,0 +1,21 @@
+use crate::backend::conformance::{BackendFactory, ConformanceReport};
+
+pub(crate) fn register<F>(report: &mut ConformanceReport, factory: &F)
+where
+    F: BackendFactory,
+{
+    let capabilities = factory.capabilities();
+
+    if capabilities.projection.header {
+        report.add_pending("projection::header_returns_header_without_payload");
+    }
+    if capabilities.projection.refs {
+        report.add_pending("projection::refs_returns_refs_without_payload");
+    }
+    if capabilities.projection.header_and_refs {
+        report.add_pending("projection::header_and_refs_returns_both_without_payload");
+    }
+    if capabilities.projection.payload {
+        report.add_pending("projection::payload_returns_payload_only");
+    }
+}

package/dist-engine-src/src/backend/conformance/pushdown.rs

@@ -0,0 +1,24 @@
+use crate::backend::{
+    conformance::{BackendFactory, ConformanceReport},
+    PredicateSupportLevel,
+};
+
+pub(crate) fn register<F>(report: &mut ConformanceReport, factory: &F)
+where
+    F: BackendFactory,
+{
+    let capabilities = factory.capabilities();
+
+    if capabilities.pushdown.key != PredicateSupportLevel::None {
+        report.add_pending("pushdown::key_support_metadata_is_truthful");
+    }
+    if capabilities.pushdown.header != PredicateSupportLevel::None {
+        report.add_pending("pushdown::header_support_metadata_is_truthful");
+    }
+    if capabilities.pushdown.refs != PredicateSupportLevel::None {
+        report.add_pending("pushdown::refs_support_metadata_is_truthful");
+    }
+    if capabilities.pushdown.object_pruning != PredicateSupportLevel::None {
+        report.add_pending("pushdown::object_pruning_requires_residual_filtering_when_inexact");
+    }
+}