@lix-js/sdk 0.6.0-preview.0 → 0.6.0-preview.2

package/dist-engine-src/src/backend/types.rs

@@ -0,0 +1,96 @@
+use async_trait::async_trait;
+
+use crate::backend::{
+    BackendKvEntryPage, BackendKvExistsBatch, BackendKvGetRequest, BackendKvKeyPage,
+    BackendKvScanRequest, BackendKvValueBatch, BackendKvValuePage, BackendKvWriteBatch,
+    BackendKvWriteStats,
+};
+use crate::LixError;
+
+#[async_trait]
+pub trait Backend: Send + Sync {
+    async fn begin_read_transaction(
+        &self,
+    ) -> Result<Box<dyn BackendReadTransaction + Send + Sync + 'static>, LixError>;
+
+    async fn begin_write_transaction(
+        &self,
+    ) -> Result<Box<dyn BackendWriteTransaction + Send + Sync + 'static>, LixError>;
+
+    /// Releases physical resources held by this backend handle.
+    ///
+    /// This is a resource lifecycle operation, not a durability boundary and
+    /// not a destructive operation. Successful write transactions are durable
+    /// when their commit returns; callers should not rely on `close` to save
+    /// data. Implementations that do not own external resources may keep the
+    /// default no-op behavior.
+    async fn close(&self) -> Result<(), LixError> {
+        Ok(())
+    }
+
+    /// Destroys the physical storage target represented by this backend.
+    ///
+    /// This is a persistence lifecycle operation, not a logical SQL operation.
+    ///
+    /// Callers should treat the backend as the authority for what constitutes
+    /// the full storage target. For example:
+    ///
+    /// - native SQLite may delete the main database file plus WAL/SHM sidecars
+    /// - wasm/opfs SQLite may clear the persisted OPFS target
+    /// - Postgres may drop or clear the configured schema/database target
+    ///
+    /// Callers must not attempt to infer or delete backend-owned physical
+    /// artifacts themselves.
+    ///
+    /// Implementations may choose not to support destroy if the backend
+    /// instance does not have enough information or authority to remove its
+    /// target.
+    async fn destroy(&self) -> Result<(), LixError> {
+        Err(LixError {
+            code: "LIX_ERROR_UNKNOWN".to_string(),
+            message: "destroy is not supported by this backend".to_string(),
+            hint: None,
+            details: None,
+        })
+    }
+}
+
+#[async_trait]
+pub trait BackendReadTransaction: Send + Sync {
+    async fn get_values(
+        &mut self,
+        request: BackendKvGetRequest,
+    ) -> Result<BackendKvValueBatch, LixError>;
+
+    async fn exists_many(
+        &mut self,
+        request: BackendKvGetRequest,
+    ) -> Result<BackendKvExistsBatch, LixError>;
+
+    async fn scan_keys(
+        &mut self,
+        request: BackendKvScanRequest,
+    ) -> Result<BackendKvKeyPage, LixError>;
+
+    async fn scan_values(
+        &mut self,
+        request: BackendKvScanRequest,
+    ) -> Result<BackendKvValuePage, LixError>;
+
+    async fn scan_entries(
+        &mut self,
+        request: BackendKvScanRequest,
+    ) -> Result<BackendKvEntryPage, LixError>;
+
+    async fn rollback(self: Box<Self>) -> Result<(), LixError>;
+}
+
+#[async_trait]
+pub trait BackendWriteTransaction: BackendReadTransaction {
+    async fn write_kv_batch(
+        &mut self,
+        batch: BackendKvWriteBatch,
+    ) -> Result<BackendKvWriteStats, LixError>;
+
+    async fn commit(self: Box<Self>) -> Result<(), LixError>;
+}

package/dist-engine-src/src/binary_cas/chunking.rs

@@ -0,0 +1,31 @@
+const FASTCDC_MIN_CHUNK_BYTES: usize = 16 * 1024;
+const FASTCDC_AVG_CHUNK_BYTES: usize = 64 * 1024;
+const FASTCDC_MAX_CHUNK_BYTES: usize = 256 * 1024;
+const SINGLE_CHUNK_FAST_PATH_MAX_BYTES: usize = 64 * 1024;
+
+#[allow(dead_code)]
+pub(crate) fn should_materialize_chunk_cas(data: &[u8]) -> bool {
+    data.len() > SINGLE_CHUNK_FAST_PATH_MAX_BYTES
+}
+
+pub(crate) fn fastcdc_chunk_ranges(data: &[u8]) -> Vec<(usize, usize)> {
+    if data.is_empty() {
+        return Vec::new();
+    }
+    if data.len() <= SINGLE_CHUNK_FAST_PATH_MAX_BYTES {
+        return vec![(0, data.len())];
+    }
+
+    fastcdc::v2020::FastCDC::new(
+        data,
+        FASTCDC_MIN_CHUNK_BYTES as u32,
+        FASTCDC_AVG_CHUNK_BYTES as u32,
+        FASTCDC_MAX_CHUNK_BYTES as u32,
+    )
+    .map(|chunk| {
+        let start = chunk.offset as usize;
+        let end = start + (chunk.length as usize);
+        (start, end)
+    })
+    .collect()
+}

package/dist-engine-src/src/binary_cas/codec.rs

@@ -0,0 +1,346 @@
+use crate::LixError;
+
+// Binary CAS physical rows:
+// - manifest:       BCM2 | kind:u8 | blob_size:u64 | kind payload
+//   - empty payload:   []
+//   - single payload:  chunk_hash:[u8;32]
+//   - chunked payload: chunk_count:u32
+// - manifest chunk: BCC1 | chunk_hash:[u8;32] | uncompressed_len:u64
+// - chunk:          BCK1 | codec:u8 | uncompressed_len:u64 | payload:[u8]
+const MANIFEST_MAGIC: &[u8; 4] = b"BCM2";
+const MANIFEST_CHUNK_MAGIC: &[u8; 4] = b"BCC1";
+const CHUNK_MAGIC: &[u8; 4] = b"BCK1";
+const MANIFEST_KIND_EMPTY: u8 = 0;
+const MANIFEST_KIND_SINGLE_CHUNK: u8 = 1;
+const MANIFEST_KIND_CHUNKED: u8 = 2;
+const CHUNK_CODEC_RAW_TAG: u8 = 0;
+const HASH_BYTES: usize = 32;
+const MANIFEST_HEADER_BYTES: usize = 4 + 1 + 8;
+const EMPTY_MANIFEST_BYTES: usize = MANIFEST_HEADER_BYTES;
+const SINGLE_CHUNK_MANIFEST_BYTES: usize = MANIFEST_HEADER_BYTES + HASH_BYTES;
+const CHUNKED_MANIFEST_BYTES: usize = MANIFEST_HEADER_BYTES + 4;
+const MANIFEST_CHUNK_BYTES: usize = 4 + HASH_BYTES + 8;
+const CHUNK_HEADER_BYTES: usize = 4 + 1 + 8;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum BinaryChunkCodec {
+    Raw,
+}
+
+impl BinaryChunkCodec {
+    fn tag(self) -> u8 {
+        match self {
+            Self::Raw => CHUNK_CODEC_RAW_TAG,
+        }
+    }
+
+    fn from_tag(tag: u8) -> Result<Self, LixError> {
+        match tag {
+            CHUNK_CODEC_RAW_TAG => Ok(Self::Raw),
+            other => Err(codec_error(format!(
+                "unsupported binary CAS chunk codec tag {other}"
+            ))),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct EncodedBinaryChunkPayload {
+    pub(crate) codec: BinaryChunkCodec,
+    pub(crate) data: Vec<u8>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) enum BinaryCasManifest {
+    Empty {
+        size_bytes: u64,
+    },
+    SingleChunk {
+        size_bytes: u64,
+        chunk_hash: [u8; HASH_BYTES],
+    },
+    Chunked {
+        size_bytes: u64,
+        chunk_count: u32,
+    },
+}
+
+impl BinaryCasManifest {
+    pub(crate) fn size_bytes(&self) -> u64 {
+        match self {
+            Self::Empty { size_bytes }
+            | Self::SingleChunk { size_bytes, .. }
+            | Self::Chunked { size_bytes, .. } => *size_bytes,
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn binary_blob_hash_hex(data: &[u8]) -> String {
+    crate::common::stable_content_fingerprint_hex(data)
+}
+
+pub(crate) fn binary_blob_hash_bytes(data: &[u8]) -> [u8; HASH_BYTES] {
+    *blake3::hash(data).as_bytes()
+}
+
+pub(crate) fn hash_hex_to_bytes(hash_hex: &str, label: &str) -> Result<[u8; HASH_BYTES], LixError> {
+    if hash_hex.len() != HASH_BYTES * 2 {
+        return Err(codec_error(format!(
+            "{label} hash must be {} hex characters, got {}",
+            HASH_BYTES * 2,
+            hash_hex.len()
+        )));
+    }
+
+    let mut out = [0u8; HASH_BYTES];
+    let bytes = hash_hex.as_bytes();
+    for index in 0..HASH_BYTES {
+        out[index] =
+            (hex_value(bytes[index * 2], label)? << 4) | hex_value(bytes[index * 2 + 1], label)?;
+    }
+    Ok(out)
+}
+
+pub(crate) fn hash_bytes_to_hex(bytes: &[u8; HASH_BYTES]) -> String {
+    blake3::Hash::from_bytes(*bytes).to_hex().to_string()
+}
+
+pub(crate) fn encode_binary_cas_manifest(manifest: &BinaryCasManifest) -> Vec<u8> {
+    let capacity = match manifest {
+        BinaryCasManifest::Empty { .. } => EMPTY_MANIFEST_BYTES,
+        BinaryCasManifest::SingleChunk { .. } => SINGLE_CHUNK_MANIFEST_BYTES,
+        BinaryCasManifest::Chunked { .. } => CHUNKED_MANIFEST_BYTES,
+    };
+    let mut out = Vec::with_capacity(capacity);
+    out.extend_from_slice(MANIFEST_MAGIC);
+    match manifest {
+        BinaryCasManifest::Empty { size_bytes } => {
+            out.push(MANIFEST_KIND_EMPTY);
+            out.extend_from_slice(&size_bytes.to_be_bytes());
+        }
+        BinaryCasManifest::SingleChunk {
+            size_bytes,
+            chunk_hash,
+        } => {
+            out.push(MANIFEST_KIND_SINGLE_CHUNK);
+            out.extend_from_slice(&size_bytes.to_be_bytes());
+            out.extend_from_slice(chunk_hash);
+        }
+        BinaryCasManifest::Chunked {
+            size_bytes,
+            chunk_count,
+        } => {
+            out.push(MANIFEST_KIND_CHUNKED);
+            out.extend_from_slice(&size_bytes.to_be_bytes());
+            out.extend_from_slice(&chunk_count.to_be_bytes());
+        }
+    }
+    out
+}
+
+pub(crate) fn decode_binary_cas_manifest(bytes: &[u8]) -> Result<BinaryCasManifest, LixError> {
+    if bytes.len() < MANIFEST_HEADER_BYTES {
+        return Err(codec_error(format!(
+            "binary CAS manifest must be at least {MANIFEST_HEADER_BYTES} bytes, got {}",
+            bytes.len()
+        )));
+    }
+    require_magic(bytes, MANIFEST_MAGIC, "binary CAS manifest")?;
+    let size_bytes = u64::from_be_bytes(bytes[5..13].try_into().expect("fixed slice"));
+    match bytes[4] {
+        MANIFEST_KIND_EMPTY => {
+            require_len(bytes, EMPTY_MANIFEST_BYTES, "binary CAS empty manifest")?;
+            Ok(BinaryCasManifest::Empty { size_bytes })
+        }
+        MANIFEST_KIND_SINGLE_CHUNK => {
+            require_len(
+                bytes,
+                SINGLE_CHUNK_MANIFEST_BYTES,
+                "binary CAS single-chunk manifest",
+            )?;
+            let chunk_hash = bytes[13..45].try_into().expect("fixed slice");
+            Ok(BinaryCasManifest::SingleChunk {
+                size_bytes,
+                chunk_hash,
+            })
+        }
+        MANIFEST_KIND_CHUNKED => {
+            require_len(bytes, CHUNKED_MANIFEST_BYTES, "binary CAS chunked manifest")?;
+            let chunk_count = u32::from_be_bytes(bytes[13..17].try_into().expect("fixed slice"));
+            Ok(BinaryCasManifest::Chunked {
+                size_bytes,
+                chunk_count,
+            })
+        }
+        other => Err(codec_error(format!(
+            "unsupported binary CAS manifest kind {other}"
+        ))),
+    }
+}
+
+pub(crate) fn encode_binary_cas_manifest_chunk(
+    chunk_hash: &[u8; HASH_BYTES],
+    chunk_size: u64,
+) -> Vec<u8> {
+    let mut out = Vec::with_capacity(MANIFEST_CHUNK_BYTES);
+    out.extend_from_slice(MANIFEST_CHUNK_MAGIC);
+    out.extend_from_slice(chunk_hash);
+    out.extend_from_slice(&chunk_size.to_be_bytes());
+    out
+}
+
+pub(crate) fn decode_binary_cas_manifest_chunk(
+    bytes: &[u8],
+) -> Result<([u8; HASH_BYTES], u64), LixError> {
+    if bytes.len() != MANIFEST_CHUNK_BYTES {
+        return Err(codec_error(format!(
+            "binary CAS manifest chunk must be {MANIFEST_CHUNK_BYTES} bytes, got {}",
+            bytes.len()
+        )));
+    }
+    require_magic(bytes, MANIFEST_CHUNK_MAGIC, "binary CAS manifest chunk")?;
+    let chunk_hash = bytes[4..36].try_into().expect("fixed slice");
+    let chunk_size = u64::from_be_bytes(bytes[36..44].try_into().expect("fixed slice"));
+    Ok((chunk_hash, chunk_size))
+}
+
+pub(crate) fn encode_binary_cas_chunk(
+    codec: BinaryChunkCodec,
+    uncompressed_len: u64,
+    payload: &[u8],
+) -> Vec<u8> {
+    let mut out = Vec::with_capacity(CHUNK_HEADER_BYTES + payload.len());
+    out.extend_from_slice(CHUNK_MAGIC);
+    out.push(codec.tag());
+    out.extend_from_slice(&uncompressed_len.to_be_bytes());
+    out.extend_from_slice(payload);
+    out
+}
+
+pub(crate) fn decode_binary_cas_chunk(
+    bytes: &[u8],
+) -> Result<(BinaryChunkCodec, u64, &[u8]), LixError> {
+    if bytes.len() < CHUNK_HEADER_BYTES {
+        return Err(codec_error(format!(
+            "binary CAS chunk must be at least {CHUNK_HEADER_BYTES} bytes, got {}",
+            bytes.len()
+        )));
+    }
+    require_magic(bytes, CHUNK_MAGIC, "binary CAS chunk")?;
+    let codec = BinaryChunkCodec::from_tag(bytes[4])?;
+    let uncompressed_len = u64::from_be_bytes(bytes[5..13].try_into().expect("fixed slice"));
+    Ok((codec, uncompressed_len, &bytes[CHUNK_HEADER_BYTES..]))
+}
+
+fn require_magic(bytes: &[u8], expected: &[u8; 4], label: &str) -> Result<(), LixError> {
+    if &bytes[..4] == expected {
+        return Ok(());
+    }
+    Err(codec_error(format!(
+        "{label} has unsupported binary format"
+    )))
+}
+
+fn require_len(bytes: &[u8], expected: usize, label: &str) -> Result<(), LixError> {
+    if bytes.len() == expected {
+        return Ok(());
+    }
+    Err(codec_error(format!(
+        "{label} must be {expected} bytes, got {}",
+        bytes.len()
+    )))
+}
+
+fn hex_value(byte: u8, label: &str) -> Result<u8, LixError> {
+    match byte {
+        b'0'..=b'9' => Ok(byte - b'0'),
+        b'a'..=b'f' => Ok(byte - b'a' + 10),
+        b'A'..=b'F' => Ok(byte - b'A' + 10),
+        _ => Err(codec_error(format!("{label} hash contains non-hex bytes"))),
+    }
+}
+
+fn codec_error(message: String) -> LixError {
+    LixError::new("LIX_ERROR_UNKNOWN", message)
+}
+
+pub(crate) fn encode_binary_chunk_payload(chunk_data: &[u8]) -> EncodedBinaryChunkPayload {
+    EncodedBinaryChunkPayload {
+        codec: BinaryChunkCodec::Raw,
+        data: chunk_data.to_vec(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn manifests_roundtrip_fixed_binary_rows() {
+        let chunk_hash = binary_blob_hash_bytes(b"chunk");
+        let cases = vec![
+            (
+                BinaryCasManifest::Empty { size_bytes: 0 },
+                EMPTY_MANIFEST_BYTES,
+            ),
+            (
+                BinaryCasManifest::SingleChunk {
+                    size_bytes: 42,
+                    chunk_hash,
+                },
+                SINGLE_CHUNK_MANIFEST_BYTES,
+            ),
+            (
+                BinaryCasManifest::Chunked {
+                    size_bytes: 42,
+                    chunk_count: 7,
+                },
+                CHUNKED_MANIFEST_BYTES,
+            ),
+        ];
+        for (manifest, expected_len) in cases {
+            let encoded = encode_binary_cas_manifest(&manifest);
+            assert_eq!(encoded.len(), expected_len);
+            assert_eq!(decode_binary_cas_manifest(&encoded).unwrap(), manifest);
+        }
+    }
+
+    #[test]
+    fn manifest_chunk_roundtrips_fixed_binary_row() {
+        let hash = binary_blob_hash_bytes(b"chunk");
+        let encoded = encode_binary_cas_manifest_chunk(&hash, 1024);
+        assert_eq!(encoded.len(), MANIFEST_CHUNK_BYTES);
+        assert_eq!(
+            decode_binary_cas_manifest_chunk(&encoded).unwrap(),
+            (hash, 1024)
+        );
+    }
+
+    #[test]
+    fn chunk_roundtrips_payload_as_remaining_bytes() {
+        let payload = b"hello payload";
+        let encoded = encode_binary_cas_chunk(BinaryChunkCodec::Raw, payload.len() as u64, payload);
+        assert_eq!(&encoded[..4], CHUNK_MAGIC);
+        let (codec, uncompressed_len, decoded_payload) = decode_binary_cas_chunk(&encoded).unwrap();
+        assert_eq!(codec, BinaryChunkCodec::Raw);
+        assert_eq!(uncompressed_len, payload.len() as u64);
+        assert_eq!(decoded_payload, payload);
+    }
+
+    #[test]
+    fn wrong_magic_is_rejected() {
+        let mut encoded = encode_binary_cas_manifest(&BinaryCasManifest::Empty { size_bytes: 0 });
+        encoded[0] = b'X';
+        let error = decode_binary_cas_manifest(&encoded).unwrap_err();
+        assert!(error.message.contains("unsupported binary format"));
+    }
+
+    #[test]
+    fn hex_hashes_roundtrip_to_32_byte_keys() {
+        let hash_hex = binary_blob_hash_hex(b"blob");
+        let hash_bytes = hash_hex_to_bytes(&hash_hex, "test").unwrap();
+        assert_eq!(hash_bytes.len(), 32);
+        assert_eq!(hash_bytes_to_hex(&hash_bytes), hash_hex);
+    }
+}

package/dist-engine-src/src/binary_cas/context.rs

@@ -0,0 +1,139 @@
+use async_trait::async_trait;
+
+use crate::binary_cas::{
+    BlobBytesBatch, BlobExistsBatch, BlobHash, BlobMetadataBatch, BlobWrite, BlobWriteReceipt,
+};
+use crate::storage::{StorageReader, StorageWriteSet};
+use crate::LixError;
+use std::collections::HashSet;
+
+#[async_trait]
+pub(crate) trait BlobDataReader: Send + Sync {
+    async fn load_bytes_many(&self, hashes: &[BlobHash]) -> Result<BlobBytesBatch, LixError>;
+}
+
+/// Long-lived Binary CAS context factory.
+///
+/// The context does not own storage. Callers explicitly provide a KV store via
+/// `reader(...)` or `writer(...)`, keeping backend and transaction ownership at
+/// the execution layer.
+pub(crate) struct BinaryCasContext;
+
+impl BinaryCasContext {
+    pub(crate) fn new() -> Self {
+        Self
+    }
+
+    /// Creates a Binary CAS reader over any storage reader.
+    ///
+    /// The reader can be a read transaction or the active write transaction
+    /// when reads must participate in transaction-local visibility.
+    pub(crate) fn reader<S>(&self, store: S) -> BinaryCasStoreReader<S>
+    where
+        S: StorageReader,
+    {
+        BinaryCasStoreReader { store }
+    }
+
+    pub(crate) fn writer<'a>(&self, writes: &'a mut StorageWriteSet) -> BinaryCasWriter<'a> {
+        BinaryCasWriter::new(writes)
+    }
+}
+
+#[async_trait]
+impl<S> BlobDataReader for BinaryCasStoreReader<S>
+where
+    S: StorageReader + Clone + Send + Sync,
+{
+    async fn load_bytes_many(&self, hashes: &[BlobHash]) -> Result<BlobBytesBatch, LixError> {
+        let mut reader = BinaryCasStoreReader {
+            store: self.store.clone(),
+        };
+        BinaryCasStoreReader::load_bytes_many(&mut reader, hashes).await
+    }
+}
+
+/// Binary CAS reader over a caller-supplied KV store.
+pub(crate) struct BinaryCasStoreReader<S> {
+    store: S,
+}
+
+impl<S> BinaryCasStoreReader<S>
+where
+    S: StorageReader,
+{
+    #[allow(dead_code)]
+    pub(crate) async fn exists_many(
+        &mut self,
+        hashes: &[BlobHash],
+    ) -> Result<BlobExistsBatch, LixError> {
+        crate::binary_cas::kv::exists_many(&mut self.store, hashes).await
+    }
+
+    #[allow(dead_code)]
+    pub(crate) async fn load_metadata_many(
+        &mut self,
+        hashes: &[BlobHash],
+    ) -> Result<BlobMetadataBatch, LixError> {
+        crate::binary_cas::kv::load_metadata_many(&mut self.store, hashes).await
+    }
+
+    pub(crate) async fn load_bytes_many(
+        &mut self,
+        hashes: &[BlobHash],
+    ) -> Result<BlobBytesBatch, LixError> {
+        crate::binary_cas::kv::load_bytes_many(&mut self.store, hashes).await
+    }
+
+    #[cfg(feature = "storage-benches")]
+    pub(crate) async fn count_blob_manifests(&mut self) -> Result<usize, LixError> {
+        crate::binary_cas::kv::count_manifests(&mut self.store).await
+    }
+}
+
+/// Transaction-scoped Binary CAS writer.
+///
+/// This type does not begin, commit, or roll back transactions. It only writes
+/// CAS data into the transaction supplied by the caller.
+pub(crate) struct BinaryCasWriter<'a> {
+    writes: &'a mut StorageWriteSet,
+    blob_hashes: HashSet<[u8; 32]>,
+    chunk_keys: HashSet<Vec<u8>>,
+}
+
+impl<'a> BinaryCasWriter<'a> {
+    fn new(writes: &'a mut StorageWriteSet) -> Self {
+        Self {
+            writes,
+            blob_hashes: HashSet::new(),
+            chunk_keys: HashSet::new(),
+        }
+    }
+
+    pub(crate) fn stage_bytes(&mut self, bytes: &[u8]) -> Result<BlobWriteReceipt, LixError> {
+        crate::binary_cas::kv::stage_blob_write(
+            self.writes,
+            &mut self.blob_hashes,
+            &mut self.chunk_keys,
+            &BlobWrite { bytes },
+        )
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn stage_many(
+        &mut self,
+        writes: &[BlobWrite<'_>],
+    ) -> Result<Vec<BlobWriteReceipt>, LixError> {
+        writes
+            .iter()
+            .map(|write| {
+                crate::binary_cas::kv::stage_blob_write(
+                    self.writes,
+                    &mut self.blob_hashes,
+                    &mut self.chunk_keys,
+                    write,
+                )
+            })
+            .collect()
+    }
+}