@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.3

package/dist-engine-src/src/tracked_state/tree.rs

@@ -1,16 +1,23 @@
-use std::{collections::BTreeMap, future::Future, ops::Range, pin::Pin};
+use std::{
+    collections::{BTreeMap, VecDeque},
+    future::Future,
+    ops::Range,
+    pin::Pin,
+};
 
 use crate::storage::{StorageReader, StorageWriteSet};
 use crate::tracked_state::codec::{
-    boundary_trigger, child_summary_from_node, decode_key, decode_node, decode_value,
-    encode_internal_node, encode_key, encode_leaf_node, encode_schema_file_prefix,
-    encode_schema_key_prefix, encode_value, ChildSummary, DecodedNode, EncodedLeafEntry,
+    boundary_trigger, child_summary_from_node, decode_key, decode_key_with_trusted_prefix,
+    decode_node, decode_node_ref, decode_value, decode_visible_value, encode_internal_node,
+    encode_internal_node_refs, encode_key, encode_leaf_node, encode_leaf_node_refs,
+    encode_schema_file_prefix, encode_schema_key_prefix, ChildSummary, ChildSummaryRef,
+    DecodedLeafNodeRef, DecodedNode, DecodedNodeRef, EncodedLeafEntry, EncodedLeafEntryRef,
     PendingChunkWrite,
 };
 use crate::tracked_state::storage;
-use crate::tracked_state::tree_types::{
-    TrackedStateApplyResult, TrackedStateKey, TrackedStateMutation, TrackedStateRootId,
-    TrackedStateTreeDiffEntry, TrackedStateTreeScanRequest, TrackedStateValue,
+use crate::tracked_state::types::{
+    TrackedStateApplyResult, TrackedStateIndexValue, TrackedStateKey, TrackedStateMutation,
+    TrackedStateRootId, TrackedStateTreeDiffEntry, TrackedStateTreeScanRequest,
     TRACKED_STATE_HASH_BYTES,
 };
 use crate::{LixError, NullableKeyFilter};
@@ -22,6 +29,11 @@ pub(crate) struct TrackedStateTreeOptions {
     pub(crate) max_chunk_bytes: usize,
 }
 
+enum MutationApply<T> {
+    Applied(TrackedStateApplyResult),
+    Fallback(T),
+}
+
 impl Default for TrackedStateTreeOptions {
     fn default() -> Self {
         Self {
@@ -61,12 +73,13 @@ impl TrackedStateTree {
         storage::load_root(store, commit_id).await
     }
 
+    #[cfg(test)]
     pub(crate) async fn get(
         &self,
         store: &mut impl StorageReader,
         root_id: &TrackedStateRootId,
         key: &TrackedStateKey,
-    ) -> Result<Option<TrackedStateValue>, LixError> {
+    ) -> Result<Option<TrackedStateIndexValue>, LixError> {
         let encoded_key = encode_key(key);
         let mut current = *root_id.as_bytes();
         loop {
@@ -102,7 +115,7 @@ impl TrackedStateTree {
         store: &mut impl StorageReader,
         root_id: &TrackedStateRootId,
         keys: &[TrackedStateKey],
-    ) -> Result<Vec<Option<TrackedStateValue>>, LixError> {
+    ) -> Result<Vec<Option<TrackedStateIndexValue>>, LixError> {
         if keys.is_empty() {
             return Ok(Vec::new());
         }
@@ -140,15 +153,23 @@ impl TrackedStateTree {
         store: &mut impl StorageReader,
         root_id: &TrackedStateRootId,
         request: &TrackedStateTreeScanRequest,
-    ) -> Result<Vec<(TrackedStateKey, TrackedStateValue)>, LixError> {
+    ) -> Result<Vec<(TrackedStateKey, TrackedStateIndexValue)>, LixError> {
         if request.limit == Some(0) {
             return Ok(Vec::new());
         }
 
         let ranges = scan_ranges(request);
+        let key_decode_hint = scan_key_decode_hint(request, &ranges);
         let mut rows = Vec::new();
-        self.scan_node(store, *root_id.as_bytes(), request, &ranges, &mut rows)
-            .await?;
+        self.scan_node(
+            store,
+            *root_id.as_bytes(),
+            request,
+            &ranges,
+            key_decode_hint,
+            &mut rows,
+        )
+        .await?;
         Ok(rows)
     }
 
@@ -215,38 +236,41 @@ impl TrackedStateTree {
         store: &mut (impl StorageReader + ?Sized),
         writes: &mut StorageWriteSet,
         base_root: Option<&TrackedStateRootId>,
-        mutations: Vec<TrackedStateMutation>,
+        mut mutations: Vec<TrackedStateMutation>,
         commit_id: Option<&str>,
     ) -> Result<TrackedStateApplyResult, LixError> {
         let mut overlay = storage::TrackedStateChunkOverlay::new();
         if let Some(root_id) = base_root {
             if mutations.len() == 1 {
-                if let Some(result) = self
+                let mutation = mutations.pop().expect("single mutation should exist");
+                match self
                     .apply_single_mutation(
                         store,
                         writes,
                         &mut overlay,
                         root_id,
-                        &mutations[0],
+                        mutation,
                         commit_id,
                     )
                     .await?
                 {
-                    return Ok(result);
+                    MutationApply::Applied(result) => return Ok(result),
+                    MutationApply::Fallback(mutation) => mutations = vec![mutation],
                 }
             } else if mutations.len() > 1 {
-                if let Some(result) = self
+                match self
                     .apply_sorted_mutations_chunker(
                         store,
                         writes,
                         &mut overlay,
                         root_id,
-                        &mutations,
+                        mutations,
                         commit_id,
                     )
                     .await?
                 {
-                    return Ok(result);
+                    MutationApply::Applied(result) => return Ok(result),
+                    MutationApply::Fallback(fallback_mutations) => mutations = fallback_mutations,
                 }
             }
         }
@@ -264,11 +288,7 @@ impl TrackedStateTree {
         // Apply in caller order so repeated writes to the same key behave like
         // normal transaction staging: the latest mutation wins.
         for mutation in mutations {
-            match mutation {
-                TrackedStateMutation::Put { key, value } => {
-                    entries.insert(encode_key(&key), encode_value(&value));
-                }
-            }
+            entries.insert(mutation.encoded_key, mutation.encoded_value);
         }
 
         let built = self.build_tree_from_entries(
@@ -301,60 +321,71 @@ impl TrackedStateTree {
         writes: &mut StorageWriteSet,
         overlay: &mut storage::TrackedStateChunkOverlay,
         root_id: &TrackedStateRootId,
-        mutation: &TrackedStateMutation,
+        mutation: TrackedStateMutation,
         commit_id: Option<&str>,
-    ) -> Result<Option<TrackedStateApplyResult>, LixError> {
-        let TrackedStateMutation::Put { key, value } = mutation;
-        let encoded_key = encode_key(key);
-        let encoded_value = encode_value(value);
-
-        if let Some(result) = self
+    ) -> Result<MutationApply<TrackedStateMutation>, LixError> {
+        let mutation = match self
             .apply_single_mutation_from_seek_path(
-                store,
-                writes,
-                overlay,
-                root_id,
-                &encoded_key,
-                &encoded_value,
-                commit_id,
+                store, writes, overlay, root_id, mutation, commit_id,
             )
             .await?
         {
-            return Ok(result);
-        }
+            MutationApply::Applied(result) => return Ok(MutationApply::Applied(result)),
+            MutationApply::Fallback(mutation) => mutation,
+        };
+
+        let TrackedStateMutation {
+            encoded_key,
+            encoded_value,
+        } = mutation;
 
         let levels = self
             .collect_summary_levels_with_overlay(store, overlay, root_id)
             .await?;
         let Some(leaves) = levels.first() else {
-            return Ok(None);
+            return Ok(MutationApply::Fallback(TrackedStateMutation {
+                encoded_key,
+                encoded_value,
+            }));
         };
         let target_leaf_index = leaves
             .iter()
             .position(|leaf| leaf.last_key.as_slice() >= encoded_key.as_slice())
             .unwrap_or_else(|| leaves.len().saturating_sub(1));
         let Some(target_leaf) = leaves.get(target_leaf_index).cloned() else {
-            return Ok(None);
+            return Ok(MutationApply::Fallback(TrackedStateMutation {
+                encoded_key,
+                encoded_value,
+            }));
         };
 
         let mut entries = self
             .load_leaf_entries_with_overlay(store, overlay, &target_leaf.child_hash)
             .await?;
-        match entries.binary_search_by(|entry| entry.key.as_slice().cmp(&encoded_key)) {
+        let mutation_entry_index = match entries
+            .binary_search_by(|entry| entry.key.as_slice().cmp(encoded_key.as_slice()))
+        {
             Ok(index) => {
-                if entries[index].value == encoded_value {
-                    return Ok(None);
+                if entries[index].value.as_slice() == encoded_value.as_slice() {
+                    return Ok(MutationApply::Fallback(TrackedStateMutation {
+                        encoded_key,
+                        encoded_value,
+                    }));
                 }
                 entries[index].value = encoded_value;
+                index
             }
-            Err(index) => entries.insert(
-                index,
-                EncodedLeafEntry {
-                    key: encoded_key.clone(),
-                    value: encoded_value,
-                },
-            ),
-        }
+            Err(index) => {
+                entries.insert(
+                    index,
+                    EncodedLeafEntry {
+                        key: encoded_key,
+                        value: encoded_value,
+                    },
+                );
+                index
+            }
+        };
 
         let mut chunks = BTreeMap::new();
         let mut suffix_entries = entries;
@@ -366,20 +397,25 @@ impl TrackedStateTree {
         // existing post-mutation leaf, then reuse the rest of the old suffix.
         loop {
             let mut candidate_chunks = BTreeMap::new();
-            let candidate_summaries =
-                self.build_leaf_level(suffix_entries.clone(), &mut candidate_chunks);
+            let candidate_summaries = self.build_leaf_level_from_refs(
+                suffix_entries.iter().map(EncodedLeafEntry::as_ref),
+                &mut candidate_chunks,
+            );
 
             if let Some((generated_resync_index, existing_resync_index)) = first_resync_index(
                 &candidate_summaries,
                 &leaves[target_leaf_index..],
-                &encoded_key,
+                suffix_entries[mutation_entry_index].key.as_slice(),
             ) {
                 for summary in &candidate_summaries[..generated_resync_index] {
                     if let Some(chunk) = candidate_chunks.remove(&summary.child_hash) {
                         chunks.entry(chunk.hash).or_insert(chunk);
                     }
                 }
-                replacement_leaves = candidate_summaries[..generated_resync_index].to_vec();
+                replacement_leaves = candidate_summaries
+                    .into_iter()
+                    .take(generated_resync_index)
+                    .collect();
                 old_leaf_count = existing_resync_index;
                 break;
             }
@@ -408,7 +444,7 @@ impl TrackedStateTree {
             old_leaf_count,
             std::mem::take(&mut replacement_leaves),
             chunks,
-            &encoded_key,
+            suffix_entries[mutation_entry_index].key.as_slice(),
         )?;
         overlay.stage_chunks(writes, &built.chunks);
         let persisted_root = if let Some(commit_id) = commit_id {
@@ -418,7 +454,7 @@ impl TrackedStateTree {
             false
         };
 
-        Ok(Some(TrackedStateApplyResult {
+        Ok(MutationApply::Applied(TrackedStateApplyResult {
             root_id: built.root_id,
             row_count: built.row_count,
             tree_height: built.tree_height,
@@ -652,48 +688,65 @@ impl TrackedStateTree {
         writes: &mut StorageWriteSet,
         overlay: &mut storage::TrackedStateChunkOverlay,
         root_id: &TrackedStateRootId,
-        mutations: &[TrackedStateMutation],
+        mutations: Vec<TrackedStateMutation>,
         commit_id: Option<&str>,
-    ) -> Result<Option<TrackedStateApplyResult>, LixError> {
+    ) -> Result<MutationApply<Vec<TrackedStateMutation>>, LixError> {
         let mut mutation_map = BTreeMap::new();
         for mutation in mutations {
-            let TrackedStateMutation::Put { key, value } = mutation;
-            mutation_map.insert(encode_key(key), encode_value(value));
+            mutation_map.insert(mutation.encoded_key, mutation.encoded_value);
         }
         if mutation_map.is_empty() {
-            return Ok(None);
+            return Ok(MutationApply::Fallback(Vec::new()));
         }
 
-        let mutations = mutation_map.into_iter().collect::<Vec<_>>();
-
         let levels = self
             .collect_summary_levels_with_overlay(store, overlay, root_id)
             .await?;
         let Some(leaves) = levels.first() else {
-            return Ok(None);
+            return Ok(MutationApply::Fallback(
+                mutation_map
+                    .into_iter()
+                    .map(|(encoded_key, encoded_value)| TrackedStateMutation {
+                        encoded_key,
+                        encoded_value,
+                    })
+                    .collect(),
+            ));
         };
 
         let base_row_count = leaves
             .iter()
             .map(|leaf| leaf.subtree_count as usize)
             .sum::<usize>();
+        let first_mutation_key = mutation_map
+            .keys()
+            .next()
+            .expect("non-empty mutation map should have first key");
         let append_only = leaves
             .last()
-            .is_some_and(|leaf| mutations[0].0.as_slice() > leaf.last_key.as_slice());
-        if !append_only && mutations.len() * 2 > base_row_count {
-            return Ok(None);
+            .is_some_and(|leaf| first_mutation_key.as_slice() > leaf.last_key.as_slice());
+        if !append_only && mutation_map.len() * 2 > base_row_count {
+            return Ok(MutationApply::Fallback(
+                mutation_map
+                    .into_iter()
+                    .map(|(encoded_key, encoded_value)| TrackedStateMutation {
+                        encoded_key,
+                        encoded_value,
+                    })
+                    .collect(),
+            ));
         }
 
+        let mut mutations = mutation_map.into_iter().collect::<VecDeque<_>>();
         let mut output_leaves = Vec::new();
         let mut chunks = BTreeMap::new();
         let mut leaf_index = 0usize;
-        let mut next_mutation_index = 0usize;
 
         while leaf_index < leaves.len() {
-            if next_mutation_index >= mutations.len()
-                || mutations[next_mutation_index].0.as_slice()
-                    > leaves[leaf_index].last_key.as_slice()
-            {
+            let current_leaf_has_mutation = mutations
+                .front()
+                .is_some_and(|(key, _)| key.as_slice() <= leaves[leaf_index].last_key.as_slice());
+            if !current_leaf_has_mutation {
                 output_leaves.push(leaves[leaf_index].clone());
                 leaf_index += 1;
                 continue;
@@ -701,7 +754,10 @@ impl TrackedStateTree {
 
             let window_start = leaf_index;
             let mut window_entries = BTreeMap::new();
-            let mut window_mutation_ceiling = mutations[next_mutation_index].0.clone();
+            let mut window_mutation_ceiling = mutations
+                .front()
+                .map(|(key, _)| key.clone())
+                .expect("window with mutation should have front mutation");
 
             loop {
                 if leaf_index < leaves.len() {
@@ -713,46 +769,45 @@ impl TrackedStateTree {
                         window_entries.insert(entry.key, entry.value);
                     }
 
-                    while next_mutation_index < mutations.len()
-                        && mutations[next_mutation_index].0.as_slice() <= leaf.last_key.as_slice()
+                    while mutations
+                        .front()
+                        .is_some_and(|(key, _)| key.as_slice() <= leaf.last_key.as_slice())
                     {
-                        let (key, value) = &mutations[next_mutation_index];
-                        window_entries.insert(key.clone(), value.clone());
+                        let (key, value) = mutations
+                            .pop_front()
+                            .expect("front mutation should be present");
                         window_mutation_ceiling = key.clone();
-                        next_mutation_index += 1;
+                        window_entries.insert(key, value);
                     }
                     leaf_index += 1;
                 }
 
-                while next_mutation_index < mutations.len() {
-                    let (key, value) = &mutations[next_mutation_index];
+                while let Some((key, _)) = mutations.front() {
                     if leaf_index < leaves.len()
                         && key.as_slice() >= leaves[leaf_index].first_key.as_slice()
                     {
                         break;
                     }
-                    window_entries.insert(key.clone(), value.clone());
+                    let (key, value) = mutations
+                        .pop_front()
+                        .expect("front mutation should be present");
                     window_mutation_ceiling = key.clone();
-                    next_mutation_index += 1;
+                    window_entries.insert(key, value);
                 }
 
-                if next_mutation_index < mutations.len()
-                    && leaf_index < leaves.len()
-                    && mutations[next_mutation_index].0.as_slice()
-                        <= leaves[leaf_index].last_key.as_slice()
+                if leaf_index < leaves.len()
+                    && mutations.front().is_some_and(|(key, _)| {
+                        key.as_slice() <= leaves[leaf_index].last_key.as_slice()
+                    })
                 {
                     continue;
                 }
 
                 let mut candidate_chunks = BTreeMap::new();
-                let candidate_leaves = self.build_leaf_level(
+                let candidate_leaves = self.build_leaf_level_from_refs(
                     window_entries
                         .iter()
-                        .map(|(key, value)| EncodedLeafEntry {
-                            key: key.clone(),
-                            value: value.clone(),
-                        })
-                        .collect(),
+                        .map(|(key, value)| EncodedLeafEntryRef { key, value }),
                     &mut candidate_chunks,
                 );
 
@@ -766,7 +821,7 @@ impl TrackedStateTree {
                             chunks.entry(chunk.hash).or_insert(chunk);
                         }
                     }
-                    output_leaves.extend_from_slice(&candidate_leaves[..generated_resync_index]);
+                    output_leaves.extend(candidate_leaves.into_iter().take(generated_resync_index));
                     leaf_index = window_start + existing_resync_index;
                     break;
                 }
@@ -779,20 +834,19 @@ impl TrackedStateTree {
             }
         }
 
-        if next_mutation_index < mutations.len() {
-            let mut entries = Vec::new();
-            for (key, value) in &mutations[next_mutation_index..] {
-                entries.push(EncodedLeafEntry {
-                    key: key.clone(),
-                    value: value.clone(),
-                });
-            }
+        if !mutations.is_empty() {
+            let entries = mutations
+                .into_iter()
+                .map(|(key, value)| EncodedLeafEntry { key, value })
+                .collect();
             output_leaves.extend(self.build_leaf_level(entries, &mut chunks));
         }
 
         let built = self.build_tree_from_leaf_summaries(output_leaves, chunks)?;
-        self.persist_built_tree(writes, overlay, built, commit_id)
-            .await
+        Ok(MutationApply::Applied(
+            self.persist_built_tree(writes, overlay, built, commit_id)
+                .await?,
+        ))
     }
 
     async fn apply_single_mutation_from_seek_path(
@@ -801,10 +855,13 @@ impl TrackedStateTree {
         writes: &mut StorageWriteSet,
         overlay: &mut storage::TrackedStateChunkOverlay,
         root_id: &TrackedStateRootId,
-        encoded_key: &[u8],
-        encoded_value: &[u8],
+        mutation: TrackedStateMutation,
         commit_id: Option<&str>,
-    ) -> Result<Option<Option<TrackedStateApplyResult>>, LixError> {
+    ) -> Result<MutationApply<TrackedStateMutation>, LixError> {
+        let TrackedStateMutation {
+            encoded_key,
+            encoded_value,
+        } = mutation;
         let mut current = *root_id.as_bytes();
         let mut path = Vec::new();
         let mut entries = loop {
@@ -817,7 +874,7 @@ impl TrackedStateTree {
                     let children = internal.children().to_vec();
                     let child_index = children
                         .iter()
-                        .position(|child| child.last_key.as_slice() >= encoded_key)
+                        .position(|child| child.last_key.as_slice() >= encoded_key.as_slice())
                         .or_else(|| (!children.is_empty()).then_some(children.len() - 1))
                         .ok_or_else(|| {
                             LixError::new(
@@ -834,21 +891,30 @@ impl TrackedStateTree {
             }
         };
 
-        match entries.binary_search_by(|entry| entry.key.as_slice().cmp(encoded_key)) {
+        let mutation_entry_index = match entries
+            .binary_search_by(|entry| entry.key.as_slice().cmp(encoded_key.as_slice()))
+        {
             Ok(index) => {
-                if entries[index].value == encoded_value {
-                    return Ok(Some(None));
+                if entries[index].value.as_slice() == encoded_value.as_slice() {
+                    return Ok(MutationApply::Fallback(TrackedStateMutation {
+                        encoded_key,
+                        encoded_value,
+                    }));
                 }
-                entries[index].value = encoded_value.to_vec();
+                entries[index].value = encoded_value;
+                index
             }
-            Err(index) => entries.insert(
-                index,
-                EncodedLeafEntry {
-                    key: encoded_key.to_vec(),
-                    value: encoded_value.to_vec(),
-                },
-            ),
-        }
+            Err(index) => {
+                entries.insert(
+                    index,
+                    EncodedLeafEntry {
+                        key: encoded_key,
+                        value: encoded_value,
+                    },
+                );
+                index
+            }
+        };
 
         let mut chunks = BTreeMap::new();
         let mut replacement_children;
@@ -856,10 +922,10 @@ impl TrackedStateTree {
 
         let Some(leaf_parent) = path.pop() else {
             let built = self.build_tree_from_entries(entries)?;
-            return self
-                .persist_built_tree(writes, overlay, built, commit_id)
-                .await
-                .map(Some);
+            return Ok(MutationApply::Applied(
+                self.persist_built_tree(writes, overlay, built, commit_id)
+                    .await?,
+            ));
         };
         let mutation_is_right_edge = leaf_parent.child_index + 1 == leaf_parent.children.len()
             && path
@@ -870,26 +936,35 @@ impl TrackedStateTree {
         let mut next_leaf_index = leaf_parent.child_index + 1;
         loop {
             let mut candidate_chunks = BTreeMap::new();
-            let candidate_leaves =
-                self.build_leaf_level(leaf_entries.clone(), &mut candidate_chunks);
+            let candidate_leaves = self.build_leaf_level_from_refs(
+                leaf_entries.iter().map(EncodedLeafEntry::as_ref),
+                &mut candidate_chunks,
+            );
             if let Some((generated_resync_index, existing_resync_index)) = first_resync_index(
                 &candidate_leaves,
                 &leaf_parent.children[leaf_parent.child_index..],
-                encoded_key,
+                leaf_entries[mutation_entry_index].key.as_slice(),
             ) {
                 for summary in &candidate_leaves[..generated_resync_index] {
                     if let Some(chunk) = candidate_chunks.remove(&summary.child_hash) {
                         chunks.entry(chunk.hash).or_insert(chunk);
                     }
                 }
-                replacement_children = candidate_leaves[..generated_resync_index].to_vec();
+                replacement_children = candidate_leaves
+                    .into_iter()
+                    .take(generated_resync_index)
+                    .collect();
                 old_child_count = existing_resync_index;
                 break;
             }
 
             if next_leaf_index >= leaf_parent.children.len() {
                 if !mutation_is_right_edge {
-                    return Ok(None);
+                    let entry = leaf_entries.remove(mutation_entry_index);
+                    return Ok(MutationApply::Fallback(TrackedStateMutation {
+                        encoded_key: entry.key,
+                        encoded_value: entry.value,
+                    }));
                 }
                 chunks.extend(candidate_chunks);
                 replacement_children = candidate_leaves;
@@ -941,10 +1016,10 @@ impl TrackedStateTree {
                     tree_height,
                     chunk_bytes,
                 };
-                return self
-                    .persist_built_tree(writes, overlay, built, commit_id)
-                    .await
-                    .map(Some);
+                return Ok(MutationApply::Applied(
+                    self.persist_built_tree(writes, overlay, built, commit_id)
+                        .await?,
+                ));
             };
 
             child_index = frame.child_index;
@@ -959,7 +1034,7 @@ impl TrackedStateTree {
         overlay: &mut storage::TrackedStateChunkOverlay,
         built: BuiltTree,
         commit_id: Option<&str>,
-    ) -> Result<Option<TrackedStateApplyResult>, LixError> {
+    ) -> Result<TrackedStateApplyResult, LixError> {
         overlay.stage_chunks(writes, &built.chunks);
         let persisted_root = if let Some(commit_id) = commit_id {
             storage::stage_root(writes, commit_id, &built.root_id);
@@ -967,14 +1042,14 @@ impl TrackedStateTree {
         } else {
             false
         };
-        Ok(Some(TrackedStateApplyResult {
+        Ok(TrackedStateApplyResult {
             root_id: built.root_id,
             row_count: built.row_count,
             tree_height: built.tree_height,
             chunk_count: built.chunks.len(),
             chunk_bytes: built.chunk_bytes,
             persisted_root,
-        }))
+        })
     }
 
     fn build_tree_from_entries(
@@ -1130,15 +1205,23 @@ impl TrackedStateTree {
         let parent_end_child_range =
             child_range_for_parent(old_children, &old_parents[parent_end])?;
         let mut window_children = Vec::new();
-        window_children.extend_from_slice(&old_children[parent_child_range.start..child_start]);
-        window_children.extend(replacement_children);
-        window_children.extend_from_slice(&old_children[old_child_end..parent_end_child_range.end]);
+        window_children.extend(
+            old_children[parent_child_range.start..child_start]
+                .iter()
+                .map(ChildSummary::as_ref),
+        );
+        window_children.extend(replacement_children.iter().map(ChildSummary::as_ref));
+        window_children.extend(
+            old_children[old_child_end..parent_end_child_range.end]
+                .iter()
+                .map(ChildSummary::as_ref),
+        );
         let mut next_parent_index = parent_end + 1;
 
         loop {
             let mut candidate_chunks = BTreeMap::new();
-            let candidate_parents = self.build_internal_level(
-                window_children.clone(),
+            let candidate_parents = self.build_internal_level_from_refs(
+                window_children.iter().copied(),
                 parent_level,
                 &mut candidate_chunks,
             );
@@ -1156,7 +1239,10 @@ impl TrackedStateTree {
                 return Ok(ParentLevelPatch {
                     parent_start,
                     old_parent_count: existing_resync_index,
-                    replacement_parents: candidate_parents[..generated_resync_index].to_vec(),
+                    replacement_parents: candidate_parents
+                        .into_iter()
+                        .take(generated_resync_index)
+                        .collect(),
                 });
             }
 
@@ -1170,7 +1256,7 @@ impl TrackedStateTree {
             }
 
             let next_range = child_range_for_parent(old_children, &old_parents[next_parent_index])?;
-            window_children.extend_from_slice(&old_children[next_range]);
+            window_children.extend(old_children[next_range].iter().map(ChildSummary::as_ref));
             next_parent_index += 1;
         }
     }
@@ -1204,6 +1290,35 @@ impl TrackedStateTree {
             .collect()
     }
 
+    fn build_leaf_level_from_refs<'a>(
+        &self,
+        entries: impl IntoIterator<Item = EncodedLeafEntryRef<'a>>,
+        chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
+    ) -> Vec<ChildSummary> {
+        let groups = chunk_leaf_entry_refs(entries, &self.options);
+        groups
+            .into_iter()
+            .map(|group| {
+                let subtree_count = group.entries.len() as u64;
+                let first_key = group
+                    .entries
+                    .first()
+                    .map(|entry| entry.key.to_vec())
+                    .unwrap_or_default();
+                let last_key = group
+                    .entries
+                    .last()
+                    .map(|entry| entry.key.to_vec())
+                    .unwrap_or_default();
+                let node = encode_leaf_node_refs(&group.entries);
+                let (chunk, summary) =
+                    child_summary_from_node(node, first_key, last_key, subtree_count);
+                chunks.entry(chunk.hash).or_insert(chunk);
+                summary
+            })
+            .collect()
+    }
+
     fn build_internal_level(
         &self,
         children: Vec<ChildSummary>,
@@ -1234,6 +1349,36 @@ impl TrackedStateTree {
             .collect()
     }
 
+    fn build_internal_level_from_refs<'a>(
+        &self,
+        children: impl IntoIterator<Item = ChildSummaryRef<'a>>,
+        level: usize,
+        chunks: &mut BTreeMap<[u8; TRACKED_STATE_HASH_BYTES], PendingChunkWrite>,
+    ) -> Vec<ChildSummary> {
+        let groups = chunk_internal_entry_refs(children, &self.options, level);
+        groups
+            .into_iter()
+            .map(|group| {
+                let subtree_count = group.children.iter().map(|child| child.subtree_count).sum();
+                let first_key = group
+                    .children
+                    .first()
+                    .map(|child| child.first_key.to_vec())
+                    .unwrap_or_default();
+                let last_key = group
+                    .children
+                    .last()
+                    .map(|child| child.last_key.to_vec())
+                    .unwrap_or_default();
+                let node = encode_internal_node_refs(&group.children);
+                let (chunk, summary) =
+                    child_summary_from_node(node, first_key, last_key, subtree_count);
+                chunks.entry(chunk.hash).or_insert(chunk);
+                summary
+            })
+            .collect()
+    }
+
     async fn collect_leaf_entries(
         &self,
         store: &mut (impl StorageReader + ?Sized),
@@ -1261,7 +1406,7 @@ impl TrackedStateTree {
         store: &mut impl StorageReader,
         root_id: &TrackedStateRootId,
         request: &TrackedStateTreeScanRequest,
-    ) -> Result<Vec<(TrackedStateKey, TrackedStateValue)>, LixError> {
+    ) -> Result<Vec<(TrackedStateKey, TrackedStateIndexValue)>, LixError> {
         self.scan(store, root_id, request).await
     }
 
@@ -1271,39 +1416,66 @@ impl TrackedStateTree {
         hash: [u8; TRACKED_STATE_HASH_BYTES],
         request: &'a TrackedStateTreeScanRequest,
         ranges: &'a [EncodedScanRange],
-        rows: &'a mut Vec<(TrackedStateKey, TrackedStateValue)>,
+        key_decode_hint: Option<ScanKeyDecodeHint<'a>>,
+        rows: &'a mut Vec<(TrackedStateKey, TrackedStateIndexValue)>,
     ) -> Pin<Box<dyn Future<Output = Result<(), LixError>> + Send + 'a>>
     where
         S: StorageReader + Send + 'a,
     {
         Box::pin(async move {
-            match self.load_node(store, &hash).await? {
-                DecodedNode::Leaf(leaf) => {
-                    for entry in leaf.entries() {
+            let bytes = self.load_node_bytes(store, &hash).await?;
+            match decode_node_ref(&bytes)? {
+                DecodedNodeRef::Leaf(leaf) => {
+                    for index in 0..leaf.len() {
                         if scan_limit_reached(request, rows.len()) {
                             break;
                         }
-                        if !encoded_key_in_scan_ranges(&entry.key, ranges) {
+                        let entry = leaf.entry(index)?.ok_or_else(|| {
+                            LixError::new(
+                                "LIX_ERROR_UNKNOWN",
+                                "tracked-state leaf entry disappeared during scan",
+                            )
+                        })?;
+                        if !encoded_key_in_scan_ranges(entry.key, ranges) {
                             continue;
                         }
-                        let key = decode_key(&entry.key)?;
-                        if !key_matches_scan_filters(request, &key) {
+                        let key = match key_decode_hint {
+                            Some(hint) => decode_key_with_trusted_prefix(
+                                entry.key,
+                                hint.schema_key,
+                                hint.file_id,
+                                hint.prefix_len,
+                            )?,
+                            None => decode_key(entry.key)?,
+                        };
+                        if key_decode_hint.is_none() && !key_matches_scan_filters(request, &key) {
                             continue;
                         }
-                        let value = decode_value(&entry.value)?;
-                        if request.matches(&key, &value) {
+                        let Some(value) =
+                            decode_visible_value(entry.value, request.include_tombstones)?
+                        else {
+                            continue;
+                        };
+                        if key_decode_hint.is_some() || request.matches(&key, &value) {
                             rows.push((key, value));
                         }
                     }
                 }
-                DecodedNode::Internal(internal) => {
+                DecodedNodeRef::Internal(internal) => {
                     for child in internal.children() {
                         if scan_limit_reached(request, rows.len()) {
                             break;
                         }
                         if child_summary_overlaps_scan_ranges(child, ranges) {
-                            self.scan_node(store, child.child_hash, request, ranges, rows)
-                                .await?;
+                            self.scan_node(
+                                store,
+                                child.child_hash,
+                                request,
+                                ranges,
+                                key_decode_hint,
+                                rows,
+                            )
+                            .await?;
                         }
                     }
                 }
@@ -1317,7 +1489,7 @@ impl TrackedStateTree {
         store: &'a mut S,
         hash: [u8; TRACKED_STATE_HASH_BYTES],
         encoded_keys: &'a [(usize, Vec<u8>)],
-        values: &'a mut [Option<TrackedStateValue>],
+        values: &'a mut [Option<TrackedStateIndexValue>],
     ) -> Pin<Box<dyn Future<Output = Result<(), LixError>> + Send + 'a>>
     where
         S: StorageReader + Send + 'a,
@@ -1327,21 +1499,22 @@ impl TrackedStateTree {
                 return Ok(());
             }
 
-            match self.load_node(store, &hash).await? {
-                DecodedNode::Leaf(leaf) => {
+            let bytes = self.load_node_bytes(store, &hash).await?;
+            match decode_node_ref(&bytes)? {
+                DecodedNodeRef::Leaf(leaf) => {
                     for (original_index, encoded_key) in encoded_keys {
-                        let Some(entry_index) = leaf
-                            .entries()
-                            .binary_search_by(|entry| entry.key.as_slice().cmp(encoded_key))
-                            .ok()
-                        else {
-                            continue;
-                        };
-                        values[*original_index] =
-                            Some(decode_value(&leaf.entries()[entry_index].value)?);
+                        if let Some(entry_index) = binary_search_leaf_key(&leaf, encoded_key)? {
+                            let entry = leaf.entry(entry_index)?.ok_or_else(|| {
+                                LixError::new(
+                                    "LIX_ERROR_UNKNOWN",
+                                    "tracked-state leaf entry disappeared during get_many",
+                                )
+                            })?;
+                            values[*original_index] = Some(decode_value(entry.value)?);
+                        }
                     }
                 }
-                DecodedNode::Internal(internal) => {
+                DecodedNodeRef::Internal(internal) => {
                     let mut start = 0usize;
                     let children = internal.children();
                     for (child_index, child) in children.iter().enumerate() {
@@ -1540,11 +1713,20 @@ impl TrackedStateTree {
         store: &mut (impl StorageReader + ?Sized),
         hash: &[u8; TRACKED_STATE_HASH_BYTES],
     ) -> Result<DecodedNode, LixError> {
+        let bytes = self.load_node_bytes(store, hash).await?;
+        decode_node(&bytes)
+    }
+
+    async fn load_node_bytes(
+        &self,
+        store: &mut (impl StorageReader + ?Sized),
+        hash: &[u8; TRACKED_STATE_HASH_BYTES],
+    ) -> Result<Vec<u8>, LixError> {
         let bytes = storage::read_chunk(store, hash).await?.ok_or_else(|| {
             LixError::new("LIX_ERROR_UNKNOWN", "tracked-state tree chunk is missing")
         })?;
         storage::verify_chunk_hash(hash, &bytes)?;
-        decode_node(&bytes)
+        Ok(bytes)
     }
 
     async fn load_node_with_overlay(
@@ -1587,6 +1769,36 @@ struct EncodedScanRange {
     end: Option<Vec<u8>>,
 }
 
+#[derive(Debug, Clone, Copy)]
+struct ScanKeyDecodeHint<'a> {
+    schema_key: &'a str,
+    file_id: Option<&'a str>,
+    prefix_len: usize,
+}
+
+fn binary_search_leaf_key(
+    leaf: &DecodedLeafNodeRef<'_>,
+    encoded_key: &[u8],
+) -> Result<Option<usize>, LixError> {
+    let mut low = 0usize;
+    let mut high = leaf.len();
+    while low < high {
+        let mid = low + (high - low) / 2;
+        let key = leaf.key(mid)?.ok_or_else(|| {
+            LixError::new(
+                "LIX_ERROR_UNKNOWN",
+                "tracked-state leaf key disappeared during binary search",
+            )
+        })?;
+        match key.cmp(encoded_key) {
+            std::cmp::Ordering::Less => low = mid + 1,
+            std::cmp::Ordering::Equal => return Ok(Some(mid)),
+            std::cmp::Ordering::Greater => high = mid,
+        }
+    }
+    Ok(None)
+}
+
 struct LeafSummaryCursor {
     stack: Vec<LeafSummaryCursorFrame>,
     current: Option<ChildSummary>,
@@ -1702,6 +1914,13 @@ struct LeafChunkAccumulator {
     value_bytes: usize,
 }
 
+#[derive(Debug, Default)]
+struct LeafChunkRefAccumulator<'a> {
+    entries: Vec<EncodedLeafEntryRef<'a>>,
+    key_bytes: usize,
+    value_bytes: usize,
+}
+
 #[derive(Debug, Default)]
 struct InternalChunkAccumulator {
     children: Vec<ChildSummary>,
@@ -1709,6 +1928,13 @@ struct InternalChunkAccumulator {
     last_key_bytes: usize,
 }
 
+#[derive(Debug, Default)]
+struct InternalChunkRefAccumulator<'a> {
+    children: Vec<ChildSummaryRef<'a>>,
+    first_key_bytes: usize,
+    last_key_bytes: usize,
+}
+
 fn chunk_leaf_entries(
     entries: Vec<EncodedLeafEntry>,
     options: &TrackedStateTreeOptions,
@@ -1719,7 +1945,7 @@ fn chunk_leaf_entries(
     let mut groups = Vec::new();
     let mut current = LeafChunkAccumulator::default();
     for entry in entries {
-        let item_size = entry.key.len() + entry.value.len();
+        let item_size = estimate_leaf_entry_size(entry.key.len(), entry.value.len());
         let projected_size = estimate_leaf_chunk_size(
             current.entries.len() + 1,
             current.key_bytes + entry.key.len(),
@@ -1758,6 +1984,56 @@ fn chunk_leaf_entries(
     groups
 }
 
+fn chunk_leaf_entry_refs<'a>(
+    entries: impl IntoIterator<Item = EncodedLeafEntryRef<'a>>,
+    options: &TrackedStateTreeOptions,
+) -> Vec<LeafChunkRefAccumulator<'a>> {
+    let mut iter = entries.into_iter().peekable();
+    if iter.peek().is_none() {
+        return vec![LeafChunkRefAccumulator::default()];
+    }
+    let mut groups = Vec::new();
+    let mut current = LeafChunkRefAccumulator::default();
+    for entry in iter {
+        let item_size = estimate_leaf_entry_size(entry.key.len(), entry.value.len());
+        let projected_size = estimate_leaf_chunk_size(
+            current.entries.len() + 1,
+            current.key_bytes + entry.key.len(),
+            current.value_bytes + entry.value.len(),
+        );
+        if !current.entries.is_empty() && projected_size > options.max_chunk_bytes {
+            groups.push(std::mem::take(&mut current));
+        }
+
+        current.key_bytes += entry.key.len();
+        current.value_bytes += entry.value.len();
+        current.entries.push(entry);
+        let current_size = estimate_leaf_chunk_size(
+            current.entries.len(),
+            current.key_bytes,
+            current.value_bytes,
+        );
+        if current_size >= options.min_chunk_bytes
+            && (current_size >= options.max_chunk_bytes
+                || current.entries.last().is_some_and(|entry| {
+                    boundary_trigger(
+                        entry.key,
+                        0,
+                        current_size,
+                        item_size,
+                        options.target_chunk_bytes,
+                    )
+                }))
+        {
+            groups.push(std::mem::take(&mut current));
+        }
+    }
+    if !current.entries.is_empty() {
+        groups.push(current);
+    }
+    groups
+}
+
 fn chunk_internal_entries(
     children: Vec<ChildSummary>,
     options: &TrackedStateTreeOptions,
@@ -1808,8 +2084,62 @@ fn chunk_internal_entries(
     groups
 }
 
+fn chunk_internal_entry_refs<'a>(
+    children: impl IntoIterator<Item = ChildSummaryRef<'a>>,
+    options: &TrackedStateTreeOptions,
+    level: usize,
+) -> Vec<InternalChunkRefAccumulator<'a>> {
+    let mut groups = Vec::new();
+    let mut current = InternalChunkRefAccumulator::default();
+    for child in children {
+        let item_size = child.first_key.len()
+            + child.last_key.len()
+            + TRACKED_STATE_HASH_BYTES
+            + std::mem::size_of::<u64>();
+        let projected_size = estimate_internal_chunk_size(
+            current.children.len() + 1,
+            current.first_key_bytes + child.first_key.len(),
+            current.last_key_bytes + child.last_key.len(),
+        );
+        if !current.children.is_empty() && projected_size > options.max_chunk_bytes {
+            groups.push(std::mem::take(&mut current));
+        }
+
+        current.first_key_bytes += child.first_key.len();
+        current.last_key_bytes += child.last_key.len();
+        current.children.push(child);
+        let current_size = estimate_internal_chunk_size(
+            current.children.len(),
+            current.first_key_bytes,
+            current.last_key_bytes,
+        );
+        if current_size >= options.min_chunk_bytes
+            && (current_size >= options.max_chunk_bytes
+                || current.children.last().is_some_and(|child| {
+                    boundary_trigger(
+                        child.first_key,
+                        level,
+                        current_size,
+                        item_size,
+                        options.target_chunk_bytes,
+                    )
+                }))
+        {
+            groups.push(std::mem::take(&mut current));
+        }
+    }
+    if !current.children.is_empty() {
+        groups.push(current);
+    }
+    groups
+}
+
 fn estimate_leaf_chunk_size(entry_count: usize, key_bytes: usize, value_bytes: usize) -> usize {
-    16 + entry_count * 8 + key_bytes + value_bytes
+    10 + entry_count * 12 + key_bytes + value_bytes
+}
+
+fn estimate_leaf_entry_size(key_bytes: usize, value_bytes: usize) -> usize {
+    12 + key_bytes + value_bytes
 }
 
 fn estimate_internal_chunk_size(
@@ -1863,7 +2193,7 @@ async fn child_summaries_are_leaves(
 
 fn decode_entry(
     entry: &EncodedLeafEntry,
-) -> Result<(TrackedStateKey, TrackedStateValue), LixError> {
+) -> Result<(TrackedStateKey, TrackedStateIndexValue), LixError> {
     Ok((decode_key(&entry.key)?, decode_value(&entry.value)?))
 }
 
@@ -2024,6 +2354,28 @@ fn scan_ranges(request: &TrackedStateTreeScanRequest) -> Vec<EncodedScanRange> {
     ranges
 }
 
+fn scan_key_decode_hint<'a>(
+    request: &'a TrackedStateTreeScanRequest,
+    ranges: &[EncodedScanRange],
+) -> Option<ScanKeyDecodeHint<'a>> {
+    if ranges.len() != 1 || request.schema_keys.len() != 1 || request.file_ids.len() != 1 {
+        return None;
+    }
+    if !request.entity_ids.is_empty() {
+        return None;
+    }
+    let file_id = match request.file_ids.first()? {
+        NullableKeyFilter::Null => None,
+        NullableKeyFilter::Value(file_id) => Some(file_id.as_str()),
+        NullableKeyFilter::Any => return None,
+    };
+    Some(ScanKeyDecodeHint {
+        schema_key: request.schema_keys.first()?.as_str(),
+        file_id,
+        prefix_len: ranges.first()?.start.len(),
+    })
+}
+
 fn prefix_scan_range(prefix: Vec<u8>) -> EncodedScanRange {
     EncodedScanRange {
         end: lexicographic_successor(&prefix),
@@ -2113,6 +2465,7 @@ mod tests {
     use crate::backend::testing::UnitTestBackend;
     use crate::entity_identity::EntityIdentity;
     use crate::storage::{StorageContext, StorageWriteTransaction};
+    use crate::tracked_state::codec::encode_value;
 
     #[tokio::test]
     async fn exact_read_roundtrips_from_stored_root() {
@@ -2129,7 +2482,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             None,
-            vec![TrackedStateMutation::put(key.clone(), value.clone())],
+            vec![mutation(&key, &value)],
             Some("commit-1"),
         )
         .await
@@ -2159,6 +2512,8 @@ mod tests {
         let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
         let tree = TrackedStateTree::new();
         let key = key("schema", None, "entity");
+        let old_value = value("change-old", Some("{\"v\":1}"));
+        let new_value = value("change-new", Some("{\"v\":2}"));
 
         let mut transaction = storage
             .begin_write_transaction()
@@ -2168,10 +2523,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             None,
-            vec![
-                TrackedStateMutation::put(key.clone(), value("change-old", Some("{\"v\":1}"))),
-                TrackedStateMutation::put(key.clone(), value("change-new", Some("{\"v\":2}"))),
-            ],
+            vec![mutation(&key, &old_value), mutation(&key, &new_value)],
             None,
         )
         .await
@@ -2187,15 +2539,12 @@ mod tests {
             .await
             .expect("row should load")
             .expect("row should exist");
-        assert_eq!(loaded.change_id, "change-new");
-        assert_eq!(
-            loaded.snapshot_ref,
-            Some(crate::json_store::JsonRef::from_hash_bytes([2; 32]))
-        );
+        assert_eq!(loaded.change_locator.change_id, "change-new");
+        assert_eq!(loaded.change_locator.source_commit_id, "commit");
     }
 
     #[tokio::test]
-    async fn scan_filters_and_hides_tombstones_by_default() {
+    async fn scan_filters_by_index_key_without_materializing_tombstones() {
         let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
         let tree = TrackedStateTree::new();
 
@@ -2208,12 +2557,9 @@ mod tests {
             transaction.as_mut(),
             None,
             vec![
-                TrackedStateMutation::put(
-                    key("schema-a", None, "visible"),
-                    value("c1", Some("{}")),
-                ),
-                TrackedStateMutation::put(key("schema-a", None, "deleted"), value("c2", None)),
-                TrackedStateMutation::put(key("schema-b", None, "other"), value("c3", Some("{}"))),
+                mutation_owned(key("schema-a", None, "visible"), value("c1", Some("{}"))),
+                mutation_owned(key("schema-a", None, "deleted"), value("c2", None)),
+                mutation_owned(key("schema-b", None, "other"), value("c3", Some("{}"))),
             ],
             None,
         )
@@ -2236,11 +2582,30 @@ mod tests {
             )
             .await
             .expect("scan should succeed");
-        assert_eq!(rows.len(), 1);
-        assert_eq!(
-            rows[0].0.entity_id.as_string().expect("identity"),
-            "visible"
-        );
+        assert_eq!(rows.len(), 2);
+        let identities = rows
+            .iter()
+            .map(|(key, _)| key.entity_id.as_single_string_owned().expect("identity"))
+            .collect::<Vec<_>>();
+        assert_eq!(identities, vec!["deleted", "visible"]);
+
+        let live_rows = tree
+            .scan(
+                &mut store,
+                &result.root_id,
+                &TrackedStateTreeScanRequest {
+                    schema_keys: vec!["schema-a".to_string()],
+                    include_tombstones: false,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("live scan should succeed");
+        let live_identities = live_rows
+            .iter()
+            .map(|(key, _)| key.entity_id.as_single_string_owned().expect("identity"))
+            .collect::<Vec<_>>();
+        assert_eq!(live_identities, vec!["visible"]);
     }
 
     #[tokio::test]
@@ -2257,19 +2622,19 @@ mod tests {
             transaction.as_mut(),
             None,
             vec![
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema-a", Some("file-a"), "entity-a"),
                     value("c1", Some("{}")),
                 ),
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema-a", Some("file-b"), "entity-a"),
                     value("c2", Some("{}")),
                 ),
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema-a", Some("file-a"), "entity-b"),
                     value("c3", Some("{}")),
                 ),
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema-b", Some("file-a"), "entity-a"),
                     value("c4", Some("{}")),
                 ),
@@ -2301,18 +2666,93 @@ mod tests {
         assert_eq!(rows.len(), 1);
         assert_eq!(rows[0].0.schema_key, "schema-a");
         assert_eq!(
-            rows[0].0.entity_id.as_string().expect("identity"),
+            rows[0]
+                .0
+                .entity_id
+                .as_single_string_owned()
+                .expect("identity"),
             "entity-a"
         );
         assert_eq!(rows[0].0.file_id.as_deref(), Some("file-a"));
     }
 
+    #[tokio::test]
+    async fn scan_schema_file_prefix_honors_tombstones_and_limit() {
+        let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
+        let tree = TrackedStateTree::new();
+
+        let mut transaction = storage
+            .begin_write_transaction()
+            .await
+            .expect("transaction should open");
+        let result = apply_mutations_for_test(
+            &tree,
+            transaction.as_mut(),
+            None,
+            vec![
+                mutation_owned(
+                    key("schema-a", Some("file-a"), "entity-a"),
+                    value("c1", Some("{}")),
+                ),
+                mutation_owned(
+                    key("schema-a", Some("file-a"), "entity-b"),
+                    value("c2", None),
+                ),
+                mutation_owned(
+                    key("schema-a", Some("file-a"), "entity-c"),
+                    value("c3", Some("{}")),
+                ),
+                mutation_owned(
+                    key("schema-a", Some("file-b"), "entity-d"),
+                    value("c4", Some("{}")),
+                ),
+            ],
+            None,
+        )
+        .await
+        .expect("mutations should apply");
+        transaction
+            .commit()
+            .await
+            .expect("transaction should commit");
+
+        let mut store = storage.clone();
+        let rows = tree
+            .scan(
+                &mut store,
+                &result.root_id,
+                &TrackedStateTreeScanRequest {
+                    schema_keys: vec!["schema-a".to_string()],
+                    file_ids: vec![crate::NullableKeyFilter::Value("file-a".to_string())],
+                    include_tombstones: false,
+                    limit: Some(2),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("scan should succeed");
+
+        assert_eq!(rows.len(), 2);
+        assert!(rows.iter().all(
+            |(key, _)| key.schema_key == "schema-a" && key.file_id.as_deref() == Some("file-a")
+        ));
+        assert_eq!(
+            rows.iter()
+                .map(|(key, _)| key.entity_id.as_single_string_owned().expect("identity"))
+                .collect::<Vec<_>>(),
+            vec!["entity-a", "entity-c"]
+        );
+    }
+
     #[tokio::test]
     async fn applying_to_base_root_reuses_existing_rows_and_overwrites_changed_rows() {
         let storage = StorageContext::new(Arc::new(UnitTestBackend::new()));
         let tree = TrackedStateTree::new();
         let unchanged_key = key("schema", None, "unchanged");
         let changed_key = key("schema", None, "changed");
+        let unchanged_value = value("c1", Some("{}"));
+        let old_changed_value = value("c2", Some("{\"old\":true}"));
+        let new_changed_value = value("c3", Some("{\"new\":true}"));
 
         let mut transaction = storage
             .begin_write_transaction()
@@ -2323,8 +2763,8 @@ mod tests {
             transaction.as_mut(),
             None,
             vec![
-                TrackedStateMutation::put(unchanged_key.clone(), value("c1", Some("{}"))),
-                TrackedStateMutation::put(changed_key.clone(), value("c2", Some("{\"old\":true}"))),
+                mutation(&unchanged_key, &unchanged_value),
+                mutation(&changed_key, &old_changed_value),
             ],
             None,
         )
@@ -2334,10 +2774,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            vec![TrackedStateMutation::put(
-                changed_key.clone(),
-                value("c3", Some("{\"new\":true}")),
-            )],
+            vec![mutation(&changed_key, &new_changed_value)],
             None,
         )
         .await
@@ -2353,6 +2790,7 @@ mod tests {
                 .await
                 .expect("unchanged read")
                 .expect("unchanged exists")
+                .change_locator
                 .change_id,
             "c1"
         );
@@ -2361,6 +2799,7 @@ mod tests {
                 .await
                 .expect("changed read")
                 .expect("changed exists")
+                .change_locator
                 .change_id,
             "c3"
         );
@@ -2373,6 +2812,9 @@ mod tests {
         let shared_key = key("schema", None, "shared");
         let branch_a_key = key("schema", None, "branch-a");
         let branch_b_key = key("schema", None, "branch-b");
+        let shared_value = value("shared-change", Some("{\"shared\":true}"));
+        let branch_a_value = value("branch-a-change", Some("{\"branch\":\"a\"}"));
+        let branch_b_value = value("branch-b-change", Some("{\"branch\":\"b\"}"));
 
         let mut transaction = storage
             .begin_write_transaction()
@@ -2382,10 +2824,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             None,
-            vec![TrackedStateMutation::put(
-                shared_key.clone(),
-                value("shared-change", Some("{\"shared\":true}")),
-            )],
+            vec![mutation(&shared_key, &shared_value)],
             Some("commit-base"),
         )
         .await
@@ -2394,10 +2833,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            vec![TrackedStateMutation::put(
-                branch_a_key.clone(),
-                value("branch-a-change", Some("{\"branch\":\"a\"}")),
-            )],
+            vec![mutation(&branch_a_key, &branch_a_value)],
             Some("commit-a"),
         )
         .await
@@ -2406,10 +2842,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            vec![TrackedStateMutation::put(
-                branch_b_key.clone(),
-                value("branch-b-change", Some("{\"branch\":\"b\"}")),
-            )],
+            vec![mutation(&branch_b_key, &branch_b_value)],
             Some("commit-b"),
         )
         .await
@@ -2455,7 +2888,7 @@ mod tests {
         });
         let rows = (0..100)
             .map(|index| {
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema", None, &format!("entity-{index:03}")),
                     value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
                 )
@@ -2475,10 +2908,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            vec![TrackedStateMutation::put(
-                changed_key.clone(),
-                changed_value.clone(),
-            )],
+            vec![mutation(&changed_key, &changed_value)],
             None,
         )
         .await
@@ -2513,7 +2943,7 @@ mod tests {
         });
         let rows = (0..100)
             .map(|index| {
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema", None, &format!("entity-{index:03}")),
                     value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
                 )
@@ -2533,10 +2963,7 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            vec![TrackedStateMutation::put(
-                inserted_key.clone(),
-                inserted_value.clone(),
-            )],
+            vec![mutation(&inserted_key, &inserted_value)],
             None,
         )
         .await
@@ -2574,7 +3001,7 @@ mod tests {
         });
         let rows = (0..100)
             .map(|index| {
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema", None, &format!("entity-{index:03}")),
                     value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
                 )
@@ -2582,7 +3009,7 @@ mod tests {
             .collect::<Vec<_>>();
         let updates = (10..25)
             .map(|index| {
-                TrackedStateMutation::put(
+                (
                     key("schema", None, &format!("entity-{index:03}")),
                     value(
                         &format!("changed-{index}"),
@@ -2603,7 +3030,10 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            updates.clone(),
+            updates
+                .iter()
+                .map(|(key, value)| mutation(key, value))
+                .collect(),
             None,
         )
         .await
@@ -2612,8 +3042,7 @@ mod tests {
             .collect_leaf_entries(&mut transaction.as_mut(), &base.root_id)
             .await
             .expect("base entries should collect");
-        for update in updates {
-            let TrackedStateMutation::Put { key, value } = update;
+        for (key, value) in updates {
             let encoded_key = encode_key(&key);
             let encoded_value = encode_value(&value);
             let index = canonical_entries
@@ -2638,7 +3067,7 @@ mod tests {
         });
         let rows = (0..100)
             .map(|index| {
-                TrackedStateMutation::put(
+                mutation_owned(
                     key("schema", None, &format!("entity-{index:03}")),
                     value(&format!("c-{index}"), Some(&format!("{{\"v\":{index}}}"))),
                 )
@@ -2648,7 +3077,7 @@ mod tests {
             .into_iter()
             .enumerate()
             .map(|(index, entity_id)| {
-                TrackedStateMutation::put(
+                (
                     key("schema", None, entity_id),
                     value(
                         &format!("inserted-{index}"),
@@ -2669,7 +3098,10 @@ mod tests {
             &tree,
             transaction.as_mut(),
             Some(&base.root_id),
-            inserts.clone(),
+            inserts
+                .iter()
+                .map(|(key, value)| mutation(key, value))
+                .collect(),
             None,
         )
         .await
@@ -2678,8 +3110,7 @@ mod tests {
             .collect_leaf_entries(&mut transaction.as_mut(), &base.root_id)
             .await
             .expect("base entries should collect");
-        for insert in inserts {
-            let TrackedStateMutation::Put { key, value } = insert;
+        for (key, value) in inserts {
             let encoded_key = encode_key(&key);
             let encoded_value = encode_value(&value);
             let index = canonical_entries
@@ -2715,6 +3146,14 @@ mod tests {
         Ok(result)
     }
 
+    fn mutation(key: &TrackedStateKey, value: &TrackedStateIndexValue) -> TrackedStateMutation {
+        TrackedStateMutation::put_encoded(encode_key(key), encode_value(value))
+    }
+
+    fn mutation_owned(key: TrackedStateKey, value: TrackedStateIndexValue) -> TrackedStateMutation {
+        mutation(&key, &value)
+    }
+
     fn key(schema_key: &str, file_id: Option<&str>, entity_id: &str) -> TrackedStateKey {
         TrackedStateKey {
             schema_key: schema_key.to_string(),
@@ -2723,22 +3162,26 @@ mod tests {
         }
     }
 
-    fn value(change_id: &str, snapshot_content: Option<&str>) -> TrackedStateValue {
-        let snapshot_ref = match snapshot_content {
-            Some("{\"v\":1}") => Some(crate::json_store::JsonRef::from_hash_bytes([1; 32])),
-            Some("{\"v\":2}") => Some(crate::json_store::JsonRef::from_hash_bytes([2; 32])),
-            Some(_) => Some(crate::json_store::JsonRef::from_hash_bytes([3; 32])),
-            None => None,
+    fn value(change_id: &str, snapshot_content: Option<&str>) -> TrackedStateIndexValue {
+        let source_ordinal = match snapshot_content {
+            Some("{\"v\":1}") => 1,
+            Some("{\"v\":2}") => 2,
+            Some(_) => 3,
+            None => 0,
         };
-        TrackedStateValue {
-            snapshot_ref,
+        TrackedStateIndexValue {
+            change_locator: crate::commit_store::ChangeLocator {
+                source_commit_id: "commit".to_string(),
+                source_pack_id: 0,
+                source_ordinal,
+                change_id: change_id.to_string(),
+            },
+            deleted: snapshot_content.is_none(),
+            snapshot_ref: snapshot_content
+                .map(|content| crate::json_store::JsonRef::for_content(content.as_bytes())),
             metadata_ref: None,
-            schema_version: "1".to_string(),
             created_at: "2026-01-01T00:00:00Z".to_string(),
             updated_at: "2026-01-01T00:00:00Z".to_string(),
-            change_id: change_id.to_string(),
-            commit_id: "commit".to_string(),
-            deleted: snapshot_content.is_none(),
         }
     }
 }