@lix-js/sdk 0.6.0-preview.3 → 0.6.0-preview.5

package/dist-engine-src/src/tracked_state/{materialization.rs → row_materialization.rs}

@@ -1,27 +1,26 @@
-use crate::entity_identity::EntityIdentity;
+use crate::entity_pk::EntityPk;
 use crate::json_store::JsonRef;
 use crate::json_store::{JsonLoadRequestRef, JsonReadScopeRef, JsonStoreContext};
-use crate::storage::StorageReader;
+use crate::storage::StorageRead;
 use crate::tracked_state::types::{TrackedStateIndexValue, TrackedStateKey};
 use crate::tracked_state::MaterializedTrackedStateRow;
 use crate::LixError;
-use std::collections::BTreeMap;
 
 /// Materializes tracked-state index entries.
 ///
-/// The durable tracked_state value is authoritative for scalar projection
-/// fields and stores the JSON refs needed for payload projections. Snapshot and
+/// The durable tracked_state value is authoritative for scalar materialization
+/// fields and stores the JSON refs needed for payload hydration. Snapshot and
 /// metadata bytes are hydrated from grouped json_store loads only when the
-/// requested projection needs them.
-pub(crate) async fn materialize_index_entries<S>(
-    store: &mut S,
+/// requested materialization needs them.
+pub(crate) async fn materialize_rows_from_index_entries<S>(
+    store: &S,
     entries: Vec<(TrackedStateKey, TrackedStateIndexValue)>,
-    projection: &TrackedMaterializationProjection,
+    materialization: &TrackedRowMaterialization,
 ) -> Result<Vec<MaterializedTrackedStateRow>, LixError>
 where
-    S: StorageReader,
+    S: StorageRead + Send + Sync,
 {
-    if !projection.snapshot_content && !projection.metadata {
+    if !materialization.snapshot_content && !materialization.metadata {
         return Ok(entries
             .into_iter()
             .map(materialize_entry_without_json)
@@ -29,45 +28,40 @@ where
     }
 
     let json_slots_per_row =
-        usize::from(projection.snapshot_content) + usize::from(projection.metadata);
+        usize::from(materialization.snapshot_content) + usize::from(materialization.metadata);
     let json_ref_capacity = entries.len().saturating_mul(json_slots_per_row);
     let mut row_plans = Vec::with_capacity(entries.len());
     let mut json_refs = Vec::with_capacity(json_ref_capacity);
     let mut json_ref_localities = Vec::with_capacity(json_ref_capacity);
     for (key, value) in entries {
-        let row_index = row_plans.len();
-        let snapshot_ref_index = projected_json_ref_index(
-            projection.snapshot_content,
+        let snapshot_ref_index = materialized_json_ref_index(
+            materialization.snapshot_content,
             value.snapshot_ref,
-            row_index,
-            value.change_locator.source_pack_id,
             &mut json_refs,
             &mut json_ref_localities,
         );
-        let metadata_ref_index = projected_json_ref_index(
-            projection.metadata,
+        let metadata_ref_index = materialized_json_ref_index(
+            materialization.metadata,
             value.metadata_ref,
-            row_index,
-            value.change_locator.source_pack_id,
             &mut json_refs,
             &mut json_ref_localities,
         );
-        row_plans.push(MaterializedTrackedStateRowPlan {
-            entity_id: key.entity_id,
+        row_plans.push(TrackedRowMaterializationPlan {
+            entity_pk: key.entity_pk,
             schema_key: key.schema_key,
             file_id: key.file_id,
             deleted: value.deleted,
             created_at: value.created_at,
             updated_at: value.updated_at,
-            change_id: value.change_locator.change_id,
-            commit_id: value.change_locator.source_commit_id,
+            change_id: value.change_id,
+            commit_id: value.commit_id,
             snapshot_ref_index,
             metadata_ref_index,
         });
     }
 
     let mut json_values =
-        load_projection_json_values(store, &json_refs, &json_ref_localities, &row_plans).await?;
+        load_materialization_json_values(store, &json_refs, &json_ref_localities).await?;
     row_plans
         .into_iter()
         .map(|plan| materialize_row_plan(plan, &json_refs, &mut json_values))
@@ -78,7 +72,7 @@ fn materialize_entry_without_json(
     (key, value): (TrackedStateKey, TrackedStateIndexValue),
 ) -> MaterializedTrackedStateRow {
     MaterializedTrackedStateRow {
-        entity_id: key.entity_id,
+        entity_pk: key.entity_pk,
         schema_key: key.schema_key,
         file_id: key.file_id,
         snapshot_content: None,
@@ -86,13 +80,13 @@ fn materialize_entry_without_json(
         deleted: value.deleted,
         created_at: value.created_at,
         updated_at: value.updated_at,
-        change_id: value.change_locator.change_id,
-        commit_id: value.change_locator.source_commit_id,
+        change_id: value.change_id,
+        commit_id: value.commit_id,
     }
 }
 
-struct MaterializedTrackedStateRowPlan {
-    entity_id: EntityIdentity,
+struct TrackedRowMaterializationPlan {
+    entity_pk: EntityPk,
     schema_key: String,
     file_id: Option<String>,
     deleted: bool,
@@ -104,11 +98,9 @@ struct MaterializedTrackedStateRowPlan {
     metadata_ref_index: Option<usize>,
 }
 
-fn projected_json_ref_index(
+fn materialized_json_ref_index(
     include: bool,
     json_ref: Option<JsonRef>,
-    row_index: usize,
-    pack_id: u32,
     json_refs: &mut Vec<JsonRef>,
     json_ref_localities: &mut Vec<JsonRefLocality>,
 ) -> Option<usize> {
@@ -117,23 +109,19 @@ fn projected_json_ref_index(
     }
     let index = json_refs.len();
     json_refs.push(json_ref?);
-    json_ref_localities.push(JsonRefLocality { row_index, pack_id });
+    json_ref_localities.push(JsonRefLocality);
     Some(index)
 }
 
-struct JsonRefLocality {
-    row_index: usize,
-    pack_id: u32,
-}
+struct JsonRefLocality;
 
-async fn load_projection_json_values<S>(
-    store: &mut S,
+async fn load_materialization_json_values<S>(
+    store: &S,
     json_refs: &[JsonRef],
     json_ref_localities: &[JsonRefLocality],
-    row_plans: &[MaterializedTrackedStateRowPlan],
 ) -> Result<Vec<Option<Vec<u8>>>, LixError>
 where
-    S: StorageReader,
+    S: StorageRead + Send + Sync,
 {
     if json_refs.len() != json_ref_localities.len() {
         return Err(LixError::new(
@@ -142,109 +130,48 @@ where
         ));
     }
 
-    let json_store = JsonStoreContext::new();
-    if let Some((commit_id, pack_id)) = single_projection_pack(json_ref_localities, row_plans)? {
-        let pack_ids = [pack_id];
-        return json_store
-            .load_bytes_many(
-                store,
-                JsonLoadRequestRef {
-                    refs: json_refs,
-                    scope: JsonReadScopeRef::CommitPacks {
-                        commit_id,
-                        pack_ids: &pack_ids,
-                    },
-                },
-            )
-            .await
-            .map(|batch| batch.into_values());
-    }
-
     let mut json_values = vec![None; json_refs.len()];
-    let mut refs_by_pack = BTreeMap::<(&str, u32), Vec<(usize, JsonRef)>>::new();
+    let mut out_of_band_indexes = Vec::new();
+    let mut out_of_band_refs = Vec::new();
     for (index, json_ref) in json_refs.iter().copied().enumerate() {
-        let locality = json_ref_localities.get(index).ok_or_else(|| {
+        let _locality = json_ref_localities.get(index).ok_or_else(|| {
             LixError::new(
                 LixError::CODE_INTERNAL_ERROR,
                 "tracked_state materialization lost JSON locality index",
             )
         })?;
-        let row_plan = row_plans.get(locality.row_index).ok_or_else(|| {
-            LixError::new(
-                LixError::CODE_INTERNAL_ERROR,
-                "tracked_state materialization lost JSON row locality index",
-            )
-        })?;
-        refs_by_pack
-            .entry((row_plan.commit_id.as_str(), locality.pack_id))
-            .or_default()
-            .push((index, json_ref));
+        out_of_band_indexes.push(index);
+        out_of_band_refs.push(json_ref);
     }
 
-    for ((commit_id, pack_id), refs) in refs_by_pack {
-        let indexes = refs.iter().map(|(index, _)| *index).collect::<Vec<_>>();
-        let refs = refs
-            .into_iter()
-            .map(|(_, json_ref)| json_ref)
-            .collect::<Vec<_>>();
-        let pack_ids = [pack_id];
-        let values = json_store
+    if !out_of_band_refs.is_empty() {
+        let values = JsonStoreContext::new()
             .load_bytes_many(
                 store,
                 JsonLoadRequestRef {
-                    refs: &refs,
-                    scope: JsonReadScopeRef::CommitPacks {
-                        commit_id: &commit_id,
-                        pack_ids: &pack_ids,
-                    },
+                    refs: &out_of_band_refs,
+                    scope: JsonReadScopeRef::OutOfBand,
                 },
             )
             .await?
             .into_values();
-        for (index, value) in indexes.into_iter().zip(values) {
-            json_values[index] = value;
+        for (index, value) in out_of_band_indexes.into_iter().zip(values) {
+            if value.is_some() {
+                json_values[index] = value;
+            }
         }
     }
-    Ok(json_values)
-}
 
-fn single_projection_pack<'a>(
-    json_ref_localities: &[JsonRefLocality],
-    row_plans: &'a [MaterializedTrackedStateRowPlan],
-) -> Result<Option<(&'a str, u32)>, LixError> {
-    let Some(first_locality) = json_ref_localities.first() else {
-        return Ok(None);
-    };
-    let first_plan = row_plans.get(first_locality.row_index).ok_or_else(|| {
-        LixError::new(
-            LixError::CODE_INTERNAL_ERROR,
-            "tracked_state materialization lost JSON row locality index",
-        )
-    })?;
-    let commit_id = first_plan.commit_id.as_str();
-    let pack_id = first_locality.pack_id;
-
-    for locality in &json_ref_localities[1..] {
-        let row_plan = row_plans.get(locality.row_index).ok_or_else(|| {
-            LixError::new(
-                LixError::CODE_INTERNAL_ERROR,
-                "tracked_state materialization lost JSON row locality index",
-            )
-        })?;
-        if row_plan.commit_id != commit_id || locality.pack_id != pack_id {
-            return Ok(None);
-        }
-    }
-    Ok(Some((commit_id, pack_id)))
+    Ok(json_values)
 }
 
 fn materialize_row_plan(
-    plan: MaterializedTrackedStateRowPlan,
+    plan: TrackedRowMaterializationPlan,
     json_refs: &[JsonRef],
     json_values: &mut [Option<Vec<u8>>],
 ) -> Result<MaterializedTrackedStateRow, LixError> {
     Ok(MaterializedTrackedStateRow {
-        entity_id: plan.entity_id,
+        entity_pk: plan.entity_pk,
         schema_key: plan.schema_key,
         file_id: plan.file_id,
         snapshot_content: materialized_json_string(
@@ -275,7 +202,7 @@ fn materialized_json_string(
             "tracked_state materialization lost JSON ref index",
         )
     })?;
-    // Each row plan owns its projected JSON slots. If this path starts
+    // Each row plan owns its materialized JSON slots. If this path starts
     // deduplicating refs, duplicate consumers must clone intentionally.
     let bytes = json_values
         .get_mut(index)
@@ -305,12 +232,12 @@ fn materialized_json_string(
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) struct TrackedMaterializationProjection {
+pub(crate) struct TrackedRowMaterialization {
     pub(crate) snapshot_content: bool,
     pub(crate) metadata: bool,
 }
 
-impl TrackedMaterializationProjection {
+impl TrackedRowMaterialization {
     pub(crate) fn full() -> Self {
         Self {
             snapshot_content: true,
@@ -333,61 +260,6 @@ impl TrackedMaterializationProjection {
 mod tests {
     use super::*;
 
-    fn row_plan(commit_id: &str) -> MaterializedTrackedStateRowPlan {
-        MaterializedTrackedStateRowPlan {
-            entity_id: EntityIdentity::single("entity"),
-            schema_key: "schema".to_string(),
-            file_id: None,
-            deleted: false,
-            created_at: "2024-01-01T00:00:00.000Z".to_string(),
-            updated_at: "2024-01-01T00:00:00.000Z".to_string(),
-            change_id: "change".to_string(),
-            commit_id: commit_id.to_string(),
-            snapshot_ref_index: None,
-            metadata_ref_index: None,
-        }
-    }
-
-    #[test]
-    fn single_projection_pack_accepts_duplicate_slots_from_same_pack() {
-        let row_plans = vec![row_plan("commit-a")];
-        let localities = vec![
-            JsonRefLocality {
-                row_index: 0,
-                pack_id: 7,
-            },
-            JsonRefLocality {
-                row_index: 0,
-                pack_id: 7,
-            },
-        ];
-
-        assert_eq!(
-            single_projection_pack(&localities, &row_plans).expect("pack detection should succeed"),
-            Some(("commit-a", 7))
-        );
-    }
-
-    #[test]
-    fn single_projection_pack_rejects_mixed_packs() {
-        let row_plans = vec![row_plan("commit-a")];
-        let localities = vec![
-            JsonRefLocality {
-                row_index: 0,
-                pack_id: 7,
-            },
-            JsonRefLocality {
-                row_index: 0,
-                pack_id: 8,
-            },
-        ];
-
-        assert_eq!(
-            single_projection_pack(&localities, &row_plans).expect("pack detection should succeed"),
-            None
-        );
-    }
-
     #[test]
     fn materialized_json_string_consumes_owned_payload_bytes() {
         let json = br#"{"value":1}"#.to_vec();