automerge-rb 0.1.1

data/vendor/automerge-rust/automerge/src/storage/columns/column_specification.rs

@@ -0,0 +1,340 @@
+/// An implementation of column specifications as specified in [1]
+///
+/// [1]: https://alexjg.github.io/automerge-storage-docs/#column-specifications
+#[derive(Eq, Hash, PartialEq, Clone, Copy)]
+pub(crate) struct ColumnSpec(u32);
+
+impl PartialOrd for ColumnSpec {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        //self.normalize().partial_cmp(&other.normalize())
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for ColumnSpec {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.normalize().cmp(&other.normalize())
+    }
+}
+
+impl ColumnSpec {
+    pub(crate) const fn new(id: ColumnId, col_type: ColumnType, deflate: bool) -> Self {
+        let mut raw = id.0 << 4;
+        raw |= col_type.as_u8() as u32;
+        if deflate {
+            raw |= 0b00001000;
+        } else {
+            raw &= 0b11110111;
+        }
+        ColumnSpec(raw)
+    }
+
+    pub(crate) const fn new_delta(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::DeltaInteger, false)
+    }
+
+    pub(crate) const fn new_integer(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::Integer, false)
+    }
+
+    pub(crate) const fn new_string(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::String, false)
+    }
+
+    pub(crate) const fn new_value(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::Value, false)
+    }
+
+    pub(crate) const fn new_value_metadata(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::ValueMetadata, false)
+    }
+
+    pub(crate) const fn new_actor(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::Actor, false)
+    }
+
+    pub(crate) const fn new_group(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::Group, false)
+    }
+
+    pub(crate) const fn new_boolean(id: ColumnId) -> Self {
+        Self::new(id, ColumnType::Boolean, false)
+    }
+
+    pub(crate) fn col_type(&self) -> ColumnType {
+        self.0.to_be_bytes()[3].into()
+    }
+
+    pub(crate) fn id(&self) -> ColumnId {
+        ColumnId(self.0 >> 4)
+    }
+
+    pub(crate) fn deflate(&self) -> bool {
+        self.0 & 0b00001000 > 0
+    }
+
+    pub(crate) fn deflated(&self) -> Self {
+        Self::new(self.id(), self.col_type(), true)
+    }
+
+    pub(crate) fn inflated(&self) -> Self {
+        Self::new(self.id(), self.col_type(), false)
+    }
+
+    pub(crate) fn normalize(&self) -> Normalized {
+        Normalized(self.0 & 0b11110111)
+    }
+}
+
+#[derive(PartialEq, PartialOrd, Ord, Eq)]
+pub(crate) struct Normalized(u32);
+
+impl std::fmt::Debug for ColumnSpec {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "ColumnSpec(id: {:?}, type: {}, deflate: {})",
+            self.id(),
+            self.col_type(),
+            self.deflate()
+        )
+    }
+}
+
+#[derive(Eq, PartialEq, Clone, Copy, PartialOrd, Ord)]
+pub(crate) struct ColumnId(u32);
+
+impl ColumnId {
+    pub(crate) const fn new(raw: u32) -> Self {
+        ColumnId(raw)
+    }
+}
+
+impl From<u32> for ColumnId {
+    fn from(raw: u32) -> Self {
+        Self(raw)
+    }
+}
+
+impl std::fmt::Debug for ColumnId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+/// The differente possible column types, as specified in [1]
+///
+/// [1]: https://alexjg.github.io/automerge-storage-docs/#column-specifications
+#[derive(Eq, PartialEq, Clone, Copy, Debug)]
+pub(crate) enum ColumnType {
+    Group,
+    Actor,
+    Integer,
+    DeltaInteger,
+    Boolean,
+    String,
+    ValueMetadata,
+    Value,
+}
+
+impl ColumnType {
+    const fn from_u8(val: u8) -> Self {
+        let type_bits = val & 0b00000111;
+        match type_bits {
+            0 => Self::Group,
+            1 => Self::Actor,
+            2 => Self::Integer,
+            3 => Self::DeltaInteger,
+            4 => Self::Boolean,
+            5 => Self::String,
+            6 => Self::ValueMetadata,
+            7 => Self::Value,
+            _ => unreachable!(),
+        }
+    }
+
+    const fn as_u8(&self) -> u8 {
+        match self {
+            ColumnType::Group => 0,
+            ColumnType::Actor => 1,
+            ColumnType::Integer => 2,
+            ColumnType::DeltaInteger => 3,
+            ColumnType::Boolean => 4,
+            ColumnType::String => 5,
+            ColumnType::ValueMetadata => 6,
+            ColumnType::Value => 7,
+        }
+    }
+}
+
+impl std::fmt::Display for ColumnType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Group => write!(f, "Group"),
+            Self::Actor => write!(f, "Actor"),
+            Self::Integer => write!(f, "Integer"),
+            Self::DeltaInteger => write!(f, "DeltaInteger"),
+            Self::Boolean => write!(f, "Boolean"),
+            Self::String => write!(f, "String"),
+            Self::ValueMetadata => write!(f, "ValueMetadata"),
+            Self::Value => write!(f, "Value"),
+        }
+    }
+}
+
+impl From<u8> for ColumnType {
+    fn from(v: u8) -> Self {
+        Self::from_u8(v)
+    }
+}
+
+impl From<ColumnType> for u8 {
+    fn from(ct: ColumnType) -> Self {
+        ct.as_u8()
+    }
+}
+
+impl From<u32> for ColumnSpec {
+    fn from(raw: u32) -> Self {
+        ColumnSpec(raw)
+    }
+}
+
+impl From<ColumnSpec> for u32 {
+    fn from(spec: ColumnSpec) -> Self {
+        spec.0
+    }
+}
+
+impl From<[u8; 4]> for ColumnSpec {
+    fn from(raw: [u8; 4]) -> Self {
+        u32::from_be_bytes(raw).into()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn column_spec_encoding() {
+        struct Scenario {
+            id: ColumnId,
+            col_type: ColumnType,
+            int_val: u32,
+        }
+
+        let scenarios = vec![
+            Scenario {
+                id: ColumnId(7),
+                col_type: ColumnType::Group,
+                int_val: 112,
+            },
+            Scenario {
+                id: ColumnId(0),
+                col_type: ColumnType::Actor,
+                int_val: 1,
+            },
+            Scenario {
+                id: ColumnId(0),
+                col_type: ColumnType::Integer,
+                int_val: 2,
+            },
+            Scenario {
+                id: ColumnId(1),
+                col_type: ColumnType::DeltaInteger,
+                int_val: 19,
+            },
+            Scenario {
+                id: ColumnId(3),
+                col_type: ColumnType::Boolean,
+                int_val: 52,
+            },
+            Scenario {
+                id: ColumnId(1),
+                col_type: ColumnType::String,
+                int_val: 21,
+            },
+            Scenario {
+                id: ColumnId(5),
+                col_type: ColumnType::ValueMetadata,
+                int_val: 86,
+            },
+            Scenario {
+                id: ColumnId(5),
+                col_type: ColumnType::Value,
+                int_val: 87,
+            },
+        ];
+
+        for (index, scenario) in scenarios.into_iter().enumerate() {
+            let spec = ColumnSpec::new(scenario.id, scenario.col_type, false);
+
+            let encoded_val = u32::from(spec);
+            if encoded_val != scenario.int_val {
+                panic!(
+                    "Scenario {} failed encoding: expected {} but got {}",
+                    index + 1,
+                    scenario.int_val,
+                    encoded_val
+                );
+            }
+
+            if spec.col_type() != scenario.col_type {
+                panic!(
+                    "Scenario {} failed col type: expected {:?} but got {:?}",
+                    index + 1,
+                    scenario.col_type,
+                    spec.col_type()
+                );
+            }
+
+            if spec.deflate() {
+                panic!(
+                    "Scenario {} failed: spec returned true for deflate, should have been false",
+                    index + 1
+                );
+            }
+
+            if spec.id() != scenario.id {
+                panic!(
+                    "Scenario {} failed id: expected {:?} but got {:?}",
+                    index + 1,
+                    scenario.id,
+                    spec.id()
+                );
+            }
+
+            let deflated = ColumnSpec::new(scenario.id, scenario.col_type, true);
+
+            if deflated.id() != spec.id() {
+                panic!("Scenario {} failed deflate id test", index + 1);
+            }
+
+            if deflated.col_type() != spec.col_type() {
+                panic!("Scenario {} failed col type test", index + 1);
+            }
+
+            if !deflated.deflate() {
+                panic!(
+                    "Scenario {} failed: when deflate bit set deflate returned false",
+                    index + 1
+                );
+            }
+
+            let expected = scenario.int_val | 0b00001000;
+            if expected != u32::from(deflated) {
+                panic!(
+                    "Scenario {} failed deflate bit test, expected {} got {}",
+                    index + 1,
+                    expected,
+                    u32::from(deflated)
+                );
+            }
+
+            if deflated.normalize() != spec.normalize() {
+                panic!("Scenario {} failed normalize test", index + 1);
+            }
+        }
+    }
+}

data/vendor/automerge-rust/automerge/src/storage/columns/raw_column.rs

@@ -0,0 +1,286 @@
+use std::collections::BTreeMap;
+use std::{io::Read, marker::PhantomData, ops::Range};
+
+use crate::storage::parse;
+
+use super::{compression, ColumnSpec};
+
+/// This is a "raw" column in the sense that it is just the column specification[1] and range. This
+/// is in contrast to [`super::Column`] which is aware of composite columns such as value columns[2] and
+/// group columns[3].
+///
+/// `RawColumn` is generally an intermediary object which is parsed into a [`super::Column`].
+///
+/// The type parameter `T` is a witness to whether this column is compressed. If `T:
+/// compression::Uncompressed` then we have proved that this column is not compressed, otherwise it
+/// may be compressed.
+///
+/// [1]: https://alexjg.github.io/automerge-storage-docs/#column-specifications
+/// [2]: https://alexjg.github.io/automerge-storage-docs/#raw-value-columns
+/// [3]: https://alexjg.github.io/automerge-storage-docs/#group-columns
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct RawColumn<T: compression::ColumnCompression> {
+    spec: ColumnSpec,
+    /// The location of the data in the column data block. Note that this range starts at the
+    /// beginning of the column data block - i.e. the `data` attribute of the first column in the
+    /// column data block will be 0 - not at the start of the chunk.
+    data: Range<usize>,
+    _phantom: PhantomData<T>,
+}
+
+impl RawColumn<compression::Uncompressed> {
+    pub(crate) fn new(spec: ColumnSpec, data: Range<usize>) -> Self {
+        Self {
+            spec: ColumnSpec::new(spec.id(), spec.col_type(), false),
+            data,
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<T: compression::ColumnCompression> RawColumn<T> {
+    pub(crate) fn spec(&self) -> ColumnSpec {
+        self.spec
+    }
+
+    pub(crate) fn data(&self) -> Range<usize> {
+        self.data.clone()
+    }
+
+    fn compress(&self, input: &[u8], out: &mut Vec<u8>, threshold: usize) -> (ColumnSpec, usize) {
+        let (spec, len) = if self.data.len() < threshold || self.spec.deflate() {
+            out.extend(&input[self.data.clone()]);
+            (self.spec, self.data.len())
+        } else {
+            let mut deflater = flate2::bufread::DeflateEncoder::new(
+                &input[self.data.clone()],
+                flate2::Compression::default(),
+            );
+            //This unwrap should be okay as we're reading and writing to in memory buffers
+            (self.spec.deflated(), deflater.read_to_end(out).unwrap())
+        };
+        (spec, len)
+    }
+
+    pub(crate) fn uncompressed(&self) -> Option<RawColumn<compression::Uncompressed>> {
+        if self.spec.deflate() {
+            None
+        } else {
+            Some(RawColumn {
+                spec: self.spec,
+                data: self.data.clone(),
+                _phantom: PhantomData,
+            })
+        }
+    }
+
+    fn decompress(
+        &self,
+        input: &[u8],
+        out: &mut Vec<u8>,
+    ) -> Result<(ColumnSpec, usize), ParseError> {
+        let len = if self.spec.deflate() {
+            let mut inflater = flate2::bufread::DeflateDecoder::new(&input[self.data.clone()]);
+            inflater.read_to_end(out).map_err(ParseError::Deflate)?
+        } else {
+            out.extend(&input[self.data.clone()]);
+            self.data.len()
+        };
+        Ok((self.spec.inflated(), len))
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct RawColumns<T: compression::ColumnCompression>(pub(crate) Vec<RawColumn<T>>);
+
+impl<T: compression::ColumnCompression> RawColumns<T> {
+    /// Returns `Some` if no column in this set of columns is marked as compressed
+    pub(crate) fn uncompressed(&self) -> Option<RawColumns<compression::Uncompressed>> {
+        let mut result = Vec::with_capacity(self.0.len());
+        for col in &self.0 {
+            if let Some(uncomp) = col.uncompressed() {
+                result.push(uncomp);
+            } else {
+                return None;
+            }
+        }
+        Some(RawColumns(result))
+    }
+
+    /// Write each column in `input` represented by `self` into `out`, possibly compressing.
+    ///
+    /// # Returns
+    /// The `RawColumns` corresponding to the data written to `out`
+    ///
+    /// # Panics
+    /// * If any of the ranges in `self` is outside the bounds of `input`
+    pub(crate) fn compress(
+        &self,
+        input: &[u8],
+        out: &mut Vec<u8>,
+        threshold: usize,
+    ) -> RawColumns<compression::Unknown> {
+        let mut result = Vec::with_capacity(self.0.len());
+        let mut start = 0;
+        for col in &self.0 {
+            let (spec, len) = col.compress(input, out, threshold);
+            result.push(RawColumn {
+                spec,
+                data: start..(start + len),
+                _phantom: PhantomData::<compression::Unknown>,
+            });
+            start += len;
+        }
+        RawColumns(result)
+    }
+
+    /// Read each column from `input` and write to `out`, decompressing any compressed columns
+    ///
+    /// # Returns
+    /// The `RawColumns` corresponding to the data written to `out`
+    ///
+    /// # Panics
+    /// * If any of the ranges in `self` is outside the bounds of `input`
+    pub(crate) fn uncompress(
+        &self,
+        input: &[u8],
+        out: &mut Vec<u8>,
+    ) -> Result<RawColumns<compression::Uncompressed>, ParseError> {
+        let mut result = Vec::with_capacity(self.0.len());
+        let mut start = 0;
+        for col in &self.0 {
+            let (spec, len) = if let Some(decomp) = col.uncompressed() {
+                out.extend(&input[decomp.data.clone()]);
+                (decomp.spec, decomp.data.len())
+            } else {
+                col.decompress(input, out)?
+            };
+            result.push(RawColumn {
+                spec,
+                data: start..(start + len),
+                _phantom: PhantomData::<compression::Uncompressed>,
+            });
+            start += len;
+        }
+        Ok(RawColumns(result))
+    }
+}
+
+impl<T: compression::ColumnCompression> FromIterator<RawColumn<T>> for RawColumns<T> {
+    fn from_iter<U: IntoIterator<Item = RawColumn<T>>>(iter: U) -> Self {
+        Self(iter.into_iter().filter(|c| !c.data.is_empty()).collect())
+    }
+}
+
+impl FromIterator<(ColumnSpec, Range<usize>)> for RawColumns<compression::Unknown> {
+    fn from_iter<T: IntoIterator<Item = (ColumnSpec, Range<usize>)>>(iter: T) -> Self {
+        Self(
+            iter.into_iter()
+                .filter_map(|(spec, data)| {
+                    if data.is_empty() {
+                        None
+                    } else {
+                        Some(RawColumn {
+                            spec,
+                            data,
+                            _phantom: PhantomData,
+                        })
+                    }
+                })
+                .collect(),
+        )
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum ParseError {
+    #[error("columns were not in normalized order")]
+    NotInNormalOrder,
+    #[error(transparent)]
+    Leb128(#[from] parse::leb128::Error),
+    #[error(transparent)]
+    Deflate(#[from] std::io::Error),
+}
+
+impl RawColumns<compression::Unknown> {
+    pub(crate) fn parse<E>(input: parse::Input<'_>) -> parse::ParseResult<'_, Self, E>
+    where
+        E: From<ParseError>,
+    {
+        let i = input;
+        let (i, num_columns) = parse::leb128_u64(i).map_err(|e| e.lift())?;
+        let (i, specs_and_lens) = parse::apply_n(
+            num_columns as usize,
+            parse::tuple2(
+                parse::map(parse::leb128_u32, ColumnSpec::from),
+                parse::leb128_u64,
+            ),
+        )(i)
+        .map_err(|e| e.lift())?;
+        let columns: Vec<RawColumn<compression::Unknown>> = specs_and_lens
+            .into_iter()
+            .scan(0_usize, |offset, (spec, len)| {
+                // Note: we use a saturating add here as len was passed over the network
+                // and so could be anything. If the addition does every saturate we would
+                // expect parsing to fail later (but at least it won't panic!).
+                let end = offset.saturating_add(len as usize);
+                let data = *offset..end;
+                *offset = end;
+                Some(RawColumn {
+                    spec,
+                    data,
+                    _phantom: PhantomData,
+                })
+            })
+            .collect::<Vec<_>>();
+        if !are_normal_sorted(&columns) {
+            return Err(parse::ParseError::Error(
+                ParseError::NotInNormalOrder.into(),
+            ));
+        }
+        Ok((i, RawColumns(columns)))
+    }
+}
+
+impl<T: compression::ColumnCompression> RawColumns<T> {
+    pub(crate) fn write(&self, out: &mut Vec<u8>) -> usize {
+        let mut written = leb128::write::unsigned(out, self.0.len() as u64).unwrap();
+        for col in &self.0 {
+            written += leb128::write::unsigned(out, u32::from(col.spec) as u64).unwrap();
+            written += leb128::write::unsigned(out, col.data.len() as u64).unwrap();
+        }
+        written
+    }
+
+    pub(crate) fn total_column_len(&self) -> usize {
+        self.0.iter().map(|c| c.data.len()).sum()
+    }
+
+    pub(crate) fn as_map(&self) -> BTreeMap<ColumnSpec, Range<usize>> {
+        self.0.iter().map(|c| (c.spec(), c.data())).collect()
+    }
+
+    pub(crate) fn bytes<'a>(&self, c: ColumnSpec, data: &'a [u8]) -> &'a [u8] {
+        debug_assert!(self.0.iter().map(|i| i.spec()).is_sorted());
+        if let Ok(index) = self.0.binary_search_by(|col| col.spec().cmp(&c)) {
+            &data[self.0[index].data().clone()]
+        } else {
+            &[]
+        }
+    }
+
+    pub(crate) fn iter(&self) -> impl Iterator<Item = &RawColumn<T>> + '_ {
+        self.0.iter()
+    }
+}
+
+fn are_normal_sorted<T: compression::ColumnCompression>(cols: &[RawColumn<T>]) -> bool {
+    if cols.len() > 1 {
+        for (i, col) in cols[1..].iter().enumerate() {
+            if col.spec.normalize() < cols[i].spec.normalize() {
+                return false;
+            }
+        }
+    }
+    true
+}