@peerbit/native-backbone 0.1.0

package/src/js_interop.rs

@@ -0,0 +1,448 @@
+use js_sys::{Array, Reflect, Uint8Array};
+use peerbit_indexer_core::wire::{self, WireError};
+use std::collections::HashSet;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsCast;
+
+/// Drops only the first `byte_len` bytes / `record_count` records so records
+/// appended while a flush was awaiting its disk write are kept for the next
+/// flush instead of being discarded.
+pub(crate) fn clear_journal_prefix(
+    journal: &mut Vec<u8>,
+    journal_record_count: &mut usize,
+    byte_len: usize,
+    record_count: usize,
+) {
+    if byte_len >= journal.len() {
+        journal.clear();
+        *journal_record_count = 0;
+        return;
+    }
+    journal.drain(..byte_len);
+    *journal_record_count = journal_record_count.saturating_sub(record_count);
+}
+
+pub(crate) fn array_from_value(value: JsValue, label: &str) -> Result<Array, JsValue> {
+    value
+        .dyn_into::<Array>()
+        .map_err(|_| JsValue::from_str(&format!("Expected {label} array")))
+}
+
+pub(crate) fn string_field(row: &Array, index: u32, label: &str) -> Result<String, JsValue> {
+    row.get(index)
+        .as_string()
+        .ok_or_else(|| JsValue::from_str(&format!("Expected {label} string")))
+}
+
+pub(crate) fn stringish_field(row: &Array, index: u32, label: &str) -> Result<String, JsValue> {
+    let value = row.get(index);
+    if let Some(value) = value.as_string() {
+        return Ok(value);
+    }
+    if let Some(value) = value.as_f64() {
+        return Ok((value as u64).to_string());
+    }
+    Err(JsValue::from_str(&format!("Expected {label} string")))
+}
+
+pub(crate) fn bool_field(row: &Array, index: u32, label: &str) -> Result<bool, JsValue> {
+    row.get(index)
+        .as_bool()
+        .ok_or_else(|| JsValue::from_str(&format!("Expected {label} boolean")))
+}
+
+pub(crate) fn usize_field(row: &Array, index: u32, label: &str) -> Result<usize, JsValue> {
+    row.get(index)
+        .as_f64()
+        .map(|value| value as usize)
+        .ok_or_else(|| JsValue::from_str(&format!("Expected {label} number")))
+}
+
+pub(crate) fn bytes_field(row: &Array, index: u32, label: &str) -> Result<Vec<u8>, JsValue> {
+    let value = row.get(index);
+    if value.is_undefined() || value.is_null() {
+        return Err(JsValue::from_str(&format!("Expected {label} bytes")));
+    }
+    Ok(Uint8Array::new(&value).to_vec())
+}
+
+pub(crate) fn trim_hashes_vec(trim_rows: &Array) -> Result<Vec<String>, JsValue> {
+    let mut hashes = Vec::with_capacity(trim_rows.length() as usize);
+    for index in 0..trim_rows.length() {
+        let row = array_from_value(trim_rows.get(index), "trim row")?;
+        hashes.push(string_field(&row, 0, "trim hash")?);
+    }
+    Ok(hashes)
+}
+
+pub(crate) fn numbers_to_rows(resolution: &str, values: &[u64]) -> Array {
+    let out = Array::new();
+    for value in values {
+        out.push(&number_to_row(resolution, *value));
+    }
+    out
+}
+
+pub(crate) fn number_to_row(resolution: &str, value: u64) -> JsValue {
+    match resolution {
+        "u64" => JsValue::from_str(&value.to_string()),
+        _ => JsValue::from_f64(value as f64),
+    }
+}
+
+pub(crate) fn strings_to_array(values: Vec<String>) -> Array {
+    let out = Array::new();
+    for value in values {
+        out.push(&JsValue::from_str(&value));
+    }
+    out
+}
+
+pub(crate) fn has_duplicate_strings(values: &[String]) -> bool {
+    let mut seen = HashSet::with_capacity(values.len());
+    values.iter().any(|value| !seen.insert(value))
+}
+
+pub(crate) fn strings_slice_to_array(values: &[String]) -> Array {
+    let out = Array::new();
+    for value in values {
+        out.push(&JsValue::from_str(value));
+    }
+    out
+}
+
+pub(crate) fn strings_from_array(values: Array) -> Result<Vec<String>, JsValue> {
+    let mut out = Vec::with_capacity(values.length() as usize);
+    for index in 0..values.length() {
+        out.push(
+            values
+                .get(index)
+                .as_string()
+                .ok_or_else(|| JsValue::from_str("Expected string array"))?,
+        );
+    }
+    Ok(out)
+}
+
+pub(crate) fn bytes_vec_from_array(values: Array) -> Result<Vec<Vec<u8>>, JsValue> {
+    let mut out = Vec::with_capacity(values.length() as usize);
+    for index in 0..values.length() {
+        let value = values.get(index);
+        if value.is_undefined() || value.is_null() {
+            return Err(JsValue::from_str("Expected bytes array"));
+        }
+        out.push(Uint8Array::new(&value).to_vec());
+    }
+    Ok(out)
+}
+
+pub(crate) fn required_bytes_from_array(
+    values: &Array,
+    index: u32,
+    field: &str,
+) -> Result<Uint8Array, JsValue> {
+    let value = values.get(index);
+    if value.is_undefined() || value.is_null() {
+        return Err(JsValue::from_str(&format!("Expected {field} bytes")));
+    }
+    Ok(Uint8Array::new(&value))
+}
+
+pub(crate) fn string_batches_from_array(
+    values: Array,
+    label: &str,
+) -> Result<Vec<Vec<String>>, JsValue> {
+    let mut out = Vec::with_capacity(values.length() as usize);
+    for index in 0..values.length() {
+        let value = values.get(index);
+        if !Array::is_array(&value) {
+            return Err(JsValue::from_str(&format!("Expected {label}")));
+        }
+        out.push(strings_from_array(Array::from(&value))?);
+    }
+    Ok(out)
+}
+
+pub(crate) fn usize_values_from_array(values: Array) -> Result<Vec<usize>, JsValue> {
+    let mut out = Vec::with_capacity(values.length() as usize);
+    for index in 0..values.length() {
+        let value = values
+            .get(index)
+            .as_f64()
+            .ok_or_else(|| JsValue::from_str("Expected unsigned integer array"))?;
+        if !value.is_finite() || value < 0.0 || value.fract() != 0.0 {
+            return Err(JsValue::from_str("Expected unsigned integer array"));
+        }
+        out.push(value as usize);
+    }
+    Ok(out)
+}
+
+pub(crate) fn ensure_same_len(left: usize, right: usize, label: &str) -> Result<(), JsValue> {
+    if left == right {
+        Ok(())
+    } else {
+        Err(JsValue::from_str(&format!(
+            "Mismatched {label} input lengths"
+        )))
+    }
+}
+
+pub(crate) fn optional_bytes_from_js(value: JsValue) -> Option<Vec<u8>> {
+    if value.is_undefined() || value.is_null() {
+        return None;
+    }
+    Some(Uint8Array::new(&value).to_vec())
+}
+
+pub(crate) fn optional_usize_from_js(
+    value: JsValue,
+    label: &str,
+) -> Result<Option<usize>, JsValue> {
+    if value.is_undefined() || value.is_null() {
+        return Ok(None);
+    }
+    value
+        .as_f64()
+        .map(|value| Some(value as usize))
+        .ok_or_else(|| JsValue::from_str(&format!("{label} must be a number")))
+}
+
+pub(crate) fn number_strings_to_array(values: &[u64]) -> Array {
+    let out = Array::new();
+    for value in values {
+        out.push(&JsValue::from_str(&value.to_string()));
+    }
+    out
+}
+
+pub(crate) fn parse_u64_string(value: &str, label: &str) -> Result<u64, JsValue> {
+    value
+        .parse::<u64>()
+        .map_err(|_| JsValue::from_str(&format!("Expected {label} u64 string")))
+}
+
+pub(crate) fn parse_optional_u64_string(value: &str, label: &str) -> Result<Option<u64>, JsValue> {
+    if value.is_empty() {
+        Ok(None)
+    } else {
+        parse_u64_string(value, label).map(Some)
+    }
+}
+
+const JOURNAL_PUT_OPERATION: u8 = 1;
+const JOURNAL_DELETE_OPERATION: u8 = 2;
+
+/// Writes the same wire format as
+/// `peerbit_indexer_core::persistence::encode_journal_put_record`, but
+/// directly into `out` — the journal push runs once per append transaction,
+/// so the intermediate payload/record allocations are avoided on purpose
+/// (see the `journal record encoding matches indexer core` parity test).
+pub(crate) fn append_journal_put_record(out: &mut Vec<u8>, key: &str, value: &[u8]) {
+    let payload_len = 1usize + 4 + key.len() + 4 + value.len();
+    let key_len = (key.len() as u32).to_le_bytes();
+    let value_len = (value.len() as u32).to_le_bytes();
+    let checksum = fnv1a_parts([
+        &[JOURNAL_PUT_OPERATION][..],
+        key_len.as_slice(),
+        key.as_bytes(),
+        value_len.as_slice(),
+        value,
+    ]);
+    out.reserve(8 + payload_len);
+    out.extend_from_slice(&(payload_len as u32).to_le_bytes());
+    out.extend_from_slice(&checksum.to_le_bytes());
+    out.push(JOURNAL_PUT_OPERATION);
+    out.extend_from_slice(&key_len);
+    out.extend_from_slice(key.as_bytes());
+    out.extend_from_slice(&value_len);
+    out.extend_from_slice(value);
+}
+
+/// Allocation-free equivalent of
+/// `peerbit_indexer_core::persistence::encode_journal_delete_record`.
+pub(crate) fn append_journal_delete_record(out: &mut Vec<u8>, key: &str) {
+    let payload_len = 1usize + 4 + key.len();
+    let key_len = (key.len() as u32).to_le_bytes();
+    let checksum = fnv1a_parts([
+        &[JOURNAL_DELETE_OPERATION][..],
+        key_len.as_slice(),
+        key.as_bytes(),
+    ]);
+    out.reserve(8 + payload_len);
+    out.extend_from_slice(&(payload_len as u32).to_le_bytes());
+    out.extend_from_slice(&checksum.to_le_bytes());
+    out.push(JOURNAL_DELETE_OPERATION);
+    out.extend_from_slice(&key_len);
+    out.extend_from_slice(key.as_bytes());
+}
+
+/// FNV-1a over concatenated parts; must stay identical to
+/// `peerbit_indexer_core::persistence::fnv1a` over the joined bytes.
+fn fnv1a_parts<const N: usize>(parts: [&[u8]; N]) -> u32 {
+    let mut hash = 0x811c9dc5u32;
+    for part in parts {
+        for byte in part {
+            hash ^= u32::from(*byte);
+            hash = hash.wrapping_mul(0x01000193);
+        }
+    }
+    hash
+}
+
+pub(crate) fn write_string(out: &mut Vec<u8>, value: &str) {
+    out.extend_from_slice(&(value.len() as u32).to_le_bytes());
+    out.extend_from_slice(value.as_bytes());
+}
+
+pub(crate) fn write_u64(out: &mut Vec<u8>, value: u64) {
+    out.extend_from_slice(&value.to_le_bytes());
+}
+
+pub(crate) fn write_u32(out: &mut Vec<u8>, value: u32) {
+    out.extend_from_slice(&value.to_le_bytes());
+}
+
+pub(crate) fn write_bytes(out: &mut Vec<u8>, value: &[u8]) {
+    out.extend_from_slice(&(value.len() as u32).to_le_bytes());
+    out.extend_from_slice(value);
+}
+
+pub(crate) fn wire_error_to_js(error: WireError) -> JsValue {
+    JsValue::from_str(&error.to_string())
+}
+
+pub(crate) fn read_u32(
+    bytes: &[u8],
+    offset: &mut usize,
+    label: &'static str,
+) -> Result<u32, JsValue> {
+    wire::read_u32(bytes, offset, label).map_err(wire_error_to_js)
+}
+
+pub(crate) fn read_u64(
+    bytes: &[u8],
+    offset: &mut usize,
+    label: &'static str,
+) -> Result<u64, JsValue> {
+    wire::read_u64(bytes, offset, label).map_err(wire_error_to_js)
+}
+
+pub(crate) fn read_encoded_string(
+    bytes: &[u8],
+    offset: &mut usize,
+    label: &'static str,
+) -> Result<String, JsValue> {
+    wire::read_encoded_string(bytes, offset, label).map_err(wire_error_to_js)
+}
+
+pub(crate) fn read_bytes(
+    bytes: &[u8],
+    offset: &mut usize,
+    label: &'static str,
+) -> Result<Vec<u8>, JsValue> {
+    wire::read_bytes(bytes, offset, label).map_err(wire_error_to_js)
+}
+
+pub(crate) fn js_get(value: &JsValue, key: &str) -> JsValue {
+    Reflect::get(value, &JsValue::from_str(key)).unwrap_or(JsValue::UNDEFINED)
+}
+
+fn js_string(value: JsValue, field: &str) -> Result<String, JsValue> {
+    value
+        .as_string()
+        .ok_or_else(|| JsValue::from_str(&format!("Missing or invalid {field}")))
+}
+
+pub(crate) fn array_strings(value: JsValue, field: &str) -> Result<Vec<String>, JsValue> {
+    if !Array::is_array(&value) {
+        return Err(JsValue::from_str(&format!("{field} must be an array")));
+    }
+    let array = Array::from(&value);
+    let mut out = Vec::with_capacity(array.length() as usize);
+    for index in 0..array.length() {
+        out.push(js_string(array.get(index), field)?);
+    }
+    Ok(out)
+}
+
+pub(crate) fn optional_string(value: JsValue) -> Option<String> {
+    if value.is_null() || value.is_undefined() {
+        None
+    } else {
+        value.as_string()
+    }
+}
+
+pub(crate) fn write_u8(out: &mut Vec<u8>, value: u8) {
+    out.push(value);
+}
+
+pub(crate) fn write_bool(out: &mut Vec<u8>, value: bool) {
+    out.push(if value { 1 } else { 0 });
+}
+
+pub(crate) fn js_error(error: impl std::fmt::Display) -> JsValue {
+    JsValue::from_str(&error.to_string())
+}
+
+pub(crate) fn decode_error(error: impl std::fmt::Display) -> JsValue {
+    JsValue::from_str(&error.to_string())
+}
+
+pub(crate) fn hash_number_u64(resolution: &str, digest: &[u8]) -> Result<u64, JsValue> {
+    match resolution {
+        "u32" => {
+            if digest.len() < 4 {
+                return Err(JsValue::from_str("hash digest must have at least 4 bytes"));
+            }
+            Ok(u32::from_le_bytes(digest[0..4].try_into().unwrap()) as u64)
+        }
+        "u64" => {
+            if digest.len() < 8 {
+                return Err(JsValue::from_str("hash digest must have at least 8 bytes"));
+            }
+            Ok(u64::from_le_bytes(digest[0..8].try_into().unwrap()))
+        }
+        _ => Err(JsValue::from_str("resolution must be u32 or u64")),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{append_journal_delete_record, append_journal_put_record, hash_number_u64};
+    use peerbit_indexer_core::persistence::{
+        encode_journal_delete_record, encode_journal_put_record,
+    };
+
+    #[test]
+    fn decodes_hash_numbers_like_shared_log_integer_helpers() {
+        let bytes = [1, 0, 0, 0, 2, 0, 0, 0];
+        assert_eq!(hash_number_u64("u32", &bytes).unwrap(), 1);
+        assert_eq!(hash_number_u64("u64", &bytes).unwrap(), 8_589_934_593);
+    }
+
+    #[test]
+    fn journal_record_encoding_matches_indexer_core() {
+        for (key, value) in [
+            ("", &[][..]),
+            ("k", &[0u8][..]),
+            ("some-journal-key", &[1u8, 2, 3, 255, 0, 42][..]),
+            ("zAbc123", &[7u8; 1200][..]),
+        ] {
+            let mut direct = vec![9u8, 9, 9];
+            append_journal_put_record(&mut direct, key, value);
+            let mut expected = vec![9u8, 9, 9];
+            expected.extend_from_slice(&encode_journal_put_record(key, value));
+            assert_eq!(direct, expected, "put record for key {key:?}");
+
+            let mut direct = Vec::new();
+            append_journal_delete_record(&mut direct, key);
+            assert_eq!(
+                direct,
+                encode_journal_delete_record(key),
+                "delete record for key {key:?}"
+            );
+        }
+    }
+}

package/src/lib.rs

@@ -0,0 +1,144 @@
+use js_sys::{Array, Uint8Array};
+use peerbit_indexer_core::planner::NativeQueryIndex;
+use peerbit_indexer_core::schema::NativeSchemaIr;
+use peerbit_indexer_core::storage::MemoryByteStorage;
+use peerbit_log_rust::{NativeEntryV0PlainBuilder, NativeLogBlockStore, NativeLogIndex};
+use peerbit_shared_log_rust::NativeSharedLogState;
+use std::collections::{HashMap, HashSet};
+use wasm_bindgen::prelude::*;
+
+mod append_tx;
+mod coordinates;
+mod documents;
+mod graph_blocks;
+mod js_interop;
+mod profile;
+mod raw_receive;
+mod shared_log_plan;
+mod sync_send;
+mod wire_sync;
+
+use crate::documents::{DocumentContextFields, DocumentPreviousSignerFact, ParsedProjectionPlan};
+use crate::profile::NativeBackboneAppendProfile;
+use crate::raw_receive::PendingRawReceiveEntry;
+
+// Native-backbone is optimized for document-store hot paths where large byte
+// fields are payloads, not query keys. Standalone indexer-rust keeps exact large
+// byte matching through the compatibility extractor.
+const NATIVE_BACKBONE_BYTE_EXACT_INDEX_LIMIT: usize = 128;
+
+#[wasm_bindgen]
+pub struct NativePeerbitBackbone {
+    resolution: String,
+    log: NativeLogIndex,
+    blocks: NativeLogBlockStore,
+    shared_log: NativeSharedLogState,
+    coordinate_index: HashSet<String>,
+    coordinate_values: MemoryByteStorage,
+    coordinate_journal: Vec<u8>,
+    coordinate_journal_record_count: usize,
+    coordinate_journal_enabled: bool,
+    document_index: NativeQueryIndex,
+    document_values: MemoryByteStorage,
+    document_journal: Vec<u8>,
+    document_journal_record_count: usize,
+    document_journal_enabled: bool,
+    document_byte_element_index_limit: usize,
+    document_key_by_head: HashMap<String, String>,
+    document_previous_signer_by_key: HashMap<String, DocumentPreviousSignerFact>,
+    document_signer_journal: Vec<u8>,
+    document_signer_journal_record_count: usize,
+    document_signer_journal_enabled: bool,
+    document_schema_ir: Option<NativeSchemaIr>,
+    document_context_head_field: Option<u32>,
+    document_context_fields: Option<DocumentContextFields>,
+    document_projection_plans: Vec<ParsedProjectionPlan>,
+    local_public_key: Vec<u8>,
+    builder: NativeEntryV0PlainBuilder,
+    pending_raw_receive_entries: HashMap<String, PendingRawReceiveEntry>,
+    append_profile_enabled: bool,
+    append_profile: NativeBackboneAppendProfile,
+}
+
+#[wasm_bindgen]
+impl NativePeerbitBackbone {
+    #[wasm_bindgen(constructor)]
+    pub fn new(
+        resolution: String,
+        clock_id: Uint8Array,
+        private_key: Uint8Array,
+        public_key: Uint8Array,
+    ) -> Result<Self, JsValue> {
+        if resolution != "u32" && resolution != "u64" {
+            return Err(JsValue::from_str("resolution must be u32 or u64"));
+        }
+        let local_public_key = public_key.to_vec();
+        Ok(Self {
+            resolution: resolution.clone(),
+            log: NativeLogIndex::new(),
+            blocks: NativeLogBlockStore::new(),
+            shared_log: NativeSharedLogState::new(resolution),
+            coordinate_index: HashSet::new(),
+            coordinate_values: MemoryByteStorage::new(),
+            coordinate_journal: Vec::new(),
+            coordinate_journal_record_count: 0,
+            coordinate_journal_enabled: false,
+            document_index: NativeQueryIndex::new(),
+            document_values: MemoryByteStorage::new(),
+            document_journal: Vec::new(),
+            document_journal_record_count: 0,
+            document_journal_enabled: false,
+            document_byte_element_index_limit: 0,
+            document_key_by_head: HashMap::new(),
+            document_previous_signer_by_key: HashMap::new(),
+            document_signer_journal: Vec::new(),
+            document_signer_journal_record_count: 0,
+            document_signer_journal_enabled: false,
+            document_schema_ir: None,
+            document_context_head_field: None,
+            document_context_fields: None,
+            document_projection_plans: Vec::new(),
+            local_public_key,
+            builder: NativeEntryV0PlainBuilder::new(clock_id, private_key, public_key)?,
+            pending_raw_receive_entries: HashMap::new(),
+            append_profile_enabled: false,
+            append_profile: NativeBackboneAppendProfile::default(),
+        })
+    }
+
+    pub fn set_append_profile_enabled(&mut self, enabled: bool) {
+        self.append_profile_enabled = enabled;
+    }
+
+    pub fn reset_append_profile(&mut self) {
+        self.append_profile = NativeBackboneAppendProfile::default();
+    }
+
+    pub fn append_profile(&self) -> Array {
+        self.append_profile.to_row()
+    }
+
+    pub fn log_len(&self) -> usize {
+        self.log.len()
+    }
+
+    pub fn block_len(&self) -> usize {
+        self.blocks.len()
+    }
+
+    pub fn has_log_entry(&self, hash: &str) -> bool {
+        self.log.has(hash)
+    }
+
+    pub fn has_block(&self, hash: &str) -> bool {
+        self.blocks.has(hash)
+    }
+
+    pub fn clear(&mut self) {
+        self.log.clear();
+        self.blocks.clear();
+        self.shared_log.clear();
+        self.clear_coordinate_core();
+        self.clear_document_core();
+    }
+}