gqlite 1.5.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aac591ce0775f004fc859b9cd453a940d39f9f80aa1535e2a37aae948b6baac
-  data.tar.gz: c66e83597f70d3c69ed412d5431530d1c2cc211507b68c18c7df74b65da017ef
+  metadata.gz: b0a8fea4c8e6f2e5eee4fd041a66f4cbe3ae35d8c3a0fc323dbea013efc14d30
+  data.tar.gz: 80af6abeddd183326704ae95cb5b00f2e41317e5a104d2a0c4d77c95cc4cdd5b
 SHA512:
-  metadata.gz: 32503eae6aec73407846b1cce09f58374445473ef59122efe87280b4b756118d202199fc07da0b2ae7dd906f457997aac7b27d47eb17e490ce9c1a648c4e76cf
-  data.tar.gz: 95f14af4c49792144d5efa84ccc906c9d49ac065d808d927858d3d3a448ece3aa3137b1ece5057bb5b3534b301dd839e159a4fb745af0265efb352255c36d5c0
+  metadata.gz: e53d19758a9efaefcb1cac12d20d43045b1ea1a58578f2dbba8a5986797b77c04f133bbfbeb9309a522c0f44dc95fd8597201bcb7e78287792a7496c30462b0e
+  data.tar.gz: 9de4b0738781537c2657c90bad7e1831064f8e783387973503971de741c2b861f651efc64733ecc406b0741dba2ba19cfb5f32fab83692b24f2be6a62b6057a2

data/ext/Cargo.toml

@@ -1,22 +1,28 @@
 [workspace]
-resolver = "2"
+resolver = "3"
 members = ["gqliterb", "gqlitedb", "graphcore"]
 
 [workspace.package]
-version = "0.8.0"
-edition = "2021"
+version = "0.11.0"
+edition = "2024"
 license = "MIT"
 homepage = "https://gqlite.org"
 repository = "https://gitlab.com/gqlite/gqlite"
 
 [workspace.dependencies]
-graphcore = { version = "0.8.0", path = "graphcore" }
-gqlitedb = { version = "0.8.0", path = "gqlitedb" }
+graphcore = { version = "0.11.0", path = "graphcore" }
+gqlitedb = { version = "0.11.0", path = "gqlitedb" }
+gqlparser = { version = "0.11.0", path = "gqlparser" }
+db-index = { version = "0.11.0", path = "db-index" }
 
 askama = { version = "0.15" }
 ccutils = { version = "0.4" }
+ciborium = "0.2"
+divan = "0.1"
+indexmap = "2"
 itertools = "0.14"
-rusqlite = { version = "0.37" }
+log = "0.4"
+rusqlite = { version = "0.38", features = ["load_extension"] }
 serde = "1"
 thiserror = "2"
 uuid = { version = "1", features = ["v4"] }

data/ext/db-index/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "db-index"
+description = "Reusable indexing algorithms for key-value storage engines"
+readme = "README.md"
+license = "MIT"
+homepage.workspace = true
+repository.workspace = true
+version = "0.11.0"
+edition = "2021"
+
+[dependencies]
+ahash = "0.8"
+arc-swap = "1"
+bincode = "1"
+bytemuck = "1"
+ciborium.workspace = true
+half = { version = "2", features = ["bytemuck"] }
+ordered-float = "5"
+parking_lot = "0.12"
+serde = { version = "1", features = ["derive"] }
+thiserror = "2"
+tracing = "0.1"
+
+[dev-dependencies]
+approx = "0.5"
+proptest = "1"
+tempfile = "3"
+rand_09 = { version = "0.9", package = "rand" }
+divan = "0.1"

data/ext/db-index/src/hnsw/error.rs

@@ -0,0 +1,66 @@
+pub type Result<T, E = Error> = std::result::Result<T, E>;
+
+const HNSW_META_SCHEMA_VERSION: u8 = 1;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error
+{
+  #[error("dimension mismatch: expected {expected}, got {actual}")]
+  DimensionMismatch
+  {
+    expected: usize, actual: usize
+  },
+
+  #[error("key not found")]
+  KeyNotFound,
+
+  #[error("key already exists")]
+  KeyAlreadyExists,
+
+  #[error("index is empty")]
+  EmptyIndex,
+
+  #[error("missing vector")]
+  MissingVector,
+
+  #[error("invalid index format: {0}")]
+  InvalidIndexFormat(String),
+
+  #[error(transparent)]
+  Io(#[from] std::io::Error),
+
+  #[error("invalid qi8 scale: {scale}")]
+  InvalidQi8Scale
+  {
+    scale: f32
+  },
+
+  #[error("Schema version is too new: found {found} maximum is {HNSW_META_SCHEMA_VERSION}")]
+  SchemaVersionTooNew
+  {
+    found: u8
+  },
+  #[error("Schema version is too old: found {found} minim is {HNSW_META_SCHEMA_VERSION}")]
+  SchemaVersionUnsupported
+  {
+    found: u8
+  },
+  #[error("Malformed metadata: {0}")]
+  MalformedMetadata(String),
+  #[error("Empty decoded adjacency")]
+  EmptyDecodedAdjency,
+  #[error("Truncated decoded adjacency")]
+  TruncatedDecodedAdjency,
+  #[error("Trailing bytes after decoding adjacency")]
+  TrailingBytesDecodedAdjency,
+  #[error("Store error: {0}")]
+  StoreError(String),
+}
+
+impl From<std::convert::Infallible> for Error
+{
+  fn from(_: std::convert::Infallible) -> Self
+  {
+    unreachable!("Infallible should never be instantiated")
+  }
+}

data/ext/db-index/src/hnsw/graph_store.rs

@@ -0,0 +1,272 @@
+use super::error::Error;
+use crate::hnsw::id::NodeId;
+
+/// Read-only graph store interface.
+///
+/// The store tracks node allocation and connectivity. All absent nodes
+/// (never-allocated or physically removed) are indistinguishable: methods
+/// return `false` or `None` consistently. This is the contract that allows
+/// stale links in surviving neighbors' adjacency lists to be safely skipped
+/// during traversal without errors, closing the ghost-node bug class.
+pub trait GraphStoreRead<K>
+{
+  type Error: Into<Error>;
+
+  /// Allocation high-water mark (== next_node_id), NOT the live count.
+  /// This is what visited-set sizing needs; after delete churn the live
+  /// NodeId space is sparse within [0, node_count).
+  fn node_count(&self) -> Result<usize, Self::Error>;
+
+  /// Entry point: the node with the highest level (and that level).
+  /// None only for an empty graph.
+  fn entry_point(&self) -> Result<Option<(NodeId, i32)>, Self::Error>;
+
+  /// False for both never-allocated and physically-removed nodes alike.
+  /// The traversal guard is `if !graph.contains(n)? { continue; }` — stale
+  /// links to removed nodes are expected and must be skipped, not errors.
+  /// Absent must return clean `false`, never an error.
+  fn contains(&self, node: NodeId) -> Result<bool, Self::Error>;
+
+  /// None for absent nodes (never-allocated or removed), mirroring contains().
+  fn key_of(&self, node: NodeId) -> Result<Option<K>, Self::Error>;
+
+  /// None for absent nodes, mirroring contains().
+  fn node_of(&self, key: &K) -> Result<Option<NodeId>, Self::Error>;
+
+  /// Level of an existing node; Err for absent nodes.
+  /// Callers check contains() first on critical paths; this may error for
+  /// never-allocated nodes if the storage backend prefers that.
+  /// Precondition: `contains(node)` is true; implementations may panic otherwise.
+  fn level(&self, node: NodeId) -> Result<i32, Self::Error>;
+
+  /// Buffer-filling neighbors for a specific level.
+  /// Clears `out` before appending; absent nodes fill nothing (no error).
+  /// The redb impl cannot return references past an access guard, and the
+  /// hot loop copies candidates into its working set anyway.
+  fn neighbors(&self, node: NodeId, level: usize, out: &mut Vec<NodeId>)
+    -> Result<(), Self::Error>;
+}
+
+/// Write-capable graph store interface.
+///
+/// Extends `GraphStoreRead<K>` with mutation operations. All writes are
+/// immediate (no soft-delete or tombstone phase). NodeIds are monotonic and
+/// never reused, even after removal, to prevent stale links from pointing at
+/// wrong live nodes.
+pub trait GraphStoreWrite<K>: GraphStoreRead<K>
+{
+  /// Allocates the next NodeId (strictly increasing, never reused),
+  /// binds key<->id bidirectionally, and records the level in one atomic
+  /// operation. The adjacency list for all levels starts empty.
+  fn insert_node(&mut self, key: K, level: i32) -> Result<NodeId, Self::Error>;
+
+  /// Sets the complete neighbors list for a node at a specific level.
+  /// Overwrites any previous neighbors at that level.
+  fn set_neighbors(
+    &mut self,
+    node: NodeId,
+    level: usize,
+    neighbors: &[NodeId],
+  ) -> Result<(), Self::Error>;
+
+  /// Sets or clears the graph entry point.
+  fn set_entry_point(&mut self, ep: Option<(NodeId, i32)>) -> Result<(), Self::Error>;
+
+  /// Physically removes all rows for the node (adjacency, key, reverse
+  /// key, vector-side handled by the caller). The next_node_id is NOT
+  /// decremented, ever. After removal, `contains()` returns false and
+  /// `key_of()` / `node_of()` return None, but stale links in surviving
+  /// neighbors' adjacency lists remain untouched — traversal must skip them
+  /// via `contains()` checks.
+  fn remove_node(&mut self, node: NodeId) -> Result<(), Self::Error>;
+}
+
+/// # Conformance suite
+///
+/// Every `GraphStoreWrite<K>` implementation must pass these tests. They are
+/// public (not `#[cfg(test)]`) so that WI 367's gqlite cross-crate tests can
+/// use them to validate the redb implementation.
+/// Test key trait allowing conformance tests to work with any key type.
+pub trait TestKey: Clone + PartialEq + std::fmt::Debug
+{
+  fn sample_a() -> Self;
+  fn sample_b() -> Self;
+  fn sample_c() -> Self;
+  fn sample_d() -> Self;
+}
+
+impl TestKey for u128
+{
+  fn sample_a() -> Self
+  {
+    10
+  }
+  fn sample_b() -> Self
+  {
+    20
+  }
+  fn sample_c() -> Self
+  {
+    30
+  }
+  fn sample_d() -> Self
+  {
+    40
+  }
+}
+
+pub fn conformance_alloc_is_monotonic_and_binds<K: TestKey, G: GraphStoreWrite<K>>(g: &mut G)
+where
+  <G as GraphStoreRead<K>>::Error: std::fmt::Debug,
+{
+  let a = g.insert_node(K::sample_a(), 0).unwrap();
+  let b = g.insert_node(K::sample_b(), 1).unwrap();
+  assert!(
+    b.as_u32() > a.as_u32(),
+    "allocation must be strictly increasing"
+  );
+  assert_eq!(
+    g.node_of(&K::sample_a()).unwrap(),
+    Some(a),
+    "key_of(node) must round-trip forward"
+  );
+  assert_eq!(
+    g.key_of(b).unwrap(),
+    Some(K::sample_b()),
+    "node_of(key) must round-trip backward"
+  );
+  assert_eq!(g.level(b).unwrap(), 1, "level must be stored and retrieved");
+  assert_eq!(
+    g.node_count().unwrap(),
+    2,
+    "node_count must track high-water mark"
+  );
+}
+
+/// Tests that neighbors are stored and retrieved per-level, and that unset
+/// levels read back empty without error.
+pub fn conformance_neighbors_roundtrip_per_level<K: TestKey, G: GraphStoreWrite<K>>(g: &mut G)
+where
+  <G as GraphStoreRead<K>>::Error: std::fmt::Debug,
+{
+  let a = g.insert_node(K::sample_a(), 2).unwrap();
+  let b = g.insert_node(K::sample_b(), 0).unwrap();
+  let c = g.insert_node(K::sample_c(), 0).unwrap();
+  g.set_neighbors(a, 0, &[b, c]).unwrap();
+  g.set_neighbors(a, 2, &[c]).unwrap();
+
+  let mut out = Vec::new();
+  g.neighbors(a, 0, &mut out).unwrap();
+  assert_eq!(out, vec![b, c], "level 0 must return set neighbors");
+
+  g.neighbors(a, 1, &mut out).unwrap();
+  assert!(out.is_empty(), "unset level must return empty, not error");
+
+  g.neighbors(a, 2, &mut out).unwrap();
+  assert_eq!(out, vec![c], "level 2 must return its neighbors");
+}
+
+/// Tests that removal is immediate (no stale keys/nodes), IDs are never
+/// reused even after removal, and stale links remain in surviving neighbors'
+/// lists (must be skipped at traversal time).
+pub fn conformance_remove_is_immediate_and_ids_not_reused<K: TestKey, G: GraphStoreWrite<K>>(
+  g: &mut G,
+) where
+  <G as GraphStoreRead<K>>::Error: std::fmt::Debug,
+{
+  let a = g.insert_node(K::sample_a(), 0).unwrap();
+  let b = g.insert_node(K::sample_b(), 0).unwrap(); // highest id at this point
+  g.set_neighbors(a, 0, &[b]).unwrap();
+
+  g.remove_node(b).unwrap();
+  assert!(
+    !g.contains(b).unwrap(),
+    "removed node must return false from contains()"
+  );
+  assert_eq!(
+    g.key_of(b).unwrap(),
+    None,
+    "removed node must return None from key_of()"
+  );
+  assert_eq!(
+    g.node_of(&K::sample_b()).unwrap(),
+    None,
+    "removed key must return None from node_of()"
+  );
+
+  // Stale link survives in a's list; traversal-side skip is the contract.
+  let mut out = Vec::new();
+  g.neighbors(a, 0, &mut out).unwrap();
+  assert_eq!(
+    out,
+    vec![b],
+    "remove_node must NOT rewrite other nodes' adjacency lists"
+  );
+
+  // The invariant this design hangs on: removing the HIGHEST id must not free it for reuse.
+  let c = g.insert_node(K::sample_c(), 0).unwrap();
+  assert!(
+    c.as_u32() > b.as_u32(),
+    "removed id must never be reallocated (new id must be > removed id)"
+  );
+  assert_eq!(
+    g.node_count().unwrap(),
+    3,
+    "node_count must be high-water mark, not live count"
+  );
+}
+
+/// Tests that entry point is stored and retrieved correctly, and can be
+/// cleared.
+pub fn conformance_entry_point_roundtrip<K: TestKey, G: GraphStoreWrite<K>>(g: &mut G)
+where
+  <G as GraphStoreRead<K>>::Error: std::fmt::Debug,
+{
+  assert_eq!(
+    g.entry_point().unwrap(),
+    None,
+    "empty graph has no entry point"
+  );
+
+  let a = g.insert_node(K::sample_a(), 3).unwrap();
+  g.set_entry_point(Some((a, 3))).unwrap();
+  assert_eq!(
+    g.entry_point().unwrap(),
+    Some((a, 3)),
+    "entry point must round-trip"
+  );
+
+  g.set_entry_point(None).unwrap();
+  assert_eq!(
+    g.entry_point().unwrap(),
+    None,
+    "entry point must be clearable"
+  );
+}
+
+/// Umbrella conformance suite: runs all conformance tests, creating a fresh
+/// store for each test via the provided closure.
+pub fn conformance_all<K: TestKey, G: GraphStoreWrite<K>>(mut fresh: impl FnMut() -> G)
+where
+  <G as GraphStoreRead<K>>::Error: std::fmt::Debug,
+{
+  {
+    let mut g = fresh();
+    conformance_alloc_is_monotonic_and_binds::<K, G>(&mut g);
+  } // g dropped here, explicitly, before the next fresh() call
+
+  {
+    let mut g = fresh();
+    conformance_neighbors_roundtrip_per_level::<K, G>(&mut g);
+  }
+
+  {
+    let mut g = fresh();
+    conformance_remove_is_immediate_and_ids_not_reused::<K, G>(&mut g);
+  }
+
+  {
+    let mut g = fresh();
+    conformance_entry_point_roundtrip::<K, G>(&mut g);
+  }
+}

data/ext/db-index/src/hnsw/id.rs

@@ -0,0 +1,36 @@
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct NodeId(u32);
+
+impl NodeId
+{
+  #[inline]
+  pub fn as_u32(self) -> u32
+  {
+    self.0
+  }
+
+  #[inline]
+  pub fn as_usize(self) -> usize
+  {
+    self.0 as usize
+  }
+
+  #[inline]
+  pub(crate) fn new(raw: u32) -> Self
+  {
+    Self(raw)
+  }
+
+  /// Reconstruct a NodeId from a raw value previously produced by
+  /// `as_u32` and persisted by a storage adapter. Adapters are the only
+  /// intended caller -- this intentionally does not enforce that `raw` was
+  /// ever actually allocated by this crate, since the only place that
+  /// matters (load-time validation) is `load_from_store` (WI 300), which
+  /// checks NodeId-set consistency across adjacency/key/vector data
+  /// regardless of where the NodeId values came from.
+  #[inline]
+  pub fn from_u32(raw: u32) -> Self
+  {
+    Self(raw)
+  }
+}