gqlite 1.5.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/ext/Cargo.toml +12 -6
  3. data/ext/db-index/Cargo.toml +29 -0
  4. data/ext/db-index/src/hnsw/error.rs +66 -0
  5. data/ext/db-index/src/hnsw/graph_store.rs +272 -0
  6. data/ext/db-index/src/hnsw/id.rs +36 -0
  7. data/ext/db-index/src/hnsw/implementation.rs +1517 -0
  8. data/ext/db-index/src/hnsw/kernels.rs +1228 -0
  9. data/ext/db-index/src/hnsw/metric.rs +244 -0
  10. data/ext/db-index/src/hnsw/scalar.rs +72 -0
  11. data/ext/db-index/src/hnsw/simple_store.rs +140 -0
  12. data/ext/db-index/src/hnsw/store.rs +472 -0
  13. data/ext/db-index/src/hnsw/vector.rs +105 -0
  14. data/ext/db-index/src/hnsw/vectors.rs +568 -0
  15. data/ext/db-index/src/hnsw.rs +42 -0
  16. data/ext/db-index/src/lib.rs +3 -0
  17. data/ext/gqlitedb/Cargo.toml +19 -8
  18. data/ext/gqlitedb/benches/common/pokec.rs +62 -2
  19. data/ext/gqlitedb/benches/pokec_divan.rs +66 -2
  20. data/ext/gqlitedb/benches/pokec_iai.rs +60 -3
  21. data/ext/gqlitedb/release.toml +2 -2
  22. data/ext/gqlitedb/src/aggregators/arithmetic.rs +3 -1
  23. data/ext/gqlitedb/src/aggregators/containers.rs +1 -1
  24. data/ext/gqlitedb/src/aggregators/stats.rs +21 -12
  25. data/ext/gqlitedb/src/capi.rs +20 -24
  26. data/ext/gqlitedb/src/compiler/expression_analyser.rs +28 -20
  27. data/ext/gqlitedb/src/compiler/variables_manager.rs +104 -38
  28. data/ext/gqlitedb/src/compiler.rs +506 -227
  29. data/ext/gqlitedb/src/connection.rs +151 -11
  30. data/ext/gqlitedb/src/consts.rs +1 -2
  31. data/ext/gqlitedb/src/error.rs +123 -61
  32. data/ext/gqlitedb/src/functions/common.rs +64 -0
  33. data/ext/gqlitedb/src/functions/containers.rs +2 -46
  34. data/ext/gqlitedb/src/functions/edge.rs +1 -1
  35. data/ext/gqlitedb/src/functions/math.rs +87 -15
  36. data/ext/gqlitedb/src/functions/node.rs +1 -1
  37. data/ext/gqlitedb/src/functions/path.rs +48 -11
  38. data/ext/gqlitedb/src/functions/scalar.rs +47 -4
  39. data/ext/gqlitedb/src/functions/string.rs +123 -1
  40. data/ext/gqlitedb/src/functions/value.rs +1 -1
  41. data/ext/gqlitedb/src/functions.rs +165 -28
  42. data/ext/gqlitedb/src/graph.rs +1 -9
  43. data/ext/gqlitedb/src/interpreter/evaluators.rs +968 -130
  44. data/ext/gqlitedb/src/interpreter/instructions.rs +39 -2
  45. data/ext/gqlitedb/src/lib.rs +5 -4
  46. data/ext/gqlitedb/src/parser/gql.pest +1 -1
  47. data/ext/gqlitedb/src/planner.rs +329 -0
  48. data/ext/gqlitedb/src/prelude.rs +3 -3
  49. data/ext/gqlitedb/src/store/pgrx.rs +1 -1
  50. data/ext/gqlitedb/src/store/postgres.rs +742 -16
  51. data/ext/gqlitedb/src/store/redb/hnsw_store.rs +702 -0
  52. data/ext/gqlitedb/src/store/redb/index.rs +274 -0
  53. data/ext/gqlitedb/src/store/redb.rs +1268 -113
  54. data/ext/gqlitedb/src/store/sqlbase/sqlmetadata.rs +28 -0
  55. data/ext/gqlitedb/src/store/sqlbase/sqlstore.rs +103 -0
  56. data/ext/gqlitedb/src/store/sqlbase.rs +146 -16
  57. data/ext/gqlitedb/src/store/sqlite.rs +576 -14
  58. data/ext/gqlitedb/src/store/vector_extract.rs +56 -0
  59. data/ext/gqlitedb/src/store.rs +140 -29
  60. data/ext/gqlitedb/src/tests/compiler.rs +207 -10
  61. data/ext/gqlitedb/src/tests/connection/postgres.rs +38 -3
  62. data/ext/gqlitedb/src/tests/connection/redb.rs +23 -0
  63. data/ext/gqlitedb/src/tests/connection/sqlite.rs +31 -0
  64. data/ext/gqlitedb/src/tests/connection.rs +61 -1
  65. data/ext/gqlitedb/src/tests/evaluators.rs +162 -7
  66. data/ext/gqlitedb/src/tests/parser.rs +54 -23
  67. data/ext/gqlitedb/src/tests/planner.rs +511 -0
  68. data/ext/gqlitedb/src/tests/store/postgres.rs +7 -0
  69. data/ext/gqlitedb/src/tests/store/redb.rs +8 -0
  70. data/ext/gqlitedb/src/tests/store/sqlite.rs +8 -0
  71. data/ext/gqlitedb/src/tests/store/vector_index/postgres.rs +182 -0
  72. data/ext/gqlitedb/src/tests/store/vector_index/redb.rs +386 -0
  73. data/ext/gqlitedb/src/tests/store/vector_index/sqlite.rs +166 -0
  74. data/ext/gqlitedb/src/tests/store/vector_index.rs +2313 -0
  75. data/ext/gqlitedb/src/tests/store.rs +78 -14
  76. data/ext/gqlitedb/src/tests/templates/ast.rs +92 -16
  77. data/ext/gqlitedb/src/tests/templates/programs.rs +14 -7
  78. data/ext/gqlitedb/src/tests.rs +15 -9
  79. data/ext/gqlitedb/src/utils.rs +6 -1
  80. data/ext/gqlitedb/src/value/compare.rs +61 -3
  81. data/ext/gqlitedb/src/value.rs +136 -7
  82. data/ext/gqlitedb/templates/sql/postgres/metadata_delete.sql +1 -0
  83. data/ext/gqlitedb/templates/sql/postgres/node_select.sql +1 -1
  84. data/ext/gqlitedb/templates/sql/sqlite/metadata_delete.sql +1 -0
  85. data/ext/gqliterb/src/lib.rs +53 -8
  86. data/ext/gqlparser/Cargo.toml +25 -0
  87. data/ext/gqlparser/README.MD +9 -0
  88. data/ext/gqlparser/benches/pokec_divan.rs +34 -0
  89. data/ext/gqlparser/src/common.rs +69 -0
  90. data/ext/gqlparser/src/gqls/ast.rs +69 -0
  91. data/ext/gqlparser/src/gqls/constraint.rs +79 -0
  92. data/ext/gqlparser/src/gqls/error.rs +22 -0
  93. data/ext/gqlparser/src/gqls/parser.rs +813 -0
  94. data/ext/gqlparser/src/gqls/prelude.rs +8 -0
  95. data/ext/gqlparser/src/gqls/properties.rs +115 -0
  96. data/ext/gqlparser/src/gqls/resolve.rs +207 -0
  97. data/ext/gqlparser/src/gqls.rs +265 -0
  98. data/ext/gqlparser/src/lib.rs +7 -0
  99. data/ext/gqlparser/src/oc/ast.rs +680 -0
  100. data/ext/gqlparser/src/oc/error.rs +172 -0
  101. data/ext/gqlparser/src/oc/lexer.rs +429 -0
  102. data/ext/gqlparser/src/oc/parser/tests.rs +2284 -0
  103. data/ext/gqlparser/src/oc/parser.rs +2005 -0
  104. data/ext/gqlparser/src/oc.rs +33 -0
  105. data/ext/gqlparser/src/prelude.rs +3 -0
  106. data/ext/graphcore/Cargo.toml +1 -0
  107. data/ext/graphcore/src/error.rs +26 -0
  108. data/ext/graphcore/src/graph.rs +177 -51
  109. data/ext/graphcore/src/lib.rs +4 -2
  110. data/ext/graphcore/src/open_cypher.rs +12 -0
  111. data/ext/graphcore/src/table.rs +50 -2
  112. data/ext/graphcore/src/timestamp.rs +127 -104
  113. data/ext/graphcore/src/value/tensor.rs +739 -0
  114. data/ext/graphcore/src/value/value_map.rs +1 -1
  115. data/ext/graphcore/src/value.rs +343 -19
  116. metadata +93 -28
  117. data/ext/gqlitedb/src/parser/ast.rs +0 -604
  118. data/ext/gqlitedb/src/parser/parser_impl.rs +0 -1213
  119. data/ext/gqlitedb/src/parser.rs +0 -4
@@ -0,0 +1,702 @@
1
+ use db_index::hnsw::id::NodeId;
2
+ use redb::{ReadableTable, Table, TableDefinition};
3
+
4
+ use crate::{graph, prelude::*};
5
+
6
+ /// Convert gqlitedb errors to hnsw_stable errors for trait bridge
7
+ fn store_error(e: impl std::fmt::Display) -> db_index::hnsw::Error
8
+ {
9
+ db_index::hnsw::Error::StoreError(e.to_string())
10
+ }
11
+
12
+ /// Simple metadata for tracking next_node_id and entry point within the dedicated meta table.
13
+ /// This is a minimal version until the full HnswMetaBlob serialization is available.
14
+ #[derive(serde::Serialize, serde::Deserialize, Clone)]
15
+ pub(crate) struct SimpleMetaBlob
16
+ {
17
+ pub next_node_id: u32,
18
+ pub entry_point: Option<u32>,
19
+ pub max_level: i32,
20
+ }
21
+
22
+ /// Manages the five table definitions for one HNSW index.
23
+ /// Table names are computed from (graph_name, index_name) at construction and stored as String fields.
24
+ pub(crate) struct HnswTableSet
25
+ {
26
+ adj_table: String,
27
+ vec_table: String,
28
+ key_table: String,
29
+ key_rev_table: String,
30
+ meta_table: String,
31
+ }
32
+
33
+ impl HnswTableSet
34
+ {
35
+ pub(crate) fn new(graph_name: &str, index_name: &str) -> Self
36
+ {
37
+ Self {
38
+ adj_table: format!("__{graph_name}__hnsw_{index_name}_adj"),
39
+ vec_table: format!("__{graph_name}__hnsw_{index_name}_vec"),
40
+ key_table: format!("__{graph_name}__hnsw_{index_name}_key"),
41
+ key_rev_table: format!("__{graph_name}__hnsw_{index_name}_rkey"),
42
+ meta_table: format!("__{graph_name}__hnsw_{index_name}_meta"),
43
+ }
44
+ }
45
+
46
+ /// Returns the table definition for adjacency layer data.
47
+ /// Uses Vec<u8> instead of &'static [u8] to allow arbitrary runtime byte slices
48
+ /// to be stored with proper lifetime semantics and ownership transfer into redb.
49
+ pub(crate) fn adj(&self) -> TableDefinition<'_, u32, Vec<u8>>
50
+ {
51
+ TableDefinition::new(&self.adj_table)
52
+ }
53
+
54
+ /// Returns the table definition for vector data.
55
+ /// Uses Vec<u8> for the same lifetime/ownership reasons as adj().
56
+ pub(crate) fn vec(&self) -> TableDefinition<'_, u32, Vec<u8>>
57
+ {
58
+ TableDefinition::new(&self.vec_table)
59
+ }
60
+
61
+ pub(crate) fn key(&self) -> TableDefinition<'_, u32, u128>
62
+ {
63
+ TableDefinition::new(&self.key_table)
64
+ }
65
+
66
+ pub(crate) fn key_rev(&self) -> TableDefinition<'_, u128, u32>
67
+ {
68
+ TableDefinition::new(&self.key_rev_table)
69
+ }
70
+
71
+ pub(crate) fn meta(&self) -> TableDefinition<'_, (), Vec<u8>>
72
+ {
73
+ TableDefinition::new(&self.meta_table)
74
+ }
75
+ }
76
+
77
+ /// Write adapter for HNSW graph operations.
78
+ /// Owns five redb Table handles directly from a WriteTransaction.
79
+ /// `GraphStoreWrite<graph::Key>` requires `&mut self` for write operations.
80
+ pub(crate) struct RedbGraphStore<'txn>
81
+ {
82
+ adj: Table<'txn, u32, Vec<u8>>,
83
+ key: Table<'txn, u32, u128>,
84
+ key_rev: Table<'txn, u128, u32>,
85
+ meta: Table<'txn, (), Vec<u8>>,
86
+ }
87
+
88
+ impl<'txn> RedbGraphStore<'txn>
89
+ {
90
+ pub(crate) fn open(txn: &'txn super::WriteTransaction, tables: &HnswTableSet) -> Result<Self>
91
+ {
92
+ let adj = txn.write_transaction.open_table(tables.adj())?;
93
+ let key = txn.write_transaction.open_table(tables.key())?;
94
+ let key_rev = txn.write_transaction.open_table(tables.key_rev())?;
95
+ let meta = txn.write_transaction.open_table(tables.meta())?;
96
+
97
+ Ok(Self {
98
+ adj,
99
+ key,
100
+ key_rev,
101
+ meta,
102
+ })
103
+ }
104
+
105
+ fn read_meta(&self) -> Result<SimpleMetaBlob>
106
+ {
107
+ let bytes = self
108
+ .meta
109
+ .get(&())?
110
+ .map(|g| g.value().to_vec())
111
+ .ok_or_else(|| {
112
+ crate::error::Error::StoreError(crate::error::StoreError::InvalidFormat(
113
+ "meta row missing".into(),
114
+ ))
115
+ })?;
116
+ ciborium::from_reader(bytes.as_slice()).map_err(|e| crate::error::Error::Internal(e.into()))
117
+ }
118
+
119
+ pub(crate) fn write_meta(&mut self, meta: &SimpleMetaBlob) -> Result<()>
120
+ {
121
+ let mut bytes = Vec::new();
122
+ ciborium::into_writer(meta, &mut bytes).map_err(|e| crate::error::Error::Internal(e.into()))?;
123
+ self.meta.insert((), bytes)?;
124
+ Ok(())
125
+ }
126
+ }
127
+
128
+ impl<'txn> db_index::hnsw::graph_store::GraphStoreRead<graph::Key> for RedbGraphStore<'txn>
129
+ {
130
+ type Error = db_index::hnsw::Error;
131
+
132
+ fn node_count(&self) -> std::result::Result<usize, db_index::hnsw::Error>
133
+ {
134
+ Ok(self.read_meta().map_err(store_error)?.next_node_id as usize)
135
+ }
136
+
137
+ fn entry_point(&self) -> std::result::Result<Option<(NodeId, i32)>, db_index::hnsw::Error>
138
+ {
139
+ let m = self.read_meta().map_err(store_error)?;
140
+ Ok(
141
+ m.entry_point
142
+ .map(|raw| (NodeId::from_u32(raw), m.max_level)),
143
+ )
144
+ }
145
+
146
+ fn contains(&self, node: NodeId) -> std::result::Result<bool, db_index::hnsw::Error>
147
+ {
148
+ Ok(self.adj.get(&node.as_u32()).map_err(store_error)?.is_some())
149
+ }
150
+
151
+ fn key_of(&self, node: NodeId) -> std::result::Result<Option<graph::Key>, db_index::hnsw::Error>
152
+ {
153
+ Ok(
154
+ self
155
+ .key
156
+ .get(&node.as_u32())
157
+ .map_err(store_error)?
158
+ .map(|g| graph::Key::new(g.value())),
159
+ )
160
+ }
161
+
162
+ fn node_of(&self, key: &graph::Key)
163
+ -> std::result::Result<Option<NodeId>, db_index::hnsw::Error>
164
+ {
165
+ Ok(
166
+ self
167
+ .key_rev
168
+ .get(&key.uuid())
169
+ .map_err(store_error)?
170
+ .map(|g| NodeId::from_u32(g.value())),
171
+ )
172
+ }
173
+
174
+ fn level(&self, node: NodeId) -> std::result::Result<i32, db_index::hnsw::Error>
175
+ {
176
+ let bytes = self
177
+ .adj
178
+ .get(&node.as_u32())
179
+ .map_err(store_error)?
180
+ .ok_or_else(|| store_error("level of absent node".to_string()))?;
181
+ Ok(bytes.value()[0] as i32)
182
+ }
183
+
184
+ fn neighbors(
185
+ &self,
186
+ node: NodeId,
187
+ level: usize,
188
+ out: &mut Vec<NodeId>,
189
+ ) -> std::result::Result<(), db_index::hnsw::Error>
190
+ {
191
+ out.clear();
192
+ let Some(bytes) = self.adj.get(&node.as_u32()).map_err(store_error)?
193
+ else
194
+ {
195
+ return Ok(());
196
+ };
197
+ let (_, layers) = db_index::hnsw::store::decode_adjacency_layers(bytes.value().as_slice())
198
+ .map_err(store_error)?;
199
+ if let Some(layer) = layers.get(level)
200
+ {
201
+ out.extend(layer.iter().map(|&id| NodeId::from_u32(id)));
202
+ }
203
+ Ok(())
204
+ }
205
+ }
206
+
207
+ impl<'txn> db_index::hnsw::graph_store::GraphStoreWrite<graph::Key> for RedbGraphStore<'txn>
208
+ {
209
+ fn insert_node(
210
+ &mut self,
211
+ key: graph::Key,
212
+ level: i32,
213
+ ) -> std::result::Result<NodeId, db_index::hnsw::Error>
214
+ {
215
+ let mut meta = self.read_meta().map_err(store_error)?;
216
+ let id = NodeId::from_u32(meta.next_node_id);
217
+ meta.next_node_id += 1; // monotonic; the sole enforcement of never-reuse
218
+ self.write_meta(&meta).map_err(store_error)?;
219
+ self
220
+ .key
221
+ .insert(id.as_u32(), key.uuid())
222
+ .map_err(store_error)?;
223
+ self
224
+ .key_rev
225
+ .insert(key.uuid(), id.as_u32())
226
+ .map_err(store_error)?;
227
+ let empty_layers = vec![Vec::new(); (level as usize) + 1];
228
+ self
229
+ .adj
230
+ .insert(
231
+ id.as_u32(),
232
+ db_index::hnsw::store::encode_adjacency(level as u8, &empty_layers),
233
+ )
234
+ .map_err(store_error)?;
235
+ Ok(id)
236
+ }
237
+
238
+ fn set_neighbors(
239
+ &mut self,
240
+ node: NodeId,
241
+ level: usize,
242
+ nbrs: &[NodeId],
243
+ ) -> std::result::Result<(), db_index::hnsw::Error>
244
+ {
245
+ let cur = self
246
+ .adj
247
+ .get(&node.as_u32())
248
+ .map_err(store_error)?
249
+ .ok_or_else(|| store_error("set_neighbors on absent node"))?;
250
+ let (lvl, mut layers) = db_index::hnsw::store::decode_adjacency_layers(cur.value().as_slice())
251
+ .map_err(store_error)?;
252
+ drop(cur);
253
+ if layers.len() <= level
254
+ {
255
+ layers.resize(level + 1, Vec::new());
256
+ }
257
+ layers[level] = nbrs.iter().map(|n| n.as_u32()).collect();
258
+ self
259
+ .adj
260
+ .insert(
261
+ node.as_u32(),
262
+ db_index::hnsw::store::encode_adjacency(lvl, &layers),
263
+ )
264
+ .map_err(store_error)?;
265
+ Ok(())
266
+ }
267
+
268
+ fn set_entry_point(
269
+ &mut self,
270
+ ep: Option<(NodeId, i32)>,
271
+ ) -> std::result::Result<(), db_index::hnsw::Error>
272
+ {
273
+ let mut meta = self.read_meta().map_err(store_error)?;
274
+ meta.entry_point = ep.map(|(n, _)| n.as_u32());
275
+ meta.max_level = ep.map_or(-1, |(_, l)| l);
276
+ self.write_meta(&meta).map_err(store_error)
277
+ }
278
+
279
+ fn remove_node(&mut self, node: NodeId) -> std::result::Result<(), db_index::hnsw::Error>
280
+ {
281
+ if let Some(k) = self.key.remove(node.as_u32()).map_err(store_error)?
282
+ {
283
+ self.key_rev.remove(k.value()).map_err(store_error)?;
284
+ }
285
+ self.adj.remove(node.as_u32()).map_err(store_error)?;
286
+ Ok(()) // next_node_id in meta untouched
287
+ }
288
+ }
289
+
290
+ /// Write adapter for HNSW vector operations.
291
+ /// Uses Mutex for interior mutability (required by VectorStoreMut trait taking &self).
292
+ /// Safe because write transactions are exclusive, so contention never occurs.
293
+ pub(crate) struct RedbVectorStore<'txn>
294
+ {
295
+ vec: std::sync::Mutex<Table<'txn, u32, Vec<u8>>>,
296
+ dim: usize,
297
+ encoding: db_index::hnsw::store::VectorEncoding,
298
+ }
299
+
300
+ impl<'txn> RedbVectorStore<'txn>
301
+ {
302
+ pub(crate) fn open(
303
+ txn: &'txn super::WriteTransaction,
304
+ tables: &HnswTableSet,
305
+ dim: usize,
306
+ encoding: db_index::hnsw::store::VectorEncoding,
307
+ ) -> Result<Self>
308
+ {
309
+ let vec = txn.write_transaction.open_table(tables.vec())?;
310
+ Ok(Self {
311
+ vec: std::sync::Mutex::new(vec),
312
+ dim,
313
+ encoding,
314
+ })
315
+ }
316
+
317
+ pub(crate) fn delete(&self, id: NodeId) -> Result<()>
318
+ {
319
+ let id_u32 = id.as_u32();
320
+ let mut table = self.vec.lock().map_err(|_| {
321
+ crate::error::Error::Internal(crate::error::InternalError::Unimplemented(
322
+ "failed to acquire table lock for vector deletion",
323
+ ))
324
+ })?;
325
+ table.remove(id_u32)?;
326
+ Ok(())
327
+ }
328
+ }
329
+
330
+ impl<'txn> db_index::hnsw::vectors::VectorStore for RedbVectorStore<'txn>
331
+ {
332
+ type Family = db_index::hnsw::vector::Dense<f32>;
333
+ type Vector<'a>
334
+ = db_index::hnsw::vectors::ArcVector<f32>
335
+ where
336
+ Self: 'a;
337
+
338
+ fn dim(&self) -> usize
339
+ {
340
+ self.dim
341
+ }
342
+
343
+ fn vector(&self, id: NodeId) -> Option<Self::Vector<'_>>
344
+ {
345
+ let id_u32 = id.as_u32();
346
+ let table = self.vec.lock().ok()?;
347
+ let bytes = table.get(&id_u32).ok()??.value().to_vec();
348
+ let floats = db_index::hnsw::store::decode_vector(&self.encoding, &bytes, self.dim)
349
+ .map_err(|_| ())
350
+ .ok()?;
351
+ Some(db_index::hnsw::vectors::ArcVector::new(
352
+ floats.into_boxed_slice(),
353
+ ))
354
+ }
355
+ }
356
+
357
+ impl<'txn> db_index::hnsw::vectors::VectorStoreMut for RedbVectorStore<'txn>
358
+ {
359
+ fn set(
360
+ &self,
361
+ id: NodeId,
362
+ vector: <Self::Family as db_index::hnsw::vector::VectorFamily>::Ref<'_>,
363
+ ) -> db_index::hnsw::error::Result<()>
364
+ {
365
+ let bytes = db_index::hnsw::store::encode_vector(&self.encoding, vector)?;
366
+ let id_u32 = id.as_u32();
367
+ let mut table = self
368
+ .vec
369
+ .lock()
370
+ .map_err(|_| db_index::hnsw::Error::StoreError("failed to acquire table lock".into()))?;
371
+ table.insert(id_u32, bytes).map_err(store_error)?;
372
+ Ok(())
373
+ }
374
+ }
375
+
376
+ /// Read-only HNSW graph store backed by redb read-only tables.
377
+ ///
378
+ /// Implements `GraphStoreRead<graph::Key>` for loading HNSW graph structure from disk.
379
+ pub(crate) struct RedbGraphStoreRead<'txn>
380
+ {
381
+ adj: redb::ReadOnlyTable<u32, Vec<u8>>,
382
+ key: redb::ReadOnlyTable<u32, u128>,
383
+ key_rev: redb::ReadOnlyTable<u128, u32>,
384
+ meta: redb::ReadOnlyTable<(), Vec<u8>>,
385
+ _marker: std::marker::PhantomData<&'txn ()>,
386
+ }
387
+
388
+ impl<'txn> RedbGraphStoreRead<'txn>
389
+ {
390
+ /// Opens read-only tables for an HNSW index.
391
+ ///
392
+ /// Returns `Ok(None)` if the adjacency table does not exist,
393
+ /// indicating the index has not yet been created (e.g., new database).
394
+ pub(crate) fn open(
395
+ txn: &'txn redb::ReadTransaction,
396
+ tables: &HnswTableSet,
397
+ ) -> Result<Option<Self>>
398
+ {
399
+ let adj = match txn.open_table(tables.adj())
400
+ {
401
+ Ok(t) => t,
402
+ Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
403
+ Err(e) => return Err(e.into()),
404
+ };
405
+
406
+ Ok(Some(Self {
407
+ adj,
408
+ key: txn.open_table(tables.key())?,
409
+ key_rev: txn.open_table(tables.key_rev())?,
410
+ meta: txn.open_table(tables.meta())?,
411
+ _marker: std::marker::PhantomData,
412
+ }))
413
+ }
414
+
415
+ fn read_meta(&self) -> Result<SimpleMetaBlob>
416
+ {
417
+ let bytes = self
418
+ .meta
419
+ .get(&())?
420
+ .map(|g| g.value().to_vec())
421
+ .ok_or_else(|| {
422
+ crate::error::Error::StoreError(crate::error::StoreError::InvalidFormat(
423
+ "meta row missing".into(),
424
+ ))
425
+ })?;
426
+ ciborium::from_reader(bytes.as_slice()).map_err(|e| crate::error::Error::Internal(e.into()))
427
+ }
428
+ }
429
+
430
+ impl<'txn> db_index::hnsw::graph_store::GraphStoreRead<graph::Key> for RedbGraphStoreRead<'txn>
431
+ {
432
+ type Error = db_index::hnsw::Error;
433
+
434
+ fn node_count(&self) -> std::result::Result<usize, db_index::hnsw::Error>
435
+ {
436
+ let count = self.read_meta().map_err(store_error)?.next_node_id as usize;
437
+ Ok(count)
438
+ }
439
+
440
+ fn entry_point(&self) -> std::result::Result<Option<(NodeId, i32)>, db_index::hnsw::Error>
441
+ {
442
+ let m = self.read_meta().map_err(store_error)?;
443
+ Ok(
444
+ m.entry_point
445
+ .map(|raw| (NodeId::from_u32(raw), m.max_level)),
446
+ )
447
+ }
448
+
449
+ fn contains(&self, node: NodeId) -> std::result::Result<bool, db_index::hnsw::Error>
450
+ {
451
+ Ok(self.adj.get(&node.as_u32()).map_err(store_error)?.is_some())
452
+ }
453
+
454
+ fn key_of(&self, node: NodeId) -> std::result::Result<Option<graph::Key>, db_index::hnsw::Error>
455
+ {
456
+ Ok(
457
+ self
458
+ .key
459
+ .get(&node.as_u32())
460
+ .map_err(store_error)?
461
+ .map(|g| graph::Key::new(g.value())),
462
+ )
463
+ }
464
+
465
+ fn node_of(&self, key: &graph::Key)
466
+ -> std::result::Result<Option<NodeId>, db_index::hnsw::Error>
467
+ {
468
+ Ok(
469
+ self
470
+ .key_rev
471
+ .get(&key.uuid())
472
+ .map_err(store_error)?
473
+ .map(|g| NodeId::from_u32(g.value())),
474
+ )
475
+ }
476
+
477
+ fn level(&self, node: NodeId) -> std::result::Result<i32, db_index::hnsw::Error>
478
+ {
479
+ let bytes = self
480
+ .adj
481
+ .get(&node.as_u32())
482
+ .map_err(store_error)?
483
+ .ok_or_else(|| store_error("level of absent node"))?;
484
+ Ok(bytes.value()[0] as i32)
485
+ }
486
+
487
+ fn neighbors(
488
+ &self,
489
+ node: NodeId,
490
+ level: usize,
491
+ out: &mut Vec<NodeId>,
492
+ ) -> std::result::Result<(), db_index::hnsw::Error>
493
+ {
494
+ out.clear();
495
+ let Some(bytes) = self.adj.get(&node.as_u32()).map_err(store_error)?
496
+ else
497
+ {
498
+ return Ok(());
499
+ };
500
+ let (_, layers) = db_index::hnsw::store::decode_adjacency_layers(bytes.value().as_slice())
501
+ .map_err(store_error)?;
502
+ if let Some(layer) = layers.get(level)
503
+ {
504
+ out.extend(layer.iter().map(|&id| NodeId::from_u32(id)));
505
+ }
506
+ Ok(())
507
+ }
508
+ }
509
+
510
+ /// Read-only HNSW vector store backed by redb read-only tables.
511
+ ///
512
+ /// Implements `VectorStore` for loading HNSW vectors from disk.
513
+ pub(crate) struct RedbVectorStoreRead<'txn>
514
+ {
515
+ vec: redb::ReadOnlyTable<u32, Vec<u8>>,
516
+ dim: usize,
517
+ encoding: db_index::hnsw::store::VectorEncoding,
518
+ _marker: std::marker::PhantomData<&'txn ()>,
519
+ }
520
+
521
+ impl<'txn> RedbVectorStoreRead<'txn>
522
+ {
523
+ /// Opens read-only tables for an HNSW vector index.
524
+ pub(crate) fn open(
525
+ txn: &'txn redb::ReadTransaction,
526
+ tables: &HnswTableSet,
527
+ dim: usize,
528
+ encoding: db_index::hnsw::store::VectorEncoding,
529
+ ) -> Result<Option<Self>>
530
+ {
531
+ let vec = match txn.open_table(tables.vec())
532
+ {
533
+ Ok(t) => t,
534
+ Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
535
+ Err(e) => return Err(e.into()),
536
+ };
537
+
538
+ Ok(Some(Self {
539
+ vec,
540
+ dim,
541
+ encoding,
542
+ _marker: std::marker::PhantomData,
543
+ }))
544
+ }
545
+ }
546
+
547
+ impl<'txn> db_index::hnsw::vectors::VectorStore for RedbVectorStoreRead<'txn>
548
+ {
549
+ type Family = db_index::hnsw::vector::Dense<f32>;
550
+ type Vector<'a>
551
+ = db_index::hnsw::vectors::ArcVector<f32>
552
+ where
553
+ Self: 'a;
554
+
555
+ fn dim(&self) -> usize
556
+ {
557
+ self.dim
558
+ }
559
+
560
+ fn vector(&self, id: NodeId) -> Option<Self::Vector<'_>>
561
+ {
562
+ let id_u32 = id.as_u32();
563
+ let bytes = self.vec.get(&id_u32).ok()??.value().to_vec();
564
+ let floats = db_index::hnsw::store::decode_vector(&self.encoding, &bytes, self.dim)
565
+ .map_err(|_| ())
566
+ .ok()?;
567
+ Some(db_index::hnsw::vectors::ArcVector::new(
568
+ floats.into_boxed_slice(),
569
+ ))
570
+ }
571
+ }
572
+
573
+ #[cfg(test)]
574
+ mod tests
575
+ {
576
+ use super::*;
577
+
578
+ #[test]
579
+ fn test_hnsw_table_set_construction_and_names()
580
+ {
581
+ let hnsw_set = HnswTableSet::new("mygraph", "embedding");
582
+
583
+ assert_eq!(hnsw_set.adj_table, "__mygraph__hnsw_embedding_adj");
584
+ assert_eq!(hnsw_set.vec_table, "__mygraph__hnsw_embedding_vec");
585
+ assert_eq!(hnsw_set.key_table, "__mygraph__hnsw_embedding_key");
586
+ assert_eq!(hnsw_set.key_rev_table, "__mygraph__hnsw_embedding_rkey");
587
+ assert_eq!(hnsw_set.meta_table, "__mygraph__hnsw_embedding_meta");
588
+ }
589
+
590
+ #[test]
591
+ fn test_hnsw_table_set_multiple_indices()
592
+ {
593
+ let index1 = HnswTableSet::new("graph1", "idx1");
594
+ let index2 = HnswTableSet::new("graph1", "idx2");
595
+ let index3 = HnswTableSet::new("graph2", "idx1");
596
+
597
+ assert_eq!(index1.adj_table, "__graph1__hnsw_idx1_adj");
598
+ assert_eq!(index2.adj_table, "__graph1__hnsw_idx2_adj");
599
+ assert_eq!(index3.adj_table, "__graph2__hnsw_idx1_adj");
600
+
601
+ assert_eq!(index1.key_rev_table, "__graph1__hnsw_idx1_rkey");
602
+ assert_eq!(index2.key_rev_table, "__graph1__hnsw_idx2_rkey");
603
+ assert_eq!(index3.key_rev_table, "__graph2__hnsw_idx1_rkey");
604
+ }
605
+ }
606
+
607
+ #[cfg(test)]
608
+ mod graph_store_tests
609
+ {
610
+ use super::*;
611
+ use crate::graph;
612
+ use db_index::hnsw::graph_store::{GraphStoreRead, GraphStoreWrite};
613
+ use db_index::hnsw::id::NodeId;
614
+ use redb::ReadableDatabase;
615
+
616
+ fn create_test_db() -> (redb::Database, ccutils::temporary::TemporaryFile)
617
+ {
618
+ let db_path = crate::tests::create_tmp_file();
619
+ let db = redb::Database::create(db_path.path()).expect("Failed to create test database");
620
+ (db, db_path)
621
+ }
622
+
623
+ #[test]
624
+ fn test_graph_store_metadata_roundtrip()
625
+ {
626
+ let (db, _db_path) = create_test_db();
627
+
628
+ // First transaction: write data
629
+ {
630
+ let write_tx = db.begin_write().expect("Failed to begin write transaction");
631
+ let txn = super::super::WriteTransaction {
632
+ write_transaction: write_tx,
633
+ };
634
+ let tables = HnswTableSet::new("g", "idx_meta_roundtrip");
635
+ let mut store = RedbGraphStore::open(&txn, &tables).unwrap();
636
+ // Initialize metadata
637
+ let meta = SimpleMetaBlob {
638
+ next_node_id: 0,
639
+ entry_point: None,
640
+ max_level: -1,
641
+ };
642
+ store.write_meta(&meta).unwrap();
643
+ let node = store.insert_node(graph::Key::new(1), 2).unwrap();
644
+ store.set_entry_point(Some((node, 2))).unwrap();
645
+ drop(store);
646
+ // Commit the transaction
647
+ txn.write_transaction.commit().unwrap();
648
+ }
649
+
650
+ // Use a read transaction to verify data
651
+ {
652
+ let read_tx = db.begin_read().expect("Failed to begin read transaction");
653
+ let tables = HnswTableSet::new("g", "idx_meta_roundtrip");
654
+ let store = RedbGraphStoreRead::open(&read_tx, &tables)
655
+ .expect("Failed to open store read")
656
+ .expect("Store does not exist");
657
+ assert_eq!(store.entry_point().unwrap(), Some((NodeId::from_u32(0), 2)));
658
+ }
659
+ }
660
+
661
+ #[test]
662
+ fn test_key_rev_bidirectional_mapping()
663
+ {
664
+ let (db, _db_path) = create_test_db();
665
+
666
+ // First transaction: write keys
667
+ {
668
+ let write_tx = db.begin_write().expect("Failed to begin write transaction");
669
+ let txn = super::super::WriteTransaction {
670
+ write_transaction: write_tx,
671
+ };
672
+ let tables = HnswTableSet::new("g", "idx_key_rev");
673
+ let mut store = RedbGraphStore::open(&txn, &tables).unwrap();
674
+ // Initialize metadata
675
+ let meta = SimpleMetaBlob {
676
+ next_node_id: 0,
677
+ entry_point: None,
678
+ max_level: -1,
679
+ };
680
+ store.write_meta(&meta).unwrap();
681
+ let key = graph::Key::new(999u128);
682
+ let node_id = store.insert_node(key, 0).unwrap();
683
+ assert_eq!(store.key_of(node_id).unwrap(), Some(key));
684
+ assert_eq!(store.node_of(&key).unwrap(), Some(node_id));
685
+ drop(store);
686
+ // Commit the transaction
687
+ txn.write_transaction.commit().unwrap();
688
+ }
689
+
690
+ // Use a read transaction to verify bidirectional mapping
691
+ {
692
+ let read_tx = db.begin_read().expect("Failed to begin read transaction");
693
+ let tables = HnswTableSet::new("g", "idx_key_rev");
694
+ let store = RedbGraphStoreRead::open(&read_tx, &tables)
695
+ .expect("Failed to open store read")
696
+ .expect("Store does not exist");
697
+ let key = graph::Key::new(999u128);
698
+ assert_eq!(store.node_of(&key).unwrap(), Some(NodeId::from_u32(0)));
699
+ assert_eq!(store.key_of(NodeId::from_u32(0)).unwrap(), Some(key));
700
+ }
701
+ }
702
+ }