gqlite 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/Cargo.toml +12 -6
- data/ext/db-index/Cargo.toml +29 -0
- data/ext/db-index/src/hnsw/error.rs +66 -0
- data/ext/db-index/src/hnsw/graph_store.rs +272 -0
- data/ext/db-index/src/hnsw/id.rs +36 -0
- data/ext/db-index/src/hnsw/implementation.rs +1517 -0
- data/ext/db-index/src/hnsw/kernels.rs +1228 -0
- data/ext/db-index/src/hnsw/metric.rs +244 -0
- data/ext/db-index/src/hnsw/scalar.rs +72 -0
- data/ext/db-index/src/hnsw/simple_store.rs +140 -0
- data/ext/db-index/src/hnsw/store.rs +472 -0
- data/ext/db-index/src/hnsw/vector.rs +105 -0
- data/ext/db-index/src/hnsw/vectors.rs +568 -0
- data/ext/db-index/src/hnsw.rs +42 -0
- data/ext/db-index/src/lib.rs +3 -0
- data/ext/gqlitedb/Cargo.toml +19 -8
- data/ext/gqlitedb/benches/common/pokec.rs +62 -2
- data/ext/gqlitedb/benches/pokec_divan.rs +66 -2
- data/ext/gqlitedb/benches/pokec_iai.rs +60 -3
- data/ext/gqlitedb/release.toml +2 -2
- data/ext/gqlitedb/src/aggregators/arithmetic.rs +3 -1
- data/ext/gqlitedb/src/aggregators/containers.rs +1 -1
- data/ext/gqlitedb/src/aggregators/stats.rs +21 -12
- data/ext/gqlitedb/src/capi.rs +20 -24
- data/ext/gqlitedb/src/compiler/expression_analyser.rs +28 -20
- data/ext/gqlitedb/src/compiler/variables_manager.rs +104 -38
- data/ext/gqlitedb/src/compiler.rs +506 -227
- data/ext/gqlitedb/src/connection.rs +151 -11
- data/ext/gqlitedb/src/consts.rs +1 -2
- data/ext/gqlitedb/src/error.rs +123 -61
- data/ext/gqlitedb/src/functions/common.rs +64 -0
- data/ext/gqlitedb/src/functions/containers.rs +2 -46
- data/ext/gqlitedb/src/functions/edge.rs +1 -1
- data/ext/gqlitedb/src/functions/math.rs +87 -15
- data/ext/gqlitedb/src/functions/node.rs +1 -1
- data/ext/gqlitedb/src/functions/path.rs +48 -11
- data/ext/gqlitedb/src/functions/scalar.rs +47 -4
- data/ext/gqlitedb/src/functions/string.rs +123 -1
- data/ext/gqlitedb/src/functions/value.rs +1 -1
- data/ext/gqlitedb/src/functions.rs +165 -28
- data/ext/gqlitedb/src/graph.rs +1 -9
- data/ext/gqlitedb/src/interpreter/evaluators.rs +968 -130
- data/ext/gqlitedb/src/interpreter/instructions.rs +39 -2
- data/ext/gqlitedb/src/lib.rs +5 -4
- data/ext/gqlitedb/src/parser/gql.pest +1 -1
- data/ext/gqlitedb/src/planner.rs +329 -0
- data/ext/gqlitedb/src/prelude.rs +3 -3
- data/ext/gqlitedb/src/store/pgrx.rs +1 -1
- data/ext/gqlitedb/src/store/postgres.rs +742 -16
- data/ext/gqlitedb/src/store/redb/hnsw_store.rs +702 -0
- data/ext/gqlitedb/src/store/redb/index.rs +274 -0
- data/ext/gqlitedb/src/store/redb.rs +1268 -113
- data/ext/gqlitedb/src/store/sqlbase/sqlmetadata.rs +28 -0
- data/ext/gqlitedb/src/store/sqlbase/sqlstore.rs +103 -0
- data/ext/gqlitedb/src/store/sqlbase.rs +146 -16
- data/ext/gqlitedb/src/store/sqlite.rs +576 -14
- data/ext/gqlitedb/src/store/vector_extract.rs +56 -0
- data/ext/gqlitedb/src/store.rs +140 -29
- data/ext/gqlitedb/src/tests/compiler.rs +207 -10
- data/ext/gqlitedb/src/tests/connection/postgres.rs +38 -3
- data/ext/gqlitedb/src/tests/connection/redb.rs +23 -0
- data/ext/gqlitedb/src/tests/connection/sqlite.rs +31 -0
- data/ext/gqlitedb/src/tests/connection.rs +61 -1
- data/ext/gqlitedb/src/tests/evaluators.rs +162 -7
- data/ext/gqlitedb/src/tests/parser.rs +54 -23
- data/ext/gqlitedb/src/tests/planner.rs +511 -0
- data/ext/gqlitedb/src/tests/store/postgres.rs +7 -0
- data/ext/gqlitedb/src/tests/store/redb.rs +8 -0
- data/ext/gqlitedb/src/tests/store/sqlite.rs +8 -0
- data/ext/gqlitedb/src/tests/store/vector_index/postgres.rs +182 -0
- data/ext/gqlitedb/src/tests/store/vector_index/redb.rs +386 -0
- data/ext/gqlitedb/src/tests/store/vector_index/sqlite.rs +166 -0
- data/ext/gqlitedb/src/tests/store/vector_index.rs +2313 -0
- data/ext/gqlitedb/src/tests/store.rs +78 -14
- data/ext/gqlitedb/src/tests/templates/ast.rs +92 -16
- data/ext/gqlitedb/src/tests/templates/programs.rs +14 -7
- data/ext/gqlitedb/src/tests.rs +15 -9
- data/ext/gqlitedb/src/utils.rs +6 -1
- data/ext/gqlitedb/src/value/compare.rs +61 -3
- data/ext/gqlitedb/src/value.rs +136 -7
- data/ext/gqlitedb/templates/sql/postgres/metadata_delete.sql +1 -0
- data/ext/gqlitedb/templates/sql/postgres/node_select.sql +1 -1
- data/ext/gqlitedb/templates/sql/sqlite/metadata_delete.sql +1 -0
- data/ext/gqliterb/src/lib.rs +53 -8
- data/ext/gqlparser/Cargo.toml +25 -0
- data/ext/gqlparser/README.MD +9 -0
- data/ext/gqlparser/benches/pokec_divan.rs +34 -0
- data/ext/gqlparser/src/common.rs +69 -0
- data/ext/gqlparser/src/gqls/ast.rs +69 -0
- data/ext/gqlparser/src/gqls/constraint.rs +79 -0
- data/ext/gqlparser/src/gqls/error.rs +22 -0
- data/ext/gqlparser/src/gqls/parser.rs +813 -0
- data/ext/gqlparser/src/gqls/prelude.rs +8 -0
- data/ext/gqlparser/src/gqls/properties.rs +115 -0
- data/ext/gqlparser/src/gqls/resolve.rs +207 -0
- data/ext/gqlparser/src/gqls.rs +265 -0
- data/ext/gqlparser/src/lib.rs +7 -0
- data/ext/gqlparser/src/oc/ast.rs +680 -0
- data/ext/gqlparser/src/oc/error.rs +172 -0
- data/ext/gqlparser/src/oc/lexer.rs +429 -0
- data/ext/gqlparser/src/oc/parser/tests.rs +2284 -0
- data/ext/gqlparser/src/oc/parser.rs +2005 -0
- data/ext/gqlparser/src/oc.rs +33 -0
- data/ext/gqlparser/src/prelude.rs +3 -0
- data/ext/graphcore/Cargo.toml +1 -0
- data/ext/graphcore/src/error.rs +26 -0
- data/ext/graphcore/src/graph.rs +177 -51
- data/ext/graphcore/src/lib.rs +4 -2
- data/ext/graphcore/src/open_cypher.rs +12 -0
- data/ext/graphcore/src/table.rs +50 -2
- data/ext/graphcore/src/timestamp.rs +127 -104
- data/ext/graphcore/src/value/tensor.rs +739 -0
- data/ext/graphcore/src/value/value_map.rs +1 -1
- data/ext/graphcore/src/value.rs +343 -19
- metadata +93 -28
- data/ext/gqlitedb/src/parser/ast.rs +0 -604
- data/ext/gqlitedb/src/parser/parser_impl.rs +0 -1213
- data/ext/gqlitedb/src/parser.rs +0 -4
|
@@ -0,0 +1,702 @@
|
|
|
1
|
+
use db_index::hnsw::id::NodeId;
|
|
2
|
+
use redb::{ReadableTable, Table, TableDefinition};
|
|
3
|
+
|
|
4
|
+
use crate::{graph, prelude::*};
|
|
5
|
+
|
|
6
|
+
/// Convert gqlitedb errors to hnsw_stable errors for trait bridge
|
|
7
|
+
fn store_error(e: impl std::fmt::Display) -> db_index::hnsw::Error
|
|
8
|
+
{
|
|
9
|
+
db_index::hnsw::Error::StoreError(e.to_string())
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/// Simple metadata for tracking next_node_id and entry point within the dedicated meta table.
|
|
13
|
+
/// This is a minimal version until the full HnswMetaBlob serialization is available.
|
|
14
|
+
#[derive(serde::Serialize, serde::Deserialize, Clone)]
|
|
15
|
+
pub(crate) struct SimpleMetaBlob
|
|
16
|
+
{
|
|
17
|
+
pub next_node_id: u32,
|
|
18
|
+
pub entry_point: Option<u32>,
|
|
19
|
+
pub max_level: i32,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/// Manages the five table definitions for one HNSW index.
|
|
23
|
+
/// Table names are computed from (graph_name, index_name) at construction and stored as String fields.
|
|
24
|
+
pub(crate) struct HnswTableSet
|
|
25
|
+
{
|
|
26
|
+
adj_table: String,
|
|
27
|
+
vec_table: String,
|
|
28
|
+
key_table: String,
|
|
29
|
+
key_rev_table: String,
|
|
30
|
+
meta_table: String,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
impl HnswTableSet
|
|
34
|
+
{
|
|
35
|
+
pub(crate) fn new(graph_name: &str, index_name: &str) -> Self
|
|
36
|
+
{
|
|
37
|
+
Self {
|
|
38
|
+
adj_table: format!("__{graph_name}__hnsw_{index_name}_adj"),
|
|
39
|
+
vec_table: format!("__{graph_name}__hnsw_{index_name}_vec"),
|
|
40
|
+
key_table: format!("__{graph_name}__hnsw_{index_name}_key"),
|
|
41
|
+
key_rev_table: format!("__{graph_name}__hnsw_{index_name}_rkey"),
|
|
42
|
+
meta_table: format!("__{graph_name}__hnsw_{index_name}_meta"),
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// Returns the table definition for adjacency layer data.
|
|
47
|
+
/// Uses Vec<u8> instead of &'static [u8] to allow arbitrary runtime byte slices
|
|
48
|
+
/// to be stored with proper lifetime semantics and ownership transfer into redb.
|
|
49
|
+
pub(crate) fn adj(&self) -> TableDefinition<'_, u32, Vec<u8>>
|
|
50
|
+
{
|
|
51
|
+
TableDefinition::new(&self.adj_table)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/// Returns the table definition for vector data.
|
|
55
|
+
/// Uses Vec<u8> for the same lifetime/ownership reasons as adj().
|
|
56
|
+
pub(crate) fn vec(&self) -> TableDefinition<'_, u32, Vec<u8>>
|
|
57
|
+
{
|
|
58
|
+
TableDefinition::new(&self.vec_table)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
pub(crate) fn key(&self) -> TableDefinition<'_, u32, u128>
|
|
62
|
+
{
|
|
63
|
+
TableDefinition::new(&self.key_table)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
pub(crate) fn key_rev(&self) -> TableDefinition<'_, u128, u32>
|
|
67
|
+
{
|
|
68
|
+
TableDefinition::new(&self.key_rev_table)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
pub(crate) fn meta(&self) -> TableDefinition<'_, (), Vec<u8>>
|
|
72
|
+
{
|
|
73
|
+
TableDefinition::new(&self.meta_table)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/// Write adapter for HNSW graph operations.
|
|
78
|
+
/// Owns five redb Table handles directly from a WriteTransaction.
|
|
79
|
+
/// `GraphStoreWrite<graph::Key>` requires `&mut self` for write operations.
|
|
80
|
+
pub(crate) struct RedbGraphStore<'txn>
|
|
81
|
+
{
|
|
82
|
+
adj: Table<'txn, u32, Vec<u8>>,
|
|
83
|
+
key: Table<'txn, u32, u128>,
|
|
84
|
+
key_rev: Table<'txn, u128, u32>,
|
|
85
|
+
meta: Table<'txn, (), Vec<u8>>,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
impl<'txn> RedbGraphStore<'txn>
|
|
89
|
+
{
|
|
90
|
+
pub(crate) fn open(txn: &'txn super::WriteTransaction, tables: &HnswTableSet) -> Result<Self>
|
|
91
|
+
{
|
|
92
|
+
let adj = txn.write_transaction.open_table(tables.adj())?;
|
|
93
|
+
let key = txn.write_transaction.open_table(tables.key())?;
|
|
94
|
+
let key_rev = txn.write_transaction.open_table(tables.key_rev())?;
|
|
95
|
+
let meta = txn.write_transaction.open_table(tables.meta())?;
|
|
96
|
+
|
|
97
|
+
Ok(Self {
|
|
98
|
+
adj,
|
|
99
|
+
key,
|
|
100
|
+
key_rev,
|
|
101
|
+
meta,
|
|
102
|
+
})
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
fn read_meta(&self) -> Result<SimpleMetaBlob>
|
|
106
|
+
{
|
|
107
|
+
let bytes = self
|
|
108
|
+
.meta
|
|
109
|
+
.get(&())?
|
|
110
|
+
.map(|g| g.value().to_vec())
|
|
111
|
+
.ok_or_else(|| {
|
|
112
|
+
crate::error::Error::StoreError(crate::error::StoreError::InvalidFormat(
|
|
113
|
+
"meta row missing".into(),
|
|
114
|
+
))
|
|
115
|
+
})?;
|
|
116
|
+
ciborium::from_reader(bytes.as_slice()).map_err(|e| crate::error::Error::Internal(e.into()))
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
pub(crate) fn write_meta(&mut self, meta: &SimpleMetaBlob) -> Result<()>
|
|
120
|
+
{
|
|
121
|
+
let mut bytes = Vec::new();
|
|
122
|
+
ciborium::into_writer(meta, &mut bytes).map_err(|e| crate::error::Error::Internal(e.into()))?;
|
|
123
|
+
self.meta.insert((), bytes)?;
|
|
124
|
+
Ok(())
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
impl<'txn> db_index::hnsw::graph_store::GraphStoreRead<graph::Key> for RedbGraphStore<'txn>
|
|
129
|
+
{
|
|
130
|
+
type Error = db_index::hnsw::Error;
|
|
131
|
+
|
|
132
|
+
fn node_count(&self) -> std::result::Result<usize, db_index::hnsw::Error>
|
|
133
|
+
{
|
|
134
|
+
Ok(self.read_meta().map_err(store_error)?.next_node_id as usize)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
fn entry_point(&self) -> std::result::Result<Option<(NodeId, i32)>, db_index::hnsw::Error>
|
|
138
|
+
{
|
|
139
|
+
let m = self.read_meta().map_err(store_error)?;
|
|
140
|
+
Ok(
|
|
141
|
+
m.entry_point
|
|
142
|
+
.map(|raw| (NodeId::from_u32(raw), m.max_level)),
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
fn contains(&self, node: NodeId) -> std::result::Result<bool, db_index::hnsw::Error>
|
|
147
|
+
{
|
|
148
|
+
Ok(self.adj.get(&node.as_u32()).map_err(store_error)?.is_some())
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
fn key_of(&self, node: NodeId) -> std::result::Result<Option<graph::Key>, db_index::hnsw::Error>
|
|
152
|
+
{
|
|
153
|
+
Ok(
|
|
154
|
+
self
|
|
155
|
+
.key
|
|
156
|
+
.get(&node.as_u32())
|
|
157
|
+
.map_err(store_error)?
|
|
158
|
+
.map(|g| graph::Key::new(g.value())),
|
|
159
|
+
)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
fn node_of(&self, key: &graph::Key)
|
|
163
|
+
-> std::result::Result<Option<NodeId>, db_index::hnsw::Error>
|
|
164
|
+
{
|
|
165
|
+
Ok(
|
|
166
|
+
self
|
|
167
|
+
.key_rev
|
|
168
|
+
.get(&key.uuid())
|
|
169
|
+
.map_err(store_error)?
|
|
170
|
+
.map(|g| NodeId::from_u32(g.value())),
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
fn level(&self, node: NodeId) -> std::result::Result<i32, db_index::hnsw::Error>
|
|
175
|
+
{
|
|
176
|
+
let bytes = self
|
|
177
|
+
.adj
|
|
178
|
+
.get(&node.as_u32())
|
|
179
|
+
.map_err(store_error)?
|
|
180
|
+
.ok_or_else(|| store_error("level of absent node".to_string()))?;
|
|
181
|
+
Ok(bytes.value()[0] as i32)
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
fn neighbors(
|
|
185
|
+
&self,
|
|
186
|
+
node: NodeId,
|
|
187
|
+
level: usize,
|
|
188
|
+
out: &mut Vec<NodeId>,
|
|
189
|
+
) -> std::result::Result<(), db_index::hnsw::Error>
|
|
190
|
+
{
|
|
191
|
+
out.clear();
|
|
192
|
+
let Some(bytes) = self.adj.get(&node.as_u32()).map_err(store_error)?
|
|
193
|
+
else
|
|
194
|
+
{
|
|
195
|
+
return Ok(());
|
|
196
|
+
};
|
|
197
|
+
let (_, layers) = db_index::hnsw::store::decode_adjacency_layers(bytes.value().as_slice())
|
|
198
|
+
.map_err(store_error)?;
|
|
199
|
+
if let Some(layer) = layers.get(level)
|
|
200
|
+
{
|
|
201
|
+
out.extend(layer.iter().map(|&id| NodeId::from_u32(id)));
|
|
202
|
+
}
|
|
203
|
+
Ok(())
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
impl<'txn> db_index::hnsw::graph_store::GraphStoreWrite<graph::Key> for RedbGraphStore<'txn>
|
|
208
|
+
{
|
|
209
|
+
fn insert_node(
|
|
210
|
+
&mut self,
|
|
211
|
+
key: graph::Key,
|
|
212
|
+
level: i32,
|
|
213
|
+
) -> std::result::Result<NodeId, db_index::hnsw::Error>
|
|
214
|
+
{
|
|
215
|
+
let mut meta = self.read_meta().map_err(store_error)?;
|
|
216
|
+
let id = NodeId::from_u32(meta.next_node_id);
|
|
217
|
+
meta.next_node_id += 1; // monotonic; the sole enforcement of never-reuse
|
|
218
|
+
self.write_meta(&meta).map_err(store_error)?;
|
|
219
|
+
self
|
|
220
|
+
.key
|
|
221
|
+
.insert(id.as_u32(), key.uuid())
|
|
222
|
+
.map_err(store_error)?;
|
|
223
|
+
self
|
|
224
|
+
.key_rev
|
|
225
|
+
.insert(key.uuid(), id.as_u32())
|
|
226
|
+
.map_err(store_error)?;
|
|
227
|
+
let empty_layers = vec![Vec::new(); (level as usize) + 1];
|
|
228
|
+
self
|
|
229
|
+
.adj
|
|
230
|
+
.insert(
|
|
231
|
+
id.as_u32(),
|
|
232
|
+
db_index::hnsw::store::encode_adjacency(level as u8, &empty_layers),
|
|
233
|
+
)
|
|
234
|
+
.map_err(store_error)?;
|
|
235
|
+
Ok(id)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
fn set_neighbors(
|
|
239
|
+
&mut self,
|
|
240
|
+
node: NodeId,
|
|
241
|
+
level: usize,
|
|
242
|
+
nbrs: &[NodeId],
|
|
243
|
+
) -> std::result::Result<(), db_index::hnsw::Error>
|
|
244
|
+
{
|
|
245
|
+
let cur = self
|
|
246
|
+
.adj
|
|
247
|
+
.get(&node.as_u32())
|
|
248
|
+
.map_err(store_error)?
|
|
249
|
+
.ok_or_else(|| store_error("set_neighbors on absent node"))?;
|
|
250
|
+
let (lvl, mut layers) = db_index::hnsw::store::decode_adjacency_layers(cur.value().as_slice())
|
|
251
|
+
.map_err(store_error)?;
|
|
252
|
+
drop(cur);
|
|
253
|
+
if layers.len() <= level
|
|
254
|
+
{
|
|
255
|
+
layers.resize(level + 1, Vec::new());
|
|
256
|
+
}
|
|
257
|
+
layers[level] = nbrs.iter().map(|n| n.as_u32()).collect();
|
|
258
|
+
self
|
|
259
|
+
.adj
|
|
260
|
+
.insert(
|
|
261
|
+
node.as_u32(),
|
|
262
|
+
db_index::hnsw::store::encode_adjacency(lvl, &layers),
|
|
263
|
+
)
|
|
264
|
+
.map_err(store_error)?;
|
|
265
|
+
Ok(())
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
fn set_entry_point(
|
|
269
|
+
&mut self,
|
|
270
|
+
ep: Option<(NodeId, i32)>,
|
|
271
|
+
) -> std::result::Result<(), db_index::hnsw::Error>
|
|
272
|
+
{
|
|
273
|
+
let mut meta = self.read_meta().map_err(store_error)?;
|
|
274
|
+
meta.entry_point = ep.map(|(n, _)| n.as_u32());
|
|
275
|
+
meta.max_level = ep.map_or(-1, |(_, l)| l);
|
|
276
|
+
self.write_meta(&meta).map_err(store_error)
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
fn remove_node(&mut self, node: NodeId) -> std::result::Result<(), db_index::hnsw::Error>
|
|
280
|
+
{
|
|
281
|
+
if let Some(k) = self.key.remove(node.as_u32()).map_err(store_error)?
|
|
282
|
+
{
|
|
283
|
+
self.key_rev.remove(k.value()).map_err(store_error)?;
|
|
284
|
+
}
|
|
285
|
+
self.adj.remove(node.as_u32()).map_err(store_error)?;
|
|
286
|
+
Ok(()) // next_node_id in meta untouched
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/// Write adapter for HNSW vector operations.
|
|
291
|
+
/// Uses Mutex for interior mutability (required by VectorStoreMut trait taking &self).
|
|
292
|
+
/// Safe because write transactions are exclusive, so contention never occurs.
|
|
293
|
+
pub(crate) struct RedbVectorStore<'txn>
|
|
294
|
+
{
|
|
295
|
+
vec: std::sync::Mutex<Table<'txn, u32, Vec<u8>>>,
|
|
296
|
+
dim: usize,
|
|
297
|
+
encoding: db_index::hnsw::store::VectorEncoding,
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
impl<'txn> RedbVectorStore<'txn>
|
|
301
|
+
{
|
|
302
|
+
pub(crate) fn open(
|
|
303
|
+
txn: &'txn super::WriteTransaction,
|
|
304
|
+
tables: &HnswTableSet,
|
|
305
|
+
dim: usize,
|
|
306
|
+
encoding: db_index::hnsw::store::VectorEncoding,
|
|
307
|
+
) -> Result<Self>
|
|
308
|
+
{
|
|
309
|
+
let vec = txn.write_transaction.open_table(tables.vec())?;
|
|
310
|
+
Ok(Self {
|
|
311
|
+
vec: std::sync::Mutex::new(vec),
|
|
312
|
+
dim,
|
|
313
|
+
encoding,
|
|
314
|
+
})
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
pub(crate) fn delete(&self, id: NodeId) -> Result<()>
|
|
318
|
+
{
|
|
319
|
+
let id_u32 = id.as_u32();
|
|
320
|
+
let mut table = self.vec.lock().map_err(|_| {
|
|
321
|
+
crate::error::Error::Internal(crate::error::InternalError::Unimplemented(
|
|
322
|
+
"failed to acquire table lock for vector deletion",
|
|
323
|
+
))
|
|
324
|
+
})?;
|
|
325
|
+
table.remove(id_u32)?;
|
|
326
|
+
Ok(())
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
impl<'txn> db_index::hnsw::vectors::VectorStore for RedbVectorStore<'txn>
|
|
331
|
+
{
|
|
332
|
+
type Family = db_index::hnsw::vector::Dense<f32>;
|
|
333
|
+
type Vector<'a>
|
|
334
|
+
= db_index::hnsw::vectors::ArcVector<f32>
|
|
335
|
+
where
|
|
336
|
+
Self: 'a;
|
|
337
|
+
|
|
338
|
+
fn dim(&self) -> usize
|
|
339
|
+
{
|
|
340
|
+
self.dim
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
fn vector(&self, id: NodeId) -> Option<Self::Vector<'_>>
|
|
344
|
+
{
|
|
345
|
+
let id_u32 = id.as_u32();
|
|
346
|
+
let table = self.vec.lock().ok()?;
|
|
347
|
+
let bytes = table.get(&id_u32).ok()??.value().to_vec();
|
|
348
|
+
let floats = db_index::hnsw::store::decode_vector(&self.encoding, &bytes, self.dim)
|
|
349
|
+
.map_err(|_| ())
|
|
350
|
+
.ok()?;
|
|
351
|
+
Some(db_index::hnsw::vectors::ArcVector::new(
|
|
352
|
+
floats.into_boxed_slice(),
|
|
353
|
+
))
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
impl<'txn> db_index::hnsw::vectors::VectorStoreMut for RedbVectorStore<'txn>
|
|
358
|
+
{
|
|
359
|
+
fn set(
|
|
360
|
+
&self,
|
|
361
|
+
id: NodeId,
|
|
362
|
+
vector: <Self::Family as db_index::hnsw::vector::VectorFamily>::Ref<'_>,
|
|
363
|
+
) -> db_index::hnsw::error::Result<()>
|
|
364
|
+
{
|
|
365
|
+
let bytes = db_index::hnsw::store::encode_vector(&self.encoding, vector)?;
|
|
366
|
+
let id_u32 = id.as_u32();
|
|
367
|
+
let mut table = self
|
|
368
|
+
.vec
|
|
369
|
+
.lock()
|
|
370
|
+
.map_err(|_| db_index::hnsw::Error::StoreError("failed to acquire table lock".into()))?;
|
|
371
|
+
table.insert(id_u32, bytes).map_err(store_error)?;
|
|
372
|
+
Ok(())
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/// Read-only HNSW graph store backed by redb read-only tables.
|
|
377
|
+
///
|
|
378
|
+
/// Implements `GraphStoreRead<graph::Key>` for loading HNSW graph structure from disk.
|
|
379
|
+
pub(crate) struct RedbGraphStoreRead<'txn>
|
|
380
|
+
{
|
|
381
|
+
adj: redb::ReadOnlyTable<u32, Vec<u8>>,
|
|
382
|
+
key: redb::ReadOnlyTable<u32, u128>,
|
|
383
|
+
key_rev: redb::ReadOnlyTable<u128, u32>,
|
|
384
|
+
meta: redb::ReadOnlyTable<(), Vec<u8>>,
|
|
385
|
+
_marker: std::marker::PhantomData<&'txn ()>,
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
impl<'txn> RedbGraphStoreRead<'txn>
|
|
389
|
+
{
|
|
390
|
+
/// Opens read-only tables for an HNSW index.
|
|
391
|
+
///
|
|
392
|
+
/// Returns `Ok(None)` if the adjacency table does not exist,
|
|
393
|
+
/// indicating the index has not yet been created (e.g., new database).
|
|
394
|
+
pub(crate) fn open(
|
|
395
|
+
txn: &'txn redb::ReadTransaction,
|
|
396
|
+
tables: &HnswTableSet,
|
|
397
|
+
) -> Result<Option<Self>>
|
|
398
|
+
{
|
|
399
|
+
let adj = match txn.open_table(tables.adj())
|
|
400
|
+
{
|
|
401
|
+
Ok(t) => t,
|
|
402
|
+
Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
|
|
403
|
+
Err(e) => return Err(e.into()),
|
|
404
|
+
};
|
|
405
|
+
|
|
406
|
+
Ok(Some(Self {
|
|
407
|
+
adj,
|
|
408
|
+
key: txn.open_table(tables.key())?,
|
|
409
|
+
key_rev: txn.open_table(tables.key_rev())?,
|
|
410
|
+
meta: txn.open_table(tables.meta())?,
|
|
411
|
+
_marker: std::marker::PhantomData,
|
|
412
|
+
}))
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
fn read_meta(&self) -> Result<SimpleMetaBlob>
|
|
416
|
+
{
|
|
417
|
+
let bytes = self
|
|
418
|
+
.meta
|
|
419
|
+
.get(&())?
|
|
420
|
+
.map(|g| g.value().to_vec())
|
|
421
|
+
.ok_or_else(|| {
|
|
422
|
+
crate::error::Error::StoreError(crate::error::StoreError::InvalidFormat(
|
|
423
|
+
"meta row missing".into(),
|
|
424
|
+
))
|
|
425
|
+
})?;
|
|
426
|
+
ciborium::from_reader(bytes.as_slice()).map_err(|e| crate::error::Error::Internal(e.into()))
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
impl<'txn> db_index::hnsw::graph_store::GraphStoreRead<graph::Key> for RedbGraphStoreRead<'txn>
|
|
431
|
+
{
|
|
432
|
+
type Error = db_index::hnsw::Error;
|
|
433
|
+
|
|
434
|
+
fn node_count(&self) -> std::result::Result<usize, db_index::hnsw::Error>
|
|
435
|
+
{
|
|
436
|
+
let count = self.read_meta().map_err(store_error)?.next_node_id as usize;
|
|
437
|
+
Ok(count)
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
fn entry_point(&self) -> std::result::Result<Option<(NodeId, i32)>, db_index::hnsw::Error>
|
|
441
|
+
{
|
|
442
|
+
let m = self.read_meta().map_err(store_error)?;
|
|
443
|
+
Ok(
|
|
444
|
+
m.entry_point
|
|
445
|
+
.map(|raw| (NodeId::from_u32(raw), m.max_level)),
|
|
446
|
+
)
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
fn contains(&self, node: NodeId) -> std::result::Result<bool, db_index::hnsw::Error>
|
|
450
|
+
{
|
|
451
|
+
Ok(self.adj.get(&node.as_u32()).map_err(store_error)?.is_some())
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
fn key_of(&self, node: NodeId) -> std::result::Result<Option<graph::Key>, db_index::hnsw::Error>
|
|
455
|
+
{
|
|
456
|
+
Ok(
|
|
457
|
+
self
|
|
458
|
+
.key
|
|
459
|
+
.get(&node.as_u32())
|
|
460
|
+
.map_err(store_error)?
|
|
461
|
+
.map(|g| graph::Key::new(g.value())),
|
|
462
|
+
)
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
fn node_of(&self, key: &graph::Key)
|
|
466
|
+
-> std::result::Result<Option<NodeId>, db_index::hnsw::Error>
|
|
467
|
+
{
|
|
468
|
+
Ok(
|
|
469
|
+
self
|
|
470
|
+
.key_rev
|
|
471
|
+
.get(&key.uuid())
|
|
472
|
+
.map_err(store_error)?
|
|
473
|
+
.map(|g| NodeId::from_u32(g.value())),
|
|
474
|
+
)
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
fn level(&self, node: NodeId) -> std::result::Result<i32, db_index::hnsw::Error>
|
|
478
|
+
{
|
|
479
|
+
let bytes = self
|
|
480
|
+
.adj
|
|
481
|
+
.get(&node.as_u32())
|
|
482
|
+
.map_err(store_error)?
|
|
483
|
+
.ok_or_else(|| store_error("level of absent node"))?;
|
|
484
|
+
Ok(bytes.value()[0] as i32)
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
fn neighbors(
|
|
488
|
+
&self,
|
|
489
|
+
node: NodeId,
|
|
490
|
+
level: usize,
|
|
491
|
+
out: &mut Vec<NodeId>,
|
|
492
|
+
) -> std::result::Result<(), db_index::hnsw::Error>
|
|
493
|
+
{
|
|
494
|
+
out.clear();
|
|
495
|
+
let Some(bytes) = self.adj.get(&node.as_u32()).map_err(store_error)?
|
|
496
|
+
else
|
|
497
|
+
{
|
|
498
|
+
return Ok(());
|
|
499
|
+
};
|
|
500
|
+
let (_, layers) = db_index::hnsw::store::decode_adjacency_layers(bytes.value().as_slice())
|
|
501
|
+
.map_err(store_error)?;
|
|
502
|
+
if let Some(layer) = layers.get(level)
|
|
503
|
+
{
|
|
504
|
+
out.extend(layer.iter().map(|&id| NodeId::from_u32(id)));
|
|
505
|
+
}
|
|
506
|
+
Ok(())
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/// Read-only HNSW vector store backed by redb read-only tables.
|
|
511
|
+
///
|
|
512
|
+
/// Implements `VectorStore` for loading HNSW vectors from disk.
|
|
513
|
+
pub(crate) struct RedbVectorStoreRead<'txn>
|
|
514
|
+
{
|
|
515
|
+
vec: redb::ReadOnlyTable<u32, Vec<u8>>,
|
|
516
|
+
dim: usize,
|
|
517
|
+
encoding: db_index::hnsw::store::VectorEncoding,
|
|
518
|
+
_marker: std::marker::PhantomData<&'txn ()>,
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
impl<'txn> RedbVectorStoreRead<'txn>
|
|
522
|
+
{
|
|
523
|
+
/// Opens read-only tables for an HNSW vector index.
|
|
524
|
+
pub(crate) fn open(
|
|
525
|
+
txn: &'txn redb::ReadTransaction,
|
|
526
|
+
tables: &HnswTableSet,
|
|
527
|
+
dim: usize,
|
|
528
|
+
encoding: db_index::hnsw::store::VectorEncoding,
|
|
529
|
+
) -> Result<Option<Self>>
|
|
530
|
+
{
|
|
531
|
+
let vec = match txn.open_table(tables.vec())
|
|
532
|
+
{
|
|
533
|
+
Ok(t) => t,
|
|
534
|
+
Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
|
|
535
|
+
Err(e) => return Err(e.into()),
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
Ok(Some(Self {
|
|
539
|
+
vec,
|
|
540
|
+
dim,
|
|
541
|
+
encoding,
|
|
542
|
+
_marker: std::marker::PhantomData,
|
|
543
|
+
}))
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
impl<'txn> db_index::hnsw::vectors::VectorStore for RedbVectorStoreRead<'txn>
|
|
548
|
+
{
|
|
549
|
+
type Family = db_index::hnsw::vector::Dense<f32>;
|
|
550
|
+
type Vector<'a>
|
|
551
|
+
= db_index::hnsw::vectors::ArcVector<f32>
|
|
552
|
+
where
|
|
553
|
+
Self: 'a;
|
|
554
|
+
|
|
555
|
+
fn dim(&self) -> usize
|
|
556
|
+
{
|
|
557
|
+
self.dim
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
fn vector(&self, id: NodeId) -> Option<Self::Vector<'_>>
|
|
561
|
+
{
|
|
562
|
+
let id_u32 = id.as_u32();
|
|
563
|
+
let bytes = self.vec.get(&id_u32).ok()??.value().to_vec();
|
|
564
|
+
let floats = db_index::hnsw::store::decode_vector(&self.encoding, &bytes, self.dim)
|
|
565
|
+
.map_err(|_| ())
|
|
566
|
+
.ok()?;
|
|
567
|
+
Some(db_index::hnsw::vectors::ArcVector::new(
|
|
568
|
+
floats.into_boxed_slice(),
|
|
569
|
+
))
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
#[cfg(test)]
|
|
574
|
+
mod tests
|
|
575
|
+
{
|
|
576
|
+
use super::*;
|
|
577
|
+
|
|
578
|
+
#[test]
|
|
579
|
+
fn test_hnsw_table_set_construction_and_names()
|
|
580
|
+
{
|
|
581
|
+
let hnsw_set = HnswTableSet::new("mygraph", "embedding");
|
|
582
|
+
|
|
583
|
+
assert_eq!(hnsw_set.adj_table, "__mygraph__hnsw_embedding_adj");
|
|
584
|
+
assert_eq!(hnsw_set.vec_table, "__mygraph__hnsw_embedding_vec");
|
|
585
|
+
assert_eq!(hnsw_set.key_table, "__mygraph__hnsw_embedding_key");
|
|
586
|
+
assert_eq!(hnsw_set.key_rev_table, "__mygraph__hnsw_embedding_rkey");
|
|
587
|
+
assert_eq!(hnsw_set.meta_table, "__mygraph__hnsw_embedding_meta");
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
#[test]
|
|
591
|
+
fn test_hnsw_table_set_multiple_indices()
|
|
592
|
+
{
|
|
593
|
+
let index1 = HnswTableSet::new("graph1", "idx1");
|
|
594
|
+
let index2 = HnswTableSet::new("graph1", "idx2");
|
|
595
|
+
let index3 = HnswTableSet::new("graph2", "idx1");
|
|
596
|
+
|
|
597
|
+
assert_eq!(index1.adj_table, "__graph1__hnsw_idx1_adj");
|
|
598
|
+
assert_eq!(index2.adj_table, "__graph1__hnsw_idx2_adj");
|
|
599
|
+
assert_eq!(index3.adj_table, "__graph2__hnsw_idx1_adj");
|
|
600
|
+
|
|
601
|
+
assert_eq!(index1.key_rev_table, "__graph1__hnsw_idx1_rkey");
|
|
602
|
+
assert_eq!(index2.key_rev_table, "__graph1__hnsw_idx2_rkey");
|
|
603
|
+
assert_eq!(index3.key_rev_table, "__graph2__hnsw_idx1_rkey");
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
#[cfg(test)]
|
|
608
|
+
mod graph_store_tests
|
|
609
|
+
{
|
|
610
|
+
use super::*;
|
|
611
|
+
use crate::graph;
|
|
612
|
+
use db_index::hnsw::graph_store::{GraphStoreRead, GraphStoreWrite};
|
|
613
|
+
use db_index::hnsw::id::NodeId;
|
|
614
|
+
use redb::ReadableDatabase;
|
|
615
|
+
|
|
616
|
+
fn create_test_db() -> (redb::Database, ccutils::temporary::TemporaryFile)
|
|
617
|
+
{
|
|
618
|
+
let db_path = crate::tests::create_tmp_file();
|
|
619
|
+
let db = redb::Database::create(db_path.path()).expect("Failed to create test database");
|
|
620
|
+
(db, db_path)
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
#[test]
|
|
624
|
+
fn test_graph_store_metadata_roundtrip()
|
|
625
|
+
{
|
|
626
|
+
let (db, _db_path) = create_test_db();
|
|
627
|
+
|
|
628
|
+
// First transaction: write data
|
|
629
|
+
{
|
|
630
|
+
let write_tx = db.begin_write().expect("Failed to begin write transaction");
|
|
631
|
+
let txn = super::super::WriteTransaction {
|
|
632
|
+
write_transaction: write_tx,
|
|
633
|
+
};
|
|
634
|
+
let tables = HnswTableSet::new("g", "idx_meta_roundtrip");
|
|
635
|
+
let mut store = RedbGraphStore::open(&txn, &tables).unwrap();
|
|
636
|
+
// Initialize metadata
|
|
637
|
+
let meta = SimpleMetaBlob {
|
|
638
|
+
next_node_id: 0,
|
|
639
|
+
entry_point: None,
|
|
640
|
+
max_level: -1,
|
|
641
|
+
};
|
|
642
|
+
store.write_meta(&meta).unwrap();
|
|
643
|
+
let node = store.insert_node(graph::Key::new(1), 2).unwrap();
|
|
644
|
+
store.set_entry_point(Some((node, 2))).unwrap();
|
|
645
|
+
drop(store);
|
|
646
|
+
// Commit the transaction
|
|
647
|
+
txn.write_transaction.commit().unwrap();
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
// Use a read transaction to verify data
|
|
651
|
+
{
|
|
652
|
+
let read_tx = db.begin_read().expect("Failed to begin read transaction");
|
|
653
|
+
let tables = HnswTableSet::new("g", "idx_meta_roundtrip");
|
|
654
|
+
let store = RedbGraphStoreRead::open(&read_tx, &tables)
|
|
655
|
+
.expect("Failed to open store read")
|
|
656
|
+
.expect("Store does not exist");
|
|
657
|
+
assert_eq!(store.entry_point().unwrap(), Some((NodeId::from_u32(0), 2)));
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
#[test]
|
|
662
|
+
fn test_key_rev_bidirectional_mapping()
|
|
663
|
+
{
|
|
664
|
+
let (db, _db_path) = create_test_db();
|
|
665
|
+
|
|
666
|
+
// First transaction: write keys
|
|
667
|
+
{
|
|
668
|
+
let write_tx = db.begin_write().expect("Failed to begin write transaction");
|
|
669
|
+
let txn = super::super::WriteTransaction {
|
|
670
|
+
write_transaction: write_tx,
|
|
671
|
+
};
|
|
672
|
+
let tables = HnswTableSet::new("g", "idx_key_rev");
|
|
673
|
+
let mut store = RedbGraphStore::open(&txn, &tables).unwrap();
|
|
674
|
+
// Initialize metadata
|
|
675
|
+
let meta = SimpleMetaBlob {
|
|
676
|
+
next_node_id: 0,
|
|
677
|
+
entry_point: None,
|
|
678
|
+
max_level: -1,
|
|
679
|
+
};
|
|
680
|
+
store.write_meta(&meta).unwrap();
|
|
681
|
+
let key = graph::Key::new(999u128);
|
|
682
|
+
let node_id = store.insert_node(key, 0).unwrap();
|
|
683
|
+
assert_eq!(store.key_of(node_id).unwrap(), Some(key));
|
|
684
|
+
assert_eq!(store.node_of(&key).unwrap(), Some(node_id));
|
|
685
|
+
drop(store);
|
|
686
|
+
// Commit the transaction
|
|
687
|
+
txn.write_transaction.commit().unwrap();
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
// Use a read transaction to verify bidirectional mapping
|
|
691
|
+
{
|
|
692
|
+
let read_tx = db.begin_read().expect("Failed to begin read transaction");
|
|
693
|
+
let tables = HnswTableSet::new("g", "idx_key_rev");
|
|
694
|
+
let store = RedbGraphStoreRead::open(&read_tx, &tables)
|
|
695
|
+
.expect("Failed to open store read")
|
|
696
|
+
.expect("Store does not exist");
|
|
697
|
+
let key = graph::Key::new(999u128);
|
|
698
|
+
assert_eq!(store.node_of(&key).unwrap(), Some(NodeId::from_u32(0)));
|
|
699
|
+
assert_eq!(store.key_of(NodeId::from_u32(0)).unwrap(), Some(key));
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
}
|