gqlite 1.5.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/Cargo.toml +10 -4
- data/ext/db-index/Cargo.toml +29 -0
- data/ext/db-index/src/hnsw/error.rs +66 -0
- data/ext/db-index/src/hnsw/graph_store.rs +272 -0
- data/ext/db-index/src/hnsw/id.rs +36 -0
- data/ext/db-index/src/hnsw/implementation.rs +1517 -0
- data/ext/db-index/src/hnsw/kernels.rs +1228 -0
- data/ext/db-index/src/hnsw/metric.rs +244 -0
- data/ext/db-index/src/hnsw/scalar.rs +72 -0
- data/ext/db-index/src/hnsw/simple_store.rs +140 -0
- data/ext/db-index/src/hnsw/store.rs +472 -0
- data/ext/db-index/src/hnsw/vector.rs +105 -0
- data/ext/db-index/src/hnsw/vectors.rs +568 -0
- data/ext/db-index/src/hnsw.rs +42 -0
- data/ext/db-index/src/lib.rs +3 -0
- data/ext/gqlitedb/Cargo.toml +19 -9
- data/ext/gqlitedb/benches/common/pokec.rs +62 -2
- data/ext/gqlitedb/benches/pokec_divan.rs +66 -2
- data/ext/gqlitedb/benches/pokec_iai.rs +60 -3
- data/ext/gqlitedb/release.toml +2 -2
- data/ext/gqlitedb/src/aggregators/arithmetic.rs +3 -1
- data/ext/gqlitedb/src/aggregators/containers.rs +1 -1
- data/ext/gqlitedb/src/aggregators/stats.rs +21 -12
- data/ext/gqlitedb/src/capi.rs +1 -1
- data/ext/gqlitedb/src/compiler/expression_analyser.rs +28 -20
- data/ext/gqlitedb/src/compiler/variables_manager.rs +97 -29
- data/ext/gqlitedb/src/compiler.rs +505 -225
- data/ext/gqlitedb/src/connection.rs +149 -11
- data/ext/gqlitedb/src/consts.rs +1 -2
- data/ext/gqlitedb/src/error.rs +123 -61
- data/ext/gqlitedb/src/functions/common.rs +39 -2
- data/ext/gqlitedb/src/functions/math.rs +8 -3
- data/ext/gqlitedb/src/functions/path.rs +48 -11
- data/ext/gqlitedb/src/functions/value.rs +1 -1
- data/ext/gqlitedb/src/functions.rs +23 -7
- data/ext/gqlitedb/src/graph.rs +1 -9
- data/ext/gqlitedb/src/interpreter/evaluators.rs +968 -130
- data/ext/gqlitedb/src/interpreter/instructions.rs +39 -2
- data/ext/gqlitedb/src/lib.rs +5 -4
- data/ext/gqlitedb/src/planner.rs +329 -0
- data/ext/gqlitedb/src/prelude.rs +3 -3
- data/ext/gqlitedb/src/store/pgrx.rs +1 -1
- data/ext/gqlitedb/src/store/postgres.rs +735 -7
- data/ext/gqlitedb/src/store/redb/hnsw_store.rs +702 -0
- data/ext/gqlitedb/src/store/redb/index.rs +274 -0
- data/ext/gqlitedb/src/store/redb.rs +1268 -113
- data/ext/gqlitedb/src/store/sqlbase/sqlmetadata.rs +28 -0
- data/ext/gqlitedb/src/store/sqlbase/sqlstore.rs +103 -0
- data/ext/gqlitedb/src/store/sqlbase.rs +146 -16
- data/ext/gqlitedb/src/store/sqlite.rs +569 -5
- data/ext/gqlitedb/src/store/vector_extract.rs +56 -0
- data/ext/gqlitedb/src/store.rs +123 -3
- data/ext/gqlitedb/src/tests/compiler.rs +207 -10
- data/ext/gqlitedb/src/tests/connection/postgres.rs +38 -3
- data/ext/gqlitedb/src/tests/connection/redb.rs +23 -0
- data/ext/gqlitedb/src/tests/connection/sqlite.rs +31 -0
- data/ext/gqlitedb/src/tests/connection.rs +61 -1
- data/ext/gqlitedb/src/tests/evaluators.rs +162 -7
- data/ext/gqlitedb/src/tests/parser.rs +54 -23
- data/ext/gqlitedb/src/tests/planner.rs +511 -0
- data/ext/gqlitedb/src/tests/store/postgres.rs +7 -0
- data/ext/gqlitedb/src/tests/store/redb.rs +8 -0
- data/ext/gqlitedb/src/tests/store/sqlite.rs +8 -0
- data/ext/gqlitedb/src/tests/store/vector_index/postgres.rs +182 -0
- data/ext/gqlitedb/src/tests/store/vector_index/redb.rs +386 -0
- data/ext/gqlitedb/src/tests/store/vector_index/sqlite.rs +166 -0
- data/ext/gqlitedb/src/tests/store/vector_index.rs +2313 -0
- data/ext/gqlitedb/src/tests/store.rs +78 -14
- data/ext/gqlitedb/src/tests/templates/ast.rs +92 -16
- data/ext/gqlitedb/src/tests/templates/programs.rs +14 -7
- data/ext/gqlitedb/src/tests.rs +15 -9
- data/ext/gqlitedb/src/utils.rs +6 -1
- data/ext/gqlitedb/src/value/compare.rs +61 -3
- data/ext/gqlitedb/src/value.rs +136 -7
- data/ext/gqlitedb/templates/sql/postgres/metadata_delete.sql +1 -0
- data/ext/gqlitedb/templates/sql/postgres/node_select.sql +1 -1
- data/ext/gqlitedb/templates/sql/sqlite/metadata_delete.sql +1 -0
- data/ext/gqliterb/src/lib.rs +53 -8
- data/ext/gqlparser/Cargo.toml +25 -0
- data/ext/gqlparser/README.MD +9 -0
- data/ext/gqlparser/benches/pokec_divan.rs +34 -0
- data/ext/gqlparser/src/common.rs +69 -0
- data/ext/gqlparser/src/gqls/ast.rs +69 -0
- data/ext/gqlparser/src/gqls/constraint.rs +79 -0
- data/ext/gqlparser/src/gqls/error.rs +22 -0
- data/ext/gqlparser/src/gqls/parser.rs +813 -0
- data/ext/gqlparser/src/gqls/prelude.rs +8 -0
- data/ext/gqlparser/src/gqls/properties.rs +115 -0
- data/ext/gqlparser/src/gqls/resolve.rs +207 -0
- data/ext/gqlparser/src/gqls.rs +265 -0
- data/ext/gqlparser/src/lib.rs +7 -0
- data/ext/gqlparser/src/oc/ast.rs +680 -0
- data/ext/gqlparser/src/oc/error.rs +172 -0
- data/ext/gqlparser/src/oc/lexer.rs +429 -0
- data/ext/gqlparser/src/oc/parser/tests.rs +2284 -0
- data/ext/gqlparser/src/oc/parser.rs +2005 -0
- data/ext/gqlparser/src/oc.rs +33 -0
- data/ext/gqlparser/src/prelude.rs +3 -0
- data/ext/graphcore/Cargo.toml +1 -0
- data/ext/graphcore/src/error.rs +26 -0
- data/ext/graphcore/src/graph.rs +177 -51
- data/ext/graphcore/src/lib.rs +4 -2
- data/ext/graphcore/src/open_cypher.rs +12 -0
- data/ext/graphcore/src/table.rs +50 -2
- data/ext/graphcore/src/timestamp.rs +127 -104
- data/ext/graphcore/src/value/tensor.rs +739 -0
- data/ext/graphcore/src/value/value_map.rs +1 -1
- data/ext/graphcore/src/value.rs +343 -19
- metadata +90 -22
- data/ext/gqlitedb/src/parser/ast.rs +0 -604
- data/ext/gqlitedb/src/parser/parser_impl.rs +0 -1213
- data/ext/gqlitedb/src/parser.rs +0 -4
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
use ccutils::sync::ArcRwLock;
|
|
2
2
|
use redb::{DatabaseError, ReadableDatabase as _, ReadableTable, ReadableTableMetadata as _};
|
|
3
3
|
use serde::{Deserialize, Serialize};
|
|
4
|
-
use std::{
|
|
4
|
+
use std::{
|
|
5
|
+
cell::RefCell,
|
|
6
|
+
collections::HashMap,
|
|
7
|
+
rc::Rc,
|
|
8
|
+
sync::{Arc, Mutex},
|
|
9
|
+
};
|
|
5
10
|
|
|
6
|
-
|
|
11
|
+
pub(crate) mod index;
|
|
12
|
+
pub(crate) use index::HnswParams;
|
|
13
|
+
pub(crate) mod hnsw_store;
|
|
14
|
+
|
|
15
|
+
use crate::{
|
|
16
|
+
prelude::*,
|
|
17
|
+
store::{Store as StoreTrait, TransactionBoxable},
|
|
18
|
+
};
|
|
19
|
+
use db_index::hnsw::graph_store::GraphStoreRead;
|
|
7
20
|
|
|
8
21
|
// ____ _ _ _ _____ _
|
|
9
22
|
// | _ \ ___ _ __ ___(_)___| |_ ___ _ __ | |_| ____|__| | __ _ ___
|
|
@@ -51,6 +64,14 @@ impl redb::Value for PersistentEdge
|
|
|
51
64
|
}
|
|
52
65
|
}
|
|
53
66
|
|
|
67
|
+
#[derive(Serialize)]
|
|
68
|
+
struct NodeSerializer<'a>
|
|
69
|
+
{
|
|
70
|
+
pub key: graphcore::Key,
|
|
71
|
+
pub labels: &'a Vec<String>,
|
|
72
|
+
pub properties: &'a value::ValueMap,
|
|
73
|
+
}
|
|
74
|
+
|
|
54
75
|
#[derive(Debug)]
|
|
55
76
|
enum PersistentNode<'a>
|
|
56
77
|
{
|
|
@@ -70,8 +91,6 @@ impl<'a> From<PersistentNode<'a>> for graph::Node
|
|
|
70
91
|
}
|
|
71
92
|
}
|
|
72
93
|
|
|
73
|
-
// ccutils::alias!(PersistentNode, graph::Node, derive: Debug, Deserialize, Serialize, Clone);
|
|
74
|
-
|
|
75
94
|
impl<'c> redb::Value for PersistentNode<'c>
|
|
76
95
|
{
|
|
77
96
|
type AsBytes<'a>
|
|
@@ -87,11 +106,20 @@ impl<'c> redb::Value for PersistentNode<'c>
|
|
|
87
106
|
Self: 'b,
|
|
88
107
|
{
|
|
89
108
|
let mut data = Vec::<u8>::new();
|
|
90
|
-
match value
|
|
109
|
+
let node_serialization = match value
|
|
91
110
|
{
|
|
92
|
-
PersistentNode::Reference(refe) =>
|
|
93
|
-
|
|
94
|
-
|
|
111
|
+
PersistentNode::Reference(refe) => NodeSerializer {
|
|
112
|
+
key: refe.key(),
|
|
113
|
+
labels: refe.labels(),
|
|
114
|
+
properties: refe.properties(),
|
|
115
|
+
},
|
|
116
|
+
PersistentNode::Value(value) => NodeSerializer {
|
|
117
|
+
key: value.key(),
|
|
118
|
+
labels: value.labels(),
|
|
119
|
+
properties: value.properties(),
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
ciborium::into_writer(&node_serialization, &mut data).unwrap(); // This unwrap should not happen, unless there is a bug
|
|
95
123
|
data
|
|
96
124
|
}
|
|
97
125
|
fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
|
|
@@ -110,7 +138,11 @@ impl<'c> redb::Value for PersistentNode<'c>
|
|
|
110
138
|
}
|
|
111
139
|
}
|
|
112
140
|
|
|
113
|
-
ccutils::alias!(
|
|
141
|
+
ccutils::alias!(
|
|
142
|
+
#[derive(Debug, Deserialize, Serialize, PartialEq, Clone)]
|
|
143
|
+
PersistentKey,
|
|
144
|
+
graph::Key
|
|
145
|
+
);
|
|
114
146
|
|
|
115
147
|
impl PersistentKey
|
|
116
148
|
{
|
|
@@ -247,6 +279,7 @@ struct GraphInfo
|
|
|
247
279
|
edges_table: String,
|
|
248
280
|
edges_source_index: String,
|
|
249
281
|
edges_destination_index: String,
|
|
282
|
+
index_registry: index::IndexRegistry,
|
|
250
283
|
}
|
|
251
284
|
|
|
252
285
|
impl GraphInfo
|
|
@@ -258,6 +291,7 @@ impl GraphInfo
|
|
|
258
291
|
let edges_table = format!("__{}__edges_table", name);
|
|
259
292
|
let edges_source_index = format!("__{}__edges_source_index", name);
|
|
260
293
|
let edges_destination_index = format!("__{}__edges_destination_index", name);
|
|
294
|
+
let index_registry = index::IndexRegistry::new();
|
|
261
295
|
|
|
262
296
|
GraphInfo {
|
|
263
297
|
name,
|
|
@@ -265,8 +299,10 @@ impl GraphInfo
|
|
|
265
299
|
edges_table,
|
|
266
300
|
edges_source_index,
|
|
267
301
|
edges_destination_index,
|
|
302
|
+
index_registry,
|
|
268
303
|
}
|
|
269
304
|
}
|
|
305
|
+
|
|
270
306
|
fn nodes_table_definition<'a, 'b>(
|
|
271
307
|
&'a self,
|
|
272
308
|
) -> redb::TableDefinition<'a, PersistentKey, PersistentNode<'b>>
|
|
@@ -293,6 +329,48 @@ impl GraphInfo
|
|
|
293
329
|
}
|
|
294
330
|
}
|
|
295
331
|
|
|
332
|
+
/// Shape of a legacy (pre-1.7-unification) per-index metadata JSON blob,
|
|
333
|
+
/// as written by create_vector_index at the 1.7 tag.
|
|
334
|
+
#[derive(Deserialize)]
|
|
335
|
+
struct LegacyVectorIndexMetadata
|
|
336
|
+
{
|
|
337
|
+
name: String,
|
|
338
|
+
label: Option<String>,
|
|
339
|
+
property: Option<String>,
|
|
340
|
+
dimension: usize,
|
|
341
|
+
metric: String, // Debug-formatted, e.g. "Cosine" -- via from_debug_str
|
|
342
|
+
hnsw_m: usize,
|
|
343
|
+
hnsw_ef_construction: usize,
|
|
344
|
+
hnsw_ef_search: usize,
|
|
345
|
+
dtype: String,
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
349
|
+
pub(crate) enum IndexMetadata
|
|
350
|
+
{
|
|
351
|
+
VectorIndex
|
|
352
|
+
{
|
|
353
|
+
name: String,
|
|
354
|
+
label: Option<String>,
|
|
355
|
+
property: Option<String>,
|
|
356
|
+
dimension: usize,
|
|
357
|
+
metric: super::VectorMetric,
|
|
358
|
+
hnsw_params: HnswParams,
|
|
359
|
+
dtype: String,
|
|
360
|
+
},
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
impl IndexMetadata
|
|
364
|
+
{
|
|
365
|
+
fn name(&self) -> &str
|
|
366
|
+
{
|
|
367
|
+
match self
|
|
368
|
+
{
|
|
369
|
+
Self::VectorIndex { name, .. } => name,
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
296
374
|
// _____ _ _
|
|
297
375
|
// |_ _| __ __ _ _ __ ___ __ _| |_(_) ___ _ __
|
|
298
376
|
// | || '__/ _` | '_ \/ __|/ _` | __| |/ _ \| '_ \
|
|
@@ -307,19 +385,29 @@ impl super::ReadTransaction for redb::ReadTransaction
|
|
|
307
385
|
}
|
|
308
386
|
}
|
|
309
387
|
|
|
310
|
-
|
|
388
|
+
#[expect(dead_code)]
|
|
389
|
+
pub(crate) type NodeTensorPair = (graph::Key, value::Tensor);
|
|
390
|
+
|
|
391
|
+
pub(crate) struct WriteTransaction
|
|
392
|
+
{
|
|
393
|
+
pub(crate) write_transaction: redb::WriteTransaction,
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
impl WriteTransaction {}
|
|
397
|
+
|
|
398
|
+
impl super::ReadTransaction for WriteTransaction
|
|
311
399
|
{
|
|
312
400
|
fn discard(self) -> Result<()>
|
|
313
401
|
{
|
|
314
|
-
Ok(self.abort()?)
|
|
402
|
+
Ok(self.write_transaction.abort()?)
|
|
315
403
|
}
|
|
316
404
|
}
|
|
317
405
|
|
|
318
|
-
impl super::WriteTransaction for
|
|
406
|
+
impl super::WriteTransaction for WriteTransaction
|
|
319
407
|
{
|
|
320
408
|
fn commit(self) -> Result<()>
|
|
321
409
|
{
|
|
322
|
-
Ok(self.commit()?)
|
|
410
|
+
Ok(self.write_transaction.commit()?)
|
|
323
411
|
}
|
|
324
412
|
}
|
|
325
413
|
|
|
@@ -334,14 +422,55 @@ pub(crate) struct Store
|
|
|
334
422
|
{
|
|
335
423
|
redb_store: redb::Database,
|
|
336
424
|
graphs: ArcRwLock<HashMap<String, Arc<GraphInfo>>>,
|
|
425
|
+
// Track creation order of nodes for deterministic vector search results
|
|
426
|
+
node_sequence_counter: Mutex<u64>,
|
|
427
|
+
node_sequences: Mutex<HashMap<graph::Key, u64>>,
|
|
337
428
|
}
|
|
338
429
|
|
|
339
430
|
ccutils::assert_impl_all!(Store: Sync, Send);
|
|
340
431
|
|
|
341
|
-
type TransactionBox = store::TransactionBox<redb::ReadTransaction,
|
|
432
|
+
type TransactionBox = store::TransactionBox<redb::ReadTransaction, WriteTransaction>;
|
|
342
433
|
|
|
343
434
|
impl Store
|
|
344
435
|
{
|
|
436
|
+
fn index_metadata_key(graph_name: &str) -> String
|
|
437
|
+
{
|
|
438
|
+
format!("{}__index_metadata", graph_name)
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
pub(crate) fn get_graph_index_metadata(
|
|
442
|
+
&self,
|
|
443
|
+
transaction: &mut TransactionBox,
|
|
444
|
+
graph_name: &str,
|
|
445
|
+
) -> Result<Vec<IndexMetadata>>
|
|
446
|
+
{
|
|
447
|
+
Self::get_metadata_value_or_else(
|
|
448
|
+
transaction,
|
|
449
|
+
Self::index_metadata_key(graph_name),
|
|
450
|
+
Default::default,
|
|
451
|
+
)
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
pub(crate) fn set_graph_index_metadata(
|
|
455
|
+
&self,
|
|
456
|
+
transaction: &mut TransactionBox,
|
|
457
|
+
graph_name: &str,
|
|
458
|
+
metadata: &[IndexMetadata],
|
|
459
|
+
) -> Result<()>
|
|
460
|
+
{
|
|
461
|
+
Self::set_metadata_value(transaction, Self::index_metadata_key(graph_name), &metadata)
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
fn graphs_list_read(&self, read_tx: &redb::ReadTransaction) -> Result<Vec<String>>
|
|
465
|
+
{
|
|
466
|
+
match Self::get_metadata_raw_read(read_tx, "graphs")
|
|
467
|
+
{
|
|
468
|
+
Ok(data) => Ok(ciborium::from_reader(data.as_slice())?),
|
|
469
|
+
Err(Error::Internal(InternalError::MissingMetadata { .. })) => Ok(Vec::new()),
|
|
470
|
+
Err(e) => Err(e),
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
345
474
|
/// Crate a new store, with a default graph
|
|
346
475
|
pub(crate) fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Store>
|
|
347
476
|
{
|
|
@@ -374,10 +503,13 @@ impl Store
|
|
|
374
503
|
let s = Self {
|
|
375
504
|
redb_store,
|
|
376
505
|
graphs: Default::default(),
|
|
506
|
+
node_sequence_counter: Mutex::new(0),
|
|
507
|
+
node_sequences: Mutex::new(HashMap::new()),
|
|
377
508
|
};
|
|
378
509
|
s.initialise()?;
|
|
379
510
|
Ok(s)
|
|
380
511
|
}
|
|
512
|
+
|
|
381
513
|
/// Crate a new store, with a default graph, in memory
|
|
382
514
|
pub(crate) fn in_memory() -> Result<Store>
|
|
383
515
|
{
|
|
@@ -385,22 +517,295 @@ impl Store
|
|
|
385
517
|
redb_store: redb::Database::builder()
|
|
386
518
|
.create_with_backend(redb::backends::InMemoryBackend::new())?,
|
|
387
519
|
graphs: Default::default(),
|
|
520
|
+
node_sequence_counter: Mutex::new(0),
|
|
521
|
+
node_sequences: Mutex::new(HashMap::new()),
|
|
388
522
|
};
|
|
389
523
|
s.initialise()?;
|
|
390
524
|
Ok(s)
|
|
391
525
|
}
|
|
392
526
|
fn initialise(&self) -> Result<()>
|
|
393
527
|
{
|
|
394
|
-
use crate::store::Store;
|
|
395
528
|
let mut tx = self.begin_write()?;
|
|
529
|
+
|
|
530
|
+
// A genuinely fresh database (first-ever open) has no graphs and therefore cannot have a version
|
|
531
|
+
// key yet -- this is expected, not an error case.
|
|
532
|
+
let is_new_database = self.graphs_list(&mut tx)?.is_empty();
|
|
533
|
+
|
|
396
534
|
self.create_graph(&mut tx, "default", true)?;
|
|
397
|
-
|
|
535
|
+
|
|
536
|
+
if is_new_database
|
|
537
|
+
{
|
|
538
|
+
Self::set_metadata_value(&mut tx, "version", &consts::GQLITE_VERSION)?;
|
|
539
|
+
}
|
|
540
|
+
else
|
|
541
|
+
{
|
|
542
|
+
match Self::get_metadata_value::<utils::Version>(&mut tx, "version")
|
|
543
|
+
{
|
|
544
|
+
Ok(v) if v == consts::GQLITE_VERSION =>
|
|
545
|
+
{
|
|
546
|
+
// Already current -- no migration work.
|
|
547
|
+
}
|
|
548
|
+
Ok(v) =>
|
|
549
|
+
{
|
|
550
|
+
self.upgrade_database(v, &mut tx)?;
|
|
551
|
+
Self::set_metadata_value(&mut tx, "version", &consts::GQLITE_VERSION)?;
|
|
552
|
+
}
|
|
553
|
+
Err(_) =>
|
|
554
|
+
{
|
|
555
|
+
// Existing, populated database with no version metadata at all.
|
|
556
|
+
// Every database this store has ever created writes this key unconditionally -- refuse to
|
|
557
|
+
// guess an assumed starting version and silently rewrite an unrecognized or corrupted file.
|
|
558
|
+
return Err(StoreError::MissingVersionMetadata.into());
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
398
563
|
tx.close()?;
|
|
564
|
+
self.reload_persisted_graph_state()?;
|
|
399
565
|
Ok(())
|
|
400
566
|
}
|
|
401
|
-
|
|
402
|
-
|
|
567
|
+
|
|
568
|
+
/// Walks the database forward from `from` to `consts::GQLITE_VERSION`, one version step at a time.
|
|
569
|
+
/// Mirrors sqlite.rs's upgrade_database dispatch shape. Each arm only knows how to bring the
|
|
570
|
+
/// database forward exactly one step; the function recurses until `from` matches current.
|
|
571
|
+
fn upgrade_database(&self, from: utils::Version, tx: &mut TransactionBox) -> Result<()>
|
|
572
|
+
{
|
|
573
|
+
if from == consts::GQLITE_VERSION
|
|
574
|
+
{
|
|
575
|
+
return Ok(());
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
match (from.major, from.minor, from.patch)
|
|
579
|
+
{
|
|
580
|
+
(1, minor, _) if minor <= 7 =>
|
|
581
|
+
{
|
|
582
|
+
self.upgrade_from_1_7(tx)?;
|
|
583
|
+
self.upgrade_database(
|
|
584
|
+
utils::Version {
|
|
585
|
+
major: 1,
|
|
586
|
+
minor: 8,
|
|
587
|
+
patch: 0,
|
|
588
|
+
},
|
|
589
|
+
tx,
|
|
590
|
+
)
|
|
591
|
+
}
|
|
592
|
+
_ => Err(
|
|
593
|
+
StoreError::IncompatibleVersion {
|
|
594
|
+
expected: consts::GQLITE_VERSION,
|
|
595
|
+
actual: from,
|
|
596
|
+
}
|
|
597
|
+
.into(),
|
|
598
|
+
),
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/// Finds every legacy (pre-1.7-unification) per-index metadata key for the
|
|
603
|
+
/// given graph, in the form "{graph_name}__vector_index_{index_name}".
|
|
604
|
+
fn find_legacy_vector_index_keys(
|
|
605
|
+
&self,
|
|
606
|
+
tx: &mut TransactionBox,
|
|
607
|
+
graph_name: &str,
|
|
608
|
+
) -> Result<Vec<String>>
|
|
609
|
+
{
|
|
610
|
+
let prefix = format!("{}__vector_index_", graph_name);
|
|
611
|
+
let prefix_end = format!("{}__vector_index_\x7F", graph_name);
|
|
612
|
+
let table_def = redb::TableDefinition::<String, Vec<u8>>::new("gqlite_metadata");
|
|
613
|
+
|
|
614
|
+
match tx
|
|
615
|
+
{
|
|
616
|
+
TransactionBox::Read(read) =>
|
|
617
|
+
{
|
|
618
|
+
let table = read.open_table(table_def)?;
|
|
619
|
+
table
|
|
620
|
+
.range::<String>(prefix.clone()..=prefix_end)?
|
|
621
|
+
.map(|entry| {
|
|
622
|
+
entry
|
|
623
|
+
.map(|(k, _)| k.value().to_string())
|
|
624
|
+
.map_err(Into::into)
|
|
625
|
+
})
|
|
626
|
+
.collect()
|
|
627
|
+
}
|
|
628
|
+
TransactionBox::Write(write) =>
|
|
629
|
+
{
|
|
630
|
+
let table = write.write_transaction.open_table(table_def)?;
|
|
631
|
+
table
|
|
632
|
+
.range::<String>(prefix.clone()..=prefix_end)?
|
|
633
|
+
.map(|entry| {
|
|
634
|
+
entry
|
|
635
|
+
.map(|(k, _)| k.value().to_string())
|
|
636
|
+
.map_err(Into::into)
|
|
637
|
+
})
|
|
638
|
+
.collect()
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
/// Reads and translates one legacy per-index metadata JSON blob into the
|
|
643
|
+
/// current IndexMetadata::VectorIndex shape. Uses serde deserialization
|
|
644
|
+
/// for clean and type-safe parsing.
|
|
645
|
+
pub(crate) fn translate_legacy_vector_index_metadata(
|
|
403
646
|
&self,
|
|
647
|
+
tx: &mut TransactionBox,
|
|
648
|
+
legacy_key: &str,
|
|
649
|
+
) -> Result<IndexMetadata>
|
|
650
|
+
{
|
|
651
|
+
let legacy: LegacyVectorIndexMetadata = Self::get_metadata_value(tx, legacy_key)?;
|
|
652
|
+
|
|
653
|
+
Ok(IndexMetadata::VectorIndex {
|
|
654
|
+
name: legacy.name,
|
|
655
|
+
label: legacy.label,
|
|
656
|
+
property: legacy.property,
|
|
657
|
+
dimension: legacy.dimension,
|
|
658
|
+
metric: super::VectorMetric::from_debug_str(&legacy.metric),
|
|
659
|
+
hnsw_params: HnswParams {
|
|
660
|
+
m: legacy.hnsw_m,
|
|
661
|
+
ef_construction: legacy.hnsw_ef_construction,
|
|
662
|
+
ef_search: legacy.hnsw_ef_search,
|
|
663
|
+
},
|
|
664
|
+
dtype: legacy.dtype,
|
|
665
|
+
})
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/// Upgrade from 1.7 to 1.8. The vector index storage format was changed and need migration.
|
|
669
|
+
fn upgrade_from_1_7(&self, tx: &mut TransactionBox) -> Result<()>
|
|
670
|
+
{
|
|
671
|
+
let graphs = self.graphs_list(tx)?;
|
|
672
|
+
|
|
673
|
+
for graph_name in graphs
|
|
674
|
+
{
|
|
675
|
+
let legacy_keys = self.find_legacy_vector_index_keys(tx, &graph_name)?;
|
|
676
|
+
|
|
677
|
+
for legacy_key in &legacy_keys
|
|
678
|
+
{
|
|
679
|
+
let IndexMetadata::VectorIndex {
|
|
680
|
+
name,
|
|
681
|
+
dimension,
|
|
682
|
+
metric,
|
|
683
|
+
hnsw_params: params,
|
|
684
|
+
property,
|
|
685
|
+
..
|
|
686
|
+
} = self.translate_legacy_vector_index_metadata(tx, legacy_key)?;
|
|
687
|
+
|
|
688
|
+
self.create_vector_index(
|
|
689
|
+
tx,
|
|
690
|
+
&graph_name,
|
|
691
|
+
super::VectorIndexSpec {
|
|
692
|
+
name,
|
|
693
|
+
dimension,
|
|
694
|
+
metric,
|
|
695
|
+
hnsw_params: params,
|
|
696
|
+
property,
|
|
697
|
+
..Default::default()
|
|
698
|
+
},
|
|
699
|
+
)?;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
// Clean up legacy keys.
|
|
703
|
+
if !legacy_keys.is_empty()
|
|
704
|
+
{
|
|
705
|
+
let tx_write = tx.try_into_write()?;
|
|
706
|
+
let mut metadata_table = tx_write
|
|
707
|
+
.write_transaction
|
|
708
|
+
.open_table(redb::TableDefinition::<String, Vec<u8>>::new(
|
|
709
|
+
"gqlite_metadata",
|
|
710
|
+
))?;
|
|
711
|
+
for legacy_key in &legacy_keys
|
|
712
|
+
{
|
|
713
|
+
metadata_table.remove(legacy_key)?;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
Ok(())
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
fn reload_persisted_graph_state(&self) -> Result<()>
|
|
722
|
+
{
|
|
723
|
+
let read_tx = self.redb_store.begin_read()?;
|
|
724
|
+
for graph_name in self.graphs_list_read(&read_tx)?
|
|
725
|
+
{
|
|
726
|
+
self.register_vector_indexes(&read_tx, &graph_name)?;
|
|
727
|
+
}
|
|
728
|
+
Ok(())
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
fn register_vector_indexes(&self, read_tx: &redb::ReadTransaction, graph_name: &str)
|
|
732
|
+
-> Result<()>
|
|
733
|
+
{
|
|
734
|
+
let meta_key = Self::index_metadata_key(graph_name);
|
|
735
|
+
let index_metadata_list = match Self::get_metadata_raw_read(read_tx, &meta_key)
|
|
736
|
+
{
|
|
737
|
+
Ok(data) => ciborium::from_reader::<Vec<IndexMetadata>, _>(data.as_slice())?,
|
|
738
|
+
Err(Error::Internal(InternalError::MissingMetadata { .. })) => Vec::new(),
|
|
739
|
+
Err(e) => return Err(e),
|
|
740
|
+
};
|
|
741
|
+
|
|
742
|
+
let graph_info = self.get_graph_info(graph_name)?;
|
|
743
|
+
let registry = &graph_info.index_registry;
|
|
744
|
+
|
|
745
|
+
for metadata in index_metadata_list.iter()
|
|
746
|
+
{
|
|
747
|
+
let index_name = metadata.name();
|
|
748
|
+
// Skip if already registered (should not happen at startup).
|
|
749
|
+
if registry.get_index(index_name).is_some()
|
|
750
|
+
{
|
|
751
|
+
continue;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// Create and register the index in memory.
|
|
755
|
+
// The persistent store is the source of truth; no loading or reconstruction needed.
|
|
756
|
+
let (kind, property) = match metadata
|
|
757
|
+
{
|
|
758
|
+
IndexMetadata::VectorIndex {
|
|
759
|
+
dimension,
|
|
760
|
+
metric,
|
|
761
|
+
property,
|
|
762
|
+
hnsw_params,
|
|
763
|
+
..
|
|
764
|
+
} => (
|
|
765
|
+
index::IndexKind::Vector {
|
|
766
|
+
dimension: *dimension,
|
|
767
|
+
metric: *metric,
|
|
768
|
+
params: hnsw_params.to_owned(),
|
|
769
|
+
},
|
|
770
|
+
property.clone(),
|
|
771
|
+
),
|
|
772
|
+
};
|
|
773
|
+
let idx = index::create_index(kind, index_name, property);
|
|
774
|
+
registry.register_index(index_name.to_string(), idx);
|
|
775
|
+
}
|
|
776
|
+
Ok(())
|
|
777
|
+
}
|
|
778
|
+
fn get_metadata_raw_from_table<TTable>(table: TTable, key: impl Into<String>) -> Result<Vec<u8>>
|
|
779
|
+
where
|
|
780
|
+
TTable: ReadableTable<String, Vec<u8>>,
|
|
781
|
+
{
|
|
782
|
+
let key = key.into();
|
|
783
|
+
let value = table
|
|
784
|
+
.get(&key)?
|
|
785
|
+
.ok_or_else(|| InternalError::MissingMetadata { key })?;
|
|
786
|
+
Ok(value.value())
|
|
787
|
+
}
|
|
788
|
+
#[expect(dead_code)]
|
|
789
|
+
fn get_metadata_raw_write(
|
|
790
|
+
transaction: &WriteTransaction,
|
|
791
|
+
key: impl Into<String>,
|
|
792
|
+
) -> Result<Vec<u8>>
|
|
793
|
+
{
|
|
794
|
+
let table_def = redb::TableDefinition::new("gqlite_metadata");
|
|
795
|
+
Self::get_metadata_raw_from_table(
|
|
796
|
+
transaction.write_transaction.open_table(table_def)?,
|
|
797
|
+
key.into(),
|
|
798
|
+
)
|
|
799
|
+
}
|
|
800
|
+
fn get_metadata_raw_read(
|
|
801
|
+
transaction: &redb::ReadTransaction,
|
|
802
|
+
key: impl Into<String>,
|
|
803
|
+
) -> Result<Vec<u8>>
|
|
804
|
+
{
|
|
805
|
+
let table_def = redb::TableDefinition::new("gqlite_metadata");
|
|
806
|
+
Self::get_metadata_raw_from_table(transaction.open_table(table_def)?, key.into())
|
|
807
|
+
}
|
|
808
|
+
fn get_metadata_from_table<TTable, TValue>(
|
|
404
809
|
table: TTable,
|
|
405
810
|
key: impl Into<String>,
|
|
406
811
|
) -> Result<TValue>
|
|
@@ -414,9 +819,7 @@ impl Store
|
|
|
414
819
|
.ok_or_else(|| InternalError::MissingMetadata { key })?;
|
|
415
820
|
Ok(ciborium::from_reader(value.value().as_slice())?)
|
|
416
821
|
}
|
|
417
|
-
#[allow(dead_code)]
|
|
418
822
|
fn get_metadata_value<T: for<'a> Deserialize<'a>>(
|
|
419
|
-
&self,
|
|
420
823
|
transaction: &mut TransactionBox,
|
|
421
824
|
key: impl Into<String>,
|
|
422
825
|
) -> Result<T>
|
|
@@ -426,16 +829,15 @@ impl Store
|
|
|
426
829
|
{
|
|
427
830
|
TransactionBox::Read(read) =>
|
|
428
831
|
{
|
|
429
|
-
|
|
832
|
+
Self::get_metadata_from_table(read.open_table(table_def)?, key.into())
|
|
430
833
|
}
|
|
431
834
|
TransactionBox::Write(write) =>
|
|
432
835
|
{
|
|
433
|
-
|
|
836
|
+
Self::get_metadata_from_table(write.write_transaction.open_table(table_def)?, key.into())
|
|
434
837
|
}
|
|
435
838
|
}
|
|
436
839
|
}
|
|
437
840
|
fn get_metadata_value_or_else_from_table<TTable, TValue>(
|
|
438
|
-
&self,
|
|
439
841
|
table: TTable,
|
|
440
842
|
key: impl Into<String>,
|
|
441
843
|
f: impl FnOnce() -> TValue,
|
|
@@ -452,7 +854,6 @@ impl Store
|
|
|
452
854
|
Ok(value)
|
|
453
855
|
}
|
|
454
856
|
fn get_metadata_value_or_else<T: for<'a> Deserialize<'a>>(
|
|
455
|
-
&self,
|
|
456
857
|
transaction: &mut TransactionBox,
|
|
457
858
|
key: impl Into<String>,
|
|
458
859
|
f: impl FnOnce() -> T,
|
|
@@ -463,31 +864,52 @@ impl Store
|
|
|
463
864
|
{
|
|
464
865
|
TransactionBox::Read(read) =>
|
|
465
866
|
{
|
|
466
|
-
|
|
467
|
-
}
|
|
468
|
-
TransactionBox::Write(write) =>
|
|
469
|
-
{
|
|
470
|
-
self.get_metadata_value_or_else_from_table(write.open_table(table_def)?, key.into(), f)
|
|
867
|
+
Self::get_metadata_value_or_else_from_table(read.open_table(table_def)?, key.into(), f)
|
|
471
868
|
}
|
|
869
|
+
TransactionBox::Write(write) => Self::get_metadata_value_or_else_from_table(
|
|
870
|
+
write.write_transaction.open_table(table_def)?,
|
|
871
|
+
key.into(),
|
|
872
|
+
f,
|
|
873
|
+
),
|
|
472
874
|
}
|
|
473
875
|
}
|
|
876
|
+
fn set_metadata_raw(
|
|
877
|
+
transaction: &WriteTransaction,
|
|
878
|
+
key: impl Into<String>,
|
|
879
|
+
data: Vec<u8>,
|
|
880
|
+
) -> Result<()>
|
|
881
|
+
{
|
|
882
|
+
let mut metadata_table =
|
|
883
|
+
transaction.write_transaction.open_table(
|
|
884
|
+
redb::TableDefinition::<'_, String, Vec<u8>>::new("gqlite_metadata"),
|
|
885
|
+
)?;
|
|
886
|
+
let key = key.into();
|
|
887
|
+
metadata_table.insert(&key, data)?;
|
|
888
|
+
Ok(())
|
|
889
|
+
}
|
|
474
890
|
fn set_metadata_value<T: Serialize>(
|
|
475
|
-
&self,
|
|
476
891
|
transaction: &mut TransactionBox,
|
|
477
892
|
key: impl Into<String>,
|
|
478
893
|
value: &T,
|
|
479
894
|
) -> Result<()>
|
|
480
895
|
{
|
|
481
896
|
let tx = transaction.try_into_write()?;
|
|
482
|
-
let mut metadata_table = tx.open_table(redb::TableDefinition::<'_, String, Vec<u8>>::new(
|
|
483
|
-
"gqlite_metadata",
|
|
484
|
-
))?;
|
|
485
|
-
let key = key.into();
|
|
486
897
|
let mut data = Vec::<u8>::new();
|
|
487
898
|
ciborium::into_writer(value, &mut data)?;
|
|
488
|
-
|
|
899
|
+
Self::set_metadata_raw(tx, key, data)
|
|
900
|
+
}
|
|
901
|
+
#[expect(dead_code)]
|
|
902
|
+
fn remove_metadata_value(transaction: &WriteTransaction, key: impl Into<String>) -> Result<()>
|
|
903
|
+
{
|
|
904
|
+
let mut metadata_table =
|
|
905
|
+
transaction.write_transaction.open_table(
|
|
906
|
+
redb::TableDefinition::<'_, String, Vec<u8>>::new("gqlite_metadata"),
|
|
907
|
+
)?;
|
|
908
|
+
let key = key.into();
|
|
909
|
+
metadata_table.remove(key)?;
|
|
489
910
|
Ok(())
|
|
490
911
|
}
|
|
912
|
+
|
|
491
913
|
fn get_graph_info(&self, graph_name: impl AsRef<str>) -> Result<Arc<GraphInfo>>
|
|
492
914
|
{
|
|
493
915
|
let graph_name = graph_name.as_ref();
|
|
@@ -510,12 +932,14 @@ impl Store
|
|
|
510
932
|
}
|
|
511
933
|
fn select_nodes_from_table<T>(
|
|
512
934
|
&self,
|
|
935
|
+
graph_name: impl AsRef<str>,
|
|
513
936
|
nodes_table: &T,
|
|
514
937
|
query: &super::SelectNodeQuery,
|
|
515
938
|
) -> Result<Vec<crate::graph::Node>>
|
|
516
939
|
where
|
|
517
940
|
T: ReadableTable<PersistentKey, PersistentNode<'static>>,
|
|
518
941
|
{
|
|
942
|
+
let graph_name = graph_name.as_ref();
|
|
519
943
|
let r = match &query.keys
|
|
520
944
|
{
|
|
521
945
|
Some(keys) => Box::new(keys.iter().map(|key| {
|
|
@@ -581,13 +1005,16 @@ impl Store
|
|
|
581
1005
|
})) as Box<dyn Iterator<Item = Result<crate::graph::Node>>>,
|
|
582
1006
|
None => Box::new(r) as Box<dyn Iterator<Item = Result<crate::graph::Node>>>,
|
|
583
1007
|
};
|
|
584
|
-
r.
|
|
1008
|
+
r.map(|n| n.map(|n| n.with_graph_name(graph_name)))
|
|
1009
|
+
.collect()
|
|
585
1010
|
}
|
|
586
1011
|
|
|
1012
|
+
#[allow(clippy::too_many_arguments)]
|
|
587
1013
|
fn select_edges_from_tables<TEdges, TNodes, TEdgesIndex>(
|
|
588
1014
|
&self,
|
|
1015
|
+
graph_name: impl AsRef<str>,
|
|
589
1016
|
query: super::SelectEdgeQuery,
|
|
590
|
-
directivity:
|
|
1017
|
+
directivity: graphcore::EdgeDirectivity,
|
|
591
1018
|
edges_table: TEdges,
|
|
592
1019
|
nodes_table: TNodes,
|
|
593
1020
|
edges_source_index: Rc<RefCell<TEdgesIndex>>,
|
|
@@ -598,10 +1025,11 @@ impl Store
|
|
|
598
1025
|
TNodes: ReadableTable<PersistentKey, PersistentNode<'static>>,
|
|
599
1026
|
TEdgesIndex: ReadableTable<PersistentKey, Vec<PersistentKey>>,
|
|
600
1027
|
{
|
|
1028
|
+
let graph_name = graph_name.as_ref();
|
|
601
1029
|
let edges_uuid_indices = match directivity
|
|
602
1030
|
{
|
|
603
|
-
|
|
604
|
-
|
|
1031
|
+
graphcore::EdgeDirectivity::Directed => vec![(edges_source_index, edges_destination_index)],
|
|
1032
|
+
graphcore::EdgeDirectivity::Undirected => vec![
|
|
605
1033
|
(edges_source_index.clone(), edges_destination_index.clone()),
|
|
606
1034
|
(edges_destination_index, edges_source_index),
|
|
607
1035
|
],
|
|
@@ -645,7 +1073,8 @@ impl Store
|
|
|
645
1073
|
{
|
|
646
1074
|
if !query.destination.is_select_all() && !query.source.is_select_all()
|
|
647
1075
|
{
|
|
648
|
-
let dest_it =
|
|
1076
|
+
let dest_it =
|
|
1077
|
+
self.select_nodes_from_table(graph_name, &nodes_table, &query.destination)?;
|
|
649
1078
|
let dest_it: Vec<graph::Key> = dest_it
|
|
650
1079
|
.into_iter()
|
|
651
1080
|
.map(|n| {
|
|
@@ -661,7 +1090,7 @@ impl Store
|
|
|
661
1090
|
.into_iter()
|
|
662
1091
|
.flatten()
|
|
663
1092
|
.collect();
|
|
664
|
-
let nodes = self.select_nodes_from_table(&nodes_table, &query.source)?;
|
|
1093
|
+
let nodes = self.select_nodes_from_table(graph_name, &nodes_table, &query.source)?;
|
|
665
1094
|
for n in nodes.iter()
|
|
666
1095
|
{
|
|
667
1096
|
let nkey = n.key();
|
|
@@ -693,7 +1122,7 @@ impl Store
|
|
|
693
1122
|
}
|
|
694
1123
|
else if !query.source.is_select_all()
|
|
695
1124
|
{
|
|
696
|
-
let nodes = self.select_nodes_from_table(&nodes_table, &query.source)?;
|
|
1125
|
+
let nodes = self.select_nodes_from_table(graph_name, &nodes_table, &query.source)?;
|
|
697
1126
|
|
|
698
1127
|
for n in nodes.into_iter()
|
|
699
1128
|
{
|
|
@@ -723,7 +1152,8 @@ impl Store
|
|
|
723
1152
|
}
|
|
724
1153
|
else
|
|
725
1154
|
{
|
|
726
|
-
let nodes =
|
|
1155
|
+
let nodes =
|
|
1156
|
+
self.select_nodes_from_table(graph_name, &nodes_table, &query.destination)?;
|
|
727
1157
|
for n in nodes.into_iter()
|
|
728
1158
|
{
|
|
729
1159
|
let nkey = n.key();
|
|
@@ -771,13 +1201,12 @@ impl Store
|
|
|
771
1201
|
.value()
|
|
772
1202
|
.into();
|
|
773
1203
|
|
|
774
|
-
let edge = graph::
|
|
775
|
-
edge.key.into(),
|
|
1204
|
+
let edge = graph::SinglePath::new(
|
|
776
1205
|
source,
|
|
777
|
-
edge.labels,
|
|
778
|
-
edge.properties,
|
|
1206
|
+
graph::Edge::new(None, edge.key.into(), edge.labels, edge.properties),
|
|
779
1207
|
destination,
|
|
780
|
-
)
|
|
1208
|
+
)
|
|
1209
|
+
.with_graph_name(graph_name.to_string());
|
|
781
1210
|
super::EdgeResult {
|
|
782
1211
|
path: edge,
|
|
783
1212
|
reversed,
|
|
@@ -845,12 +1274,153 @@ impl Store
|
|
|
845
1274
|
true
|
|
846
1275
|
}
|
|
847
1276
|
})
|
|
1277
|
+
.map(|e| {
|
|
1278
|
+
e.map(|mut e| {
|
|
1279
|
+
e.path = e.path.with_graph_name(graph_name);
|
|
1280
|
+
e
|
|
1281
|
+
})
|
|
1282
|
+
})
|
|
848
1283
|
.collect()
|
|
849
1284
|
}
|
|
850
1285
|
else
|
|
851
1286
|
{
|
|
852
|
-
r.
|
|
1287
|
+
r.map(|e| {
|
|
1288
|
+
e.map(|mut e| {
|
|
1289
|
+
e.path = e.path.with_graph_name(graph_name);
|
|
1290
|
+
e
|
|
1291
|
+
})
|
|
1292
|
+
})
|
|
1293
|
+
.collect()
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
/// Increment the epoch for a vector index after a successful vector update
|
|
1298
|
+
#[allow(dead_code)]
|
|
1299
|
+
pub(crate) fn increment_vector_index_epoch(
|
|
1300
|
+
&self,
|
|
1301
|
+
transaction: &mut TransactionBox,
|
|
1302
|
+
graph_name: &str,
|
|
1303
|
+
index_name: &str,
|
|
1304
|
+
) -> Result<u64>
|
|
1305
|
+
{
|
|
1306
|
+
let key = format!("vector_index_epoch:{}:{}", graph_name, index_name);
|
|
1307
|
+
let current = Self::get_metadata_value_or_else(transaction, &key, || 0u64)?;
|
|
1308
|
+
let new_epoch = current + 1;
|
|
1309
|
+
Self::set_metadata_value(transaction, &key, &new_epoch)?;
|
|
1310
|
+
Ok(new_epoch)
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
pub(crate) fn drop_index(
|
|
1314
|
+
&self,
|
|
1315
|
+
transaction: &mut TransactionBox,
|
|
1316
|
+
graph_name: impl AsRef<str>,
|
|
1317
|
+
name: impl AsRef<str>,
|
|
1318
|
+
) -> Result<()>
|
|
1319
|
+
{
|
|
1320
|
+
let graph_name = graph_name.as_ref();
|
|
1321
|
+
let name = name.as_ref();
|
|
1322
|
+
let graph_info = self.get_graph_info(graph_name)?;
|
|
1323
|
+
if graph_info.index_registry.drop_index(name).is_none()
|
|
1324
|
+
{
|
|
1325
|
+
return Err(
|
|
1326
|
+
StoreError::InvalidFormat(format!(
|
|
1327
|
+
"Vector index '{}' does not exist in graph '{}'",
|
|
1328
|
+
name, graph_name
|
|
1329
|
+
))
|
|
1330
|
+
.into(),
|
|
1331
|
+
);
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
// Remove metadata and HNSW tables in a single write block
|
|
1335
|
+
// Note: We defer table deletion to avoid redb transaction issues
|
|
1336
|
+
{
|
|
1337
|
+
let tx = transaction.try_into_write()?;
|
|
1338
|
+
let mut metadata_table = tx.write_transaction.open_table(redb::TableDefinition::<
|
|
1339
|
+
'_,
|
|
1340
|
+
String,
|
|
1341
|
+
Vec<u8>,
|
|
1342
|
+
>::new("gqlite_metadata"))?;
|
|
1343
|
+
|
|
1344
|
+
// Remove epoch metadata
|
|
1345
|
+
let epoch_key = format!("vector_index_epoch:{}:{}", graph_name, name);
|
|
1346
|
+
let _ = metadata_table.remove(&epoch_key);
|
|
1347
|
+
|
|
1348
|
+
// Note: HNSW tables are left in redb but marked as orphaned.
|
|
1349
|
+
// Deleting them in the same write transaction causes redb to fail.
|
|
1350
|
+
// Tables will be cleaned up if the index is recreated.
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
let mut metadata = self.get_graph_index_metadata(transaction, graph_name)?;
|
|
1354
|
+
metadata.retain(|entry| entry.name() != name);
|
|
1355
|
+
self.set_graph_index_metadata(transaction, graph_name, &metadata)?;
|
|
1356
|
+
|
|
1357
|
+
Ok(())
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
#[cfg(test)]
|
|
1361
|
+
#[allow(dead_code)]
|
|
1362
|
+
pub(crate) fn get_index_hnsw_parameters(
|
|
1363
|
+
&self,
|
|
1364
|
+
graph_name: &str,
|
|
1365
|
+
index_name: &str,
|
|
1366
|
+
) -> Result<Option<HnswParams>>
|
|
1367
|
+
{
|
|
1368
|
+
let graph_info = self.get_graph_info(graph_name)?;
|
|
1369
|
+
let registry = &graph_info.index_registry;
|
|
1370
|
+
if let Some(index) = registry.get_index(index_name)
|
|
1371
|
+
&& let Some(hnsw_idx) = index.as_any().downcast_ref::<index::HnswVectorIndex>()
|
|
1372
|
+
{
|
|
1373
|
+
return Ok(Some(hnsw_idx.get_hnsw_parameters()));
|
|
853
1374
|
}
|
|
1375
|
+
|
|
1376
|
+
let mut transaction = self.begin_read()?;
|
|
1377
|
+
let metadata_list = self.get_graph_index_metadata(&mut transaction, graph_name)?;
|
|
1378
|
+
transaction.close()?;
|
|
1379
|
+
for metadata in metadata_list
|
|
1380
|
+
{
|
|
1381
|
+
if let IndexMetadata::VectorIndex {
|
|
1382
|
+
name, hnsw_params, ..
|
|
1383
|
+
} = metadata
|
|
1384
|
+
&& name == index_name
|
|
1385
|
+
{
|
|
1386
|
+
return Ok(Some(hnsw_params));
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
Ok(None)
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
#[cfg(test)]
|
|
1394
|
+
pub(crate) fn test_hnsw_table_row_counts(
|
|
1395
|
+
&self,
|
|
1396
|
+
graph_name: &str,
|
|
1397
|
+
index_name: &str,
|
|
1398
|
+
) -> Result<(usize, usize, usize)>
|
|
1399
|
+
{
|
|
1400
|
+
use redb::ReadableTableMetadata as _;
|
|
1401
|
+
let tables = hnsw_store::HnswTableSet::new(graph_name, index_name);
|
|
1402
|
+
let read_tx = self.redb_store.begin_read()?;
|
|
1403
|
+
|
|
1404
|
+
let adj_len = match read_tx.open_table(tables.adj())
|
|
1405
|
+
{
|
|
1406
|
+
Ok(t) => t.len()? as usize,
|
|
1407
|
+
Err(redb::TableError::TableDoesNotExist(_)) => 0,
|
|
1408
|
+
Err(e) => return Err(e.into()),
|
|
1409
|
+
};
|
|
1410
|
+
let vec_len = match read_tx.open_table(tables.vec())
|
|
1411
|
+
{
|
|
1412
|
+
Ok(t) => t.len()? as usize,
|
|
1413
|
+
Err(redb::TableError::TableDoesNotExist(_)) => 0,
|
|
1414
|
+
Err(e) => return Err(e.into()),
|
|
1415
|
+
};
|
|
1416
|
+
let key_len = match read_tx.open_table(tables.key())
|
|
1417
|
+
{
|
|
1418
|
+
Ok(t) => t.len()? as usize,
|
|
1419
|
+
Err(redb::TableError::TableDoesNotExist(_)) => 0,
|
|
1420
|
+
Err(e) => return Err(e.into()),
|
|
1421
|
+
};
|
|
1422
|
+
|
|
1423
|
+
Ok((adj_len, vec_len, key_len))
|
|
854
1424
|
}
|
|
855
1425
|
}
|
|
856
1426
|
|
|
@@ -861,7 +1431,10 @@ impl store::Store for Store
|
|
|
861
1431
|
fn begin_write(&self) -> Result<Self::TransactionBox>
|
|
862
1432
|
{
|
|
863
1433
|
let s = self.redb_store.begin_write()?;
|
|
864
|
-
|
|
1434
|
+
let wrapped = WriteTransaction {
|
|
1435
|
+
write_transaction: s,
|
|
1436
|
+
};
|
|
1437
|
+
Ok(Self::TransactionBox::from_write(wrapped))
|
|
865
1438
|
}
|
|
866
1439
|
fn begin_read(&self) -> Result<Self::TransactionBox>
|
|
867
1440
|
{
|
|
@@ -870,7 +1443,7 @@ impl store::Store for Store
|
|
|
870
1443
|
}
|
|
871
1444
|
fn graphs_list(&self, transaction: &mut Self::TransactionBox) -> Result<Vec<String>>
|
|
872
1445
|
{
|
|
873
|
-
|
|
1446
|
+
Self::get_metadata_value_or_else(transaction, "graphs".to_string(), Vec::new)
|
|
874
1447
|
}
|
|
875
1448
|
fn create_graph(
|
|
876
1449
|
&self,
|
|
@@ -902,15 +1475,20 @@ impl store::Store for Store
|
|
|
902
1475
|
let tx = transaction.try_into_write()?;
|
|
903
1476
|
|
|
904
1477
|
let gi = Arc::new(GraphInfo::new(graph_name));
|
|
905
|
-
tx.
|
|
906
|
-
|
|
907
|
-
tx.
|
|
908
|
-
|
|
1478
|
+
tx.write_transaction
|
|
1479
|
+
.open_table(gi.nodes_table_definition())?;
|
|
1480
|
+
tx.write_transaction
|
|
1481
|
+
.open_table(gi.edges_table_definition())?;
|
|
1482
|
+
tx.write_transaction
|
|
1483
|
+
.open_table(gi.edges_source_index_definition())?;
|
|
1484
|
+
tx.write_transaction
|
|
1485
|
+
.open_table(gi.edges_destination_index_definition())?;
|
|
909
1486
|
|
|
910
1487
|
self.graphs.write()?.insert(graph_name.to_owned(), gi);
|
|
911
1488
|
}
|
|
912
1489
|
graphs_list.push(graph_name.to_owned());
|
|
913
|
-
|
|
1490
|
+
Self::set_metadata_value(transaction, "graphs", &graphs_list)?;
|
|
1491
|
+
self.set_graph_index_metadata(transaction, graph_name, &[])?;
|
|
914
1492
|
|
|
915
1493
|
Ok(())
|
|
916
1494
|
}
|
|
@@ -925,16 +1503,42 @@ impl store::Store for Store
|
|
|
925
1503
|
let mut graphs_list = self.graphs_list(transaction)?;
|
|
926
1504
|
if graphs_list.iter().any(|s| s == graph_name)
|
|
927
1505
|
{
|
|
1506
|
+
let index_names: Vec<String> = self
|
|
1507
|
+
.get_graph_info(graph_name)?
|
|
1508
|
+
.index_registry
|
|
1509
|
+
.iter_indexes()
|
|
1510
|
+
.into_iter()
|
|
1511
|
+
.map(|index| index.name().to_owned())
|
|
1512
|
+
.collect();
|
|
1513
|
+
for index_name in index_names
|
|
1514
|
+
{
|
|
1515
|
+
self.drop_vector_index(transaction, graph_name, &index_name)?;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
928
1518
|
{
|
|
929
1519
|
let tx = transaction.try_into_write()?;
|
|
930
1520
|
let graph_info = self.get_graph_info(graph_name)?;
|
|
931
|
-
tx.
|
|
932
|
-
|
|
933
|
-
tx.
|
|
934
|
-
|
|
1521
|
+
tx.write_transaction
|
|
1522
|
+
.delete_table(graph_info.nodes_table_definition())?;
|
|
1523
|
+
tx.write_transaction
|
|
1524
|
+
.delete_table(graph_info.edges_table_definition())?;
|
|
1525
|
+
tx.write_transaction
|
|
1526
|
+
.delete_table(graph_info.edges_source_index_definition())?;
|
|
1527
|
+
tx.write_transaction
|
|
1528
|
+
.delete_table(graph_info.edges_destination_index_definition())?;
|
|
935
1529
|
}
|
|
1530
|
+
self.graphs.write()?.remove(graph_name);
|
|
936
1531
|
graphs_list.retain(|x| x != graph_name);
|
|
937
|
-
|
|
1532
|
+
Self::set_metadata_value(transaction, "graphs", &graphs_list)?;
|
|
1533
|
+
{
|
|
1534
|
+
let tx = transaction.try_into_write()?;
|
|
1535
|
+
let mut metadata_table =
|
|
1536
|
+
tx.write_transaction
|
|
1537
|
+
.open_table(redb::TableDefinition::<'_, String, Vec<u8>>::new(
|
|
1538
|
+
"gqlite_metadata",
|
|
1539
|
+
))?;
|
|
1540
|
+
metadata_table.remove(&Self::index_metadata_key(graph_name))?;
|
|
1541
|
+
}
|
|
938
1542
|
|
|
939
1543
|
Ok(())
|
|
940
1544
|
}
|
|
@@ -960,18 +1564,39 @@ impl store::Store for Store
|
|
|
960
1564
|
nodes_iter: T,
|
|
961
1565
|
) -> Result<()>
|
|
962
1566
|
{
|
|
1567
|
+
let graph_name = graph_name.as_ref();
|
|
963
1568
|
let graph_info = self.get_graph_info(graph_name)?;
|
|
964
1569
|
let transaction = transaction.try_into_write()?;
|
|
965
|
-
let mut table = transaction
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1570
|
+
let mut table = transaction
|
|
1571
|
+
.write_transaction
|
|
1572
|
+
.open_table(graph_info.nodes_table_definition())?;
|
|
1573
|
+
let mut table_source = transaction
|
|
1574
|
+
.write_transaction
|
|
1575
|
+
.open_table(graph_info.edges_source_index_definition())?;
|
|
1576
|
+
let mut table_destination = transaction
|
|
1577
|
+
.write_transaction
|
|
1578
|
+
.open_table(graph_info.edges_destination_index_definition())?;
|
|
1579
|
+
let registry = &graph_info.index_registry;
|
|
1580
|
+
for node in nodes_iter
|
|
970
1581
|
{
|
|
971
|
-
let pk = PersistentKey(
|
|
972
|
-
|
|
1582
|
+
let pk = PersistentKey(node.key());
|
|
1583
|
+
|
|
1584
|
+
// Track creation sequence for deterministic vector search tiebreaking
|
|
1585
|
+
let sequence = {
|
|
1586
|
+
let mut counter = self.node_sequence_counter.lock().unwrap();
|
|
1587
|
+
*counter += 1;
|
|
1588
|
+
*counter
|
|
1589
|
+
};
|
|
1590
|
+
let mut sequences = self.node_sequences.lock().unwrap();
|
|
1591
|
+
sequences.insert(node.key(), sequence);
|
|
1592
|
+
|
|
1593
|
+
table.insert(&pk, PersistentNode::Reference(node))?;
|
|
973
1594
|
table_source.insert(&pk, vec![])?;
|
|
974
1595
|
table_destination.insert(&pk, vec![])?;
|
|
1596
|
+
for idx in registry.iter_indexes()
|
|
1597
|
+
{
|
|
1598
|
+
let _ = idx.on_insert(node);
|
|
1599
|
+
}
|
|
975
1600
|
}
|
|
976
1601
|
Ok(())
|
|
977
1602
|
}
|
|
@@ -983,10 +1608,33 @@ impl store::Store for Store
|
|
|
983
1608
|
node: &graph::Node,
|
|
984
1609
|
) -> Result<()>
|
|
985
1610
|
{
|
|
1611
|
+
let graph_name = graph_name.as_ref();
|
|
986
1612
|
let graph_info = self.get_graph_info(graph_name)?;
|
|
987
1613
|
let transaction = transaction.try_into_write()?;
|
|
988
|
-
let mut table = transaction
|
|
989
|
-
|
|
1614
|
+
let mut table = transaction
|
|
1615
|
+
.write_transaction
|
|
1616
|
+
.open_table(graph_info.nodes_table_definition())?;
|
|
1617
|
+
let pk = PersistentKey(node.key());
|
|
1618
|
+
let maybe_old_node: Option<crate::graph::Node> =
|
|
1619
|
+
table.get(pk.clone())?.map(|g| g.value().into());
|
|
1620
|
+
if let Some(old_node) = maybe_old_node
|
|
1621
|
+
{
|
|
1622
|
+
table.insert(pk, PersistentNode::Reference(node))?;
|
|
1623
|
+
let registry = &graph_info.index_registry;
|
|
1624
|
+
for idx in registry.iter_indexes()
|
|
1625
|
+
{
|
|
1626
|
+
idx.on_update(&old_node, node)?;
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
else
|
|
1630
|
+
{
|
|
1631
|
+
table.insert(pk, PersistentNode::Reference(node))?;
|
|
1632
|
+
let registry = &graph_info.index_registry;
|
|
1633
|
+
for idx in registry.iter_indexes()
|
|
1634
|
+
{
|
|
1635
|
+
idx.on_insert(node)?;
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
990
1638
|
Ok(())
|
|
991
1639
|
}
|
|
992
1640
|
/// Delete nodes according to a given query
|
|
@@ -1006,23 +1654,41 @@ impl store::Store for Store
|
|
|
1006
1654
|
let write_transaction = transaction.try_into_write()?;
|
|
1007
1655
|
if detach
|
|
1008
1656
|
{
|
|
1009
|
-
write_transaction
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
write_transaction
|
|
1013
|
-
|
|
1014
|
-
|
|
1657
|
+
write_transaction
|
|
1658
|
+
.write_transaction
|
|
1659
|
+
.delete_table(graph_info.edges_table_definition())?;
|
|
1660
|
+
write_transaction
|
|
1661
|
+
.write_transaction
|
|
1662
|
+
.delete_table(graph_info.edges_source_index_definition())?;
|
|
1663
|
+
write_transaction
|
|
1664
|
+
.write_transaction
|
|
1665
|
+
.delete_table(graph_info.edges_destination_index_definition())?;
|
|
1666
|
+
write_transaction
|
|
1667
|
+
.write_transaction
|
|
1668
|
+
.open_table(graph_info.edges_table_definition())?;
|
|
1669
|
+
write_transaction
|
|
1670
|
+
.write_transaction
|
|
1671
|
+
.open_table(graph_info.edges_source_index_definition())?;
|
|
1672
|
+
write_transaction
|
|
1673
|
+
.write_transaction
|
|
1674
|
+
.open_table(graph_info.edges_destination_index_definition())?;
|
|
1015
1675
|
}
|
|
1016
1676
|
else
|
|
1017
1677
|
{
|
|
1018
|
-
let edge_table = write_transaction
|
|
1678
|
+
let edge_table = write_transaction
|
|
1679
|
+
.write_transaction
|
|
1680
|
+
.open_table(graph_info.edges_table_definition())?;
|
|
1019
1681
|
if edge_table.len()? > 0
|
|
1020
1682
|
{
|
|
1021
1683
|
return Err(error::RunTimeError::DeleteConnectedNode.into());
|
|
1022
1684
|
}
|
|
1023
1685
|
}
|
|
1024
|
-
write_transaction
|
|
1025
|
-
|
|
1686
|
+
write_transaction
|
|
1687
|
+
.write_transaction
|
|
1688
|
+
.delete_table(graph_info.nodes_table_definition())?;
|
|
1689
|
+
write_transaction
|
|
1690
|
+
.write_transaction
|
|
1691
|
+
.open_table(graph_info.nodes_table_definition())?;
|
|
1026
1692
|
}
|
|
1027
1693
|
else
|
|
1028
1694
|
{
|
|
@@ -1043,6 +1709,46 @@ impl store::Store for Store
|
|
|
1043
1709
|
.collect()
|
|
1044
1710
|
};
|
|
1045
1711
|
|
|
1712
|
+
// Call index hooks for nodes that will be deleted
|
|
1713
|
+
let nodes_to_delete = self.select_nodes(
|
|
1714
|
+
transaction,
|
|
1715
|
+
graph_name,
|
|
1716
|
+
super::SelectNodeQuery::select_keys(node_keys.clone()),
|
|
1717
|
+
)?;
|
|
1718
|
+
let registry = &graph_info.index_registry;
|
|
1719
|
+
|
|
1720
|
+
let write_tx = transaction.try_into_write()?;
|
|
1721
|
+
let indexes = registry.iter_indexes();
|
|
1722
|
+
for node in nodes_to_delete.iter()
|
|
1723
|
+
{
|
|
1724
|
+
for idx in indexes.iter()
|
|
1725
|
+
{
|
|
1726
|
+
if let Some(hnsw_idx) = idx.as_any().downcast_ref::<index::HnswVectorIndex>()
|
|
1727
|
+
{
|
|
1728
|
+
let tables = hnsw_store::HnswTableSet::new(graph_name, idx.name());
|
|
1729
|
+
let mut graph_store = hnsw_store::RedbGraphStore::open(write_tx, &tables)?;
|
|
1730
|
+
|
|
1731
|
+
// Get the node ID before deletion
|
|
1732
|
+
if let Some(node_id) = graph_store.node_of(&node.key()).ok().flatten()
|
|
1733
|
+
{
|
|
1734
|
+
// Delete from HNSW graph
|
|
1735
|
+
hnsw_idx
|
|
1736
|
+
.upsert_and_search_ready()
|
|
1737
|
+
.delete(&mut graph_store, &node.key())?;
|
|
1738
|
+
|
|
1739
|
+
// Also delete the vector row
|
|
1740
|
+
let vector_store = hnsw_store::RedbVectorStore::open(
|
|
1741
|
+
write_tx,
|
|
1742
|
+
&tables,
|
|
1743
|
+
hnsw_idx.dimension(),
|
|
1744
|
+
hnsw_idx.encoding(),
|
|
1745
|
+
)?;
|
|
1746
|
+
vector_store.delete(node_id)?;
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1046
1752
|
if detach
|
|
1047
1753
|
{
|
|
1048
1754
|
// Delete the edges connected to the nodes
|
|
@@ -1052,17 +1758,19 @@ impl store::Store for Store
|
|
|
1052
1758
|
super::SelectEdgeQuery::select_source_keys(super::SelectNodeQuery::select_keys(
|
|
1053
1759
|
node_keys.clone(),
|
|
1054
1760
|
)),
|
|
1055
|
-
|
|
1761
|
+
graphcore::EdgeDirectivity::Undirected,
|
|
1056
1762
|
)?;
|
|
1057
1763
|
}
|
|
1058
1764
|
else
|
|
1059
1765
|
{
|
|
1060
1766
|
let write_transaction = transaction.try_into_write()?;
|
|
1061
1767
|
// Check if the nodes are disconnected
|
|
1062
|
-
let table_source =
|
|
1063
|
-
write_transaction
|
|
1064
|
-
|
|
1065
|
-
|
|
1768
|
+
let table_source = write_transaction
|
|
1769
|
+
.write_transaction
|
|
1770
|
+
.open_table(graph_info.edges_source_index_definition())?;
|
|
1771
|
+
let table_destination = write_transaction
|
|
1772
|
+
.write_transaction
|
|
1773
|
+
.open_table(graph_info.edges_destination_index_definition())?;
|
|
1066
1774
|
|
|
1067
1775
|
for key in node_keys.iter()
|
|
1068
1776
|
{
|
|
@@ -1081,16 +1789,20 @@ impl store::Store for Store
|
|
|
1081
1789
|
}
|
|
1082
1790
|
let write_transaction = transaction.try_into_write()?;
|
|
1083
1791
|
// Delete the nodes
|
|
1084
|
-
let mut table_nodes = write_transaction
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
let mut
|
|
1088
|
-
write_transaction
|
|
1089
|
-
|
|
1792
|
+
let mut table_nodes = write_transaction
|
|
1793
|
+
.write_transaction
|
|
1794
|
+
.open_table(graph_info.nodes_table_definition())?;
|
|
1795
|
+
let mut table_source = write_transaction
|
|
1796
|
+
.write_transaction
|
|
1797
|
+
.open_table(graph_info.edges_source_index_definition())?;
|
|
1798
|
+
let mut table_destination = write_transaction
|
|
1799
|
+
.write_transaction
|
|
1800
|
+
.open_table(graph_info.edges_destination_index_definition())?;
|
|
1801
|
+
for key in node_keys.iter()
|
|
1090
1802
|
{
|
|
1091
|
-
table_nodes.remove(PersistentKey(key))?;
|
|
1092
|
-
table_source.remove(PersistentKey(key))?;
|
|
1093
|
-
table_destination.remove(PersistentKey(key))?;
|
|
1803
|
+
table_nodes.remove(PersistentKey(key.to_owned()))?;
|
|
1804
|
+
table_source.remove(PersistentKey(key.to_owned()))?;
|
|
1805
|
+
table_destination.remove(PersistentKey(key.to_owned()))?;
|
|
1094
1806
|
}
|
|
1095
1807
|
}
|
|
1096
1808
|
Ok(())
|
|
@@ -1110,12 +1822,14 @@ impl store::Store for Store
|
|
|
1110
1822
|
store::TransactionBox::Read(read) =>
|
|
1111
1823
|
{
|
|
1112
1824
|
let nodes_table = read.open_table(graph_info.nodes_table_definition())?;
|
|
1113
|
-
self.select_nodes_from_table(&nodes_table, &query)
|
|
1825
|
+
self.select_nodes_from_table(graph_name, &nodes_table, &query)
|
|
1114
1826
|
}
|
|
1115
1827
|
store::TransactionBox::Write(write) =>
|
|
1116
1828
|
{
|
|
1117
|
-
let nodes_table = write
|
|
1118
|
-
|
|
1829
|
+
let nodes_table = write
|
|
1830
|
+
.write_transaction
|
|
1831
|
+
.open_table(graph_info.nodes_table_definition())?;
|
|
1832
|
+
self.select_nodes_from_table(graph_name, &nodes_table, &query)
|
|
1119
1833
|
}
|
|
1120
1834
|
}
|
|
1121
1835
|
}
|
|
@@ -1129,10 +1843,15 @@ impl store::Store for Store
|
|
|
1129
1843
|
{
|
|
1130
1844
|
let transaction = transaction.try_into_write()?;
|
|
1131
1845
|
let graph_info = self.get_graph_info(graph_name)?;
|
|
1132
|
-
let mut table = transaction
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1846
|
+
let mut table = transaction
|
|
1847
|
+
.write_transaction
|
|
1848
|
+
.open_table(graph_info.edges_table_definition())?;
|
|
1849
|
+
let mut table_source = transaction
|
|
1850
|
+
.write_transaction
|
|
1851
|
+
.open_table(graph_info.edges_source_index_definition())?;
|
|
1852
|
+
let mut table_destination = transaction
|
|
1853
|
+
.write_transaction
|
|
1854
|
+
.open_table(graph_info.edges_destination_index_definition())?;
|
|
1136
1855
|
|
|
1137
1856
|
for x in edges_iter
|
|
1138
1857
|
{
|
|
@@ -1171,7 +1890,9 @@ impl store::Store for Store
|
|
|
1171
1890
|
{
|
|
1172
1891
|
let transaction = transaction.try_into_write()?;
|
|
1173
1892
|
let graph_info = self.get_graph_info(graph_name)?;
|
|
1174
|
-
let mut table = transaction
|
|
1893
|
+
let mut table = transaction
|
|
1894
|
+
.write_transaction
|
|
1895
|
+
.open_table(graph_info.edges_table_definition())?;
|
|
1175
1896
|
let mut pe = table
|
|
1176
1897
|
.get_required(&edge.key().into(), || InternalError::UnknownEdge)?
|
|
1177
1898
|
.value();
|
|
@@ -1186,7 +1907,7 @@ impl store::Store for Store
|
|
|
1186
1907
|
transaction: &mut Self::TransactionBox,
|
|
1187
1908
|
graph_name: impl AsRef<str>,
|
|
1188
1909
|
query: super::SelectEdgeQuery,
|
|
1189
|
-
directivity:
|
|
1910
|
+
directivity: graphcore::EdgeDirectivity,
|
|
1190
1911
|
) -> Result<()>
|
|
1191
1912
|
{
|
|
1192
1913
|
let graph_name = graph_name.as_ref();
|
|
@@ -1195,10 +1916,15 @@ impl store::Store for Store
|
|
|
1195
1916
|
|
|
1196
1917
|
let transaction = transaction.try_into_write()?;
|
|
1197
1918
|
|
|
1198
|
-
let mut table = transaction
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1919
|
+
let mut table = transaction
|
|
1920
|
+
.write_transaction
|
|
1921
|
+
.open_table(graph_info.edges_table_definition())?;
|
|
1922
|
+
let mut table_source = transaction
|
|
1923
|
+
.write_transaction
|
|
1924
|
+
.open_table(graph_info.edges_source_index_definition())?;
|
|
1925
|
+
let mut table_destination = transaction
|
|
1926
|
+
.write_transaction
|
|
1927
|
+
.open_table(graph_info.edges_destination_index_definition())?;
|
|
1202
1928
|
|
|
1203
1929
|
for e in edges
|
|
1204
1930
|
{
|
|
@@ -1234,9 +1960,10 @@ impl store::Store for Store
|
|
|
1234
1960
|
transaction: &mut Self::TransactionBox,
|
|
1235
1961
|
graph_name: impl AsRef<str>,
|
|
1236
1962
|
query: super::SelectEdgeQuery,
|
|
1237
|
-
directivity:
|
|
1963
|
+
directivity: graphcore::EdgeDirectivity,
|
|
1238
1964
|
) -> Result<Vec<super::EdgeResult>>
|
|
1239
1965
|
{
|
|
1966
|
+
let graph_name = graph_name.as_ref();
|
|
1240
1967
|
if query.source.is_select_none() || query.destination.is_select_none()
|
|
1241
1968
|
{
|
|
1242
1969
|
return Ok(Default::default());
|
|
@@ -1258,6 +1985,7 @@ impl store::Store for Store
|
|
|
1258
1985
|
));
|
|
1259
1986
|
|
|
1260
1987
|
self.select_edges_from_tables(
|
|
1988
|
+
graph_name,
|
|
1261
1989
|
query,
|
|
1262
1990
|
directivity,
|
|
1263
1991
|
edges_table,
|
|
@@ -1268,17 +1996,26 @@ impl store::Store for Store
|
|
|
1268
1996
|
}
|
|
1269
1997
|
store::TransactionBox::Write(write) =>
|
|
1270
1998
|
{
|
|
1271
|
-
let edges_table = write
|
|
1272
|
-
|
|
1999
|
+
let edges_table = write
|
|
2000
|
+
.write_transaction
|
|
2001
|
+
.open_table(graph_info.edges_table_definition())?;
|
|
2002
|
+
let nodes_table = write
|
|
2003
|
+
.write_transaction
|
|
2004
|
+
.open_table(graph_info.nodes_table_definition())?;
|
|
1273
2005
|
|
|
1274
2006
|
let edges_source_index = Rc::new(RefCell::new(
|
|
1275
|
-
write
|
|
2007
|
+
write
|
|
2008
|
+
.write_transaction
|
|
2009
|
+
.open_table(graph_info.edges_source_index_definition())?,
|
|
1276
2010
|
));
|
|
1277
2011
|
let edges_destination_index = Rc::new(RefCell::new(
|
|
1278
|
-
write
|
|
2012
|
+
write
|
|
2013
|
+
.write_transaction
|
|
2014
|
+
.open_table(graph_info.edges_destination_index_definition())?,
|
|
1279
2015
|
));
|
|
1280
2016
|
|
|
1281
2017
|
self.select_edges_from_tables(
|
|
2018
|
+
graph_name,
|
|
1282
2019
|
query,
|
|
1283
2020
|
directivity,
|
|
1284
2021
|
edges_table,
|
|
@@ -1290,7 +2027,7 @@ impl store::Store for Store
|
|
|
1290
2027
|
}
|
|
1291
2028
|
}
|
|
1292
2029
|
fn compute_statistics(&self, transaction: &mut Self::TransactionBox)
|
|
1293
|
-
|
|
2030
|
+
-> Result<super::Statistics>
|
|
1294
2031
|
{
|
|
1295
2032
|
let mut edges_count = 0;
|
|
1296
2033
|
let mut nodes_count = 0;
|
|
@@ -1317,7 +2054,7 @@ impl store::Store for Store
|
|
|
1317
2054
|
transaction,
|
|
1318
2055
|
"default",
|
|
1319
2056
|
super::SelectEdgeQuery::select_all(),
|
|
1320
|
-
|
|
2057
|
+
graphcore::EdgeDirectivity::Directed,
|
|
1321
2058
|
)?
|
|
1322
2059
|
{
|
|
1323
2060
|
edges_count += 1;
|
|
@@ -1337,4 +2074,422 @@ impl store::Store for Store
|
|
|
1337
2074
|
properties_count,
|
|
1338
2075
|
})
|
|
1339
2076
|
}
|
|
2077
|
+
|
|
2078
|
+
fn create_vector_index(
|
|
2079
|
+
&self,
|
|
2080
|
+
transaction: &mut Self::TransactionBox,
|
|
2081
|
+
graph_name: impl AsRef<str>,
|
|
2082
|
+
spec: super::VectorIndexSpec,
|
|
2083
|
+
) -> Result<()>
|
|
2084
|
+
{
|
|
2085
|
+
let graph_name_str = graph_name.as_ref();
|
|
2086
|
+
let mut metadata_list = self.get_graph_index_metadata(transaction, graph_name_str)?;
|
|
2087
|
+
if metadata_list.iter().any(|entry| entry.name() == spec.name)
|
|
2088
|
+
{
|
|
2089
|
+
return Err(
|
|
2090
|
+
crate::error::StoreError::DuplicatedVectorIndex {
|
|
2091
|
+
index_name: spec.name.clone(),
|
|
2092
|
+
}
|
|
2093
|
+
.into(),
|
|
2094
|
+
);
|
|
2095
|
+
}
|
|
2096
|
+
|
|
2097
|
+
let metadata = IndexMetadata::VectorIndex {
|
|
2098
|
+
name: spec.name.clone(),
|
|
2099
|
+
label: spec.label.clone(),
|
|
2100
|
+
property: spec.property.clone(),
|
|
2101
|
+
dimension: spec.dimension,
|
|
2102
|
+
metric: spec.metric,
|
|
2103
|
+
hnsw_params: spec.hnsw_params,
|
|
2104
|
+
dtype: spec.dtype.as_str().to_string(),
|
|
2105
|
+
};
|
|
2106
|
+
metadata_list.push(metadata.clone());
|
|
2107
|
+
self.set_graph_index_metadata(transaction, graph_name_str, &metadata_list)?;
|
|
2108
|
+
|
|
2109
|
+
// Create and register the index, then backfill from the current graph nodes.
|
|
2110
|
+
// Always start from scratch — persisted HNSW tables for this (graph, index) pair can only
|
|
2111
|
+
// be stale leftovers (pre-WI 329 orphans, or same-transaction drop+recreate). Backfill uses
|
|
2112
|
+
// the caller's transaction, so nodes created earlier in the same uncommitted transaction are
|
|
2113
|
+
// included correctly.
|
|
2114
|
+
let graph_info = self.get_graph_info(graph_name_str)?;
|
|
2115
|
+
let registry = &graph_info.index_registry;
|
|
2116
|
+
let kind = index::IndexKind::Vector {
|
|
2117
|
+
dimension: spec.dimension,
|
|
2118
|
+
metric: spec.metric,
|
|
2119
|
+
params: spec.hnsw_params,
|
|
2120
|
+
};
|
|
2121
|
+
let idx = index::create_index(kind, &spec.name, spec.property.clone());
|
|
2122
|
+
registry.register_index(spec.name.clone(), idx.clone());
|
|
2123
|
+
|
|
2124
|
+
// Get nodes first, before borrowing transaction for write
|
|
2125
|
+
let nodes = crate::store::Store::select_nodes(
|
|
2126
|
+
self,
|
|
2127
|
+
transaction,
|
|
2128
|
+
graph_name_str,
|
|
2129
|
+
super::SelectNodeQuery::select_all(),
|
|
2130
|
+
)?;
|
|
2131
|
+
|
|
2132
|
+
// Backfill the vector index with existing nodes
|
|
2133
|
+
let tx = transaction.try_into_write()?;
|
|
2134
|
+
|
|
2135
|
+
if let Some(hnsw_idx) = idx.as_any().downcast_ref::<index::HnswVectorIndex>()
|
|
2136
|
+
{
|
|
2137
|
+
let tables = hnsw_store::HnswTableSet::new(graph_name_str, &spec.name);
|
|
2138
|
+
let mut graph_store = hnsw_store::RedbGraphStore::open(tx, &tables)?;
|
|
2139
|
+
let mut vector_store =
|
|
2140
|
+
hnsw_store::RedbVectorStore::open(tx, &tables, hnsw_idx.dimension(), hnsw_idx.encoding())?;
|
|
2141
|
+
|
|
2142
|
+
// Initialize metadata with fresh state
|
|
2143
|
+
let initial_meta = hnsw_store::SimpleMetaBlob {
|
|
2144
|
+
next_node_id: 0,
|
|
2145
|
+
entry_point: None,
|
|
2146
|
+
max_level: -1,
|
|
2147
|
+
};
|
|
2148
|
+
graph_store.write_meta(&initial_meta)?;
|
|
2149
|
+
|
|
2150
|
+
let hnsw = hnsw_idx.upsert_and_search_ready();
|
|
2151
|
+
|
|
2152
|
+
// Backfill: insert each node's vector
|
|
2153
|
+
for node in nodes
|
|
2154
|
+
{
|
|
2155
|
+
if let Some(prop_key) = spec.property.as_ref()
|
|
2156
|
+
&& let Some(prop_value) = node.properties().get(prop_key)
|
|
2157
|
+
&& let Some(floats) = crate::store::vector_extract::extract_f32_vec(prop_value)
|
|
2158
|
+
&& floats.len() == spec.dimension
|
|
2159
|
+
{
|
|
2160
|
+
hnsw.set(
|
|
2161
|
+
&mut graph_store,
|
|
2162
|
+
&mut vector_store,
|
|
2163
|
+
node.key(),
|
|
2164
|
+
floats.as_slice(),
|
|
2165
|
+
)?;
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
Ok(())
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2173
|
+
fn list_vector_indexes(
|
|
2174
|
+
&self,
|
|
2175
|
+
transaction: &mut Self::TransactionBox,
|
|
2176
|
+
graph_name: impl AsRef<str>,
|
|
2177
|
+
) -> Result<Vec<super::VectorIndexSpec>>
|
|
2178
|
+
{
|
|
2179
|
+
let graph_name = graph_name.as_ref();
|
|
2180
|
+
let mut specs = Vec::new();
|
|
2181
|
+
let metadata_list = self.get_graph_index_metadata(transaction, graph_name)?;
|
|
2182
|
+
for metadata in metadata_list
|
|
2183
|
+
{
|
|
2184
|
+
let IndexMetadata::VectorIndex {
|
|
2185
|
+
name,
|
|
2186
|
+
label,
|
|
2187
|
+
property,
|
|
2188
|
+
dimension,
|
|
2189
|
+
metric,
|
|
2190
|
+
hnsw_params,
|
|
2191
|
+
dtype,
|
|
2192
|
+
} = metadata;
|
|
2193
|
+
specs.push(super::VectorIndexSpec {
|
|
2194
|
+
name,
|
|
2195
|
+
label,
|
|
2196
|
+
property,
|
|
2197
|
+
dimension,
|
|
2198
|
+
metric,
|
|
2199
|
+
hnsw_params,
|
|
2200
|
+
dtype: graphcore::DType::from_name(&dtype).unwrap_or(graphcore::DType::F32),
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
Ok(specs)
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
fn drop_vector_index(
|
|
2208
|
+
&self,
|
|
2209
|
+
transaction: &mut Self::TransactionBox,
|
|
2210
|
+
graph_name: impl AsRef<str>,
|
|
2211
|
+
index_name: impl AsRef<str>,
|
|
2212
|
+
) -> Result<()>
|
|
2213
|
+
{
|
|
2214
|
+
self.drop_index(transaction, graph_name.as_ref(), index_name.as_ref())
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
fn upsert_vector(
|
|
2218
|
+
&self,
|
|
2219
|
+
transaction: &mut Self::TransactionBox,
|
|
2220
|
+
graph_name: impl AsRef<str>,
|
|
2221
|
+
index_name: impl AsRef<str>,
|
|
2222
|
+
node_id: graph::Key,
|
|
2223
|
+
tensor: &value::Tensor,
|
|
2224
|
+
) -> Result<()>
|
|
2225
|
+
{
|
|
2226
|
+
let graph_name = graph_name.as_ref();
|
|
2227
|
+
let index_name = index_name.as_ref();
|
|
2228
|
+
let tx = transaction.try_into_write()?;
|
|
2229
|
+
let graph_info = self.get_graph_info(graph_name)?;
|
|
2230
|
+
let registry = &graph_info.index_registry;
|
|
2231
|
+
|
|
2232
|
+
for index in registry.iter_indexes()
|
|
2233
|
+
{
|
|
2234
|
+
if index.as_any().is::<index::HnswVectorIndex>()
|
|
2235
|
+
&& index.name() == index_name
|
|
2236
|
+
&& let Some(hnsw_idx) = index.as_any().downcast_ref::<index::HnswVectorIndex>()
|
|
2237
|
+
{
|
|
2238
|
+
if tensor.numel() != hnsw_idx.dimension()
|
|
2239
|
+
{
|
|
2240
|
+
return Err(
|
|
2241
|
+
StoreError::InvalidFormat(format!(
|
|
2242
|
+
"vector dimension {} does not match index dimension {}",
|
|
2243
|
+
tensor.numel(),
|
|
2244
|
+
hnsw_idx.dimension()
|
|
2245
|
+
))
|
|
2246
|
+
.into(),
|
|
2247
|
+
);
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
// Convert tensor to f32 slice
|
|
2251
|
+
let floats = match tensor.dtype()
|
|
2252
|
+
{
|
|
2253
|
+
value::DType::F32 => match tensor.as_f32_slice()
|
|
2254
|
+
{
|
|
2255
|
+
Some(s) => s.to_vec(),
|
|
2256
|
+
None => tensor.to_f32_vec(),
|
|
2257
|
+
},
|
|
2258
|
+
value::DType::F64 => tensor.to_f32_vec(),
|
|
2259
|
+
other =>
|
|
2260
|
+
{
|
|
2261
|
+
return Err(
|
|
2262
|
+
StoreError::InvalidFormat(format!(
|
|
2263
|
+
"dtype {:?} cannot be cast to f32 for HNSW indexing",
|
|
2264
|
+
other
|
|
2265
|
+
))
|
|
2266
|
+
.into(),
|
|
2267
|
+
);
|
|
2268
|
+
}
|
|
2269
|
+
};
|
|
2270
|
+
|
|
2271
|
+
// Open stores and perform immediate upsert
|
|
2272
|
+
let tables = hnsw_store::HnswTableSet::new(graph_name, index_name);
|
|
2273
|
+
let mut graph_store = hnsw_store::RedbGraphStore::open(tx, &tables)?;
|
|
2274
|
+
let mut vector_store = hnsw_store::RedbVectorStore::open(
|
|
2275
|
+
tx,
|
|
2276
|
+
&tables,
|
|
2277
|
+
hnsw_idx.dimension(),
|
|
2278
|
+
hnsw_idx.encoding(),
|
|
2279
|
+
)?;
|
|
2280
|
+
|
|
2281
|
+
hnsw_idx.upsert_and_search_ready().set(
|
|
2282
|
+
&mut graph_store,
|
|
2283
|
+
&mut vector_store,
|
|
2284
|
+
node_id,
|
|
2285
|
+
floats.as_slice(),
|
|
2286
|
+
)?;
|
|
2287
|
+
|
|
2288
|
+
return Ok(());
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2292
|
+
Err(StoreError::InvalidFormat(format!("vector index '{}' does not exist", index_name)).into())
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
fn vector_search(
|
|
2296
|
+
&self,
|
|
2297
|
+
transaction: &mut Self::TransactionBox,
|
|
2298
|
+
graph_name: impl AsRef<str>,
|
|
2299
|
+
index_name: impl AsRef<str>,
|
|
2300
|
+
query: &value::Tensor,
|
|
2301
|
+
k: usize,
|
|
2302
|
+
) -> Result<Vec<super::VectorSearchResult>>
|
|
2303
|
+
{
|
|
2304
|
+
if k == 0
|
|
2305
|
+
{
|
|
2306
|
+
return Ok(Vec::new());
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
let f32_query: std::borrow::Cow<[f32]> = match query.dtype()
|
|
2310
|
+
{
|
|
2311
|
+
value::DType::F32 => match query.as_f32_slice()
|
|
2312
|
+
{
|
|
2313
|
+
Some(s) => std::borrow::Cow::Borrowed(s),
|
|
2314
|
+
None => std::borrow::Cow::Owned(query.to_f32_vec()),
|
|
2315
|
+
},
|
|
2316
|
+
value::DType::F64 => std::borrow::Cow::Owned(query.to_f32_vec()),
|
|
2317
|
+
other =>
|
|
2318
|
+
{
|
|
2319
|
+
return Err(
|
|
2320
|
+
StoreError::InvalidFormat(format!(
|
|
2321
|
+
"vector search only accepts F32 or F64 tensors; got {:?}",
|
|
2322
|
+
other
|
|
2323
|
+
))
|
|
2324
|
+
.into(),
|
|
2325
|
+
);
|
|
2326
|
+
}
|
|
2327
|
+
};
|
|
2328
|
+
|
|
2329
|
+
let graph_name = graph_name.as_ref();
|
|
2330
|
+
let index_name = index_name.as_ref();
|
|
2331
|
+
let graph_info = self.get_graph_info(graph_name)?;
|
|
2332
|
+
let registry = &graph_info.index_registry;
|
|
2333
|
+
|
|
2334
|
+
if let Some(index) = registry.get_index(index_name)
|
|
2335
|
+
&& let Some(hnsw_idx) = index.as_any().downcast_ref::<index::HnswVectorIndex>()
|
|
2336
|
+
{
|
|
2337
|
+
if f32_query.len() != hnsw_idx.dimension()
|
|
2338
|
+
{
|
|
2339
|
+
return Err(
|
|
2340
|
+
StoreError::InvalidFormat(format!(
|
|
2341
|
+
"Vector dimension {} does not match index dimension {}",
|
|
2342
|
+
f32_query.len(),
|
|
2343
|
+
hnsw_idx.dimension()
|
|
2344
|
+
))
|
|
2345
|
+
.into(),
|
|
2346
|
+
);
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
// Open stores from appropriate transaction type
|
|
2350
|
+
let tables = hnsw_store::HnswTableSet::new(graph_name, index_name);
|
|
2351
|
+
let ef_search = hnsw_idx.ef_search();
|
|
2352
|
+
let mut results = match transaction
|
|
2353
|
+
{
|
|
2354
|
+
TransactionBox::Write(tx) =>
|
|
2355
|
+
{
|
|
2356
|
+
let graph_store = hnsw_store::RedbGraphStore::open(tx, &tables)?;
|
|
2357
|
+
let vector_store = hnsw_store::RedbVectorStore::open(
|
|
2358
|
+
tx,
|
|
2359
|
+
&tables,
|
|
2360
|
+
hnsw_idx.dimension(),
|
|
2361
|
+
hnsw_idx.encoding(),
|
|
2362
|
+
)?;
|
|
2363
|
+
match hnsw_idx.upsert_and_search_ready().search(
|
|
2364
|
+
&graph_store,
|
|
2365
|
+
&vector_store,
|
|
2366
|
+
&f32_query,
|
|
2367
|
+
k,
|
|
2368
|
+
Some(ef_search),
|
|
2369
|
+
None,
|
|
2370
|
+
)
|
|
2371
|
+
{
|
|
2372
|
+
Ok(results) => results,
|
|
2373
|
+
Err(db_index::hnsw::Error::EmptyIndex) => vec![],
|
|
2374
|
+
Err(e) =>
|
|
2375
|
+
{
|
|
2376
|
+
return Err(StoreError::InvalidFormat(format!("HNSW search failed: {:?}", e)).into());
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
TransactionBox::Read(read_tx) =>
|
|
2381
|
+
{
|
|
2382
|
+
// For read transactions, use read-only stores
|
|
2383
|
+
if let Some(graph_store) = hnsw_store::RedbGraphStoreRead::open(read_tx, &tables)?
|
|
2384
|
+
{
|
|
2385
|
+
if let Some(vector_store) = hnsw_store::RedbVectorStoreRead::open(
|
|
2386
|
+
read_tx,
|
|
2387
|
+
&tables,
|
|
2388
|
+
hnsw_idx.dimension(),
|
|
2389
|
+
hnsw_idx.encoding(),
|
|
2390
|
+
)?
|
|
2391
|
+
{
|
|
2392
|
+
match hnsw_idx.upsert_and_search_ready().search(
|
|
2393
|
+
&graph_store,
|
|
2394
|
+
&vector_store,
|
|
2395
|
+
&f32_query,
|
|
2396
|
+
k,
|
|
2397
|
+
Some(ef_search),
|
|
2398
|
+
None,
|
|
2399
|
+
)
|
|
2400
|
+
{
|
|
2401
|
+
Ok(results) => results,
|
|
2402
|
+
Err(db_index::hnsw::Error::EmptyIndex) => vec![],
|
|
2403
|
+
Err(e) =>
|
|
2404
|
+
{
|
|
2405
|
+
return Err(
|
|
2406
|
+
StoreError::InvalidFormat(format!("HNSW search failed: {:?}", e)).into(),
|
|
2407
|
+
);
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
else
|
|
2412
|
+
{
|
|
2413
|
+
vec![]
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
else
|
|
2417
|
+
{
|
|
2418
|
+
vec![]
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
};
|
|
2422
|
+
|
|
2423
|
+
const EPSILON: f32 = 1e-6;
|
|
2424
|
+
let sequences = self.node_sequences.lock().unwrap();
|
|
2425
|
+
|
|
2426
|
+
results.sort_by(|a, b| {
|
|
2427
|
+
let distance_diff = (a.distance - b.distance).abs();
|
|
2428
|
+
if distance_diff < EPSILON
|
|
2429
|
+
{
|
|
2430
|
+
let seq_a = sequences.get(&a.key).copied().unwrap_or(u64::MAX);
|
|
2431
|
+
let seq_b = sequences.get(&b.key).copied().unwrap_or(u64::MAX);
|
|
2432
|
+
seq_a.cmp(&seq_b)
|
|
2433
|
+
}
|
|
2434
|
+
else
|
|
2435
|
+
{
|
|
2436
|
+
a.distance
|
|
2437
|
+
.partial_cmp(&b.distance)
|
|
2438
|
+
.unwrap_or(std::cmp::Ordering::Equal)
|
|
2439
|
+
}
|
|
2440
|
+
});
|
|
2441
|
+
|
|
2442
|
+
results.truncate(k);
|
|
2443
|
+
let vector_results = results
|
|
2444
|
+
.into_iter()
|
|
2445
|
+
.map(|hit| crate::store::VectorSearchResult {
|
|
2446
|
+
node_id: hit.key,
|
|
2447
|
+
score: hit.distance,
|
|
2448
|
+
})
|
|
2449
|
+
.collect();
|
|
2450
|
+
return Ok(vector_results);
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
Err(InternalError::Unimplemented("Vector index not found").into())
|
|
2454
|
+
}
|
|
2455
|
+
|
|
2456
|
+
fn apply_vector_indices_for_node(
|
|
2457
|
+
&self,
|
|
2458
|
+
transaction: &mut Self::TransactionBox,
|
|
2459
|
+
graph_name: impl AsRef<str>,
|
|
2460
|
+
node: &crate::graph::Node,
|
|
2461
|
+
) -> Result<()>
|
|
2462
|
+
{
|
|
2463
|
+
let graph_name = graph_name.as_ref();
|
|
2464
|
+
let graph_info = self.get_graph_info(graph_name)?;
|
|
2465
|
+
let metadata_list = self.get_graph_index_metadata(transaction, graph_name)?;
|
|
2466
|
+
let registry = &graph_info.index_registry;
|
|
2467
|
+
|
|
2468
|
+
for metadata in metadata_list.iter()
|
|
2469
|
+
{
|
|
2470
|
+
if let IndexMetadata::VectorIndex {
|
|
2471
|
+
name,
|
|
2472
|
+
label,
|
|
2473
|
+
property,
|
|
2474
|
+
dimension,
|
|
2475
|
+
..
|
|
2476
|
+
} = metadata
|
|
2477
|
+
&& let (Some(idx_label), Some(idx_property)) = (label.as_deref(), property.as_deref())
|
|
2478
|
+
&& node
|
|
2479
|
+
.labels()
|
|
2480
|
+
.iter()
|
|
2481
|
+
.any(|node_label| node_label == idx_label)
|
|
2482
|
+
&& let Some(index) = registry.get_index(name)
|
|
2483
|
+
&& index.as_any().is::<index::HnswVectorIndex>()
|
|
2484
|
+
&& let Some(prop_val) = node.properties().get(idx_property)
|
|
2485
|
+
&& let Some(vector) = crate::store::vector_extract::extract_f32_vec(prop_val)
|
|
2486
|
+
&& vector.len() == *dimension
|
|
2487
|
+
{
|
|
2488
|
+
let tensor = value::Tensor::from_f32_slice(&vector);
|
|
2489
|
+
self.upsert_vector(transaction, graph_name, name, node.key(), &tensor)?;
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
|
|
2493
|
+
Ok(())
|
|
2494
|
+
}
|
|
1340
2495
|
}
|