@harperfast/hnsw 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -0
- package/DESIGN.md +95 -9
- package/README.md +16 -6
- package/build.mjs +7 -6
- package/index.d.ts +42 -1
- package/index.js +39 -18
- package/package.json +8 -3
- package/src/bin/bench.rs +1 -1
- package/src/format.rs +241 -19
- package/src/graph.rs +263 -69
- package/src/insert.rs +141 -40
- package/src/invalidate.rs +336 -0
- package/src/lib.rs +3 -1
- package/src/napi.rs +92 -4
- package/src/search.rs +413 -87
- package/prebuilds/darwin-arm64/hnsw-plane.node +0 -0
- package/prebuilds/linux-arm64/hnsw-plane.node +0 -0
- package/prebuilds/linux-x64/hnsw-plane.node +0 -0
- package/prebuilds/win32-x64/hnsw-plane.node +0 -0
package/src/napi.rs
CHANGED
|
@@ -28,6 +28,69 @@ impl ScratchPool {
|
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
#[napi(object)]
|
|
32
|
+
pub struct InvalidationOutcome {
|
|
33
|
+
pub in_band: bool,
|
|
34
|
+
pub sidecar: bool,
|
|
35
|
+
pub in_band_error: Option<String>,
|
|
36
|
+
pub sidecar_error: Option<String>,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
impl From<crate::invalidate::Invalidation> for InvalidationOutcome {
|
|
40
|
+
fn from(outcome: crate::invalidate::Invalidation) -> Self {
|
|
41
|
+
InvalidationOutcome {
|
|
42
|
+
in_band: outcome.in_band.is_ok(),
|
|
43
|
+
sidecar: outcome.sidecar.is_ok(),
|
|
44
|
+
in_band_error: outcome.in_band.err().map(|e| e.to_string()),
|
|
45
|
+
sidecar_error: outcome.sidecar.err().map(|e| e.to_string()),
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/// Make the plane file at `path` unadoptable, durably, through a temporary handle released
|
|
51
|
+
/// before this returns: the in-band latch (watermark 0 on every handle, every later open
|
|
52
|
+
/// refused) and the fsync'd `.stale` sidecar. Both markers are attempted; throws only when
|
|
53
|
+
/// neither became durable, leaving the file exactly as found. Synchronous: three small
|
|
54
|
+
/// fsyncs on a cold path. Use invalidatePlaneAsync where the caller can await.
|
|
55
|
+
#[napi]
|
|
56
|
+
pub fn invalidate_plane(path: String) -> Result<InvalidationOutcome> {
|
|
57
|
+
crate::invalidate::invalidate_plane(std::path::Path::new(&path))
|
|
58
|
+
.map(InvalidationOutcome::from)
|
|
59
|
+
.map_err(|e| Error::from_reason(e.to_string()))
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
pub struct InvalidateTask {
|
|
63
|
+
path: String,
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
#[napi]
|
|
67
|
+
impl Task for InvalidateTask {
|
|
68
|
+
type Output = InvalidationOutcome;
|
|
69
|
+
type JsValue = InvalidationOutcome;
|
|
70
|
+
|
|
71
|
+
fn compute(&mut self) -> Result<Self::Output> {
|
|
72
|
+
crate::invalidate::invalidate_plane(std::path::Path::new(&self.path))
|
|
73
|
+
.map(InvalidationOutcome::from)
|
|
74
|
+
.map_err(|e| Error::from_reason(e.to_string()))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
|
78
|
+
Ok(output)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/// invalidatePlane on the libuv thread pool, for callers that can await the fsyncs.
|
|
83
|
+
#[napi(ts_return_type = "Promise<InvalidationOutcome>")]
|
|
84
|
+
pub fn invalidate_plane_async(path: String) -> AsyncTask<InvalidateTask> {
|
|
85
|
+
AsyncTask::new(InvalidateTask { path })
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// The sidecar convention checked at attach: `<plane path>.stale`.
|
|
89
|
+
#[napi]
|
|
90
|
+
pub fn stale_path_for(path: String) -> String {
|
|
91
|
+
crate::invalidate::stale_path_for(std::path::Path::new(&path)).to_string_lossy().into_owned()
|
|
92
|
+
}
|
|
93
|
+
|
|
31
94
|
#[napi(object)]
|
|
32
95
|
pub struct SearchHit {
|
|
33
96
|
pub id: u32,
|
|
@@ -92,7 +155,7 @@ impl Task for PredicateSearchTask {
|
|
|
92
155
|
dispatch: Box::new(move |ids: Vec<u32>| {
|
|
93
156
|
let tx = tx.clone();
|
|
94
157
|
let ids_echo = ids.clone();
|
|
95
|
-
tsfn.call_with_return_value(
|
|
158
|
+
let status = tsfn.call_with_return_value(
|
|
96
159
|
ids,
|
|
97
160
|
ThreadsafeFunctionCallMode::NonBlocking,
|
|
98
161
|
move |ret: Uint8Array| {
|
|
@@ -102,6 +165,10 @@ impl Task for PredicateSearchTask {
|
|
|
102
165
|
Ok(())
|
|
103
166
|
},
|
|
104
167
|
);
|
|
168
|
+
// a closing or saturated queue drops the callback without invoking it, so this
|
|
169
|
+
// batch will never answer; reporting it lets the drain finish on the batches
|
|
170
|
+
// that will, instead of holding teardown for the full deadline
|
|
171
|
+
status == Status::Ok
|
|
105
172
|
}),
|
|
106
173
|
rx,
|
|
107
174
|
};
|
|
@@ -240,9 +307,6 @@ impl Plane {
|
|
|
240
307
|
id, self.graph.file.max_nodes
|
|
241
308
|
)));
|
|
242
309
|
}
|
|
243
|
-
if (id as u64) >= self.graph.file.max_nodes {
|
|
244
|
-
return Err(Error::from_reason(format!("id {} exceeds plane capacity {}", id, self.graph.file.max_nodes)));
|
|
245
|
-
}
|
|
246
310
|
if !(scale as f32).is_finite() || !(inv_mag as f32).is_finite() {
|
|
247
311
|
return Err(Error::from_reason("scale/invMag must be finite"));
|
|
248
312
|
}
|
|
@@ -483,4 +547,28 @@ impl Plane {
|
|
|
483
547
|
pub fn flush(&self, watermark: Option<f64>) -> Result<()> {
|
|
484
548
|
self.graph.file.flush_with_watermark(watermark.map(|w| w as u64)).map_err(|e| Error::from_reason(e.to_string()))
|
|
485
549
|
}
|
|
550
|
+
|
|
551
|
+
/// Durably mark this plane invalidated in band: set the one-way latch, zero the watermark,
|
|
552
|
+
/// and msync the header page alone, so a host disabling a plane it cannot delete has the
|
|
553
|
+
/// mark on disk before it writes any out-of-band tombstone. Synchronous by design — it is
|
|
554
|
+
/// a 4 KB msync, not the whole-mapping writeback `flush` performs.
|
|
555
|
+
#[napi]
|
|
556
|
+
pub fn invalidate(&self) -> Result<()> {
|
|
557
|
+
self.graph.file.invalidate().map_err(|e| Error::from_reason(e.to_string()))
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/// invalidatePlane through this handle: the in-band mark via this mapping (no second
|
|
561
|
+
/// open, no second registry slot) and the `.stale` sidecar next to the path it opened.
|
|
562
|
+
#[napi]
|
|
563
|
+
pub fn invalidate_file(&self) -> Result<InvalidationOutcome> {
|
|
564
|
+
crate::invalidate::invalidate_file(&self.graph.file)
|
|
565
|
+
.map(InvalidationOutcome::from)
|
|
566
|
+
.map_err(|e| Error::from_reason(e.to_string()))
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/// Whether the plane was invalidated (by any handle) since this one opened.
|
|
570
|
+
#[napi]
|
|
571
|
+
pub fn invalidated(&self) -> bool {
|
|
572
|
+
self.graph.file.invalidated()
|
|
573
|
+
}
|
|
486
574
|
}
|