faiss 0.5.0 → 0.5.1

data/vendor/faiss/faiss/svs/IndexSVSVamana.h

@@ -0,0 +1,137 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <faiss/Index.h>
+#include <faiss/svs/IndexSVSFaissUtils.h>
+
+#include <svs/runtime/api_defs.h>
+
+#include <iostream>
+
+namespace faiss {
+
+struct SearchParametersSVSVamana : public SearchParameters {
+    size_t search_window_size = 0;
+    size_t search_buffer_capacity = 0;
+};
+
+// redefinition for swig export
+enum SVSStorageKind {
+    SVS_FP32,
+    SVS_FP16,
+    SVS_SQI8,
+    SVS_LVQ4x0,
+    SVS_LVQ4x4,
+    SVS_LVQ4x8,
+    SVS_LeanVec4x4,
+    SVS_LeanVec4x8,
+    SVS_LeanVec8x8,
+};
+
+inline svs_runtime::StorageKind to_svs_storage_kind(SVSStorageKind kind) {
+    switch (kind) {
+        case SVS_FP32:
+            return svs_runtime::StorageKind::FP32;
+        case SVS_FP16:
+            return svs_runtime::StorageKind::FP16;
+        case SVS_SQI8:
+            return svs_runtime::StorageKind::SQI8;
+        case SVS_LVQ4x0:
+            return svs_runtime::StorageKind::LVQ4x0;
+        case SVS_LVQ4x4:
+            return svs_runtime::StorageKind::LVQ4x4;
+        case SVS_LVQ4x8:
+            return svs_runtime::StorageKind::LVQ4x8;
+        case SVS_LeanVec4x4:
+            return svs_runtime::StorageKind::LeanVec4x4;
+        case SVS_LeanVec4x8:
+            return svs_runtime::StorageKind::LeanVec4x8;
+        case SVS_LeanVec8x8:
+            return svs_runtime::StorageKind::LeanVec8x8;
+        default:
+            FAISS_ASSERT(!"not supported SVS storage kind");
+    }
+}
+
+struct IndexSVSVamana : Index {
+    size_t graph_max_degree;
+    size_t prune_to;
+    float alpha = 1.2;
+    size_t search_window_size = 10;
+    size_t search_buffer_capacity = 10;
+    size_t construction_window_size = 40;
+    size_t max_candidate_pool_size = 200;
+    bool use_full_search_history = true;
+
+    SVSStorageKind storage_kind;
+
+    IndexSVSVamana();
+
+    IndexSVSVamana(
+            idx_t d,
+            size_t degree,
+            MetricType metric = METRIC_L2,
+            SVSStorageKind storage = SVSStorageKind::SVS_FP32);
+
+    ~IndexSVSVamana() override;
+
+    // static member that exposes whether or not LVQ/LeanVec are enabled for
+    // this build and runtime.
+    static bool is_lvq_leanvec_enabled();
+
+    void add(idx_t n, const float* x) override;
+
+    void search(
+            idx_t n,
+            const float* x,
+            idx_t k,
+            float* distances,
+            idx_t* labels,
+            const SearchParameters* params = nullptr) const override;
+
+    void range_search(
+            idx_t n,
+            const float* x,
+            float radius,
+            RangeSearchResult* result,
+            const SearchParameters* params = nullptr) const override;
+
+    size_t remove_ids(const IDSelector& sel) override;
+
+    void reset() override;
+
+    /* Serialization and deserialization helpers */
+    void serialize_impl(std::ostream& out) const;
+    virtual void deserialize_impl(std::istream& in);
+
+    /* The actual SVS implementation */
+    svs_runtime::DynamicVamanaIndex* impl{nullptr};
+
+   protected:
+    /* Initializes the implementation*/
+    virtual void create_impl();
+};
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp

@@ -0,0 +1,39 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <faiss/svs/IndexSVSVamanaLVQ.h>
+
+namespace faiss {
+
+IndexSVSVamanaLVQ::IndexSVSVamanaLVQ() : IndexSVSVamana() {
+    storage_kind = SVSStorageKind::SVS_LVQ4x0;
+}
+
+IndexSVSVamanaLVQ::IndexSVSVamanaLVQ(
+        idx_t d,
+        size_t degree,
+        MetricType metric,
+        SVSStorageKind storage)
+        : IndexSVSVamana(d, degree, metric, storage) {}
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h

@@ -0,0 +1,42 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <faiss/svs/IndexSVSVamana.h>
+
+namespace faiss {
+
+struct IndexSVSVamanaLVQ : IndexSVSVamana {
+    IndexSVSVamanaLVQ();
+
+    IndexSVSVamanaLVQ(
+            idx_t d,
+            size_t degree,
+            MetricType metric = METRIC_L2,
+            SVSStorageKind storage = SVSStorageKind::SVS_LVQ4x4);
+
+    ~IndexSVSVamanaLVQ() override = default;
+};
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp

@@ -0,0 +1,149 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <faiss/svs/IndexSVSFaissUtils.h>
+#include <faiss/svs/IndexSVSVamanaLeanVec.h>
+
+#include <svs/runtime/dynamic_vamana_index.h>
+#include <svs/runtime/training.h>
+#include <svs/runtime/vamana_index.h>
+
+#include <memory>
+#include <span>
+#include "faiss/svs/IndexSVSVamana.h"
+
+namespace faiss {
+
+IndexSVSVamanaLeanVec::IndexSVSVamanaLeanVec() : IndexSVSVamana() {
+    is_trained = false;
+    storage_kind = SVSStorageKind::SVS_LeanVec4x4;
+}
+
+IndexSVSVamanaLeanVec::IndexSVSVamanaLeanVec(
+        idx_t d,
+        size_t degree,
+        MetricType metric,
+        size_t leanvec_dims,
+        SVSStorageKind storage_kind)
+        : IndexSVSVamana(d, degree, metric, storage_kind) {
+    is_trained = false;
+    leanvec_d = leanvec_dims == 0 ? d / 2 : leanvec_dims;
+}
+
+IndexSVSVamanaLeanVec::~IndexSVSVamanaLeanVec() {
+    if (training_data) {
+        auto status = svs_runtime::LeanVecTrainingData::destroy(training_data);
+        FAISS_ASSERT(status.ok());
+        training_data = nullptr;
+    }
+    IndexSVSVamana::~IndexSVSVamana();
+}
+
+void IndexSVSVamanaLeanVec::add(idx_t n, const float* x) {
+    FAISS_THROW_IF_MSG(
+            !is_trained, "Index not trained: call train() before add().");
+    IndexSVSVamana::add(n, x);
+}
+
+void IndexSVSVamanaLeanVec::train(idx_t n, const float* x) {
+    FAISS_THROW_IF_MSG(
+            training_data || impl, "Index already trained or contains data.");
+
+    FAISS_THROW_IF_NOT_MSG(
+            IndexSVSVamana::is_lvq_leanvec_enabled(),
+            "LVQ/LeanVec support not available on this platform or build");
+
+    auto status = svs_runtime::LeanVecTrainingData::build(
+            &training_data, d, n, x, leanvec_d);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    FAISS_THROW_IF_NOT_MSG(
+            training_data, "Failed to build leanvec training info.");
+    is_trained = true;
+}
+
+void IndexSVSVamanaLeanVec::serialize_training_data(std::ostream& out) const {
+    FAISS_THROW_IF_NOT_MSG(
+            training_data, "Cannot serialize: Training data not initialized.");
+
+    auto status = training_data->save(out);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+}
+
+void IndexSVSVamanaLeanVec::deserialize_training_data(std::istream& in) {
+    svs_runtime::LeanVecTrainingData* tdata = nullptr;
+    auto status = svs_runtime::LeanVecTrainingData::load(&tdata, in);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    FAISS_THROW_IF_NOT_MSG(tdata, "Failed to load leanvec training data.");
+    training_data = tdata;
+}
+
+void IndexSVSVamanaLeanVec::create_impl() {
+    ntotal = 0;
+    auto svs_metric = to_svs_metric(metric_type);
+    auto svs_storage_kind = to_svs_storage_kind(storage_kind);
+    auto build_params = svs_runtime::VamanaIndex::BuildParams{
+            .graph_max_degree = graph_max_degree,
+            .prune_to = prune_to,
+            .alpha = alpha,
+            .construction_window_size = construction_window_size,
+            .max_candidate_pool_size = max_candidate_pool_size,
+            .use_full_search_history = use_full_search_history,
+    };
+    auto search_params = svs_runtime::VamanaIndex::SearchParams{
+            .search_window_size = search_window_size,
+            .search_buffer_capacity = search_buffer_capacity,
+    };
+    auto status = svs_runtime::Status_Ok;
+    if (training_data) {
+        status = svs_runtime::DynamicVamanaIndexLeanVec::build(
+                &impl,
+                d,
+                svs_metric,
+                svs_storage_kind,
+                training_data,
+                build_params,
+                search_params);
+    } else {
+        status = svs_runtime::DynamicVamanaIndexLeanVec::build(
+                &impl,
+                d,
+                svs_metric,
+                svs_storage_kind,
+                leanvec_d,
+                build_params,
+                search_params);
+    }
+
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    FAISS_THROW_IF_NOT(impl);
+}
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h

@@ -0,0 +1,58 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <faiss/svs/IndexSVSVamana.h>
+
+namespace faiss {
+
+struct IndexSVSVamanaLeanVec : IndexSVSVamana {
+    IndexSVSVamanaLeanVec();
+
+    IndexSVSVamanaLeanVec(
+            idx_t d,
+            size_t degree,
+            MetricType metric = METRIC_L2,
+            size_t leanvec_dims = 0,
+            SVSStorageKind storage = SVSStorageKind::SVS_LeanVec4x4);
+
+    ~IndexSVSVamanaLeanVec() override;
+
+    void add(idx_t n, const float* x) override;
+
+    void train(idx_t n, const float* x) override;
+
+    void serialize_training_data(std::ostream& out) const;
+    void deserialize_training_data(std::istream& in);
+
+    size_t leanvec_d;
+
+    /* Training information */
+    svs_runtime::LeanVecTrainingData* training_data{nullptr};
+
+   protected:
+    void create_impl() override;
+};
+
+} // namespace faiss

data/vendor/faiss/faiss/utils/distances.cpp

@@ -321,7 +321,6 @@ void exhaustive_L2sqr_blas_default_impl(
                        ip_block.get(),
                        &nyi);
             }
-#pragma omp parallel for
             for (int64_t i = i0; i < i1; i++) {
                 float* ip_line = ip_block.get() + (i - i0) * (j1 - j0);
 
@@ -423,7 +422,6 @@ void exhaustive_L2sqr_blas_cmax_avx2(
                        ip_block.get(),
                        &nyi);
             }
-#pragma omp parallel for
             for (int64_t i = i0; i < i1; i++) {
                 float* ip_line = ip_block.get() + (i - i0) * (j1 - j0);
 
@@ -633,7 +631,6 @@ void exhaustive_L2sqr_blas_cmax_sve(
                        ip_block.get(),
                        &nyi);
             }
-#pragma omp parallel for
             for (int64_t i = i0; i < i1; i++) {
                 const size_t count = j1 - j0;
                 float* ip_line = ip_block.get() + (i - i0) * count;

data/vendor/faiss/faiss/utils/utils.cpp

@@ -127,6 +127,10 @@ std::string get_compile_options() {
     options += "GENERIC ";
 #endif
 
+#ifdef FAISS_ENABLE_SVS
+    options += "SVS ";
+#endif
+
     options += ref_gpu_compile_options();
 
     return options;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Andrew Kane
@@ -249,6 +249,8 @@ files:
 - vendor/faiss/faiss/impl/NNDescent.h
 - vendor/faiss/faiss/impl/NSG.cpp
 - vendor/faiss/faiss/impl/NSG.h
+- vendor/faiss/faiss/impl/Panorama.cpp
+- vendor/faiss/faiss/impl/Panorama.h
 - vendor/faiss/faiss/impl/PanoramaStats.cpp
 - vendor/faiss/faiss/impl/PanoramaStats.h
 - vendor/faiss/faiss/impl/PolysemousTraining.cpp
@@ -259,10 +261,14 @@ files:
 - vendor/faiss/faiss/impl/ProductQuantizer.cpp
 - vendor/faiss/faiss/impl/ProductQuantizer.h
 - vendor/faiss/faiss/impl/Quantizer.h
+- vendor/faiss/faiss/impl/RaBitQStats.cpp
+- vendor/faiss/faiss/impl/RaBitQStats.h
 - vendor/faiss/faiss/impl/RaBitQUtils.cpp
 - vendor/faiss/faiss/impl/RaBitQUtils.h
 - vendor/faiss/faiss/impl/RaBitQuantizer.cpp
 - vendor/faiss/faiss/impl/RaBitQuantizer.h
+- vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp
+- vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.h
 - vendor/faiss/faiss/impl/ResidualQuantizer.cpp
 - vendor/faiss/faiss/impl/ResidualQuantizer.h
 - vendor/faiss/faiss/impl/ResultHandler.h
@@ -296,6 +302,8 @@ files:
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h
 - vendor/faiss/faiss/impl/simd_result_handlers.h
+- vendor/faiss/faiss/impl/svs_io.cpp
+- vendor/faiss/faiss/impl/svs_io.h
 - vendor/faiss/faiss/impl/zerocopy_io.cpp
 - vendor/faiss/faiss/impl/zerocopy_io.h
 - vendor/faiss/faiss/index_factory.cpp
@@ -313,6 +321,15 @@ files:
 - vendor/faiss/faiss/invlists/OnDiskInvertedLists.h
 - vendor/faiss/faiss/python/python_callbacks.cpp
 - vendor/faiss/faiss/python/python_callbacks.h
+- vendor/faiss/faiss/svs/IndexSVSFaissUtils.h
+- vendor/faiss/faiss/svs/IndexSVSFlat.cpp
+- vendor/faiss/faiss/svs/IndexSVSFlat.h
+- vendor/faiss/faiss/svs/IndexSVSVamana.cpp
+- vendor/faiss/faiss/svs/IndexSVSVamana.h
+- vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp
+- vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h
+- vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp
+- vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h
 - vendor/faiss/faiss/utils/AlignedTable.h
 - vendor/faiss/faiss/utils/Heap.cpp
 - vendor/faiss/faiss/utils/Heap.h