faiss 0.6.0 → 0.6.1

data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp

@@ -0,0 +1,350 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2026 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <faiss/svs/IndexSVSFaissUtils.h>
+#include <faiss/svs/IndexSVSIVF.h>
+
+#include <faiss/Index.h>
+
+#include <svs/runtime/api_defs.h>
+#include <svs/runtime/dynamic_ivf_index.h>
+#include <svs/runtime/ivf_index.h>
+
+#include <cstddef>
+#include <numeric>
+#include <span>
+#include <type_traits>
+#include <vector>
+
+namespace faiss {
+namespace {
+svs_runtime::IVFIndex::SearchParams make_ivf_search_parameters(
+        const IndexSVSIVF& index,
+        const SearchParameters* params) {
+    FAISS_THROW_IF_NOT(index.impl);
+
+    auto n_probes = index.n_probes;
+    auto k_reorder = index.k_reorder;
+
+    if (auto svs_params = dynamic_cast<const SearchParametersSVSIVF*>(params)) {
+        if (svs_params->n_probes > 0) {
+            n_probes = svs_params->n_probes;
+        }
+        if (svs_params->k_reorder > 0) {
+            k_reorder = svs_params->k_reorder;
+        }
+    }
+
+    return {n_probes, k_reorder};
+}
+} // namespace
+
+IndexSVSIVF::IndexSVSIVF() = default;
+
+IndexSVSIVF::IndexSVSIVF(
+        idx_t d,
+        size_t nlist,
+        MetricType metric,
+        SVSStorageKind storage,
+        bool is_static)
+        : Index(d, metric),
+          num_centroids{nlist},
+          is_static{is_static},
+          storage_kind{storage} {
+    is_trained = false;
+
+    // Validate the requested storage kind is available in current runtime.
+    auto svs_storage = to_svs_storage_kind(storage_kind);
+    auto status = is_static
+            ? svs_runtime::IVFIndex::check_storage_kind(svs_storage)
+            : svs_runtime::DynamicIVFIndex::check_storage_kind(svs_storage);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+}
+
+bool IndexSVSIVF::is_lvq_leanvec_enabled() {
+    auto lvq = to_svs_storage_kind(SVS_LVQ4x0);
+    auto status = svs_runtime::IVFIndex::check_storage_kind(lvq);
+    if (!status.ok()) {
+        return false;
+    }
+    status = svs_runtime::DynamicIVFIndex::check_storage_kind(lvq);
+    if (!status.ok()) {
+        return false;
+    }
+    auto leanvec = to_svs_storage_kind(SVS_LeanVec4x4);
+    status = svs_runtime::IVFIndex::check_storage_kind(leanvec);
+    if (!status.ok()) {
+        return false;
+    }
+    status = svs_runtime::DynamicIVFIndex::check_storage_kind(leanvec);
+    if (!status.ok()) {
+        return false;
+    }
+    return true;
+}
+
+IndexSVSIVF::~IndexSVSIVF() {
+    if (impl) {
+        svs_runtime::Status status;
+        if (is_static) {
+            status = svs_runtime::IVFIndex::destroy(impl);
+        } else {
+            status = svs_runtime::DynamicIVFIndex::destroy(
+                    static_cast<svs_runtime::DynamicIVFIndex*>(impl));
+        }
+        FAISS_ASSERT(status.ok());
+        impl = nullptr;
+    }
+}
+
+void IndexSVSIVF::train(idx_t n, const float* x) {
+    FAISS_THROW_IF_MSG(impl, "Index already trained or contains data.");
+    create_impl(n, x);
+    is_trained = true;
+}
+
+void IndexSVSIVF::add(idx_t n, const float* x) {
+    FAISS_THROW_IF_MSG(
+            is_static,
+            "Static IVF index does not support add() after initial index "
+            "creation. All data must be provided during train().");
+    FAISS_THROW_IF_MSG(
+            !is_trained, "Index not trained: call train() before add().");
+    FAISS_THROW_IF_NOT(impl);
+
+    auto* dyn = dynamic_impl();
+    std::vector<size_t> labels(n);
+    std::iota(labels.begin(), labels.end(), ntotal);
+
+    auto status = dyn->add(n, labels.data(), x);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    ntotal += n;
+}
+
+void IndexSVSIVF::reset() {
+    if (impl) {
+        svs_runtime::Status status;
+        if (is_static) {
+            status = svs_runtime::IVFIndex::destroy(impl);
+        } else {
+            status = svs_runtime::DynamicIVFIndex::destroy(
+                    static_cast<svs_runtime::DynamicIVFIndex*>(impl));
+        }
+        FAISS_ASSERT(status.ok());
+        impl = nullptr;
+    }
+    is_trained = false;
+    ntotal = 0;
+}
+
+void IndexSVSIVF::search(
+        idx_t n,
+        const float* x,
+        idx_t k,
+        float* distances,
+        idx_t* labels,
+        const SearchParameters* params) const {
+    if (!impl) {
+        for (idx_t i = 0; i < n * k; ++i) {
+            distances[i] = std::numeric_limits<float>::infinity();
+            labels[i] = -1;
+        }
+        return;
+    }
+    FAISS_THROW_IF_NOT(k > 0);
+    FAISS_THROW_IF_NOT(is_trained);
+
+    // Sync thread settings to the SVS backend before searching
+    {
+        size_t current_threads = 0;
+        impl->get_num_threads(&current_threads);
+        if (num_threads > 0 && current_threads != num_threads) {
+            auto st = impl->set_num_threads(num_threads);
+            if (!st.ok()) {
+                FAISS_THROW_MSG(st.message());
+            }
+        }
+    }
+
+    auto sp = make_ivf_search_parameters(*this, params);
+    auto status = impl->search(
+            static_cast<size_t>(n),
+            x,
+            static_cast<size_t>(k),
+            distances,
+            convert_output_buffer<size_t>(labels, static_cast<size_t>(n * k)),
+            &sp);
+
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+}
+
+size_t IndexSVSIVF::remove_ids(const IDSelector& sel) {
+    FAISS_THROW_IF_MSG(
+            is_static,
+            "Static IVF index does not support remove_ids(). "
+            "The index is immutable after creation.");
+    FAISS_THROW_IF_NOT(impl);
+    auto* dyn = dynamic_impl();
+    auto id_filter = FaissIDFilter{sel};
+    size_t removed = 0;
+    auto status = dyn->remove_selected(&removed, id_filter);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    ntotal -= removed;
+    return removed;
+}
+
+void IndexSVSIVF::create_impl(idx_t n, const float* x) {
+    FAISS_THROW_IF_NOT(!impl);
+    ntotal = 0;
+    auto svs_metric = to_svs_metric(metric_type);
+    auto svs_storage_kind = to_svs_storage_kind(storage_kind);
+    auto build_params = svs_runtime::IVFIndex::BuildParams{
+            .num_centroids = num_centroids,
+            .minibatch_size = minibatch_size,
+            .num_iterations = num_iterations,
+            .is_hierarchical = is_hierarchical,
+            .training_fraction = training_fraction,
+            .hierarchical_level1_clusters = hierarchical_level1_clusters,
+            .seed = seed,
+    };
+    auto search_params = svs_runtime::IVFIndex::SearchParams{
+            .n_probes = n_probes,
+            .k_reorder = k_reorder,
+    };
+
+    svs_runtime::Status status;
+    if (is_static) {
+        status = svs_runtime::IVFIndex::build(
+                &impl,
+                d,
+                svs_metric,
+                svs_storage_kind,
+                static_cast<size_t>(n),
+                x,
+                build_params,
+                search_params,
+                num_threads,
+                intra_query_threads);
+    } else {
+        std::vector<size_t> labels(n);
+        std::iota(labels.begin(), labels.end(), 0);
+
+        svs_runtime::DynamicIVFIndex* dyn_impl = nullptr;
+        status = svs_runtime::DynamicIVFIndex::build(
+                &dyn_impl,
+                d,
+                svs_metric,
+                svs_storage_kind,
+                static_cast<size_t>(n),
+                x,
+                labels.data(),
+                build_params,
+                search_params,
+                num_threads,
+                intra_query_threads);
+        impl = dyn_impl;
+    }
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    FAISS_THROW_IF_NOT(impl);
+
+    // Force reinitialization of SVS search infrastructure after build.
+    // Same workaround as in deserialize_impl — the move chain through
+    // the orchestrator wrappers can leave matmul_results_ inconsistent
+    // when threads > centroids.
+    size_t current_threads = 0;
+    impl->get_num_threads(&current_threads);
+    auto st = impl->set_num_threads(current_threads);
+    if (!st.ok()) {
+        FAISS_THROW_MSG(st.message());
+    }
+
+    ntotal = n;
+}
+
+void IndexSVSIVF::serialize_impl(std::ostream& out) const {
+    FAISS_THROW_IF_NOT_MSG(
+            impl, "Cannot serialize: SVS IVF index not initialized.");
+
+    auto status = impl->save(out);
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+}
+
+void IndexSVSIVF::deserialize_impl(std::istream& in) {
+    FAISS_THROW_IF_MSG(
+            impl, "Cannot deserialize: SVS IVF index already loaded.");
+    auto svs_metric = to_svs_metric(metric_type);
+    auto svs_storage_kind = to_svs_storage_kind(storage_kind);
+
+    svs_runtime::Status status;
+    if (is_static) {
+        status = svs_runtime::IVFIndex::load(
+                &impl,
+                in,
+                svs_metric,
+                svs_storage_kind,
+                num_threads,
+                intra_query_threads);
+    } else {
+        svs_runtime::DynamicIVFIndex* dyn_impl = nullptr;
+        status = svs_runtime::DynamicIVFIndex::load(
+                &dyn_impl,
+                in,
+                svs_metric,
+                svs_storage_kind,
+                num_threads,
+                intra_query_threads);
+        impl = dyn_impl;
+    }
+    if (!status.ok()) {
+        FAISS_THROW_MSG(status.message());
+    }
+    FAISS_THROW_IF_NOT_MSG(impl, "SVS IVF load returned ok but impl is null");
+
+    size_t current_threads = 0;
+    impl->get_num_threads(&current_threads);
+    auto st = impl->set_num_threads(current_threads);
+    if (!st.ok()) {
+        FAISS_THROW_MSG(st.message());
+    }
+}
+
+svs_runtime::DynamicIVFIndex* IndexSVSIVF::dynamic_impl() const {
+    FAISS_THROW_IF_MSG(
+            is_static, "Operation not supported on a static IVF index.");
+    FAISS_THROW_IF_NOT(impl);
+    return static_cast<svs_runtime::DynamicIVFIndex*>(impl);
+}
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSIVF.h

@@ -0,0 +1,128 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2026 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <faiss/Index.h>
+#include <faiss/svs/IndexSVSFaissUtils.h>
+#include <faiss/svs/IndexSVSVamana.h> // for SVSStorageKind, to_svs_storage_kind
+
+#include <svs/runtime/api_defs.h>
+#include <svs/runtime/dynamic_ivf_index.h>
+#include <svs/runtime/ivf_index.h>
+
+#include <iostream>
+
+namespace faiss {
+
+struct SearchParametersSVSIVF : public SearchParameters {
+    size_t n_probes = 0;
+    float k_reorder = 0;
+};
+
+struct IndexSVSIVF : Index {
+    /// Number of centroids / clusters
+    size_t num_centroids = 1000;
+    /// Minibatch size for k-means clustering
+    size_t minibatch_size = 10000;
+    /// Number of iterations for k-means clustering
+    size_t num_iterations = 10;
+    /// Whether to use hierarchical clustering
+    bool is_hierarchical = true;
+    /// Fraction of data to use for training (0.0 to 1.0)
+    float training_fraction = 0.1f;
+    /// Number of level-1 clusters for hierarchical clustering
+    size_t hierarchical_level1_clusters = 0;
+    /// Random seed for clustering
+    size_t seed = 42;
+
+    /// Number of probes for search
+    size_t n_probes = 10;
+    /// Reranking multiplier for compressed datasets
+    float k_reorder = 1.0f;
+
+    /// Number of threads for inter-query parallelism (0 = use all available)
+    size_t num_threads = 0;
+    /// Number of threads for intra-query parallelism (cluster exploration).
+    /// NOTE: Must be set before train() or deserialize_impl(); changes after
+    /// index creation are not propagated to the SVS backend. This is a
+    /// limitation of the current SVS runtime API and will be fixed in a
+    /// future SVS version.
+    size_t intra_query_threads = 1;
+
+    /// Whether this is a static (immutable) IVF index
+    bool is_static = false;
+
+    SVSStorageKind storage_kind = SVSStorageKind::SVS_FP32;
+
+    IndexSVSIVF();
+
+    IndexSVSIVF(
+            idx_t d,
+            size_t nlist,
+            MetricType metric = METRIC_L2,
+            SVSStorageKind storage = SVSStorageKind::SVS_FP32,
+            bool is_static = false);
+
+    ~IndexSVSIVF() override;
+
+    IndexSVSIVF(const IndexSVSIVF&) = delete;
+    IndexSVSIVF& operator=(const IndexSVSIVF&) = delete;
+
+    // static member that exposes whether or not LVQ/LeanVec are enabled for
+    // this build and runtime.
+    static bool is_lvq_leanvec_enabled();
+
+    void train(idx_t n, const float* x) override;
+
+    void add(idx_t n, const float* x) override;
+
+    void search(
+            idx_t n,
+            const float* x,
+            idx_t k,
+            float* distances,
+            idx_t* labels,
+            const SearchParameters* params = nullptr) const override;
+
+    size_t remove_ids(const IDSelector& sel) override;
+
+    void reset() override;
+
+    /* Serialization and deserialization helpers */
+    void serialize_impl(std::ostream& out) const;
+    virtual void deserialize_impl(std::istream& in);
+
+    /* The actual SVS implementation (IVFIndex is the base for both
+       static and dynamic variants) */
+    svs_runtime::IVFIndex* impl{nullptr};
+
+   protected:
+    /* Initializes the implementation from training data */
+    virtual void create_impl(idx_t n, const float* x);
+
+    /* Returns the dynamic impl pointer, throwing if static */
+    svs_runtime::DynamicIVFIndex* dynamic_impl() const;
+};
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSIVFLVQ.cpp

@@ -0,0 +1,40 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2026 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <faiss/svs/IndexSVSIVFLVQ.h>
+
+namespace faiss {
+
+IndexSVSIVFLVQ::IndexSVSIVFLVQ() : IndexSVSIVF() {
+    storage_kind = SVSStorageKind::SVS_LVQ4x0;
+}
+
+IndexSVSIVFLVQ::IndexSVSIVFLVQ(
+        idx_t d,
+        size_t nlist,
+        MetricType metric,
+        SVSStorageKind storage,
+        bool is_static)
+        : IndexSVSIVF(d, nlist, metric, storage, is_static) {}
+
+} // namespace faiss

data/vendor/faiss/faiss/svs/IndexSVSIVFLVQ.h

@@ -0,0 +1,43 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2026 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <faiss/svs/IndexSVSIVF.h>
+
+namespace faiss {
+
+struct IndexSVSIVFLVQ : IndexSVSIVF {
+    IndexSVSIVFLVQ();
+
+    IndexSVSIVFLVQ(
+            idx_t d,
+            size_t nlist,
+            MetricType metric = METRIC_L2,
+            SVSStorageKind storage = SVSStorageKind::SVS_LVQ4x0,
+            bool is_static = false);
+
+    ~IndexSVSIVFLVQ() override = default;
+};
+
+} // namespace faiss