faiss 0.6.2 → 0.6.4

data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp

@@ -7,6 +7,7 @@
 
 // -*- c++ -*-
 
+#include <cmath>
 #include <cstring>
 #include <memory>
 
@@ -206,6 +207,206 @@ const float kLloydMaxBoundaries8[] = {
      2.8413202220f,  2.9594262622f,  3.0937618168f,  3.2505157199f,
      3.4405881450f,  3.6864032383f,  4.0502898282f
 };
+const float kLloydMaxCentroids5[] = {
+    -3.260726295605043f, -2.691115579554310f, -2.317736402126149f,
+    -2.028725991363304f, -1.787231211885846f, -1.576226389073775f,
+    -1.386338935362625f, -1.211803212032400f, -1.048782381365585f,
+    -0.894564395854440f, -0.747135131789057f, -0.604933168939543f,
+    -0.466699175119721f, -0.331378051429876f, -0.198051689203879f,
+    -0.065889622349093f,  0.065889622349093f,  0.198051689203879f,
+     0.331378051429876f,  0.466699175119721f,  0.604933168939543f,
+     0.747135131789057f,  0.894564395854440f,  1.048782381365585f,
+     1.211803212032400f,  1.386338935362625f,  1.576226389073775f,
+     1.787231211885846f,  2.028725991363304f,  2.317736402126149f,
+     2.691115579554310f,  3.260726295605043f
+};
+const float kLloydMaxBoundaries5[] = {
+    -2.975920937579676f, -2.504425990840229f,
+    -2.173231196744727f, -1.907978601624575f,
+    -1.681728800479811f, -1.481282662218200f,
+    -1.299071073697513f, -1.130292796698992f,
+    -0.971673388610013f, -0.820849763821748f,
+    -0.676034150364300f, -0.535816172029632f,
+    -0.399038613274799f, -0.264714870316877f,
+    -0.131970655776486f,  0.000000000000000f,
+     0.131970655776486f,  0.264714870316877f,
+     0.399038613274799f,  0.535816172029632f,
+     0.676034150364300f,  0.820849763821748f,
+     0.971673388610013f,  1.130292796698992f,
+     1.299071073697513f,  1.481282662218200f,
+     1.681728800479811f,  1.907978601624575f,
+     2.173231196744727f,  2.504425990840229f,
+     2.975920937579676f
+};
+const float kLloydMaxCentroids6[] = {
+    -3.744069023696475f, -3.240416640324168f, -2.917391465309850f,
+    -2.672261701410258f, -2.471294740528467f, -2.298972741297400f,
+    -2.146803022259123f, -2.009604894545278f, -1.883971899229356f,
+    -1.767537148129260f, -1.658584711429843f, -1.555827465952835f,
+    -1.458272959944728f, -1.365137897182360f, -1.275791622351997f,
+    -1.189717571132746f, -1.106485959691858f, -1.025734313393752f,
+    -0.947153093015629f, -0.870474868055092f, -0.795466052902551f,
+    -0.721920472069418f, -0.649654245231535f, -0.578501846064579f,
+    -0.508312758753803f, -0.438948800917774f, -0.370281933281155f,
+    -0.302192289440361f, -0.234566569768735f, -0.167296609901720f,
+    -0.100278121713920f, -0.033409455880258f,  0.033409455880258f,
+     0.100278121713920f,  0.167296609901720f,  0.234566569768735f,
+     0.302192289440361f,  0.370281933281155f,  0.438948800917774f,
+     0.508312758753803f,  0.578501846064579f,  0.649654245231535f,
+     0.721920472069418f,  0.795466052902551f,  0.870474868055092f,
+     0.947153093015629f,  1.025734313393752f,  1.106485959691858f,
+     1.189717571132746f,  1.275791622351997f,  1.365137897182360f,
+     1.458272959944728f,  1.555827465952835f,  1.658584711429843f,
+     1.767537148129260f,  1.883971899229356f,  2.009604894545278f,
+     2.146803022259123f,  2.298972741297400f,  2.471294740528467f,
+     2.672261701410258f,  2.917391465309850f,  3.240416640324168f,
+     3.744069023696475f
+};
+const float kLloydMaxBoundaries6[] = {
+    -3.492242832010322f, -3.078904052817009f,
+    -2.794826583360054f, -2.571778220969362f,
+    -2.385133740912933f, -2.222887881778261f,
+    -2.078203958402201f, -1.946788396887317f,
+    -1.825754523679308f, -1.713060929779552f,
+    -1.607206088691339f, -1.507050212948781f,
+    -1.411705428563544f, -1.320464759767178f,
+    -1.232754596742371f, -1.148101765412302f,
+    -1.066110136542805f, -0.986443703204690f,
+    -0.908813980535361f, -0.832970460478822f,
+    -0.758693262485985f, -0.685787358650477f,
+    -0.614078045648057f, -0.543407302409191f,
+    -0.473630779835788f, -0.404615367099464f,
+    -0.336237111360758f, -0.268379429604548f,
+    -0.200931589835227f, -0.133787365807820f,
+    -0.066843788797089f,  0.000000000000000f,
+     0.066843788797089f,  0.133787365807820f,
+     0.200931589835227f,  0.268379429604548f,
+     0.336237111360758f,  0.404615367099464f,
+     0.473630779835788f,  0.543407302409191f,
+     0.614078045648057f,  0.685787358650477f,
+     0.758693262485985f,  0.832970460478822f,
+     0.908813980535361f,  0.986443703204690f,
+     1.066110136542805f,  1.148101765412302f,
+     1.232754596742371f,  1.320464759767178f,
+     1.411705428563544f,  1.507050212948781f,
+     1.607206088691339f,  1.713060929779552f,
+     1.825754523679308f,  1.946788396887317f,
+     2.078203958402201f,  2.222887881778261f,
+     2.385133740912933f,  2.571778220969362f,
+     2.794826583360054f,  3.078904052817009f,
+     3.492242832010322f
+};
+const float kLloydMaxCentroids7[] = {
+    -4.189521933023523f, -3.734857105369156f, -3.447381010593710f,
+    -3.231896182843021f, -3.057216102842374f, -2.909021527386642f,
+    -2.779491911054078f, -2.663867022558051f, -2.559023499137449f,
+    -2.462795908452321f, -2.373620316421171f, -2.290326870492530f,
+    -2.212016460501213f, -2.137983242734970f, -2.067662153838450f,
+    -2.000593243383757f, -1.936396720412283f, -1.874755326807296f,
+    -1.815400993379865f, -1.758105160675647f, -1.702671162415673f,
+    -1.648928306223807f, -1.596727786313424f, -1.545938700893484f,
+    -1.496445137501057f, -1.448144252238147f, -1.400944106526214f,
+    -1.354762105115604f, -1.309523700469555f, -1.265161378699170f,
+    -1.221613876387012f, -1.178825265520504f, -1.136744262343403f,
+    -1.095323771921318f, -1.054520418037026f, -1.014294067830043f,
+    -0.974607484216044f, -0.935426075762634f, -0.896717615963322f,
+    -0.858451939611374f, -0.820600783245502f, -0.783137537563140f,
+    -0.746037126679468f, -0.709275946093977f, -0.672831567493634f,
+    -0.636682661398141f, -0.600808970989236f, -0.565191195643323f,
+    -0.529810843625619f, -0.494650181610316f, -0.459692230045422f,
+    -0.424920552397901f, -0.390319263305967f, -0.355873075050329f,
+    -0.321567103923158f, -0.287386806226095f, -0.253318071908346f,
+    -0.219347083403316f, -0.185460257086808f, -0.151644263013162f,
+    -0.117885968250325f, -0.084172419896719f, -0.050490753968962f,
+    -0.016828143177728f,  0.016828143177728f,  0.050490753968962f,
+     0.084172419896719f,  0.117885968250325f,  0.151644263013162f,
+     0.185460257086808f,  0.219347083403316f,  0.253318071908346f,
+     0.287386806226095f,  0.321567103923158f,  0.355873075050329f,
+     0.390319263305967f,  0.424920552397901f,  0.459692230045422f,
+     0.494650181610316f,  0.529810843625619f,  0.565191195643323f,
+     0.600808970989236f,  0.636682661398141f,  0.672831567493634f,
+     0.709275946093977f,  0.746037126679468f,  0.783137537563140f,
+     0.820600783245502f,  0.858451939611374f,  0.896717615963322f,
+     0.935426075762634f,  0.974607484216044f,  1.014294067830043f,
+     1.054520418037026f,  1.095323771921318f,  1.136744262343403f,
+     1.178825265520504f,  1.221613876387012f,  1.265161378699170f,
+     1.309523700469555f,  1.354762105115604f,  1.400944106526214f,
+     1.448144252238147f,  1.496445137501057f,  1.545938700893484f,
+     1.596727786313424f,  1.648928306223807f,  1.702671162415673f,
+     1.758105160675647f,  1.815400993379865f,  1.874755326807296f,
+     1.936396720412283f,  2.000593243383757f,  2.067662153838450f,
+     2.137983242734970f,  2.212016460501213f,  2.290326870492530f,
+     2.373620316421171f,  2.462795908452321f,  2.559023499137449f,
+     2.663867022558051f,  2.779491911054078f,  2.909021527386642f,
+     3.057216102842374f,  3.231896182843021f,  3.447381010593710f,
+     3.734857105369156f,  4.189521933023523f
+};
+const float kLloydMaxBoundaries7[] = {
+    -3.962189519196340f, -3.591119057981433f,
+    -3.339638596718365f, -3.144556142842697f,
+    -2.983118815114508f, -2.844256719220360f,
+    -2.721679466806065f, -2.611445260847750f,
+    -2.510909703794885f, -2.418208112436746f,
+    -2.331973593456850f, -2.251171665496871f,
+    -2.174999851618091f, -2.102822698286710f,
+    -2.034127698611103f, -1.968494981898020f,
+    -1.905576023609790f, -1.845078160093581f,
+    -1.786753077027756f, -1.730388161545660f,
+    -1.675799734319740f, -1.622828046268615f,
+    -1.571333243603454f, -1.521191919197270f,
+    -1.472294694869602f, -1.424544179382180f,
+    -1.377853105820909f, -1.332142902792580f,
+    -1.287342539584363f, -1.243387627543091f,
+    -1.200219570953758f, -1.157784763931954f,
+    -1.116034017132360f, -1.074922094979172f,
+    -1.034407242933534f, -0.994450776023043f,
+    -0.955016779989339f, -0.916071845862978f,
+    -0.877584777787348f, -0.839526361428438f,
+    -0.801869160404321f, -0.764587332121304f,
+    -0.727656536386722f, -0.691053756793806f,
+    -0.654757114445887f, -0.618745816193689f,
+    -0.583000083316280f, -0.547501019634471f,
+    -0.512230512617967f, -0.477171205827869f,
+    -0.442306391221661f, -0.407619907851934f,
+    -0.373096169178148f, -0.338720089486743f,
+    -0.304476955074627f, -0.270352439067221f,
+    -0.236332577655831f, -0.202403670245062f,
+    -0.168552260049985f, -0.134765115631744f,
+    -0.101029194073522f, -0.067331586932840f,
+    -0.033659448573345f,  0.000000000000000f,
+     0.033659448573345f,  0.067331586932840f,
+     0.101029194073522f,  0.134765115631744f,
+     0.168552260049985f,  0.202403670245062f,
+     0.236332577655831f,  0.270352439067221f,
+     0.304476955074627f,  0.338720089486743f,
+     0.373096169178148f,  0.407619907851934f,
+     0.442306391221661f,  0.477171205827869f,
+     0.512230512617967f,  0.547501019634471f,
+     0.583000083316280f,  0.618745816193689f,
+     0.654757114445887f,  0.691053756793806f,
+     0.727656536386722f,  0.764587332121304f,
+     0.801869160404321f,  0.839526361428438f,
+     0.877584777787348f,  0.916071845862978f,
+     0.955016779989339f,  0.994450776023043f,
+     1.034407242933534f,  1.074922094979172f,
+     1.116034017132360f,  1.157784763931954f,
+     1.200219570953758f,  1.243387627543091f,
+     1.287342539584363f,  1.332142902792580f,
+     1.377853105820909f,  1.424544179382180f,
+     1.472294694869602f,  1.521191919197270f,
+     1.571333243603454f,  1.622828046268615f,
+     1.675799734319740f,  1.730388161545660f,
+     1.786753077027756f,  1.845078160093581f,
+     1.905576023609790f,  1.968494981898020f,
+     2.034127698611103f,  2.102822698286710f,
+     2.174999851618091f,  2.251171665496871f,
+     2.331973593456850f,  2.418208112436746f,
+     2.510909703794885f,  2.611445260847750f,
+     2.721679466806065f,  2.844256719220360f,
+     2.983118815114508f,  3.144556142842697f,
+     3.339638596718365f,  3.591119057981433f,
+     3.962189519196340f
+};
 // clang-format on
 
 struct LloydMaxTable {
@@ -219,20 +420,35 @@ const LloydMaxTable kLloydMaxTables[] = {
         {kLloydMaxCentroids2, kLloydMaxBoundaries2}, // 2
         {kLloydMaxCentroids3, kLloydMaxBoundaries3}, // 3
         {kLloydMaxCentroids4, kLloydMaxBoundaries4}, // 4
-        {nullptr, nullptr},                          // 5 (unused)
-        {nullptr, nullptr},                          // 6 (unused)
-        {nullptr, nullptr},                          // 7 (unused)
+        {kLloydMaxCentroids5, kLloydMaxBoundaries5}, // 5
+        {kLloydMaxCentroids6, kLloydMaxBoundaries6}, // 6
+        {kLloydMaxCentroids7, kLloydMaxBoundaries7}, // 7
         {kLloydMaxCentroids8, kLloydMaxBoundaries8}, // 8
 };
 
-void populate_lloyd_max_trained(size_t mse_bits, std::vector<float>& trained) {
+// The tables are Lloyd-Max optimal for N(0, 1) input. Callers whose input has
+// a different standard deviation pass it as `scale` to stretch the table.
+void populate_lloyd_max_trained(
+        size_t mse_bits,
+        std::vector<float>& trained,
+        float scale = 1.0f) {
     FAISS_THROW_IF_NOT(mse_bits >= 1 && mse_bits <= 8);
-    FAISS_THROW_IF_NOT(kLloydMaxTables[mse_bits].centroids != nullptr);
+    FAISS_THROW_IF_NOT(kLloydMaxTables[mse_bits].centroids);
     size_t k = size_t(1) << mse_bits;
     const auto& t = kLloydMaxTables[mse_bits];
     trained.resize(k + (k - 1));
-    std::copy(t.centroids, t.centroids + k, trained.begin());
-    std::copy(t.boundaries, t.boundaries + k - 1, trained.begin() + k);
+    for (size_t i = 0; i < k; i++) {
+        trained[i] = t.centroids[i] * scale;
+    }
+    for (size_t i = 0; i + 1 < k; i++) {
+        trained[k + i] = t.boundaries[i] * scale;
+    }
+}
+
+// Component scale of a unit-norm vector in R^d.
+float unit_norm_component_scale(size_t d) {
+    FAISS_THROW_IF_NOT(d > 0);
+    return 1.0f / std::sqrt(static_cast<float>(d));
 }
 
 } // namespace
@@ -251,14 +467,17 @@ ScalarQuantizer::ScalarQuantizer() {}
 void ScalarQuantizer::set_derived_sizes() {
     switch (qtype) {
         case QT_1bit_tqmse:
+        case QT_1bit_eden:
             code_size = (d + 7) / 8;
             bits = 1;
             break;
         case QT_2bit_tqmse:
+        case QT_2bit_eden:
             code_size = (d * 2 + 7) / 8;
             bits = 2;
             break;
         case QT_3bit_tqmse:
+        case QT_3bit_eden:
             code_size = (d * 3 + 7) / 8;
             bits = 3;
             break;
@@ -267,19 +486,30 @@ void ScalarQuantizer::set_derived_sizes() {
         case QT_8bit_direct:
         case QT_8bit_direct_signed:
         case QT_8bit_tqmse:
+        case QT_8bit_eden:
             code_size = d;
             bits = 8;
             break;
         case QT_4bit:
         case QT_4bit_uniform:
         case QT_4bit_tqmse:
+        case QT_4bit_eden:
             code_size = (d + 1) / 2;
             bits = 4;
             break;
+        case QT_5bit_eden:
+            code_size = (d * 5 + 7) / 8;
+            bits = 5;
+            break;
         case QT_6bit:
+        case QT_6bit_eden:
             code_size = (d * 6 + 7) / 8;
             bits = 6;
             break;
+        case QT_7bit_eden:
+            code_size = (d * 7 + 7) / 8;
+            bits = 7;
+            break;
         case QT_fp16:
             code_size = d * 2;
             bits = 16;
@@ -334,7 +564,7 @@ void ScalarQuantizer::train(size_t n, const float* x) {
         case QT_4bit_uniform:
         case QT_8bit_uniform:
             FAISS_THROW_IF_NOT(n > 0);
-            FAISS_THROW_IF_NOT(x != nullptr);
+            FAISS_THROW_IF_NOT(x);
             train_Uniform(
                     rangestat,
                     rangestat_arg,
@@ -347,7 +577,7 @@ void ScalarQuantizer::train(size_t n, const float* x) {
         case QT_8bit:
         case QT_6bit:
             FAISS_THROW_IF_NOT(n > 0);
-            FAISS_THROW_IF_NOT(x != nullptr);
+            FAISS_THROW_IF_NOT(x);
             train_NonUniform(
                     rangestat,
                     rangestat_arg,
@@ -363,20 +593,35 @@ void ScalarQuantizer::train(size_t n, const float* x) {
         case QT_8bit_direct_signed:
             // no training necessary
             break;
+        case QT_1bit_eden:
+        case QT_2bit_eden:
+        case QT_3bit_eden:
+        case QT_4bit_eden:
+        case QT_5bit_eden:
+        case QT_6bit_eden:
+        case QT_7bit_eden:
+        case QT_8bit_eden:
+            populate_lloyd_max_trained(bits, trained);
+            break;
         case QT_1bit_tqmse:
-            populate_lloyd_max_trained(1, trained);
+            populate_lloyd_max_trained(
+                    1, trained, unit_norm_component_scale(d));
             break;
         case QT_2bit_tqmse:
-            populate_lloyd_max_trained(2, trained);
+            populate_lloyd_max_trained(
+                    2, trained, unit_norm_component_scale(d));
             break;
         case QT_3bit_tqmse:
-            populate_lloyd_max_trained(3, trained);
+            populate_lloyd_max_trained(
+                    3, trained, unit_norm_component_scale(d));
             break;
         case QT_4bit_tqmse:
-            populate_lloyd_max_trained(4, trained);
+            populate_lloyd_max_trained(
+                    4, trained, unit_norm_component_scale(d));
             break;
         case QT_8bit_tqmse:
-            populate_lloyd_max_trained(8, trained);
+            populate_lloyd_max_trained(
+                    8, trained, unit_norm_component_scale(d));
             break;
         case QT_2bit_tq:
         case QT_3bit_tq:
@@ -390,7 +635,6 @@ void ScalarQuantizer::train(size_t n, const float* x) {
             trained.push_back(seed_f[0]);
             trained.push_back(seed_f[1]);
             trained.push_back(static_cast<float>(turboq_refine.qjl_type));
-            turboq_refine.init_projection(d);
             break;
         }
         default:
@@ -398,36 +642,15 @@ void ScalarQuantizer::train(size_t n, const float* x) {
     }
 }
 
-void ScalarQuantizer::TurboQuantRefine::init_projection(size_t d) {
-    if (use_fwht()) {
-        padded_d = 1;
-        while (padded_d < d) {
-            padded_d <<= 1;
-        }
-        fwht_signs.resize(padded_d);
-        RandomGenerator rng(seed);
-        for (size_t i = 0; i < padded_d; i++) {
-            fwht_signs[i] = (rng.rand_int(2) == 0) ? 1.0f : -1.0f;
-        }
-    } else {
-        rr_matrix.resize(d * d);
-        float_randn(rr_matrix.data(), d * d, seed);
-        matrix_qr(static_cast<int>(d), static_cast<int>(d), rr_matrix.data());
-    }
-}
-
 ScalarQuantizer::SQuantizer* ScalarQuantizer::select_quantizer() const {
-    return with_simd_level_spr([&]<SIMDLevel SL>() -> SQuantizer* {
-        if constexpr (SL != SIMDLevel::NONE) {
-            auto* q = scalar_quantizer::sq_select_quantizer<SL>(
-                    qtype, d, trained);
-            if (q) {
-                return q;
-            }
-        }
-        return scalar_quantizer::sq_select_quantizer<SIMDLevel::NONE>(
-                qtype, d, trained);
-    });
+    // A SIMD level's factory returns nullptr when the dimension is
+    // incompatible (e.g. AVX-512 needs d % 16 == 0); the dispatcher then falls
+    // back to the next-lower level (AVX-512 -> AVX2 -> scalar).
+    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR>(
+            [&]<SIMDLevel SL>() -> SQuantizer* {
+                return scalar_quantizer::sq_select_quantizer<SL>(
+                        qtype, d, trained);
+            });
 }
 
 void ScalarQuantizer::compute_codes(const float* x, uint8_t* codes, size_t n)
@@ -438,7 +661,7 @@ void ScalarQuantizer::compute_codes(const float* x, uint8_t* codes, size_t n)
     std::unique_ptr<SQuantizer> squant(select_quantizer());
 
     memset(codes, 0, code_size * n);
-#pragma omp parallel for
+#pragma omp parallel for if (n > 100)
     for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         squant->encode_vector(x + i * d, codes + i * code_size);
     }
@@ -451,7 +674,7 @@ void ScalarQuantizer::decode(const uint8_t* codes, float* x, size_t n) const {
     }
     std::unique_ptr<SQuantizer> squant(select_quantizer());
 
-#pragma omp parallel for
+#pragma omp parallel for if (n > 100)
     for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         squant->decode_vector(codes + i * code_size, x + i * d);
     }
@@ -460,17 +683,11 @@ void ScalarQuantizer::decode(const uint8_t* codes, float* x, size_t n) const {
 ScalarQuantizer::SQDistanceComputer* ScalarQuantizer::get_distance_computer(
         MetricType metric) const {
     FAISS_THROW_IF_NOT(metric == METRIC_L2 || metric == METRIC_INNER_PRODUCT);
-    return with_simd_level_spr([&]<SIMDLevel SL>() -> SQDistanceComputer* {
-        if constexpr (SL != SIMDLevel::NONE) {
-            auto* dc = scalar_quantizer::sq_select_distance_computer<SL>(
-                    metric, qtype, d, trained);
-            if (dc) {
-                return dc;
-            }
-        }
-        return scalar_quantizer::sq_select_distance_computer<SIMDLevel::NONE>(
-                metric, qtype, d, trained);
-    });
+    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR>(
+            [&]<SIMDLevel SL>() -> SQDistanceComputer* {
+                return scalar_quantizer::sq_select_distance_computer<SL>(
+                        metric, qtype, d, trained);
+            });
 }
 
 InvertedListScanner* ScalarQuantizer::select_InvertedListScanner(
@@ -479,33 +696,19 @@ InvertedListScanner* ScalarQuantizer::select_InvertedListScanner(
         bool store_pairs,
         const IDSelector* sel,
         bool by_residual) const {
-    return with_simd_level_spr([&]<SIMDLevel SL>() -> InvertedListScanner* {
-        if constexpr (SL != SIMDLevel::NONE) {
-            auto* s = scalar_quantizer::sq_select_InvertedListScanner<SL>(
-                    qtype,
-                    mt,
-                    d,
-                    code_size,
-                    trained,
-                    quantizer,
-                    store_pairs,
-                    sel,
-                    by_residual);
-            if (s) {
-                return s;
-            }
-        }
-        return scalar_quantizer::sq_select_InvertedListScanner<SIMDLevel::NONE>(
-                qtype,
-                mt,
-                d,
-                code_size,
-                trained,
-                quantizer,
-                store_pairs,
-                sel,
-                by_residual);
-    });
+    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR>(
+            [&]<SIMDLevel SL>() -> InvertedListScanner* {
+                return scalar_quantizer::sq_select_InvertedListScanner<SL>(
+                        qtype,
+                        mt,
+                        d,
+                        code_size,
+                        trained,
+                        quantizer,
+                        store_pairs,
+                        sel,
+                        by_residual);
+            });
 }
 
 } // namespace faiss

data/vendor/faiss/faiss/impl/ScalarQuantizer.h

@@ -45,6 +45,14 @@ struct ScalarQuantizer : Quantizer {
         QT_3bit_tq,    ///< Full TurboQuant (2-bit MSE + 1-bit QJL + factors)
         QT_4bit_tq,    ///< Full TurboQuant (3-bit MSE + 1-bit QJL + factors)
         QT_5bit_tq,    ///< Full TurboQuant (4-bit MSE + 1-bit QJL + factors)
+        QT_1bit_eden,  ///< EDEN Lloyd-Max scalar code, 1 bit per component
+        QT_2bit_eden,  ///< EDEN Lloyd-Max scalar code, 2 bits per component
+        QT_3bit_eden,  ///< EDEN Lloyd-Max scalar code, 3 bits per component
+        QT_4bit_eden,  ///< EDEN Lloyd-Max scalar code, 4 bits per component
+        QT_5bit_eden,  ///< EDEN Lloyd-Max scalar code, 5 bits per component
+        QT_6bit_eden,  ///< EDEN Lloyd-Max scalar code, 6 bits per component
+        QT_7bit_eden,  ///< EDEN Lloyd-Max scalar code, 7 bits per component
+        QT_8bit_eden,  ///< EDEN Lloyd-Max scalar code, 8 bits per component
         QT_count
     };
 
@@ -135,6 +143,18 @@ struct ScalarQuantizer : Quantizer {
         float distance_to_code(const uint8_t* code) final {
             return query_to_code(code);
         }
+
+        void distance_to_code_batch_4(
+                const uint8_t* c1,
+                const uint8_t* c2,
+                const uint8_t* c3,
+                const uint8_t* c4,
+                float& d1,
+                float& d2,
+                float& d3,
+                float& d4) override {
+            query_to_codes_batch_4(c1, c2, c3, c4, d1, d2, d3, d4);
+        }
     };
 
     /// TurboQuant full (QT_*_tq) refinement state, isolated from the
@@ -157,18 +177,9 @@ struct ScalarQuantizer : Quantizer {
             return s;
         }
 
+        /// The selected projection is built from `trained` by the quantizer.
         uint8_t qjl_type = 0;
         uint64_t seed = 42;
-        size_t padded_d = 0;
-        std::vector<float> fwht_signs;
-        std::vector<float> rr_matrix;
-        size_t nb_bits_lo = 0;
-        size_t n_hi_dims = 0;
-
-        void init_projection(size_t d);
-        bool use_fwht() const {
-            return qjl_type == 0;
-        }
 
         struct DistanceComputer : SQDistanceComputer {
             virtual void configure(uint8_t qb, bool int_qjl) = 0;
@@ -179,6 +190,11 @@ struct ScalarQuantizer : Quantizer {
         };
     };
 
+    static_assert(
+            sizeof(TurboQuantRefine) <= 16,
+            "keep this a small config struct -- do not add projection buffers "
+            "here (T287092602)");
+
     TurboQuantRefine turboq_refine;
 
     SQDistanceComputer* get_distance_computer(

data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h

@@ -70,8 +70,8 @@ void ThreadedIndex<IndexT>::addIndex(IndexT* index) {
 
         // Make sure this index is not duplicated
         for (auto& p : indices_) {
-            FAISS_THROW_IF_NOT_MSG(
-                    p.first != index,
+            FAISS_THROW_IF_MSG(
+                    p.first == index,
                     "addIndex: attempting to add index "
                     "that is already in the collection");
         }

data/vendor/faiss/faiss/impl/VisitedTable.cpp

@@ -15,8 +15,12 @@ namespace faiss {
 // advance() is O(1) except every 250 calls, which are O(size).
 // The hash set strategy is a constant factor slower for get()/set(),
 // but O(1) to construct and O(visits) to advance.
-// A size of ~1M seems to be the threshold where the hash set wins.
-size_t visited_table_hashset_threshold = 500000;
+// 10M is only a current estimated threshold, not a proven crossover: we are not
+// sure the array still wins at 10M. The point where the array stops paying off
+// varies by dataset (it shifts with dimension, working-set / cache pressure,
+// etc.), so this is a coarse default that should eventually be replaced by
+// smarter per-index tuning.
+size_t visited_table_hashset_threshold = 10000000;
 
 std::unique_ptr<VisitedTable> VisitedTable::create(
         size_t size,
@@ -29,6 +33,22 @@ std::unique_ptr<VisitedTable> VisitedTable::create(
     return std::make_unique<VisitedTableVector>(size);
 }
 
+VisitedTable& VisitedTable::get_reusable(
+        size_t size,
+        std::optional<bool> use_hashset) {
+    bool use_set =
+            use_hashset.value_or(size >= visited_table_hashset_threshold);
+    if (use_set) {
+        thread_local VisitedTableSet tls_set;
+        tls_set.advance();
+        return tls_set;
+    }
+    thread_local VisitedTableVector tls_vec(0);
+    tls_vec.ensure_size(size);
+    tls_vec.advance();
+    return tls_vec;
+}
+
 void VisitedTableVector::advance() {
     if (visno < 254) {
         // 254 rather than 255 because sometimes we use visno and visno+1

data/vendor/faiss/faiss/impl/VisitedTable.h

@@ -48,6 +48,18 @@ struct VisitedTable {
     static std::unique_ptr<VisitedTable> create(
             size_t size,
             std::optional<bool> use_hashset = std::nullopt);
+
+    /// Returns a thread-local, reusable table sized for at least `size` and
+    /// reset to a clean state. Unlike create(), it does not allocate on each
+    /// call: the O(size) versioned array is allocated once per thread and
+    /// reused across searches, avoiding a per-search alloc+zero of the whole
+    /// array when a static index is searched repeatedly.
+    ///
+    /// The returned reference is owned by thread-local storage: do not delete
+    /// it and do not use it beyond the current search on the calling thread.
+    static VisitedTable& get_reusable(
+            size_t size,
+            std::optional<bool> use_hashset = std::nullopt);
 };
 
 /// Set-based implementation using unordered_set.
@@ -87,6 +99,14 @@ struct VisitedTableVector FAISS_FINAL : VisitedTable {
 
     explicit VisitedTableVector(size_t size) : visited(size, 0) {}
 
+    /// Grow so indices in [0, size) are valid; new slots read as unvisited.
+    /// Never shrinks, so capacity is retained when the table is reused.
+    void ensure_size(size_t size) {
+        if (visited.size() < size) {
+            visited.resize(size, 0);
+        }
+    }
+
     bool set(size_t no) final {
         if (visited[no] == visno) {
             return false;