faiss 0.6.4 → 0.6.5

data/vendor/faiss/faiss/utils/distances.cpp

@@ -26,6 +26,7 @@
 #include <faiss/utils/distances_dispatch.h>
 #include <faiss/utils/distances_fused/distances_fused.h>
 #include <faiss/utils/simd_impl/exhaustive_L2sqr_blas_cmax.h>
+#include <faiss/utils/utils.h>
 
 #ifndef FINTEGER
 #define FINTEGER long
@@ -361,10 +362,11 @@ void exhaustive_L2sqr_seq(
     }
 }
 
-/** Find the nearest neighbors for nx queries in a set of ny vectors */
-template <class BlockResultHandler>
-void exhaustive_inner_product_blas(
-        const float* x,
+/** Find the nearest neighbors for nx queries (see Fp32QueryTiles) in a set
+ * of ny vectors */
+template <class BlockResultHandler, class QueryTiles>
+void exhaustive_inner_product_blas_tiles(
+        QueryTiles& queries,
         const float* y,
         size_t d,
         size_t nx,
@@ -386,6 +388,8 @@ void exhaustive_inner_product_blas(
             i1 = nx;
         }
 
+        const float* xt = queries.tile(i0, i1, nullptr);
+
         res.begin_multiple(i0, i1);
 
         for (size_t j0 = 0; j0 < ny; j0 += bs_y) {
@@ -405,7 +409,7 @@ void exhaustive_inner_product_blas(
                        &one,
                        y + j0 * d,
                        &di,
-                       x + i0 * d,
+                       xt,
                        &di,
                        &zero,
                        ip_block.get(),
@@ -421,9 +425,9 @@ void exhaustive_inner_product_blas(
 
 // distance correction is an operator that can be applied to transform
 // the distances
-template <class BlockResultHandler>
-void exhaustive_L2sqr_blas_default_impl(
-        const float* x,
+template <class BlockResultHandler, class QueryTiles>
+void exhaustive_L2sqr_blas_tiles(
+        QueryTiles& queries,
         const float* y,
         size_t d,
         size_t nx,
@@ -443,7 +447,7 @@ void exhaustive_L2sqr_blas_default_impl(
     std::unique_ptr<float[]> x_norms(new float[nx]);
     std::unique_ptr<float[]> del2;
 
-    fvec_norms_L2sqr(x_norms.get(), x, d, nx);
+    queries.prepare_norms(nx, x_norms.get());
 
     if (!y_norms) {
         float* y_norms2 = new float[ny];
@@ -458,6 +462,8 @@ void exhaustive_L2sqr_blas_default_impl(
             i1 = nx;
         }
 
+        const float* xt = queries.tile(i0, i1, x_norms.get());
+
         res.begin_multiple(i0, i1);
 
         for (size_t j0 = 0; j0 < ny; j0 += bs_y) {
@@ -477,7 +483,7 @@ void exhaustive_L2sqr_blas_default_impl(
                        &one,
                        y + j0 * d,
                        &di,
-                       x + i0 * d,
+                       xt,
                        &di,
                        &zero,
                        ip_block.get(),
@@ -510,6 +516,31 @@ void exhaustive_L2sqr_blas_default_impl(
     }
 }
 
+template <class BlockResultHandler>
+void exhaustive_inner_product_blas(
+        const float* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        BlockResultHandler& res) {
+    Fp32QueryTiles queries{x, d};
+    exhaustive_inner_product_blas_tiles(queries, y, d, nx, ny, res);
+}
+
+template <class BlockResultHandler>
+void exhaustive_L2sqr_blas_default_impl(
+        const float* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        BlockResultHandler& res,
+        const float* y_norms = nullptr) {
+    Fp32QueryTiles queries{x, d};
+    exhaustive_L2sqr_blas_tiles(queries, y, d, nx, ny, res, y_norms);
+}
+
 template <class BlockResultHandler>
 void exhaustive_L2sqr_blas(
         const float* x,
@@ -556,6 +587,44 @@ void exhaustive_L2sqr_blas<Top1BlockResultHandler<CMax<float, int64_t>>>(
             });
 }
 
+/// exhaustive_L2sqr_blas for packed IEEE binary16 queries
+template <class BlockResultHandler>
+void exhaustive_L2sqr_blas_fp16(
+        const uint16_t* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        BlockResultHandler& res,
+        const float* y_norms) {
+    Fp16QueryTiles queries(x, d, nx);
+    exhaustive_L2sqr_blas_tiles(queries, y, d, nx, ny, res, y_norms);
+}
+
+template <>
+void exhaustive_L2sqr_blas_fp16<Top1BlockResultHandler<CMax<float, int64_t>>>(
+        const uint16_t* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        Top1BlockResultHandler<CMax<float, int64_t>>& res,
+        const float* y_norms) {
+    with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_SVE>(
+            [&]<SIMDLevel SL>() {
+                if constexpr (
+                        SL == SIMDLevel::AVX2 || SL == SIMDLevel::AVX512 ||
+                        SL == SIMDLevel::ARM_SVE) {
+                    exhaustive_L2sqr_blas_cmax_fp16<SL>(
+                            x, y, d, nx, ny, res, y_norms);
+                } else {
+                    Fp16QueryTiles queries(x, d, nx);
+                    exhaustive_L2sqr_blas_tiles(
+                            queries, y, d, nx, ny, res, y_norms);
+                }
+            });
+}
+
 struct Run_search_inner_product {
     using T = void;
     template <class BlockResultHandler>
@@ -598,6 +667,35 @@ struct Run_search_L2sqr {
     }
 };
 
+// fp16 consumers only run the BLAS kernels, see use_fp16_blas_kernels
+struct Run_search_inner_product_fp16 {
+    using T = void;
+    template <class BlockResultHandler>
+    void f(BlockResultHandler& res,
+           const uint16_t* x,
+           const float* y,
+           size_t d,
+           size_t nx,
+           size_t ny) {
+        Fp16QueryTiles queries(x, d, nx);
+        exhaustive_inner_product_blas_tiles(queries, y, d, nx, ny, res);
+    }
+};
+
+struct Run_search_L2sqr_fp16 {
+    using T = void;
+    template <class BlockResultHandler>
+    void f(BlockResultHandler& res,
+           const uint16_t* x,
+           const float* y,
+           size_t d,
+           size_t nx,
+           size_t ny,
+           const float* y_norm2) {
+        exhaustive_L2sqr_blas_fp16(x, y, d, nx, ny, res, y_norm2);
+    }
+};
+
 } // anonymous namespace
 
 /*******************************************************
@@ -630,7 +728,7 @@ static void knn_db_parallel_impl(
     int nt = omp_get_max_threads();
     const size_t bs_y = distance_compute_blas_database_bs;
 
-    // Per-thread result heaps: nt threads x nx queries x k results
+    // Result heaps: nt segments x nx queries x k results
     std::vector<T> all_dis(static_cast<size_t>(nt) * nx * k);
     std::vector<TI> all_ids(static_cast<size_t>(nt) * nx * k);
 
@@ -649,9 +747,8 @@ static void knn_db_parallel_impl(
         }
     }
 
-#pragma omp parallel num_threads(nt)
-    {
-        int tid = omp_get_thread_num();
+#pragma omp parallel for
+    for (int tid = 0; tid < nt; tid++) {
         size_t j_begin = static_cast<size_t>(tid) * ny / nt;
         size_t j_end = static_cast<size_t>(tid + 1) * ny / nt;
         size_t local_ny = j_end - j_begin;
@@ -734,7 +831,7 @@ static void knn_db_parallel_impl(
         }
     }
 
-    // Merge per-thread heaps into output, parallelized over queries
+    // Merge per-segment heaps into output, parallelized over queries
 #pragma omp parallel for
     for (int64_t i = 0; i < static_cast<int64_t>(nx); i++) {
         heap_heapify<C>(k, vals + i * k, ids + i * k);
@@ -894,6 +991,120 @@ void knn_L2sqr(
     knn_L2sqr(x, y, d, nx, ny, res->k, res->val, res->ids, y_norm2, sel);
 }
 
+namespace {
+
+/** Whether fp16 queries go to the BLAS kernels directly.
+ *
+ * Other cases widen chunks of queries and run the fp32 search: selectors,
+ * sequential and database-parallel searches, and small d, where the fused
+ * top-1 kernels need fp32 rows.
+ */
+bool use_fp16_blas_kernels(
+        size_t d,
+        size_t nx,
+        size_t ny,
+        const IDSelector* sel) {
+    return !sel && ny > 0 && d > 32 &&
+            nx * d >= static_cast<size_t>(distance_compute_blas_threshold) &&
+            !should_use_db_parallel(nx, ny, sel);
+}
+
+/// widens chunks of queries and calls search(x_chunk, n_chunk, i0) on them
+template <class Fp32Search>
+void search_fp16_by_chunks(
+        const uint16_t* x,
+        size_t d,
+        size_t nx,
+        const Fp32Search& search) {
+    // a multiple of the query tile, so the fp32 kernels see the same tiles
+    const size_t bs_x = std::max(distance_compute_blas_query_bs, 1);
+    const size_t chunk = bs_x *
+            std::max<size_t>(1, (size_t(8) << 20) / (bs_x * d * sizeof(float)));
+    std::vector<float> buffer(std::min(nx, chunk) * d);
+    for (size_t i0 = 0; i0 < nx; i0 += chunk) {
+        const size_t i1 = std::min(nx, i0 + chunk);
+        fp16_to_fp32((i1 - i0) * d, x + i0 * d, buffer.data());
+        search(buffer.data(), i1 - i0, i0);
+    }
+}
+
+} // namespace
+
+void knn_inner_product_fp16(
+        const uint16_t* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        size_t k,
+        float* vals,
+        int64_t* ids,
+        const IDSelector* sel) {
+    if (!use_fp16_blas_kernels(d, nx, ny, sel)) {
+        search_fp16_by_chunks(
+                x, d, nx, [&](const float* xc, size_t nc, size_t i0) {
+                    knn_inner_product(
+                            xc,
+                            y,
+                            d,
+                            nc,
+                            ny,
+                            k,
+                            vals + i0 * k,
+                            ids + i0 * k,
+                            sel);
+                });
+        return;
+    }
+    Run_search_inner_product_fp16 r;
+    dispatch_knn_ResultHandler(
+            nx,
+            vals,
+            ids,
+            k,
+            METRIC_INNER_PRODUCT,
+            nullptr,
+            r,
+            x,
+            y,
+            d,
+            nx,
+            ny);
+}
+
+void knn_L2sqr_fp16(
+        const uint16_t* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        size_t k,
+        float* vals,
+        int64_t* ids,
+        const float* y_norm2,
+        const IDSelector* sel) {
+    if (!use_fp16_blas_kernels(d, nx, ny, sel)) {
+        search_fp16_by_chunks(
+                x, d, nx, [&](const float* xc, size_t nc, size_t i0) {
+                    knn_L2sqr(
+                            xc,
+                            y,
+                            d,
+                            nc,
+                            ny,
+                            k,
+                            vals + i0 * k,
+                            ids + i0 * k,
+                            y_norm2,
+                            sel);
+                });
+        return;
+    }
+    Run_search_L2sqr_fp16 r;
+    dispatch_knn_ResultHandler(
+            nx, vals, ids, k, METRIC_L2, nullptr, r, x, y, d, nx, ny, y_norm2);
+}
+
 /***************************************************************************
  * Range search
  ***************************************************************************/

data/vendor/faiss/faiss/utils/distances.h

@@ -455,6 +455,36 @@ void knn_L2sqr(
         const float* y_norm2 = nullptr,
         const IDSelector* sel = nullptr);
 
+/** knn_inner_product for packed IEEE binary16 queries (x: nx * d uint16_t).
+ *
+ * The BLAS kernels widen each query tile once and reuse it for all database
+ * tiles, so there is no fp32 copy of the queries. Results are the same as
+ * knn_inner_product on the widened queries.
+ */
+void knn_inner_product_fp16(
+        const uint16_t* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        size_t k,
+        float* distances,
+        int64_t* indexes,
+        const IDSelector* sel = nullptr);
+
+/// knn_L2sqr for packed IEEE binary16 queries, see knn_inner_product_fp16
+void knn_L2sqr_fp16(
+        const uint16_t* x,
+        const float* y,
+        size_t d,
+        size_t nx,
+        size_t ny,
+        size_t k,
+        float* distances,
+        int64_t* indexes,
+        const float* y_norm2 = nullptr,
+        const IDSelector* sel = nullptr);
+
 /** Find the max inner product neighbors for nx queries in a set of ny vectors
  * indexed by ids. May be useful for re-ranking a pre-selected vector list
  *
@@ -600,4 +630,8 @@ int fvec_madd_and_argmin(
         const float* b,
         float* c);
 
+/// sum of x[j] for every j < d whose bit is set in bits (LSB-first per byte)
+template <SIMDLevel>
+float fvec_masked_sum(const float* x, const uint8_t* bits, size_t d);
+
 } // namespace faiss

data/vendor/faiss/faiss/utils/distances_dispatch.h

@@ -184,7 +184,7 @@ inline void fvec_sub_dispatch(
         const float* a,
         const float* b,
         float* c) {
-    with_simd_level_256bit(
+    with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_NO_AVX512>(
             [&]<SIMDLevel level>() { fvec_sub<level>(d, a, b, c); });
 }
 
@@ -193,7 +193,7 @@ inline void fvec_add_dispatch(
         const float* a,
         const float* b,
         float* c) {
-    with_simd_level_256bit(
+    with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_NO_AVX512>(
             [&]<SIMDLevel level>() { fvec_add<level>(d, a, b, c); });
 }
 
@@ -202,7 +202,7 @@ inline void fvec_add_scalar_dispatch(
         const float* a,
         float b,
         float* c) {
-    with_simd_level_256bit(
+    with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_NO_AVX512>(
             [&]<SIMDLevel level>() { fvec_add<level>(d, a, b, c); });
 }
 
@@ -214,7 +214,9 @@ inline void compute_PQ_dis_tables_dsub2_dispatch(
         const float* x,
         bool is_inner_product,
         float* dis_tables) {
-    with_simd_level_256bit([&]<SIMDLevel level>() {
+    constexpr int dsub2_mask =
+            AVAILABLE_SIMD_LEVELS_AVX2_NEON | (1 << int(SIMDLevel::RISCV_RVV));
+    with_selected_simd_levels<dsub2_mask>([&]<SIMDLevel level>() {
         compute_PQ_dis_tables_dsub2<level>(
                 d, ksub, centroids, nx, x, is_inner_product, dis_tables);
     });

data/vendor/faiss/faiss/utils/distances_fused/avx512.cpp

@@ -7,10 +7,9 @@
 
 // -*- c++ -*-
 
+#include <faiss/impl/platform_macros.h>
 #include <faiss/utils/distances_fused/distances_fused.h>
 
-#ifdef COMPILE_SIMD_AVX512
-
 #include <immintrin.h>
 
 namespace faiss {
@@ -264,7 +263,7 @@ void exhaustive_L2sqr_fused_cmax(
 
     const idx_t nx_p = (nx / NX_POINTS_PER_LOOP) * NX_POINTS_PER_LOOP;
     // the main loop.
-#pragma omp parallel for schedule(dynamic)
+#pragma omp parallel for schedule(FAISS_OMP_DYNAMIC)
     for (idx_t i = 0; i < nx_p; i += NX_POINTS_PER_LOOP) {
         kernel<DIM, NX_POINTS_PER_LOOP, NY_POINTS_PER_LOOP>(
                 x, y, y_transposed.data(), ny, res, y_norms, i);
@@ -343,5 +342,3 @@ bool exhaustive_L2sqr_fused_cmax<SIMDLevel::AVX512>(
 }
 
 } // namespace faiss
-
-#endif // COMPILE_SIMD_AVX512

data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp

@@ -24,20 +24,6 @@ bool exhaustive_L2sqr_fused_cmax<SIMDLevel::NONE>(
     return false;
 }
 
-#ifdef COMPILE_SIMD_RISCV_RVV
-template <>
-bool exhaustive_L2sqr_fused_cmax<SIMDLevel::RISCV_RVV>(
-        const float*,
-        const float*,
-        size_t,
-        size_t,
-        size_t,
-        Top1BlockResultHandler<CMax<float, int64_t>>&,
-        const float*) {
-    return false;
-}
-#endif // COMPILE_SIMD_RISCV_RVV
-
 bool exhaustive_L2sqr_fused_cmax(
         const float* x,
         const float* y,

data/vendor/faiss/faiss/utils/distances_fused/rvv.cpp

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <faiss/utils/distances_fused/distances_fused.h>
+
+namespace faiss {
+
+// Scalar fallback: no fused kernel available.
+template <>
+bool exhaustive_L2sqr_fused_cmax<SIMDLevel::RISCV_RVV>(
+        const float*,
+        const float*,
+        size_t,
+        size_t,
+        size_t,
+        Top1BlockResultHandler<CMax<float, int64_t>>&,
+        const float*) {
+    return false;
+}
+
+} // namespace faiss

data/vendor/faiss/faiss/utils/distances_fused/simdlib_based.cpp

@@ -7,8 +7,6 @@
 
 // AVX2 compilation unit for the simdlib-based fused distance kernel.
 
-#ifdef COMPILE_SIMD_AVX2
-
 #include <faiss/impl/simdlib/simdlib_avx2.h>
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
 #include <faiss/utils/distances_fused/simdlib_kernel-inl.h>
@@ -71,5 +69,3 @@ bool exhaustive_L2sqr_fused_cmax<SIMDLevel::AVX2>(
 }
 
 } // namespace faiss
-
-#endif // COMPILE_SIMD_AVX2

data/vendor/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp

@@ -7,8 +7,6 @@
 
 // ARM NEON compilation unit for the simdlib-based fused distance kernel.
 
-#ifdef COMPILE_SIMD_ARM_NEON
-
 #include <faiss/impl/simdlib/simdlib_neon.h>
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
 #include <faiss/utils/distances_fused/simdlib_kernel-inl.h>
@@ -53,5 +51,3 @@ bool exhaustive_L2sqr_fused_cmax<SIMDLevel::ARM_NEON>(
 }
 
 } // namespace faiss
-
-#endif // COMPILE_SIMD_ARM_NEON

data/vendor/faiss/faiss/utils/distances_fused/simdlib_kernel-inl.h

@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <faiss/impl/platform_macros.h>
 #include <faiss/utils/distances_fused/distances_fused.h>
 
 namespace faiss {
@@ -267,7 +268,7 @@ void exhaustive_L2sqr_fused_cmax(
 
     const size_t nx_p = (nx / NX_POINTS_PER_LOOP) * NX_POINTS_PER_LOOP;
     // the main loop.
-#pragma omp parallel for schedule(dynamic)
+#pragma omp parallel for schedule(FAISS_OMP_DYNAMIC)
     for (int64_t i = 0; i < static_cast<int64_t>(nx_p);
          i += NX_POINTS_PER_LOOP) {
         kernel<DIM, NX_POINTS_PER_LOOP, NY_POINTS_PER_LOOP, SL>(

data/vendor/faiss/faiss/utils/distances_simd.cpp

@@ -8,6 +8,7 @@
 // -*- c++ -*-
 
 #include <faiss/utils/distances.h>
+#include <faiss/utils/simd_impl/masked_sum.h>
 
 #include <cmath>
 #include <cstring>
@@ -42,6 +43,14 @@ void fvec_madd<SIMDLevel::NONE>(
     }
 }
 
+template <>
+float fvec_masked_sum<SIMDLevel::NONE>(
+        const float* x,
+        const uint8_t* bits,
+        size_t d) {
+    return masked_sum<SIMDLevel::NONE>(x, bits, d);
+}
+
 template <>
 void fvec_L2sqr_ny_transposed<SIMDLevel::NONE>(
         float* dis,

data/vendor/faiss/faiss/utils/fp16-fp16c.h

@@ -12,6 +12,8 @@
 
 namespace faiss {
 
+namespace {
+
 inline uint16_t encode_fp16(float x) {
     __m128 xf = _mm_set1_ps(x);
     __m128i xi =
@@ -25,4 +27,6 @@ inline float decode_fp16(uint16_t x) {
     return _mm_cvtss_f32(xf);
 }
 
+} // namespace
+
 } // namespace faiss

data/vendor/faiss/faiss/utils/fp16-inl.h

@@ -27,63 +27,35 @@ inline uint32_t intbits(float f) {
     return *(uint32_t*)fptr;
 }
 
-} // namespace
-
 inline uint16_t encode_fp16(float f) {
-    // via Fabian "ryg" Giesen.
-    // https://gist.github.com/2156668
-    uint32_t sign_mask = 0x80000000u;
-    int32_t o;
+    // Rounds to nearest even, as the F16C and Arm conversions do, so that
+    // every SIMD level encodes the same bits. Via Fabian "ryg" Giesen
+    // (float_to_half_fast3_rtne): https://gist.github.com/2156668
+    const uint32_t f32infty = 255u << 23;
+    const uint32_t f16max = (127u + 16u) << 23;
+    const uint32_t denorm_magic = ((127u - 15u) + (23u - 10u) + 1u) << 23;
+    const uint32_t sign_mask = 0x80000000u;
 
     uint32_t fint = intbits(f);
-    uint32_t sign = fint & sign_mask;
+    const uint32_t sign = fint & sign_mask;
     fint ^= sign;
 
-    // NOTE all the integer compares in this function can be safely
-    // compiled into signed compares since all operands are below
-    // 0x80000000. Important if you want fast straight SSE2 code (since
-    // there's no unsigned PCMPGTD).
-
-    // Inf or NaN (all exponent bits set)
-    // NaN->qNaN and Inf->Inf
-    // unconditional assignment here, will override with right value for
-    // the regular case below.
-    uint32_t f32infty = 255u << 23;
-    o = (fint > f32infty) ? 0x7e00u : 0x7c00u;
-
-    // (De)normalized number or zero
-    // update fint unconditionally to save the blending; we don't need it
-    // anymore for the Inf/NaN case anyway.
-
-    const uint32_t round_mask = ~0xfffu;
-    const uint32_t magic = 15u << 23;
-
-    // Shift exponent down, denormalize if necessary.
-    // NOTE This represents half-float denormals using single
-    // precision denormals.  The main reason to do this is that
-    // there's no shift with per-lane variable shifts in SSE*, which
-    // we'd otherwise need. It has some funky side effects though:
-    // - This conversion will actually respect the FTZ (Flush To Zero)
-    //   flag in MXCSR - if it's set, no half-float denormals will be
-    //   generated. I'm honestly not sure whether this is good or
-    //   bad. It's definitely interesting.
-    // - If the underlying HW doesn't support denormals (not an issue
-    //   with Intel CPUs, but might be a problem on GPUs or PS3 SPUs),
-    //   you will always get flush-to-zero behavior. This is bad,
-    //   unless you're on a CPU where you don't care.
-    // - Denormals tend to be slow. FP32 denormals are rare in
-    //   practice outside of things like recursive filters in DSP -
-    //   not a typical half-float application. Whether FP16 denormals
-    //   are rare in practice, I don't know. Whatever slow path your
-    //   HW may or may not have for denormals, this may well hit it.
-    float fscale = floatbits(fint & round_mask) * floatbits(magic);
-    fscale = std::min(fscale, floatbits((31u << 23) - 0x1000u));
-    int32_t fint2 = intbits(fscale) - round_mask;
-
-    if (fint < f32infty)
-        o = fint2 >> 13; // Take the bits!
-
-    return (o | (sign >> 16));
+    uint32_t o;
+    if (fint >= f16max) {
+        // Inf->Inf. NaN->qNaN with the top bits of the payload, as F16C.
+        o = (fint > f32infty) ? 0x7e00u | ((fint >> 13) & 0x3ffu) : 0x7c00u;
+    } else if (fint < (113u << 23)) {
+        // The result is subnormal or zero. Adding a magic value aligns the 10
+        // mantissa bits at the bottom of the float, and the float addition
+        // rounds to nearest even.
+        o = intbits(floatbits(fint) + floatbits(denorm_magic)) - denorm_magic;
+    } else {
+        const uint32_t mant_odd = (fint >> 13) & 1;
+        // rebias the exponent, and round to nearest even
+        fint += ((15u - 127u) << 23) + 0xfffu + mant_odd;
+        o = fint >> 13;
+    }
+    return static_cast<uint16_t>(o | (sign >> 16));
 }
 
 inline float decode_fp16(uint16_t h) {
@@ -105,4 +77,6 @@ inline float decode_fp16(uint16_t h) {
     return floatbits(((exp == shifted_exp) ? infnan_val : reg_val) | sign_bit);
 }
 
+} // namespace
+
 } // namespace faiss