faiss 0.1.7 → 0.2.3

data/vendor/faiss/faiss/gpu/impl/InterleavedCodes.h

@@ -12,40 +12,46 @@
 
 // Utilities for bit packing and unpacking CPU non-interleaved and GPU
 // interleaved by 32 encodings
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 // Unpacks arbitrary bitwidth codes to a whole number of bytes per code
 // The layout of the input is (v0 d0)(v0 d1) ... (v0 dD)(v1 d0) ...
 // (bit packed)
 // The layout of the output is the same (byte packed to roundUp(bitsPerCode, 8)
 // / 8 bytes)
-std::vector<uint8_t> unpackNonInterleaved(std::vector<uint8_t> data,
-                                          int numVecs,
-                                          int dims,
-                                          int bitsPerCode);
+std::vector<uint8_t> unpackNonInterleaved(
+        std::vector<uint8_t> data,
+        int numVecs,
+        int dims,
+        int bitsPerCode);
 
 // Unpacks arbitrary bitwidth codes to a whole number of bytes per scalar code
 // The layout of the input is (v0 d0)(v1 d0) ... (v31 d0)(v0 d1) ...
 // (bit packed)
 // The layout of the input is (v0 d0)(v0 d1) ... (v0 dD)(v1 d0) ...
 // (byte packed)
-std::vector<uint8_t> unpackInterleaved(std::vector<uint8_t> data,
-                                       int numVecs,
-                                       int dims,
-                                       int bitsPerCode);
+std::vector<uint8_t> unpackInterleaved(
+        std::vector<uint8_t> data,
+        int numVecs,
+        int dims,
+        int bitsPerCode);
 
 // Packs data in the byte packed non-interleaved form to bit packed
 // non-interleaved form
-std::vector<uint8_t> packNonInterleaved(std::vector<uint8_t> data,
-                                        int numVecs,
-                                        int dims,
-                                        int bitsPerCode);
+std::vector<uint8_t> packNonInterleaved(
+        std::vector<uint8_t> data,
+        int numVecs,
+        int dims,
+        int bitsPerCode);
 
 // Packs data in the byte packed non-interleaved form to bit packed
 // interleaved form
-std::vector<uint8_t> packInterleaved(std::vector<uint8_t> data,
-                                     int numVecs,
-                                     int dims,
-                                     int bitsPerCode);
-
-} } // namespace
+std::vector<uint8_t> packInterleaved(
+        std::vector<uint8_t> data,
+        int numVecs,
+        int dims,
+        int bitsPerCode);
+
+} // namespace gpu
+} // namespace faiss

data/vendor/faiss/faiss/gpu/impl/RemapIndices.cpp

@@ -5,39 +5,41 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #include <faiss/gpu/impl/RemapIndices.h>
 #include <faiss/impl/FaissAssert.h>
 
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 // Utility function to translate (list id, offset) to a user index on
 // the CPU. In a cpp in order to use OpenMP
 void ivfOffsetToUserIndex(
-  Index::idx_t* indices,
-  int numLists,
-  int queries,
-  int k,
-  const std::vector<std::vector<Index::idx_t>>& listOffsetToUserIndex) {
-  FAISS_ASSERT(numLists == listOffsetToUserIndex.size());
+        Index::idx_t* indices,
+        int numLists,
+        int queries,
+        int k,
+        const std::vector<std::vector<Index::idx_t>>& listOffsetToUserIndex) {
+    FAISS_ASSERT(numLists == listOffsetToUserIndex.size());
 
 #pragma omp parallel for
-  for (int q = 0; q < queries; ++q) {
-    for (int r = 0; r < k; ++r) {
-      auto offsetIndex = indices[q * k + r];
+    for (int q = 0; q < queries; ++q) {
+        for (int r = 0; r < k; ++r) {
+            auto offsetIndex = indices[q * k + r];
 
-      if (offsetIndex < 0) continue;
+            if (offsetIndex < 0)
+                continue;
 
-      int listId = (int) (offsetIndex >> 32);
-      int listOffset = (int) (offsetIndex & 0xffffffff);
+            int listId = (int)(offsetIndex >> 32);
+            int listOffset = (int)(offsetIndex & 0xffffffff);
 
-      FAISS_ASSERT(listId < numLists);
-      auto& listIndices = listOffsetToUserIndex[listId];
+            FAISS_ASSERT(listId < numLists);
+            auto& listIndices = listOffsetToUserIndex[listId];
 
-      FAISS_ASSERT(listOffset < listIndices.size());
-      indices[q * k + r] = listIndices[listOffset];
+            FAISS_ASSERT(listOffset < listIndices.size());
+            indices[q * k + r] = listIndices[listOffset];
+        }
     }
-  }
 }
 
-} } // namespace
+} // namespace gpu
+} // namespace faiss

data/vendor/faiss/faiss/gpu/impl/RemapIndices.h

@@ -5,21 +5,22 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
 #include <faiss/Index.h>
 #include <vector>
 
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 /// Utility function to translate (list id, offset) to a user index on
 /// the CPU. In a cpp in order to use OpenMP.
 void ivfOffsetToUserIndex(
-  Index::idx_t* indices,
-  int numLists,
-  int queries,
-  int k,
-  const std::vector<std::vector<Index::idx_t>>& listOffsetToUserIndex);
+        Index::idx_t* indices,
+        int numLists,
+        int queries,
+        int k,
+        const std::vector<std::vector<Index::idx_t>>& listOffsetToUserIndex);
 
-} } // namespace
+} // namespace gpu
+} // namespace faiss

data/vendor/faiss/faiss/gpu/perf/IndexWrapper-inl.h

@@ -5,67 +5,62 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #include <faiss/impl/FaissAssert.h>
 
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 template <typename GpuIndex>
 IndexWrapper<GpuIndex>::IndexWrapper(
-  int numGpus,
-  std::function<std::unique_ptr<GpuIndex>(GpuResourcesProvider*, int)> init) {
-  FAISS_ASSERT(numGpus <= faiss::gpu::getNumDevices());
-  for (int i = 0; i < numGpus; ++i) {
-    auto res = std::unique_ptr<faiss::gpu::StandardGpuResources>(
-      new StandardGpuResources);
+        int numGpus,
+        std::function<std::unique_ptr<GpuIndex>(GpuResourcesProvider*, int)>
+                init) {
+    FAISS_ASSERT(numGpus <= faiss::gpu::getNumDevices());
+    for (int i = 0; i < numGpus; ++i) {
+        auto res = std::unique_ptr<faiss::gpu::StandardGpuResources>(
+                new StandardGpuResources);
 
-    subIndex.emplace_back(init(res.get(), i));
-    resources.emplace_back(std::move(res));
-  }
+        subIndex.emplace_back(init(res.get(), i));
+        resources.emplace_back(std::move(res));
+    }
 
-  if (numGpus > 1) {
-    // create proxy
-    replicaIndex =
-      std::unique_ptr<faiss::IndexReplicas>(new faiss::IndexReplicas);
+    if (numGpus > 1) {
+        // create proxy
+        replicaIndex =
+                std::unique_ptr<faiss::IndexReplicas>(new faiss::IndexReplicas);
 
-    for (auto& index : subIndex) {
-      replicaIndex->addIndex(index.get());
+        for (auto& index : subIndex) {
+            replicaIndex->addIndex(index.get());
+        }
     }
-  }
 }
 
 template <typename GpuIndex>
-faiss::Index*
-IndexWrapper<GpuIndex>::getIndex() {
-  if ((bool) replicaIndex) {
-    return replicaIndex.get();
-  } else {
-    FAISS_ASSERT(!subIndex.empty());
-    return subIndex.front().get();
-  }
+faiss::Index* IndexWrapper<GpuIndex>::getIndex() {
+    if ((bool)replicaIndex) {
+        return replicaIndex.get();
+    } else {
+        FAISS_ASSERT(!subIndex.empty());
+        return subIndex.front().get();
+    }
 }
 
 template <typename GpuIndex>
-void
-IndexWrapper<GpuIndex>::runOnIndices(std::function<void(GpuIndex*)> f) {
-
-  if ((bool) replicaIndex) {
-    replicaIndex->runOnIndex(
-      [f](int, faiss::Index* index) {
-        f(dynamic_cast<GpuIndex*>(index));
-      });
-  } else {
-    FAISS_ASSERT(!subIndex.empty());
-    f(subIndex.front().get());
-  }
+void IndexWrapper<GpuIndex>::runOnIndices(std::function<void(GpuIndex*)> f) {
+    if ((bool)replicaIndex) {
+        replicaIndex->runOnIndex([f](int, faiss::Index* index) {
+            f(dynamic_cast<GpuIndex*>(index));
+        });
+    } else {
+        FAISS_ASSERT(!subIndex.empty());
+        f(subIndex.front().get());
+    }
 }
 
 template <typename GpuIndex>
-void
-IndexWrapper<GpuIndex>::setNumProbes(int nprobe) {
-  runOnIndices([nprobe](GpuIndex* index) {
-      index->setNumProbes(nprobe);
-    });
+void IndexWrapper<GpuIndex>::setNumProbes(int nprobe) {
+    runOnIndices([nprobe](GpuIndex* index) { index->setNumProbes(nprobe); });
 }
 
-} }
+} // namespace gpu
+} // namespace faiss

data/vendor/faiss/faiss/gpu/perf/IndexWrapper.h

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
 #include <faiss/IndexReplicas.h>
@@ -14,26 +13,29 @@
 #include <memory>
 #include <vector>
 
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 // If we want to run multi-GPU, create a proxy to wrap the indices.
 // If we don't want multi-GPU, don't involve the proxy, so it doesn't
 // affect the timings.
 template <typename GpuIndex>
 struct IndexWrapper {
-  std::vector<std::unique_ptr<faiss::gpu::StandardGpuResources>> resources;
-  std::vector<std::unique_ptr<GpuIndex>> subIndex;
-  std::unique_ptr<faiss::IndexReplicas> replicaIndex;
-
-  IndexWrapper(
-    int numGpus,
-    std::function<std::unique_ptr<GpuIndex>(GpuResourcesProvider*, int)> init);
-  faiss::Index* getIndex();
-
-  void runOnIndices(std::function<void(GpuIndex*)> f);
-  void setNumProbes(int nprobe);
+    std::vector<std::unique_ptr<faiss::gpu::StandardGpuResources>> resources;
+    std::vector<std::unique_ptr<GpuIndex>> subIndex;
+    std::unique_ptr<faiss::IndexReplicas> replicaIndex;
+
+    IndexWrapper(
+            int numGpus,
+            std::function<std::unique_ptr<GpuIndex>(GpuResourcesProvider*, int)>
+                    init);
+    faiss::Index* getIndex();
+
+    void runOnIndices(std::function<void(GpuIndex*)> f);
+    void setNumProbes(int nprobe);
 };
 
-} }
+} // namespace gpu
+} // namespace faiss
 
 #include <faiss/gpu/perf/IndexWrapper-inl.h>

data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp

@@ -5,14 +5,13 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
-#include <faiss/utils/random.h>
 #include <faiss/Clustering.h>
 #include <faiss/gpu/GpuIndexFlat.h>
 #include <faiss/gpu/StandardGpuResources.h>
 #include <faiss/gpu/perf/IndexWrapper.h>
 #include <faiss/gpu/utils/DeviceUtils.h>
 #include <faiss/gpu/utils/Timer.h>
+#include <faiss/utils/random.h>
 #include <gflags/gflags.h>
 #include <memory>
 #include <vector>
@@ -29,87 +28,94 @@ DEFINE_bool(transposed, false, "transposed vector storage");
 DEFINE_bool(verbose, false, "turn on clustering logging");
 DEFINE_int64(seed, -1, "specify random seed");
 DEFINE_int32(num_gpus, 1, "number of gpus to use");
-DEFINE_int64(min_paging_size, -1, "minimum size to use CPU -> GPU paged copies");
+DEFINE_int64(
+        min_paging_size,
+        -1,
+        "minimum size to use CPU -> GPU paged copies");
 DEFINE_int64(pinned_mem, -1, "pinned memory allocation to use");
 DEFINE_int32(max_points, -1, "max points per centroid");
 
 using namespace faiss::gpu;
 
 int main(int argc, char** argv) {
-  gflags::ParseCommandLineFlags(&argc, &argv, true);
-
-  cudaProfilerStop();
-
-  auto seed = FLAGS_seed != -1L ? FLAGS_seed : time(nullptr);
-  printf("using seed %ld\n", seed);
-
-  std::vector<float> vecs((size_t) FLAGS_num * FLAGS_dim);
-  faiss::float_rand(vecs.data(), vecs.size(), seed);
-
-  printf("K-means metric %s dim %d centroids %d num train %d niter %d\n",
-         FLAGS_L2_metric ? "L2" : "IP",
-         FLAGS_dim, FLAGS_k, FLAGS_num, FLAGS_niter);
-  printf("float16 math %s\n", FLAGS_use_float16 ? "enabled" : "disabled");
-  printf("transposed storage %s\n", FLAGS_transposed ? "enabled" : "disabled");
-  printf("verbose %s\n", FLAGS_verbose ? "enabled" : "disabled");
-
-  auto initFn = [](faiss::gpu::GpuResourcesProvider* res, int dev) ->
-    std::unique_ptr<faiss::gpu::GpuIndexFlat> {
-    if (FLAGS_pinned_mem >= 0) {
-      ((faiss::gpu::StandardGpuResources*) res)->setPinnedMemory(
-        FLAGS_pinned_mem);
-    }
-
-    GpuIndexFlatConfig config;
-    config.device = dev;
-    config.useFloat16 = FLAGS_use_float16;
-    config.storeTransposed = FLAGS_transposed;
-
-    auto p = std::unique_ptr<faiss::gpu::GpuIndexFlat>(
-      FLAGS_L2_metric ?
-      (faiss::gpu::GpuIndexFlat*)
-      new faiss::gpu::GpuIndexFlatL2(res, FLAGS_dim, config) :
-      (faiss::gpu::GpuIndexFlat*)
-      new faiss::gpu::GpuIndexFlatIP(res, FLAGS_dim, config));
-
-    if (FLAGS_min_paging_size >= 0) {
-      p->setMinPagingSize(FLAGS_min_paging_size);
+    gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+    cudaProfilerStop();
+
+    auto seed = FLAGS_seed != -1L ? FLAGS_seed : time(nullptr);
+    printf("using seed %ld\n", seed);
+
+    std::vector<float> vecs((size_t)FLAGS_num * FLAGS_dim);
+    faiss::float_rand(vecs.data(), vecs.size(), seed);
+
+    printf("K-means metric %s dim %d centroids %d num train %d niter %d\n",
+           FLAGS_L2_metric ? "L2" : "IP",
+           FLAGS_dim,
+           FLAGS_k,
+           FLAGS_num,
+           FLAGS_niter);
+    printf("float16 math %s\n", FLAGS_use_float16 ? "enabled" : "disabled");
+    printf("transposed storage %s\n",
+           FLAGS_transposed ? "enabled" : "disabled");
+    printf("verbose %s\n", FLAGS_verbose ? "enabled" : "disabled");
+
+    auto initFn = [](faiss::gpu::GpuResourcesProvider* res,
+                     int dev) -> std::unique_ptr<faiss::gpu::GpuIndexFlat> {
+        if (FLAGS_pinned_mem >= 0) {
+            ((faiss::gpu::StandardGpuResources*)res)
+                    ->setPinnedMemory(FLAGS_pinned_mem);
+        }
+
+        GpuIndexFlatConfig config;
+        config.device = dev;
+        config.useFloat16 = FLAGS_use_float16;
+        config.storeTransposed = FLAGS_transposed;
+
+        auto p = std::unique_ptr<faiss::gpu::GpuIndexFlat>(
+                FLAGS_L2_metric
+                        ? (faiss::gpu::GpuIndexFlat*)new faiss::gpu::
+                                  GpuIndexFlatL2(res, FLAGS_dim, config)
+                        : (faiss::gpu::GpuIndexFlat*)new faiss::gpu::
+                                  GpuIndexFlatIP(res, FLAGS_dim, config));
+
+        if (FLAGS_min_paging_size >= 0) {
+            p->setMinPagingSize(FLAGS_min_paging_size);
+        }
+        return p;
+    };
+
+    IndexWrapper<faiss::gpu::GpuIndexFlat> gpuIndex(FLAGS_num_gpus, initFn);
+
+    CUDA_VERIFY(cudaProfilerStart());
+    faiss::gpu::synchronizeAllDevices();
+
+    float gpuTime = 0.0f;
+
+    faiss::ClusteringParameters cp;
+    cp.niter = FLAGS_niter;
+    cp.verbose = FLAGS_verbose;
+
+    if (FLAGS_max_points > 0) {
+        cp.max_points_per_centroid = FLAGS_max_points;
     }
-    return p;
-  };
-
-  IndexWrapper<faiss::gpu::GpuIndexFlat> gpuIndex(FLAGS_num_gpus, initFn);
-
-  CUDA_VERIFY(cudaProfilerStart());
-  faiss::gpu::synchronizeAllDevices();
 
-  float gpuTime = 0.0f;
+    faiss::Clustering kmeans(FLAGS_dim, FLAGS_k, cp);
 
-  faiss::ClusteringParameters cp;
-  cp.niter = FLAGS_niter;
-  cp.verbose = FLAGS_verbose;
+    // Time k-means
+    {
+        CpuTimer timer;
 
-  if (FLAGS_max_points > 0) {
-    cp.max_points_per_centroid = FLAGS_max_points;
-  }
+        kmeans.train(FLAGS_num, vecs.data(), *(gpuIndex.getIndex()));
 
-  faiss::Clustering kmeans(FLAGS_dim, FLAGS_k, cp);
-
-  // Time k-means
-  {
-    CpuTimer timer;
-
-    kmeans.train(FLAGS_num, vecs.data(), *(gpuIndex.getIndex()));
-
-    // There is a device -> host copy above, so no need to time
-    // additional synchronization with the GPU
-    gpuTime = timer.elapsedMilliseconds();
-  }
+        // There is a device -> host copy above, so no need to time
+        // additional synchronization with the GPU
+        gpuTime = timer.elapsedMilliseconds();
+    }
 
-  CUDA_VERIFY(cudaProfilerStop());
-  printf("k-means time %.3f ms\n", gpuTime);
+    CUDA_VERIFY(cudaProfilerStop());
+    printf("k-means time %.3f ms\n", gpuTime);
 
-  CUDA_VERIFY(cudaDeviceSynchronize());
+    CUDA_VERIFY(cudaDeviceSynchronize());
 
-  return 0;
+    return 0;
 }