faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/gpu/test/TestUtils.h

@@ -5,20 +5,20 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
-#include <faiss/impl/FaissAssert.h>
 #include <faiss/Index.h>
+#include <faiss/impl/FaissAssert.h>
 #include <faiss/invlists/InvertedLists.h>
-#include <initializer_list>
 #include <gtest/gtest.h>
 #include <cstring>
+#include <initializer_list>
 #include <memory>
 #include <string>
 #include <vector>
 
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 /// Generates and displays a new seed for the test
 void newTestSeed();
@@ -40,18 +40,18 @@ bool randBool();
 /// initializer_list
 template <typename T>
 T randSelect(std::initializer_list<T> vals) {
-  FAISS_ASSERT(vals.size() > 0);
-  int sel = randVal(0, vals.size());
-
-  int i = 0;
-  for (auto v : vals) {
-    if (i++ == sel) {
-      return v;
+    FAISS_ASSERT(vals.size() > 0);
+    int sel = randVal(0, vals.size());
+
+    int i = 0;
+    for (auto v : vals) {
+        if (i++ == sel) {
+            return v;
+        }
     }
-  }
 
-  // should not get here
-  return *vals.begin();
+    // should not get here
+    return *vals.begin();
 }
 
 /// Generates a collection of random vectors in the range [0, 1]
@@ -62,65 +62,81 @@ std::vector<unsigned char> randBinaryVecs(size_t num, size_t dim);
 
 /// Compare two indices via query for similarity, with a user-specified set of
 /// query vectors
-void compareIndices(const std::vector<float>& queryVecs,
-                    faiss::Index& refIndex,
-                    faiss::Index& testIndex,
-                    int numQuery, int dim, int k,
-                    const std::string& configMsg,
-                    float maxRelativeError = 6e-5f,
-                    float pctMaxDiff1 = 0.1f,
-                    float pctMaxDiffN = 0.005f);
+void compareIndices(
+        const std::vector<float>& queryVecs,
+        faiss::Index& refIndex,
+        faiss::Index& testIndex,
+        int numQuery,
+        int dim,
+        int k,
+        const std::string& configMsg,
+        float maxRelativeError = 6e-5f,
+        float pctMaxDiff1 = 0.1f,
+        float pctMaxDiffN = 0.005f);
 
 /// Compare two indices via query for similarity, generating random query
 /// vectors
-void compareIndices(faiss::Index& refIndex,
-                    faiss::Index& testIndex,
-                    int numQuery, int dim, int k,
-                    const std::string& configMsg,
-                    float maxRelativeError = 6e-5f,
-                    float pctMaxDiff1 = 0.1f,
-                    float pctMaxDiffN = 0.005f);
+void compareIndices(
+        faiss::Index& refIndex,
+        faiss::Index& testIndex,
+        int numQuery,
+        int dim,
+        int k,
+        const std::string& configMsg,
+        float maxRelativeError = 6e-5f,
+        float pctMaxDiff1 = 0.1f,
+        float pctMaxDiffN = 0.005f);
 
 /// Display specific differences in the two (distance, index) lists
-void compareLists(const float* refDist,
-                  const faiss::Index::idx_t* refInd,
-                  const float* testDist,
-                  const faiss::Index::idx_t* testInd,
-                  int dim1, int dim2,
-                  const std::string& configMsg,
-                  bool printBasicStats, bool printDiffs, bool assertOnErr,
-                  float maxRelativeError = 6e-5f,
-                  float pctMaxDiff1 = 0.1f,
-                  float pctMaxDiffN = 0.005f);
+void compareLists(
+        const float* refDist,
+        const faiss::Index::idx_t* refInd,
+        const float* testDist,
+        const faiss::Index::idx_t* testInd,
+        int dim1,
+        int dim2,
+        const std::string& configMsg,
+        bool printBasicStats,
+        bool printDiffs,
+        bool assertOnErr,
+        float maxRelativeError = 6e-5f,
+        float pctMaxDiff1 = 0.1f,
+        float pctMaxDiffN = 0.005f);
 
 /// Compare IVF lists between a CPU and GPU index
 template <typename A, typename B>
 void testIVFEquality(A& cpuIndex, B& gpuIndex) {
-  // Ensure equality of the inverted lists
-  EXPECT_EQ(cpuIndex.nlist, gpuIndex.nlist);
+    // Ensure equality of the inverted lists
+    EXPECT_EQ(cpuIndex.nlist, gpuIndex.nlist);
 
-  for (int i = 0; i < cpuIndex.nlist; ++i) {
-    auto cpuLists = cpuIndex.invlists;
+    for (int i = 0; i < cpuIndex.nlist; ++i) {
+        auto cpuLists = cpuIndex.invlists;
 
-    // Code equality
-    EXPECT_EQ(cpuLists->list_size(i), gpuIndex.getListLength(i));
-    std::vector<uint8_t> cpuCodes(cpuLists->list_size(i) * cpuLists->code_size);
+        // Code equality
+        EXPECT_EQ(cpuLists->list_size(i), gpuIndex.getListLength(i));
+        std::vector<uint8_t> cpuCodes(
+                cpuLists->list_size(i) * cpuLists->code_size);
 
-    auto sc = faiss::InvertedLists::ScopedCodes(cpuLists, i);
-    std::memcpy(cpuCodes.data(), sc.get(),
+        auto sc = faiss::InvertedLists::ScopedCodes(cpuLists, i);
+        std::memcpy(
+                cpuCodes.data(),
+                sc.get(),
                 cpuLists->list_size(i) * cpuLists->code_size);
 
-    auto gpuCodes = gpuIndex.getListVectorData(i, false);
-    EXPECT_EQ(cpuCodes, gpuCodes);
+        auto gpuCodes = gpuIndex.getListVectorData(i, false);
+        EXPECT_EQ(cpuCodes, gpuCodes);
 
-    // Index equality
-    std::vector<Index::idx_t> cpuIndices(cpuLists->list_size(i));
+        // Index equality
+        std::vector<Index::idx_t> cpuIndices(cpuLists->list_size(i));
 
-    auto si = faiss::InvertedLists::ScopedIds(cpuLists, i);
-    std::memcpy(cpuIndices.data(), si.get(),
+        auto si = faiss::InvertedLists::ScopedIds(cpuLists, i);
+        std::memcpy(
+                cpuIndices.data(),
+                si.get(),
                 cpuLists->list_size(i) * sizeof(faiss::Index::idx_t));
-    EXPECT_EQ(cpuIndices, gpuIndex.getListIndices(i));
-  }
+        EXPECT_EQ(cpuIndices, gpuIndex.getListIndices(i));
+    }
 }
 
-} }
+} // namespace gpu
+} // namespace faiss

data/vendor/faiss/faiss/gpu/test/demo_ivfpq_indexing_gpu.cpp

@@ -12,24 +12,19 @@
 
 #include <sys/time.h>
 
-
-#include <faiss/gpu/StandardGpuResources.h>
 #include <faiss/gpu/GpuIndexIVFPQ.h>
+#include <faiss/gpu/StandardGpuResources.h>
 
 #include <faiss/gpu/GpuAutoTune.h>
 #include <faiss/index_io.h>
 
-double elapsed ()
-{
+double elapsed() {
     struct timeval tv;
-    gettimeofday (&tv, NULL);
-    return  tv.tv_sec + tv.tv_usec * 1e-6;
+    gettimeofday(&tv, NULL);
+    return tv.tv_sec + tv.tv_usec * 1e-6;
 }
 
-
-int main ()
-{
-
+int main() {
     double t0 = elapsed();
 
     // dimension of the vectors to index
@@ -48,47 +43,48 @@ int main ()
             elapsed() - t0, d, nb, nt, dev_no);
     */
     // a reasonable number of centroids to index nb vectors
-    int ncentroids = int (4 * sqrt (nb));
+    int ncentroids = int(4 * sqrt(nb));
 
     faiss::gpu::StandardGpuResources resources;
 
-
     // the coarse quantizer should not be dealloced before the index
     // 4 = nb of bytes per code (d must be a multiple of this)
     // 8 = nb of bits per sub-code (almost always 8)
     faiss::gpu::GpuIndexIVFPQConfig config;
     config.device = dev_no;
 
-    faiss::gpu::GpuIndexIVFPQ index (
-      &resources, d, ncentroids, 4, 8, faiss::METRIC_L2, config);
+    faiss::gpu::GpuIndexIVFPQ index(
+            &resources, d, ncentroids, 4, 8, faiss::METRIC_L2, config);
 
     std::mt19937 rng;
 
     { // training
-        printf ("[%.3f s] Generating %ld vectors in %dD for training\n",
-                elapsed() - t0, nt, d);
+        printf("[%.3f s] Generating %ld vectors in %dD for training\n",
+               elapsed() - t0,
+               nt,
+               d);
 
-        std::vector <float> trainvecs (nt * d);
+        std::vector<float> trainvecs(nt * d);
         std::uniform_real_distribution<> distrib;
         for (size_t i = 0; i < nt * d; i++) {
             trainvecs[i] = distrib(rng);
         }
 
-        printf ("[%.3f s] Training the index\n",
-                elapsed() - t0);
+        printf("[%.3f s] Training the index\n", elapsed() - t0);
         index.verbose = true;
 
-        index.train (nt, trainvecs.data());
+        index.train(nt, trainvecs.data());
     }
 
     { // I/O demo
-        const char *outfilename = "/tmp/index_trained.faissindex";
-        printf ("[%.3f s] storing the pre-trained index to %s\n",
-                elapsed() - t0, outfilename);
+        const char* outfilename = "/tmp/index_trained.faissindex";
+        printf("[%.3f s] storing the pre-trained index to %s\n",
+               elapsed() - t0,
+               outfilename);
 
-        faiss::Index * cpu_index = faiss::gpu::index_gpu_to_cpu (&index);
+        faiss::Index* cpu_index = faiss::gpu::index_gpu_to_cpu(&index);
 
-        write_index (cpu_index, outfilename);
+        write_index(cpu_index, outfilename);
 
         delete cpu_index;
     }
@@ -97,64 +93,65 @@ int main ()
     std::vector<float> queries;
 
     { // populating the database
-        printf ("[%.3f s] Building a dataset of %ld vectors to index\n",
-                elapsed() - t0, nb);
+        printf("[%.3f s] Building a dataset of %ld vectors to index\n",
+               elapsed() - t0,
+               nb);
 
-        std::vector <float> database (nb * d);
+        std::vector<float> database(nb * d);
         std::uniform_real_distribution<> distrib;
         for (size_t i = 0; i < nb * d; i++) {
             database[i] = distrib(rng);
         }
 
-        printf ("[%.3f s] Adding the vectors to the index\n",
-                elapsed() - t0);
+        printf("[%.3f s] Adding the vectors to the index\n", elapsed() - t0);
 
-        index.add (nb, database.data());
+        index.add(nb, database.data());
 
-        printf ("[%.3f s] done\n", elapsed() - t0);
+        printf("[%.3f s] done\n", elapsed() - t0);
 
         // remember a few elements from the database as queries
         int i0 = 1234;
         int i1 = 1243;
 
         nq = i1 - i0;
-        queries.resize (nq * d);
+        queries.resize(nq * d);
         for (int i = i0; i < i1; i++) {
             for (int j = 0; j < d; j++) {
-                queries [(i - i0) * d  + j]  = database [i * d + j];
+                queries[(i - i0) * d + j] = database[i * d + j];
             }
         }
-
     }
 
     { // searching the database
         int k = 5;
-        printf ("[%.3f s] Searching the %d nearest neighbors "
-                "of %ld vectors in the index\n",
-                elapsed() - t0, k, nq);
+        printf("[%.3f s] Searching the %d nearest neighbors "
+               "of %ld vectors in the index\n",
+               elapsed() - t0,
+               k,
+               nq);
 
-        std::vector<faiss::Index::idx_t> nns (k * nq);
-        std::vector<float>               dis (k * nq);
+        std::vector<faiss::Index::idx_t> nns(k * nq);
+        std::vector<float> dis(k * nq);
 
-        index.search (nq, queries.data(), k, dis.data(), nns.data());
+        index.search(nq, queries.data(), k, dis.data(), nns.data());
 
-        printf ("[%.3f s] Query results (vector ids, then distances):\n",
-                elapsed() - t0);
+        printf("[%.3f s] Query results (vector ids, then distances):\n",
+               elapsed() - t0);
 
         for (int i = 0; i < nq; i++) {
-            printf ("query %2d: ", i);
+            printf("query %2d: ", i);
             for (int j = 0; j < k; j++) {
-                printf ("%7ld ", nns[j + i * k]);
+                printf("%7ld ", nns[j + i * k]);
             }
-            printf ("\n     dis: ");
+            printf("\n     dis: ");
             for (int j = 0; j < k; j++) {
-                printf ("%7g ", dis[j + i * k]);
+                printf("%7g ", dis[j + i * k]);
             }
-            printf ("\n");
+            printf("\n");
         }
 
-        printf ("note that the nearest neighbor is not at "
-                "distance 0 due to quantization errors\n");
+        printf("note that the nearest neighbor is not at "
+               "distance 0 due to quantization errors\n");
     }
 
     return 0;

data/vendor/faiss/faiss/gpu/utils/DeviceUtils.h

@@ -5,15 +5,15 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
-#include <faiss/impl/FaissAssert.h>
-#include <cuda_runtime.h>
 #include <cublas_v2.h>
+#include <cuda_runtime.h>
+#include <faiss/impl/FaissAssert.h>
 #include <vector>
 
-namespace faiss { namespace gpu {
+namespace faiss {
+namespace gpu {
 
 /// Returns the current thread-local GPU device
 int getCurrentDevice();
@@ -78,114 +78,121 @@ int getMaxKSelection();
 /// RAII object to set the current device, and restore the previous
 /// device upon destruction
 class DeviceScope {
- public:
-  explicit DeviceScope(int device);
-  ~DeviceScope();
+   public:
+    explicit DeviceScope(int device);
+    ~DeviceScope();
 
- private:
-  int prevDevice_;
+   private:
+    int prevDevice_;
 };
 
 /// RAII object to manage a cublasHandle_t
 class CublasHandleScope {
- public:
-  CublasHandleScope();
-  ~CublasHandleScope();
+   public:
+    CublasHandleScope();
+    ~CublasHandleScope();
 
-  cublasHandle_t get() { return blasHandle_; }
+    cublasHandle_t get() {
+        return blasHandle_;
+    }
 
- private:
-  cublasHandle_t blasHandle_;
+   private:
+    cublasHandle_t blasHandle_;
 };
 
 // RAII object to manage a cudaEvent_t
 class CudaEvent {
- public:
-  /// Creates an event and records it in this stream
-  explicit CudaEvent(cudaStream_t stream, bool timer = false);
-  CudaEvent(const CudaEvent& event) = delete;
-  CudaEvent(CudaEvent&& event) noexcept;
-  ~CudaEvent();
-
-  inline cudaEvent_t get() { return event_; }
+   public:
+    /// Creates an event and records it in this stream
+    explicit CudaEvent(cudaStream_t stream, bool timer = false);
+    CudaEvent(const CudaEvent& event) = delete;
+    CudaEvent(CudaEvent&& event) noexcept;
+    ~CudaEvent();
+
+    inline cudaEvent_t get() {
+        return event_;
+    }
 
-  /// Wait on this event in this stream
-  void streamWaitOnEvent(cudaStream_t stream);
+    /// Wait on this event in this stream
+    void streamWaitOnEvent(cudaStream_t stream);
 
-  /// Have the CPU wait for the completion of this event
-  void cpuWaitOnEvent();
+    /// Have the CPU wait for the completion of this event
+    void cpuWaitOnEvent();
 
-  CudaEvent& operator=(CudaEvent&& event) noexcept;
-  CudaEvent& operator=(CudaEvent& event) = delete;
+    CudaEvent& operator=(CudaEvent&& event) noexcept;
+    CudaEvent& operator=(CudaEvent& event) = delete;
 
- private:
-  cudaEvent_t event_;
+   private:
+    cudaEvent_t event_;
 };
 
 /// Wrapper to test return status of CUDA functions
-#define CUDA_VERIFY(X)                                                  \
-  do {                                                                  \
-    auto err__ = (X);                                                   \
-    FAISS_ASSERT_FMT(err__ == cudaSuccess, "CUDA error %d %s",          \
-                     (int) err__, cudaGetErrorString(err__));           \
-  } while (0)
+#define CUDA_VERIFY(X)                      \
+    do {                                    \
+        auto err__ = (X);                   \
+        FAISS_ASSERT_FMT(                   \
+                err__ == cudaSuccess,       \
+                "CUDA error %d %s",         \
+                (int)err__,                 \
+                cudaGetErrorString(err__)); \
+    } while (0)
 
 /// Wrapper to synchronously probe for CUDA errors
 // #define FAISS_GPU_SYNC_ERROR 1
 
 #ifdef FAISS_GPU_SYNC_ERROR
-#define CUDA_TEST_ERROR()                       \
-  do {                                          \
-    CUDA_VERIFY(cudaDeviceSynchronize());       \
-  } while (0)
+#define CUDA_TEST_ERROR()                     \
+    do {                                      \
+        CUDA_VERIFY(cudaDeviceSynchronize()); \
+    } while (0)
 #else
-#define CUDA_TEST_ERROR()                       \
-  do {                                          \
-    CUDA_VERIFY(cudaGetLastError());            \
-  } while (0)
+#define CUDA_TEST_ERROR()                \
+    do {                                 \
+        CUDA_VERIFY(cudaGetLastError()); \
+    } while (0)
 #endif
 
 /// Call for a collection of streams to wait on
 template <typename L1, typename L2>
 void streamWaitBase(const L1& listWaiting, const L2& listWaitOn) {
-  // For all the streams we are waiting on, create an event
-  std::vector<cudaEvent_t> events;
-  for (auto& stream : listWaitOn) {
-    cudaEvent_t event;
-    CUDA_VERIFY(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));
-    CUDA_VERIFY(cudaEventRecord(event, stream));
-    events.push_back(event);
-  }
-
-  // For all the streams that are waiting, issue a wait
-  for (auto& stream : listWaiting) {
-    for (auto& event : events) {
-      CUDA_VERIFY(cudaStreamWaitEvent(stream, event, 0));
+    // For all the streams we are waiting on, create an event
+    std::vector<cudaEvent_t> events;
+    for (auto& stream : listWaitOn) {
+        cudaEvent_t event;
+        CUDA_VERIFY(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));
+        CUDA_VERIFY(cudaEventRecord(event, stream));
+        events.push_back(event);
+    }
+
+    // For all the streams that are waiting, issue a wait
+    for (auto& stream : listWaiting) {
+        for (auto& event : events) {
+            CUDA_VERIFY(cudaStreamWaitEvent(stream, event, 0));
+        }
     }
-  }
 
-  for (auto& event : events) {
-    CUDA_VERIFY(cudaEventDestroy(event));
-  }
+    for (auto& event : events) {
+        CUDA_VERIFY(cudaEventDestroy(event));
+    }
 }
 
 /// These versions allow usage of initializer_list as arguments, since
 /// otherwise {...} doesn't have a type
 template <typename L1>
-void streamWait(const L1& a,
-                const std::initializer_list<cudaStream_t>& b) {
-  streamWaitBase(a, b);
+void streamWait(const L1& a, const std::initializer_list<cudaStream_t>& b) {
+    streamWaitBase(a, b);
 }
 
 template <typename L2>
-void streamWait(const std::initializer_list<cudaStream_t>& a,
-                const L2& b) {
-  streamWaitBase(a, b);
+void streamWait(const std::initializer_list<cudaStream_t>& a, const L2& b) {
+    streamWaitBase(a, b);
 }
 
-inline void streamWait(const std::initializer_list<cudaStream_t>& a,
-                       const std::initializer_list<cudaStream_t>& b) {
-  streamWaitBase(a, b);
+inline void streamWait(
+        const std::initializer_list<cudaStream_t>& a,
+        const std::initializer_list<cudaStream_t>& b) {
+    streamWaitBase(a, b);
 }
 
-} } // namespace
+} // namespace gpu
+} // namespace faiss