faiss 0.1.0 → 0.1.1

data/vendor/faiss/gpu/test/TestUtils.cpp

@@ -0,0 +1,315 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+
+#include <faiss/gpu/test/TestUtils.h>
+#include <faiss/utils/random.h>
+#include <cmath>
+#include <gtest/gtest.h>
+#include <set>
+#include <sstream>
+#include <time.h>
+#include <unordered_map>
+
+namespace faiss { namespace gpu {
+
+inline float relativeError(float a, float b) {
+  return std::abs(a - b) / (0.5f * (std::abs(a) + std::abs(b)));
+}
+
+// This seed is also used for the faiss float_rand API; in a test it
+// is all within a single thread, so it is ok
+long s_seed = 1;
+
+void newTestSeed() {
+  struct timespec t;
+  clock_gettime(CLOCK_REALTIME, &t);
+
+  setTestSeed(t.tv_nsec);
+}
+
+void setTestSeed(long seed) {
+  printf("testing with random seed %ld\n", seed);
+
+  srand48(seed);
+  s_seed = seed;
+}
+
+int randVal(int a, int b) {
+  EXPECT_GE(a, 0);
+  EXPECT_LE(a, b);
+
+  return a + (lrand48() % (b + 1 - a));
+}
+
+bool randBool() {
+  return randSelect<bool>({true, false});
+}
+
+std::vector<float> randVecs(size_t num, size_t dim) {
+  std::vector<float> v(num * dim);
+
+  faiss::float_rand(v.data(), v.size(), s_seed);
+  // unfortunately we generate separate sets of vectors, and don't
+  // want the same values
+  ++s_seed;
+
+  return v;
+}
+
+std::vector<unsigned char> randBinaryVecs(size_t num, size_t dim) {
+  std::vector<unsigned char> v(num * (dim / 8));
+
+  faiss::byte_rand(v.data(), v.size(), s_seed);
+  // unfortunately we generate separate sets of vectors, and don't
+  // want the same values
+  ++s_seed;
+
+  return v;
+}
+
+void compareIndices(
+    const std::vector<float>& queryVecs,
+    faiss::Index& refIndex,
+    faiss::Index& testIndex,
+    int numQuery,
+    int /*dim*/,
+    int k,
+    const std::string& configMsg,
+    float maxRelativeError,
+    float pctMaxDiff1,
+    float pctMaxDiffN) {
+  // Compare
+  std::vector<float> refDistance(numQuery * k, 0);
+  std::vector<faiss::Index::idx_t> refIndices(numQuery * k, -1);
+  refIndex.search(numQuery, queryVecs.data(),
+                  k, refDistance.data(), refIndices.data());
+
+  std::vector<float> testDistance(numQuery * k, 0);
+  std::vector<faiss::Index::idx_t> testIndices(numQuery * k, -1);
+  testIndex.search(numQuery, queryVecs.data(),
+                   k, testDistance.data(), testIndices.data());
+
+  faiss::gpu::compareLists(refDistance.data(),
+                           refIndices.data(),
+                           testDistance.data(),
+                           testIndices.data(),
+                           numQuery, k,
+                           configMsg,
+                           true, false, true,
+                           maxRelativeError, pctMaxDiff1, pctMaxDiffN);
+}
+
+void compareIndices(faiss::Index& refIndex,
+                    faiss::Index& testIndex,
+                    int numQuery, int dim, int k,
+                    const std::string& configMsg,
+                    float maxRelativeError,
+                    float pctMaxDiff1,
+                    float pctMaxDiffN) {
+  auto queryVecs = faiss::gpu::randVecs(numQuery, dim);
+
+  compareIndices(queryVecs,
+                 refIndex,
+                 testIndex,
+                 numQuery, dim, k,
+                 configMsg,
+                 maxRelativeError,
+                 pctMaxDiff1,
+                 pctMaxDiffN);
+}
+
+template <typename T>
+inline T lookup(const T* p, int i, int j, int /*dim1*/, int dim2) {
+  return p[i * dim2 + j];
+}
+
+void compareLists(const float* refDist,
+                  const faiss::Index::idx_t* refInd,
+                  const float* testDist,
+                  const faiss::Index::idx_t* testInd,
+                  int dim1, int dim2,
+                  const std::string& configMsg,
+                  bool printBasicStats, bool printDiffs, bool assertOnErr,
+                  float maxRelativeError,
+                  float pctMaxDiff1,
+                  float pctMaxDiffN) {
+
+  float maxAbsErr = 0.0f;
+  for (int i = 0; i < dim1 * dim2; ++i) {
+    maxAbsErr = std::max(maxAbsErr, std::abs(refDist[i] - testDist[i]));
+  }
+  int numResults = dim1 * dim2;
+
+  // query -> {index -> result position}
+  std::vector<std::unordered_map<faiss::Index::idx_t, int>> refIndexMap;
+
+  for (int query = 0; query < dim1; ++query) {
+    std::unordered_map<faiss::Index::idx_t, int> indices;
+
+    for (int result = 0; result < dim2; ++result) {
+      indices[lookup(refInd, query, result, dim1, dim2)] = result;
+    }
+
+    refIndexMap.emplace_back(std::move(indices));
+  }
+
+  // See how far off the indices are
+  // Keep track of the difference for each entry
+  std::vector<std::vector<int>> indexDiffs;
+
+  int diff1 = 0; // index differs by 1
+  int diffN = 0; // index differs by >1
+  int diffInf = 0; // index not found in the other
+  int nonUniqueIndices = 0;
+
+  double avgDiff = 0.0;
+  int maxDiff = 0;
+  float maxRelErr = 0.0f;
+
+  for (int query = 0; query < dim1; ++query) {
+    std::vector<int> diffs;
+    std::set<faiss::Index::idx_t> uniqueIndices;
+
+    auto& indices = refIndexMap[query];
+
+    for (int result = 0; result < dim2; ++result) {
+      auto t = lookup(testInd, query, result, dim1, dim2);
+
+      // All indices reported within a query should be unique; this is
+      // a serious error if is otherwise the case.
+      // If -1 is reported (no result due to IVF partitioning or not enough
+      // entries in the index), then duplicates are allowed, but both the
+      // reference and test must have -1 in the same position.
+      if (t == -1) {
+        EXPECT_EQ(lookup(refInd, query, result, dim1, dim2), t);
+      } else {
+        bool uniqueIndex = uniqueIndices.count(t) == 0;
+        if (assertOnErr) {
+          EXPECT_TRUE(uniqueIndex) << configMsg
+                                   << " " << query
+                                   << " " << result
+                                   << " " << t;
+        }
+
+        if (!uniqueIndex) {
+          ++nonUniqueIndices;
+        } else {
+          uniqueIndices.insert(t);
+        }
+
+        auto it = indices.find(t);
+        if (it != indices.end()) {
+          int diff = std::abs(result - it->second);
+          diffs.push_back(diff);
+
+          if (diff == 1) {
+            ++diff1;
+            maxDiff = std::max(diff, maxDiff);
+          } else if (diff > 1) {
+            ++diffN;
+            maxDiff = std::max(diff, maxDiff);
+          }
+
+          avgDiff += (double) diff;
+        } else {
+          ++diffInf;
+          diffs.push_back(-1);
+          // don't count this for maxDiff
+        }
+      }
+
+      auto refD = lookup(refDist, query, result, dim1, dim2);
+      auto testD = lookup(testDist, query, result, dim1, dim2);
+
+      float relErr = relativeError(refD, testD);
+
+      if (assertOnErr) {
+        EXPECT_LE(relErr, maxRelativeError) << configMsg
+                                            << " (" << query << ", " << result
+                                            << ") refD: " << refD
+                                            << " testD: " << testD;
+      }
+
+      maxRelErr = std::max(maxRelErr, relErr);
+    }
+
+    indexDiffs.emplace_back(std::move(diffs));
+  }
+
+  if (assertOnErr) {
+    EXPECT_LE((float) (diff1 + diffN + diffInf),
+              (float) numResults * pctMaxDiff1) << configMsg;
+
+    // Don't count diffInf because that could be diff1 as far as we
+    // know
+    EXPECT_LE((float) diffN, (float) numResults * pctMaxDiffN) << configMsg;
+  }
+
+  avgDiff /= (double) numResults;
+
+  if (printBasicStats) {
+    if (!configMsg.empty()) {
+      printf("Config\n"
+             "----------------------------\n"
+             "%s\n",
+             configMsg.c_str());
+    }
+
+    printf("Result error and differences\n"
+           "----------------------------\n"
+           "max abs diff %.7f rel diff %.7f\n"
+           "idx diff avg: %.5g max: %d\n"
+           "idx diff of 1:      %d (%.3f%% of queries)\n"
+           "idx diff of >1:     %d (%.3f%% of queries)\n"
+           "idx diff not found: %d (%.3f%% of queries)"
+           " [typically a last element inversion]\n"
+           "non-unique indices: %d (a serious error if >0)\n",
+           maxAbsErr, maxRelErr,
+           avgDiff, maxDiff,
+           diff1, 100.0f * (float) diff1 / (float) numResults,
+           diffN, 100.0f * (float) diffN / (float) numResults,
+           diffInf, 100.0f * (float) diffInf / (float) numResults,
+           nonUniqueIndices);
+  }
+
+  if (printDiffs) {
+    printf("differences:\n");
+    printf("==================\n");
+    for (int query = 0; query < dim1; ++query) {
+      for (int result = 0; result < dim2; ++result) {
+        long refI = lookup(refInd, query, result, dim1, dim2);
+        long testI = lookup(testInd, query, result, dim1, dim2);
+
+        if (refI != testI) {
+          float refD = lookup(refDist, query, result, dim1, dim2);
+          float testD = lookup(testDist, query, result, dim1, dim2);
+
+          float maxDist = std::max(refD, testD);
+          float delta = std::abs(refD - testD);
+
+          float relErr = delta / maxDist;
+
+          if (refD == testD) {
+            printf("(%d, %d [%d]) (ref %ld tst %ld dist ==)\n",
+                   query, result,
+                   indexDiffs[query][result],
+                   refI, testI);
+          } else {
+            printf("(%d, %d [%d]) (ref %ld tst %ld abs %.8f "
+                   "rel %.8f ref %a tst %a)\n",
+                   query, result,
+                   indexDiffs[query][result],
+                   refI, testI, delta, relErr, refD, testD);
+          }
+        }
+      }
+    }
+  }
+}
+
+} }

data/vendor/faiss/gpu/test/TestUtils.h

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+
+#pragma once
+
+#include <faiss/impl/FaissAssert.h>
+#include <faiss/Index.h>
+#include <initializer_list>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace faiss { namespace gpu {
+
+/// Generates and displays a new seed for the test
+void newTestSeed();
+
+/// Uses an explicit seed for the test
+void setTestSeed(long seed);
+
+/// Returns the relative error in difference between a and b
+/// (|a - b| / (0.5 * (|a| + |b|))
+float relativeError(float a, float b);
+
+/// Generates a random integer in the range [a, b]
+int randVal(int a, int b);
+
+/// Generates a random bool
+bool randBool();
+
+/// Select a random value from the given list of values provided as an
+/// initializer_list
+template <typename T>
+T randSelect(std::initializer_list<T> vals) {
+  FAISS_ASSERT(vals.size() > 0);
+  int sel = randVal(0, vals.size());
+
+  int i = 0;
+  for (auto v : vals) {
+    if (i++ == sel) {
+      return v;
+    }
+  }
+
+  // should not get here
+  return *vals.begin();
+}
+
+/// Generates a collection of random vectors in the range [0, 1]
+std::vector<float> randVecs(size_t num, size_t dim);
+
+/// Generates a collection of random bit vectors
+std::vector<unsigned char> randBinaryVecs(size_t num, size_t dim);
+
+/// Compare two indices via query for similarity, with a user-specified set of
+/// query vectors
+void compareIndices(const std::vector<float>& queryVecs,
+                    faiss::Index& refIndex,
+                    faiss::Index& testIndex,
+                    int numQuery, int dim, int k,
+                    const std::string& configMsg,
+                    float maxRelativeError = 6e-5f,
+                    float pctMaxDiff1 = 0.1f,
+                    float pctMaxDiffN = 0.005f);
+
+/// Compare two indices via query for similarity, generating random query
+/// vectors
+void compareIndices(faiss::Index& refIndex,
+                    faiss::Index& testIndex,
+                    int numQuery, int dim, int k,
+                    const std::string& configMsg,
+                    float maxRelativeError = 6e-5f,
+                    float pctMaxDiff1 = 0.1f,
+                    float pctMaxDiffN = 0.005f);
+
+/// Display specific differences in the two (distance, index) lists
+void compareLists(const float* refDist,
+                  const faiss::Index::idx_t* refInd,
+                  const float* testDist,
+                  const faiss::Index::idx_t* testInd,
+                  int dim1, int dim2,
+                  const std::string& configMsg,
+                  bool printBasicStats, bool printDiffs, bool assertOnErr,
+                  float maxRelativeError = 6e-5f,
+                  float pctMaxDiff1 = 0.1f,
+                  float pctMaxDiffN = 0.005f);
+
+} }

data/vendor/faiss/gpu/test/demo_ivfpq_indexing_gpu.cpp

@@ -0,0 +1,159 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved
+
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+#include <sys/time.h>
+
+
+#include <faiss/gpu/StandardGpuResources.h>
+#include <faiss/gpu/GpuIndexIVFPQ.h>
+
+#include <faiss/gpu/GpuAutoTune.h>
+#include <faiss/index_io.h>
+
+double elapsed ()
+{
+    struct timeval tv;
+    gettimeofday (&tv, NULL);
+    return  tv.tv_sec + tv.tv_usec * 1e-6;
+}
+
+
+int main ()
+{
+
+    double t0 = elapsed();
+
+    // dimension of the vectors to index
+    int d = 128;
+
+    // size of the database we plan to index
+    size_t nb = 200 * 1000;
+
+    // make a set of nt training vectors in the unit cube
+    // (could be the database)
+    size_t nt = 100 * 1000;
+
+    int dev_no = 0;
+    /*
+    printf ("[%.3f s] Begin d=%d nb=%ld nt=%nt dev_no=%d\n",
+            elapsed() - t0, d, nb, nt, dev_no);
+    */
+    // a reasonable number of centroids to index nb vectors
+    int ncentroids = int (4 * sqrt (nb));
+
+    faiss::gpu::StandardGpuResources resources;
+
+
+    // the coarse quantizer should not be dealloced before the index
+    // 4 = nb of bytes per code (d must be a multiple of this)
+    // 8 = nb of bits per sub-code (almost always 8)
+    faiss::gpu::GpuIndexIVFPQConfig config;
+    config.device = dev_no;
+
+    faiss::gpu::GpuIndexIVFPQ index (
+      &resources, d, ncentroids, 4, 8, faiss::METRIC_L2, config);
+
+    { // training
+        printf ("[%.3f s] Generating %ld vectors in %dD for training\n",
+                elapsed() - t0, nt, d);
+
+        std::vector <float> trainvecs (nt * d);
+        for (size_t i = 0; i < nt * d; i++) {
+            trainvecs[i] = drand48();
+        }
+
+        printf ("[%.3f s] Training the index\n",
+                elapsed() - t0);
+        index.verbose = true;
+
+        index.train (nt, trainvecs.data());
+    }
+
+    { // I/O demo
+        const char *outfilename = "/tmp/index_trained.faissindex";
+        printf ("[%.3f s] storing the pre-trained index to %s\n",
+                elapsed() - t0, outfilename);
+
+        faiss::Index * cpu_index = faiss::gpu::index_gpu_to_cpu (&index);
+
+        write_index (cpu_index, outfilename);
+
+        delete cpu_index;
+    }
+
+    size_t nq;
+    std::vector<float> queries;
+
+    { // populating the database
+        printf ("[%.3f s] Building a dataset of %ld vectors to index\n",
+                elapsed() - t0, nb);
+
+        std::vector <float> database (nb * d);
+        for (size_t i = 0; i < nb * d; i++) {
+            database[i] = drand48();
+        }
+
+        printf ("[%.3f s] Adding the vectors to the index\n",
+                elapsed() - t0);
+
+        index.add (nb, database.data());
+
+        printf ("[%.3f s] done\n", elapsed() - t0);
+
+        // remember a few elements from the database as queries
+        int i0 = 1234;
+        int i1 = 1243;
+
+        nq = i1 - i0;
+        queries.resize (nq * d);
+        for (int i = i0; i < i1; i++) {
+            for (int j = 0; j < d; j++) {
+                queries [(i - i0) * d  + j]  = database [i * d + j];
+            }
+        }
+
+    }
+
+    { // searching the database
+        int k = 5;
+        printf ("[%.3f s] Searching the %d nearest neighbors "
+                "of %ld vectors in the index\n",
+                elapsed() - t0, k, nq);
+
+        std::vector<faiss::Index::idx_t> nns (k * nq);
+        std::vector<float>               dis (k * nq);
+
+        index.search (nq, queries.data(), k, dis.data(), nns.data());
+
+        printf ("[%.3f s] Query results (vector ids, then distances):\n",
+                elapsed() - t0);
+
+        for (int i = 0; i < nq; i++) {
+            printf ("query %2d: ", i);
+            for (int j = 0; j < k; j++) {
+                printf ("%7ld ", nns[j + i * k]);
+            }
+            printf ("\n     dis: ");
+            for (int j = 0; j < k; j++) {
+                printf ("%7g ", dis[j + i * k]);
+            }
+            printf ("\n");
+        }
+
+        printf ("note that the nearest neighbor is not at "
+                "distance 0 due to quantization errors\n");
+    }
+
+    return 0;
+}