faiss 0.2.0 → 0.2.1

data/vendor/faiss/faiss/impl/ScalarQuantizer.h

@@ -12,7 +12,6 @@
 #include <faiss/IndexIVF.h>
 #include <faiss/impl/AuxIndexStructures.h>
 
-
 namespace faiss {
 
 /**
@@ -22,15 +21,14 @@ namespace faiss {
  */
 
 struct ScalarQuantizer {
-
     enum QuantizerType {
-        QT_8bit,             ///< 8 bits per component
-        QT_4bit,             ///< 4 bits per component
-        QT_8bit_uniform,     ///< same, shared range for all dimensions
+        QT_8bit,         ///< 8 bits per component
+        QT_4bit,         ///< 4 bits per component
+        QT_8bit_uniform, ///< same, shared range for all dimensions
         QT_4bit_uniform,
         QT_fp16,
-        QT_8bit_direct,      ///< fast indexing of uint8s
-        QT_6bit,             ///< 6 bits per component
+        QT_8bit_direct, ///< fast indexing of uint8s
+        QT_6bit,        ///< 6 bits per component
     };
 
     QuantizerType qtype;
@@ -41,10 +39,10 @@ struct ScalarQuantizer {
 
     // rangestat_arg.
     enum RangeStat {
-        RS_minmax,           ///< [min - rs*(max-min), max + rs*(max-min)]
-        RS_meanstd,          ///< [mean - std * rs, mean + std * rs]
-        RS_quantiles,        ///< [Q(rs), Q(1-rs)]
-        RS_optim,            ///< alternate optimization of reconstruction error
+        RS_minmax,    ///< [min - rs*(max-min), max + rs*(max-min)]
+        RS_meanstd,   ///< [mean - std * rs, mean + std * rs]
+        RS_quantiles, ///< [Q(rs), Q(1-rs)]
+        RS_optim,     ///< alternate optimization of reconstruction error
     };
 
     RangeStat rangestat;
@@ -62,29 +60,35 @@ struct ScalarQuantizer {
     /// trained values (including the range)
     std::vector<float> trained;
 
-    ScalarQuantizer (size_t d, QuantizerType qtype);
-    ScalarQuantizer ();
+    ScalarQuantizer(size_t d, QuantizerType qtype);
+    ScalarQuantizer();
 
     /// updates internal values based on qtype and d
-    void set_derived_sizes ();
+    void set_derived_sizes();
 
-    void train (size_t n, const float *x);
+    void train(size_t n, const float* x);
 
     /// Used by an IVF index to train based on the residuals
-    void train_residual (size_t n,
-                         const float *x,
-                         Index *quantizer,
-                         bool by_residual,
-                         bool verbose);
-
-    /// same as compute_code for several vectors
-    void compute_codes (const float * x,
-                        uint8_t * codes,
-                        size_t n) const ;
-
-    /// decode a vector from a given code (or n vectors if third argument)
-    void decode (const uint8_t *code, float *x, size_t n) const;
-
+    void train_residual(
+            size_t n,
+            const float* x,
+            Index* quantizer,
+            bool by_residual,
+            bool verbose);
+
+    /** Encode a set of vectors
+     *
+     * @param x      vectors to encode, size n * d
+     * @param codes  output codes, size n * code_size
+     */
+    void compute_codes(const float* x, uint8_t* codes, size_t n) const;
+
+    /** Decode a set of vectors
+     *
+     * @param codes  codes to decode, size n * code_size
+     * @param x      output vectors, size n * d
+     */
+    void decode(const uint8_t* code, float* x, size_t n) const;
 
     /*****************************************************
      * Objects that provide methods for encoding/decoding, distance
@@ -93,34 +97,32 @@ struct ScalarQuantizer {
 
     struct Quantizer {
         // encodes one vector. Assumes code is filled with 0s on input!
-        virtual void encode_vector(const float *x, uint8_t *code) const = 0;
-        virtual void decode_vector(const uint8_t *code, float *x) const = 0;
+        virtual void encode_vector(const float* x, uint8_t* code) const = 0;
+        virtual void decode_vector(const uint8_t* code, float* x) const = 0;
 
         virtual ~Quantizer() {}
     };
 
-    Quantizer * select_quantizer() const;
+    Quantizer* select_quantizer() const;
 
-    struct SQDistanceComputer: DistanceComputer {
-
-        const float *q;
-        const uint8_t *codes;
+    struct SQDistanceComputer : DistanceComputer {
+        const float* q;
+        const uint8_t* codes;
         size_t code_size;
 
-        SQDistanceComputer (): q(nullptr), codes (nullptr), code_size (0)
-        {}
+        SQDistanceComputer() : q(nullptr), codes(nullptr), code_size(0) {}
 
+        virtual float query_to_code(const uint8_t* code) const = 0;
     };
 
-    SQDistanceComputer *get_distance_computer (MetricType metric = METRIC_L2)
-        const;
-
-    InvertedListScanner *select_InvertedListScanner
-        (MetricType mt, const Index *quantizer, bool store_pairs,
-         bool by_residual=false) const;
+    SQDistanceComputer* get_distance_computer(
+            MetricType metric = METRIC_L2) const;
 
+    InvertedListScanner* select_InvertedListScanner(
+            MetricType mt,
+            const Index* quantizer,
+            bool store_pairs,
+            bool by_residual = false) const;
 };
 
-
-
 } // namespace faiss

data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h

@@ -13,180 +13,178 @@ namespace faiss {
 
 template <typename IndexT>
 ThreadedIndex<IndexT>::ThreadedIndex(bool threaded)
-    // 0 is default dimension
-    : ThreadedIndex(0, threaded) {
-}
+        // 0 is default dimension
+        : ThreadedIndex(0, threaded) {}
 
 template <typename IndexT>
 ThreadedIndex<IndexT>::ThreadedIndex(int d, bool threaded)
-    : IndexT(d),
-      own_fields(false),
-      isThreaded_(threaded) {
-  }
+        : IndexT(d), own_fields(false), isThreaded_(threaded) {}
 
 template <typename IndexT>
 ThreadedIndex<IndexT>::~ThreadedIndex() {
-  for (auto& p : indices_) {
-    if (isThreaded_) {
-      // should have worker thread
-      FAISS_ASSERT((bool) p.second);
-
-      // This will also flush all pending work
-      p.second->stop();
-      p.second->waitForThreadExit();
-    } else {
-      // should not have worker thread
-      FAISS_ASSERT(!(bool) p.second);
-    }
-
-    if (own_fields) {
-      delete p.first;
+    for (auto& p : indices_) {
+        if (isThreaded_) {
+            // should have worker thread
+            FAISS_ASSERT((bool)p.second);
+
+            // This will also flush all pending work
+            p.second->stop();
+            p.second->waitForThreadExit();
+        } else {
+            // should not have worker thread
+            FAISS_ASSERT(!(bool)p.second);
+        }
+
+        if (own_fields) {
+            delete p.first;
+        }
     }
-  }
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::addIndex(IndexT* index) {
-  // We inherit the dimension from the first index added to us if we don't have
-  // a set dimension
-  if (indices_.empty() && this->d == 0) {
-    this->d = index->d;
-  }
-
-  // The new index must match our set dimension
-  FAISS_THROW_IF_NOT_FMT(this->d == index->d,
-                         "addIndex: dimension mismatch for "
-                         "newly added index; expecting dim %d, "
-                         "new index has dim %d",
-                         this->d, index->d);
-
-  if (!indices_.empty()) {
-    auto& existing = indices_.front().first;
-
-    FAISS_THROW_IF_NOT_MSG(index->metric_type == existing->metric_type,
-                           "addIndex: newly added index is "
-                           "of different metric type than old index");
-
-    // Make sure this index is not duplicated
-    for (auto& p : indices_) {
-      FAISS_THROW_IF_NOT_MSG(p.first != index,
-                             "addIndex: attempting to add index "
-                             "that is already in the collection");
+    // We inherit the dimension from the first index added to us if we don't
+    // have a set dimension
+    if (indices_.empty() && this->d == 0) {
+        this->d = index->d;
+    }
+
+    // The new index must match our set dimension
+    FAISS_THROW_IF_NOT_FMT(
+            this->d == index->d,
+            "addIndex: dimension mismatch for "
+            "newly added index; expecting dim %d, "
+            "new index has dim %d",
+            this->d,
+            index->d);
+
+    if (!indices_.empty()) {
+        auto& existing = indices_.front().first;
+
+        FAISS_THROW_IF_NOT_MSG(
+                index->metric_type == existing->metric_type,
+                "addIndex: newly added index is "
+                "of different metric type than old index");
+
+        // Make sure this index is not duplicated
+        for (auto& p : indices_) {
+            FAISS_THROW_IF_NOT_MSG(
+                    p.first != index,
+                    "addIndex: attempting to add index "
+                    "that is already in the collection");
+        }
     }
-  }
 
-  indices_.emplace_back(
-    std::make_pair(
-      index,
-      std::unique_ptr<WorkerThread>(isThreaded_ ?
-                                    new WorkerThread : nullptr)));
+    indices_.emplace_back(std::make_pair(
+            index,
+            std::unique_ptr<WorkerThread>(
+                    isThreaded_ ? new WorkerThread : nullptr)));
 
-  onAfterAddIndex(index);
+    onAfterAddIndex(index);
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::removeIndex(IndexT* index) {
-  for (auto it = indices_.begin(); it != indices_.end(); ++it) {
-    if (it->first == index) {
-      // This is our index; stop the worker thread before removing it,
-      // to ensure that it has finished before function exit
-      if (isThreaded_) {
-        // should have worker thread
-        FAISS_ASSERT((bool) it->second);
-        it->second->stop();
-        it->second->waitForThreadExit();
-      } else {
-        // should not have worker thread
-        FAISS_ASSERT(!(bool) it->second);
-      }
-
-      indices_.erase(it);
-      onAfterRemoveIndex(index);
-
-      if (own_fields) {
-        delete index;
-      }
-
-      return;
+    for (auto it = indices_.begin(); it != indices_.end(); ++it) {
+        if (it->first == index) {
+            // This is our index; stop the worker thread before removing it,
+            // to ensure that it has finished before function exit
+            if (isThreaded_) {
+                // should have worker thread
+                FAISS_ASSERT((bool)it->second);
+                it->second->stop();
+                it->second->waitForThreadExit();
+            } else {
+                // should not have worker thread
+                FAISS_ASSERT(!(bool)it->second);
+            }
+
+            indices_.erase(it);
+            onAfterRemoveIndex(index);
+
+            if (own_fields) {
+                delete index;
+            }
+
+            return;
+        }
     }
-  }
 
-  // could not find our index
-  FAISS_THROW_MSG("IndexReplicas::removeIndex: index not found");
+    // could not find our index
+    FAISS_THROW_MSG("IndexReplicas::removeIndex: index not found");
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::runOnIndex(std::function<void(int, IndexT*)> f) {
-  if (isThreaded_) {
-    std::vector<std::future<bool>> v;
-
-    for (int i = 0; i < this->indices_.size(); ++i) {
-      auto& p = this->indices_[i];
-      auto indexPtr = p.first;
-      v.emplace_back(p.second->add([f, i, indexPtr](){ f(i, indexPtr); }));
-    }
+    if (isThreaded_) {
+        std::vector<std::future<bool>> v;
 
-    waitAndHandleFutures(v);
-  } else {
-    // Multiple exceptions may be thrown; gather them as we encounter them,
-    // while letting everything else run to completion
-    std::vector<std::pair<int, std::exception_ptr>> exceptions;
+        for (int i = 0; i < this->indices_.size(); ++i) {
+            auto& p = this->indices_[i];
+            auto indexPtr = p.first;
+            v.emplace_back(
+                    p.second->add([f, i, indexPtr]() { f(i, indexPtr); }));
+        }
 
-    for (int i = 0; i < this->indices_.size(); ++i) {
-      auto& p = this->indices_[i];
-      try {
-        f(i, p.first);
-      } catch (...) {
-        exceptions.emplace_back(std::make_pair(i, std::current_exception()));
-      }
+        waitAndHandleFutures(v);
+    } else {
+        // Multiple exceptions may be thrown; gather them as we encounter them,
+        // while letting everything else run to completion
+        std::vector<std::pair<int, std::exception_ptr>> exceptions;
+
+        for (int i = 0; i < this->indices_.size(); ++i) {
+            auto& p = this->indices_[i];
+            try {
+                f(i, p.first);
+            } catch (...) {
+                exceptions.emplace_back(
+                        std::make_pair(i, std::current_exception()));
+            }
+        }
+
+        handleExceptions(exceptions);
     }
-
-    handleExceptions(exceptions);
-  }
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::runOnIndex(
-  std::function<void(int, const IndexT*)> f) const {
-  const_cast<ThreadedIndex<IndexT>*>(this)->runOnIndex(
-    [f](int i, IndexT* idx){ f(i, idx); });
+        std::function<void(int, const IndexT*)> f) const {
+    const_cast<ThreadedIndex<IndexT>*>(this)->runOnIndex(
+            [f](int i, IndexT* idx) { f(i, idx); });
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::reset() {
-  runOnIndex([](int, IndexT* index){ index->reset(); });
-  this->ntotal = 0;
-  this->is_trained = false;
+    runOnIndex([](int, IndexT* index) { index->reset(); });
+    this->ntotal = 0;
+    this->is_trained = false;
 }
 
 template <typename IndexT>
-void
-ThreadedIndex<IndexT>::onAfterAddIndex(IndexT* index) {
-}
+void ThreadedIndex<IndexT>::onAfterAddIndex(IndexT* index) {}
 
 template <typename IndexT>
-void
-ThreadedIndex<IndexT>::onAfterRemoveIndex(IndexT* index) {
-}
+void ThreadedIndex<IndexT>::onAfterRemoveIndex(IndexT* index) {}
 
 template <typename IndexT>
-void
-ThreadedIndex<IndexT>::waitAndHandleFutures(std::vector<std::future<bool>>& v) {
-  // Blocking wait for completion for all of the indices, capturing any
-  // exceptions that are generated
-  std::vector<std::pair<int, std::exception_ptr>> exceptions;
-
-  for (int i = 0; i < v.size(); ++i) {
-    auto& fut = v[i];
-
-    try {
-      fut.get();
-    } catch (...) {
-      exceptions.emplace_back(std::make_pair(i, std::current_exception()));
+void ThreadedIndex<IndexT>::waitAndHandleFutures(
+        std::vector<std::future<bool>>& v) {
+    // Blocking wait for completion for all of the indices, capturing any
+    // exceptions that are generated
+    std::vector<std::pair<int, std::exception_ptr>> exceptions;
+
+    for (int i = 0; i < v.size(); ++i) {
+        auto& fut = v[i];
+
+        try {
+            fut.get();
+        } catch (...) {
+            exceptions.emplace_back(
+                    std::make_pair(i, std::current_exception()));
+        }
     }
-  }
 
-  handleExceptions(exceptions);
+    handleExceptions(exceptions);
 }
 
-} // namespace
+} // namespace faiss