faiss 0.1.7 → 0.2.3

data/vendor/faiss/faiss/impl/ScalarQuantizer.h

@@ -12,7 +12,6 @@
 #include <faiss/IndexIVF.h>
 #include <faiss/impl/AuxIndexStructures.h>
 
-
 namespace faiss {
 
 /**
@@ -22,15 +21,14 @@ namespace faiss {
  */
 
 struct ScalarQuantizer {
-
     enum QuantizerType {
-        QT_8bit,             ///< 8 bits per component
-        QT_4bit,             ///< 4 bits per component
-        QT_8bit_uniform,     ///< same, shared range for all dimensions
+        QT_8bit,         ///< 8 bits per component
+        QT_4bit,         ///< 4 bits per component
+        QT_8bit_uniform, ///< same, shared range for all dimensions
         QT_4bit_uniform,
         QT_fp16,
-        QT_8bit_direct,      ///< fast indexing of uint8s
-        QT_6bit,             ///< 6 bits per component
+        QT_8bit_direct, ///< fast indexing of uint8s
+        QT_6bit,        ///< 6 bits per component
     };
 
     QuantizerType qtype;
@@ -41,10 +39,10 @@ struct ScalarQuantizer {
 
     // rangestat_arg.
     enum RangeStat {
-        RS_minmax,           ///< [min - rs*(max-min), max + rs*(max-min)]
-        RS_meanstd,          ///< [mean - std * rs, mean + std * rs]
-        RS_quantiles,        ///< [Q(rs), Q(1-rs)]
-        RS_optim,            ///< alternate optimization of reconstruction error
+        RS_minmax,    ///< [min - rs*(max-min), max + rs*(max-min)]
+        RS_meanstd,   ///< [mean - std * rs, mean + std * rs]
+        RS_quantiles, ///< [Q(rs), Q(1-rs)]
+        RS_optim,     ///< alternate optimization of reconstruction error
     };
 
     RangeStat rangestat;
@@ -62,29 +60,35 @@ struct ScalarQuantizer {
     /// trained values (including the range)
     std::vector<float> trained;
 
-    ScalarQuantizer (size_t d, QuantizerType qtype);
-    ScalarQuantizer ();
+    ScalarQuantizer(size_t d, QuantizerType qtype);
+    ScalarQuantizer();
 
     /// updates internal values based on qtype and d
-    void set_derived_sizes ();
+    void set_derived_sizes();
 
-    void train (size_t n, const float *x);
+    void train(size_t n, const float* x);
 
     /// Used by an IVF index to train based on the residuals
-    void train_residual (size_t n,
-                         const float *x,
-                         Index *quantizer,
-                         bool by_residual,
-                         bool verbose);
-
-    /// same as compute_code for several vectors
-    void compute_codes (const float * x,
-                        uint8_t * codes,
-                        size_t n) const ;
-
-    /// decode a vector from a given code (or n vectors if third argument)
-    void decode (const uint8_t *code, float *x, size_t n) const;
-
+    void train_residual(
+            size_t n,
+            const float* x,
+            Index* quantizer,
+            bool by_residual,
+            bool verbose);
+
+    /** Encode a set of vectors
+     *
+     * @param x      vectors to encode, size n * d
+     * @param codes  output codes, size n * code_size
+     */
+    void compute_codes(const float* x, uint8_t* codes, size_t n) const;
+
+    /** Decode a set of vectors
+     *
+     * @param codes  codes to decode, size n * code_size
+     * @param x      output vectors, size n * d
+     */
+    void decode(const uint8_t* code, float* x, size_t n) const;
 
     /*****************************************************
      * Objects that provide methods for encoding/decoding, distance
@@ -93,34 +97,32 @@ struct ScalarQuantizer {
 
     struct Quantizer {
         // encodes one vector. Assumes code is filled with 0s on input!
-        virtual void encode_vector(const float *x, uint8_t *code) const = 0;
-        virtual void decode_vector(const uint8_t *code, float *x) const = 0;
+        virtual void encode_vector(const float* x, uint8_t* code) const = 0;
+        virtual void decode_vector(const uint8_t* code, float* x) const = 0;
 
         virtual ~Quantizer() {}
     };
 
-    Quantizer * select_quantizer() const;
+    Quantizer* select_quantizer() const;
 
-    struct SQDistanceComputer: DistanceComputer {
-
-        const float *q;
-        const uint8_t *codes;
+    struct SQDistanceComputer : DistanceComputer {
+        const float* q;
+        const uint8_t* codes;
         size_t code_size;
 
-        SQDistanceComputer (): q(nullptr), codes (nullptr), code_size (0)
-        {}
+        SQDistanceComputer() : q(nullptr), codes(nullptr), code_size(0) {}
 
+        virtual float query_to_code(const uint8_t* code) const = 0;
     };
 
-    SQDistanceComputer *get_distance_computer (MetricType metric = METRIC_L2)
-        const;
-
-    InvertedListScanner *select_InvertedListScanner
-        (MetricType mt, const Index *quantizer, bool store_pairs,
-         bool by_residual=false) const;
+    SQDistanceComputer* get_distance_computer(
+            MetricType metric = METRIC_L2) const;
 
+    InvertedListScanner* select_InvertedListScanner(
+            MetricType mt,
+            const Index* quantizer,
+            bool store_pairs,
+            bool by_residual = false) const;
 };
 
-
-
 } // namespace faiss

data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h

@@ -13,180 +13,178 @@ namespace faiss {
 
 template <typename IndexT>
 ThreadedIndex<IndexT>::ThreadedIndex(bool threaded)
-    // 0 is default dimension
-    : ThreadedIndex(0, threaded) {
-}
+        // 0 is default dimension
+        : ThreadedIndex(0, threaded) {}
 
 template <typename IndexT>
 ThreadedIndex<IndexT>::ThreadedIndex(int d, bool threaded)
-    : IndexT(d),
-      own_fields(false),
-      isThreaded_(threaded) {
-  }
+        : IndexT(d), own_fields(false), isThreaded_(threaded) {}
 
 template <typename IndexT>
 ThreadedIndex<IndexT>::~ThreadedIndex() {
-  for (auto& p : indices_) {
-    if (isThreaded_) {
-      // should have worker thread
-      FAISS_ASSERT((bool) p.second);
-
-      // This will also flush all pending work
-      p.second->stop();
-      p.second->waitForThreadExit();
-    } else {
-      // should not have worker thread
-      FAISS_ASSERT(!(bool) p.second);
-    }
-
-    if (own_fields) {
-      delete p.first;
+    for (auto& p : indices_) {
+        if (isThreaded_) {
+            // should have worker thread
+            FAISS_ASSERT((bool)p.second);
+
+            // This will also flush all pending work
+            p.second->stop();
+            p.second->waitForThreadExit();
+        } else {
+            // should not have worker thread
+            FAISS_ASSERT(!(bool)p.second);
+        }
+
+        if (own_fields) {
+            delete p.first;
+        }
     }
-  }
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::addIndex(IndexT* index) {
-  // We inherit the dimension from the first index added to us if we don't have
-  // a set dimension
-  if (indices_.empty() && this->d == 0) {
-    this->d = index->d;
-  }
-
-  // The new index must match our set dimension
-  FAISS_THROW_IF_NOT_FMT(this->d == index->d,
-                         "addIndex: dimension mismatch for "
-                         "newly added index; expecting dim %d, "
-                         "new index has dim %d",
-                         this->d, index->d);
-
-  if (!indices_.empty()) {
-    auto& existing = indices_.front().first;
-
-    FAISS_THROW_IF_NOT_MSG(index->metric_type == existing->metric_type,
-                           "addIndex: newly added index is "
-                           "of different metric type than old index");
-
-    // Make sure this index is not duplicated
-    for (auto& p : indices_) {
-      FAISS_THROW_IF_NOT_MSG(p.first != index,
-                             "addIndex: attempting to add index "
-                             "that is already in the collection");
+    // We inherit the dimension from the first index added to us if we don't
+    // have a set dimension
+    if (indices_.empty() && this->d == 0) {
+        this->d = index->d;
+    }
+
+    // The new index must match our set dimension
+    FAISS_THROW_IF_NOT_FMT(
+            this->d == index->d,
+            "addIndex: dimension mismatch for "
+            "newly added index; expecting dim %d, "
+            "new index has dim %d",
+            this->d,
+            index->d);
+
+    if (!indices_.empty()) {
+        auto& existing = indices_.front().first;
+
+        FAISS_THROW_IF_NOT_MSG(
+                index->metric_type == existing->metric_type,
+                "addIndex: newly added index is "
+                "of different metric type than old index");
+
+        // Make sure this index is not duplicated
+        for (auto& p : indices_) {
+            FAISS_THROW_IF_NOT_MSG(
+                    p.first != index,
+                    "addIndex: attempting to add index "
+                    "that is already in the collection");
+        }
     }
-  }
 
-  indices_.emplace_back(
-    std::make_pair(
-      index,
-      std::unique_ptr<WorkerThread>(isThreaded_ ?
-                                    new WorkerThread : nullptr)));
+    indices_.emplace_back(std::make_pair(
+            index,
+            std::unique_ptr<WorkerThread>(
+                    isThreaded_ ? new WorkerThread : nullptr)));
 
-  onAfterAddIndex(index);
+    onAfterAddIndex(index);
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::removeIndex(IndexT* index) {
-  for (auto it = indices_.begin(); it != indices_.end(); ++it) {
-    if (it->first == index) {
-      // This is our index; stop the worker thread before removing it,
-      // to ensure that it has finished before function exit
-      if (isThreaded_) {
-        // should have worker thread
-        FAISS_ASSERT((bool) it->second);
-        it->second->stop();
-        it->second->waitForThreadExit();
-      } else {
-        // should not have worker thread
-        FAISS_ASSERT(!(bool) it->second);
-      }
-
-      indices_.erase(it);
-      onAfterRemoveIndex(index);
-
-      if (own_fields) {
-        delete index;
-      }
-
-      return;
+    for (auto it = indices_.begin(); it != indices_.end(); ++it) {
+        if (it->first == index) {
+            // This is our index; stop the worker thread before removing it,
+            // to ensure that it has finished before function exit
+            if (isThreaded_) {
+                // should have worker thread
+                FAISS_ASSERT((bool)it->second);
+                it->second->stop();
+                it->second->waitForThreadExit();
+            } else {
+                // should not have worker thread
+                FAISS_ASSERT(!(bool)it->second);
+            }
+
+            indices_.erase(it);
+            onAfterRemoveIndex(index);
+
+            if (own_fields) {
+                delete index;
+            }
+
+            return;
+        }
     }
-  }
 
-  // could not find our index
-  FAISS_THROW_MSG("IndexReplicas::removeIndex: index not found");
+    // could not find our index
+    FAISS_THROW_MSG("IndexReplicas::removeIndex: index not found");
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::runOnIndex(std::function<void(int, IndexT*)> f) {
-  if (isThreaded_) {
-    std::vector<std::future<bool>> v;
-
-    for (int i = 0; i < this->indices_.size(); ++i) {
-      auto& p = this->indices_[i];
-      auto indexPtr = p.first;
-      v.emplace_back(p.second->add([f, i, indexPtr](){ f(i, indexPtr); }));
-    }
+    if (isThreaded_) {
+        std::vector<std::future<bool>> v;
 
-    waitAndHandleFutures(v);
-  } else {
-    // Multiple exceptions may be thrown; gather them as we encounter them,
-    // while letting everything else run to completion
-    std::vector<std::pair<int, std::exception_ptr>> exceptions;
+        for (int i = 0; i < this->indices_.size(); ++i) {
+            auto& p = this->indices_[i];
+            auto indexPtr = p.first;
+            v.emplace_back(
+                    p.second->add([f, i, indexPtr]() { f(i, indexPtr); }));
+        }
 
-    for (int i = 0; i < this->indices_.size(); ++i) {
-      auto& p = this->indices_[i];
-      try {
-        f(i, p.first);
-      } catch (...) {
-        exceptions.emplace_back(std::make_pair(i, std::current_exception()));
-      }
+        waitAndHandleFutures(v);
+    } else {
+        // Multiple exceptions may be thrown; gather them as we encounter them,
+        // while letting everything else run to completion
+        std::vector<std::pair<int, std::exception_ptr>> exceptions;
+
+        for (int i = 0; i < this->indices_.size(); ++i) {
+            auto& p = this->indices_[i];
+            try {
+                f(i, p.first);
+            } catch (...) {
+                exceptions.emplace_back(
+                        std::make_pair(i, std::current_exception()));
+            }
+        }
+
+        handleExceptions(exceptions);
     }
-
-    handleExceptions(exceptions);
-  }
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::runOnIndex(
-  std::function<void(int, const IndexT*)> f) const {
-  const_cast<ThreadedIndex<IndexT>*>(this)->runOnIndex(
-    [f](int i, IndexT* idx){ f(i, idx); });
+        std::function<void(int, const IndexT*)> f) const {
+    const_cast<ThreadedIndex<IndexT>*>(this)->runOnIndex(
+            [f](int i, IndexT* idx) { f(i, idx); });
 }
 
 template <typename IndexT>
 void ThreadedIndex<IndexT>::reset() {
-  runOnIndex([](int, IndexT* index){ index->reset(); });
-  this->ntotal = 0;
-  this->is_trained = false;
+    runOnIndex([](int, IndexT* index) { index->reset(); });
+    this->ntotal = 0;
+    this->is_trained = false;
 }
 
 template <typename IndexT>
-void
-ThreadedIndex<IndexT>::onAfterAddIndex(IndexT* index) {
-}
+void ThreadedIndex<IndexT>::onAfterAddIndex(IndexT* index) {}
 
 template <typename IndexT>
-void
-ThreadedIndex<IndexT>::onAfterRemoveIndex(IndexT* index) {
-}
+void ThreadedIndex<IndexT>::onAfterRemoveIndex(IndexT* index) {}
 
 template <typename IndexT>
-void
-ThreadedIndex<IndexT>::waitAndHandleFutures(std::vector<std::future<bool>>& v) {
-  // Blocking wait for completion for all of the indices, capturing any
-  // exceptions that are generated
-  std::vector<std::pair<int, std::exception_ptr>> exceptions;
-
-  for (int i = 0; i < v.size(); ++i) {
-    auto& fut = v[i];
-
-    try {
-      fut.get();
-    } catch (...) {
-      exceptions.emplace_back(std::make_pair(i, std::current_exception()));
+void ThreadedIndex<IndexT>::waitAndHandleFutures(
+        std::vector<std::future<bool>>& v) {
+    // Blocking wait for completion for all of the indices, capturing any
+    // exceptions that are generated
+    std::vector<std::pair<int, std::exception_ptr>> exceptions;
+
+    for (int i = 0; i < v.size(); ++i) {
+        auto& fut = v[i];
+
+        try {
+            fut.get();
+        } catch (...) {
+            exceptions.emplace_back(
+                    std::make_pair(i, std::current_exception()));
+        }
     }
-  }
 
-  handleExceptions(exceptions);
+    handleExceptions(exceptions);
 }
 
-} // namespace
+} // namespace faiss

data/vendor/faiss/faiss/impl/ThreadedIndex.h

@@ -19,62 +19,68 @@ namespace faiss {
 /// The interface to this class itself is not thread safe
 template <typename IndexT>
 class ThreadedIndex : public IndexT {
- public:
-  explicit ThreadedIndex(bool threaded);
-  explicit ThreadedIndex(int d, bool threaded);
-
-  ~ThreadedIndex() override;
-
-  /// override an index that is managed by ourselves.
-  /// WARNING: once an index is added, it becomes unsafe to touch it from any
-  /// other thread than that on which is managing it, until we are shut
-  /// down. Use runOnIndex to perform work on it instead.
-  void addIndex(IndexT* index);
-
-  /// Remove an index that is managed by ourselves.
-  /// This will flush all pending work on that index, and then shut
-  /// down its managing thread, and will remove the index.
-  void removeIndex(IndexT* index);
-
-  /// Run a function on all indices, in the thread that the index is
-  /// managed in.
-  /// Function arguments are (index in collection, index pointer)
-  void runOnIndex(std::function<void(int, IndexT*)> f);
-  void runOnIndex(std::function<void(int, const IndexT*)> f) const;
-
-  /// faiss::Index API
-  /// All indices receive the same call
-  void reset() override;
-
-  /// Returns the number of sub-indices
-  int count() const { return indices_.size(); }
-
-  /// Returns the i-th sub-index
-  IndexT* at(int i) { return indices_[i].first; }
-
-  /// Returns the i-th sub-index (const version)
-  const IndexT* at(int i) const { return indices_[i].first; }
-
-  /// Whether or not we are responsible for deleting our contained indices
-  bool own_fields;
-
- protected:
-  /// Called just after an index is added
-  virtual void onAfterAddIndex(IndexT* index);
-
-  /// Called just after an index is removed
-  virtual void onAfterRemoveIndex(IndexT* index);
-
-protected:
-  static void waitAndHandleFutures(std::vector<std::future<bool>>& v);
-
-  /// Collection of Index instances, with their managing worker thread if any
-  std::vector<std::pair<IndexT*, std::unique_ptr<WorkerThread>>> indices_;
-
-  /// Is this index multi-threaded?
-  bool isThreaded_;
+   public:
+    explicit ThreadedIndex(bool threaded);
+    explicit ThreadedIndex(int d, bool threaded);
+
+    ~ThreadedIndex() override;
+
+    /// override an index that is managed by ourselves.
+    /// WARNING: once an index is added, it becomes unsafe to touch it from any
+    /// other thread than that on which is managing it, until we are shut
+    /// down. Use runOnIndex to perform work on it instead.
+    void addIndex(IndexT* index);
+
+    /// Remove an index that is managed by ourselves.
+    /// This will flush all pending work on that index, and then shut
+    /// down its managing thread, and will remove the index.
+    void removeIndex(IndexT* index);
+
+    /// Run a function on all indices, in the thread that the index is
+    /// managed in.
+    /// Function arguments are (index in collection, index pointer)
+    void runOnIndex(std::function<void(int, IndexT*)> f);
+    void runOnIndex(std::function<void(int, const IndexT*)> f) const;
+
+    /// faiss::Index API
+    /// All indices receive the same call
+    void reset() override;
+
+    /// Returns the number of sub-indices
+    int count() const {
+        return indices_.size();
+    }
+
+    /// Returns the i-th sub-index
+    IndexT* at(int i) {
+        return indices_[i].first;
+    }
+
+    /// Returns the i-th sub-index (const version)
+    const IndexT* at(int i) const {
+        return indices_[i].first;
+    }
+
+    /// Whether or not we are responsible for deleting our contained indices
+    bool own_fields;
+
+   protected:
+    /// Called just after an index is added
+    virtual void onAfterAddIndex(IndexT* index);
+
+    /// Called just after an index is removed
+    virtual void onAfterRemoveIndex(IndexT* index);
+
+   protected:
+    static void waitAndHandleFutures(std::vector<std::future<bool>>& v);
+
+    /// Collection of Index instances, with their managing worker thread if any
+    std::vector<std::pair<IndexT*, std::unique_ptr<WorkerThread>>> indices_;
+
+    /// Is this index multi-threaded?
+    bool isThreaded_;
 };
 
-} // namespace
+} // namespace faiss
 
 #include <faiss/impl/ThreadedIndex-inl.h>