faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/index_factory.h

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
-
 #pragma once
 
 #include <faiss/Index.h>
@@ -16,10 +14,14 @@ namespace faiss {
 
 /** Build and index with the sequence of processing steps described in
  *  the string. */
-Index *index_factory (int d, const char *description,
-                      MetricType metric = METRIC_L2);
+Index* index_factory(
+        int d,
+        const char* description,
+        MetricType metric = METRIC_L2);
 
-IndexBinary *index_binary_factory (int d, const char *description);
+/// set to > 0 to get more logs from index_factory
+FAISS_API extern int index_factory_verbose;
 
+IndexBinary* index_binary_factory(int d, const char* description);
 
-}
+} // namespace faiss

data/vendor/faiss/faiss/index_io.h

@@ -12,10 +12,9 @@
 #ifndef FAISS_INDEX_IO_H
 #define FAISS_INDEX_IO_H
 
-
 #include <cstdio>
-#include <typeinfo>
 #include <string>
+#include <typeinfo>
 #include <vector>
 
 /** I/O functions can read/write to a filename, a file handle or to an
@@ -36,13 +35,13 @@ struct IOReader;
 struct IOWriter;
 struct InvertedLists;
 
-void write_index (const Index *idx, const char *fname);
-void write_index (const Index *idx, FILE *f);
-void write_index (const Index *idx, IOWriter *writer);
+void write_index(const Index* idx, const char* fname);
+void write_index(const Index* idx, FILE* f);
+void write_index(const Index* idx, IOWriter* writer);
 
-void write_index_binary (const IndexBinary *idx, const char *fname);
-void write_index_binary (const IndexBinary *idx, FILE *f);
-void write_index_binary (const IndexBinary *idx, IOWriter *writer);
+void write_index_binary(const IndexBinary* idx, const char* fname);
+void write_index_binary(const IndexBinary* idx, FILE* f);
+void write_index_binary(const IndexBinary* idx, IOWriter* writer);
 
 // The read_index flags are implemented only for a subset of index types.
 const int IO_FLAG_READ_ONLY = 2;
@@ -51,32 +50,30 @@ const int IO_FLAG_READ_ONLY = 2;
 const int IO_FLAG_ONDISK_SAME_DIR = 4;
 // don't load IVF data to RAM, only list sizes
 const int IO_FLAG_SKIP_IVF_DATA = 8;
-// try to memmap data (useful to load an ArrayInvertedLists as an OnDiskInvertedLists)
+// try to memmap data (useful to load an ArrayInvertedLists as an
+// OnDiskInvertedLists)
 const int IO_FLAG_MMAP = IO_FLAG_SKIP_IVF_DATA | 0x646f0000;
 
+Index* read_index(const char* fname, int io_flags = 0);
+Index* read_index(FILE* f, int io_flags = 0);
+Index* read_index(IOReader* reader, int io_flags = 0);
 
-Index *read_index (const char *fname, int io_flags = 0);
-Index *read_index (FILE * f, int io_flags = 0);
-Index *read_index (IOReader *reader, int io_flags = 0);
+IndexBinary* read_index_binary(const char* fname, int io_flags = 0);
+IndexBinary* read_index_binary(FILE* f, int io_flags = 0);
+IndexBinary* read_index_binary(IOReader* reader, int io_flags = 0);
 
-IndexBinary *read_index_binary (const char *fname, int io_flags = 0);
-IndexBinary *read_index_binary (FILE * f, int io_flags = 0);
-IndexBinary *read_index_binary (IOReader *reader, int io_flags = 0);
+void write_VectorTransform(const VectorTransform* vt, const char* fname);
+VectorTransform* read_VectorTransform(const char* fname);
 
-void write_VectorTransform (const VectorTransform *vt, const char *fname);
-VectorTransform *read_VectorTransform (const char *fname);
+ProductQuantizer* read_ProductQuantizer(const char* fname);
+ProductQuantizer* read_ProductQuantizer(IOReader* reader);
 
-ProductQuantizer * read_ProductQuantizer (const char*fname);
-ProductQuantizer * read_ProductQuantizer (IOReader *reader);
-
-void write_ProductQuantizer (const ProductQuantizer*pq, const char *fname);
-void write_ProductQuantizer (const ProductQuantizer*pq, IOWriter *f);
-
-void write_InvertedLists (const InvertedLists *ils, IOWriter *f);
-InvertedLists *read_InvertedLists (IOReader *reader, int io_flags = 0);
+void write_ProductQuantizer(const ProductQuantizer* pq, const char* fname);
+void write_ProductQuantizer(const ProductQuantizer* pq, IOWriter* f);
 
+void write_InvertedLists(const InvertedLists* ils, IOWriter* f);
+InvertedLists* read_InvertedLists(IOReader* reader, int io_flags = 0);
 
 } // namespace faiss
 
-
 #endif

data/vendor/faiss/faiss/invlists/BlockInvertedLists.cpp

@@ -12,81 +12,80 @@
 #include <faiss/impl/io.h>
 #include <faiss/impl/io_macros.h>
 
-
 namespace faiss {
 
-BlockInvertedLists::BlockInvertedLists (
-        size_t nlist, size_t n_per_block,
-        size_t block_size):
-    InvertedLists (nlist, InvertedLists::INVALID_CODE_SIZE),
-    n_per_block(n_per_block), block_size(block_size)
-{
-    ids.resize (nlist);
-    codes.resize (nlist);
+BlockInvertedLists::BlockInvertedLists(
+        size_t nlist,
+        size_t n_per_block,
+        size_t block_size)
+        : InvertedLists(nlist, InvertedLists::INVALID_CODE_SIZE),
+          n_per_block(n_per_block),
+          block_size(block_size) {
+    ids.resize(nlist);
+    codes.resize(nlist);
 }
 
-BlockInvertedLists::BlockInvertedLists ():
-    InvertedLists (0, InvertedLists::INVALID_CODE_SIZE),
-    n_per_block(0), block_size(0)
-{}
-
-
-size_t BlockInvertedLists::add_entries (
-           size_t list_no, size_t n_entry,
-           const idx_t* ids_in, const uint8_t *code)
-{
-    if (n_entry == 0) return 0;
-    FAISS_THROW_IF_NOT (list_no < nlist);
-    size_t o = ids [list_no].size();
-    FAISS_THROW_IF_NOT (o == 0); // not clear how we should handle subsequent adds
-    ids [list_no].resize (o + n_entry);
-    memcpy (&ids[list_no][o], ids_in, sizeof (ids_in[0]) * n_entry);
+BlockInvertedLists::BlockInvertedLists()
+        : InvertedLists(0, InvertedLists::INVALID_CODE_SIZE),
+          n_per_block(0),
+          block_size(0) {}
+
+size_t BlockInvertedLists::add_entries(
+        size_t list_no,
+        size_t n_entry,
+        const idx_t* ids_in,
+        const uint8_t* code) {
+    if (n_entry == 0)
+        return 0;
+    FAISS_THROW_IF_NOT(list_no < nlist);
+    size_t o = ids[list_no].size();
+    FAISS_THROW_IF_NOT(
+            o == 0); // not clear how we should handle subsequent adds
+    ids[list_no].resize(o + n_entry);
+    memcpy(&ids[list_no][o], ids_in, sizeof(ids_in[0]) * n_entry);
 
     // copy whole blocks
     size_t n_block = (n_entry + n_per_block - 1) / n_per_block;
-    codes [list_no].resize (n_block * block_size);
-    memcpy (&codes[list_no][o * code_size], code, n_block * block_size);
+    codes[list_no].resize(n_block * block_size);
+    memcpy(&codes[list_no][o * code_size], code, n_block * block_size);
     return o;
 }
 
-size_t BlockInvertedLists::list_size(size_t list_no) const
-{
-    assert (list_no < nlist);
+size_t BlockInvertedLists::list_size(size_t list_no) const {
+    assert(list_no < nlist);
     return ids[list_no].size();
 }
 
-const uint8_t * BlockInvertedLists::get_codes (size_t list_no) const
-{
-    assert (list_no < nlist);
+const uint8_t* BlockInvertedLists::get_codes(size_t list_no) const {
+    assert(list_no < nlist);
     return codes[list_no].get();
 }
 
-const InvertedLists::idx_t * BlockInvertedLists::get_ids (size_t list_no) const
-{
-    assert (list_no < nlist);
+const InvertedLists::idx_t* BlockInvertedLists::get_ids(size_t list_no) const {
+    assert(list_no < nlist);
     return ids[list_no].data();
 }
 
-void BlockInvertedLists::resize (size_t list_no, size_t new_size)
-{
-    ids[list_no].resize (new_size);
+void BlockInvertedLists::resize(size_t list_no, size_t new_size) {
+    ids[list_no].resize(new_size);
     size_t prev_nbytes = codes[list_no].size();
     size_t n_block = (new_size + n_per_block - 1) / n_per_block;
     size_t new_nbytes = n_block * block_size;
-    codes[list_no].resize (new_nbytes);
+    codes[list_no].resize(new_nbytes);
     if (prev_nbytes < new_nbytes) {
         // set new elements to 0
-        memset(
-            codes[list_no].data() + prev_nbytes, 0,
-            new_nbytes - prev_nbytes
-        );
+        memset(codes[list_no].data() + prev_nbytes,
+               0,
+               new_nbytes - prev_nbytes);
     }
 }
 
-void BlockInvertedLists::update_entries (
-      size_t , size_t , size_t ,
-      const idx_t *, const uint8_t *)
-{
+void BlockInvertedLists::update_entries(
+        size_t,
+        size_t,
+        size_t,
+        const idx_t*,
+        const uint8_t*) {
     FAISS_THROW_MSG("not impemented");
     /*
     assert (list_no < nlist);
@@ -96,29 +95,25 @@ void BlockInvertedLists::update_entries (
     */
 }
 
-
-BlockInvertedLists::~BlockInvertedLists ()
-{}
+BlockInvertedLists::~BlockInvertedLists() {}
 
 /**************************************************
  * IO hook implementation
  **************************************************/
 
-BlockInvertedListsIOHook::BlockInvertedListsIOHook():
-    InvertedListsIOHook("ilbl", typeid(BlockInvertedLists).name())
-{}
-
+BlockInvertedListsIOHook::BlockInvertedListsIOHook()
+        : InvertedListsIOHook("ilbl", typeid(BlockInvertedLists).name()) {}
 
-void BlockInvertedListsIOHook::write(const InvertedLists *ils_in, IOWriter *f) const
-{
-    uint32_t h = fourcc ("ilbl");
-    WRITE1 (h);
-    const BlockInvertedLists *il =
-        dynamic_cast<const BlockInvertedLists*> (ils_in);
-    WRITE1 (il->nlist);
-    WRITE1 (il->code_size);
-    WRITE1 (il->n_per_block);
-    WRITE1 (il->block_size);
+void BlockInvertedListsIOHook::write(const InvertedLists* ils_in, IOWriter* f)
+        const {
+    uint32_t h = fourcc("ilbl");
+    WRITE1(h);
+    const BlockInvertedLists* il =
+            dynamic_cast<const BlockInvertedLists*>(ils_in);
+    WRITE1(il->nlist);
+    WRITE1(il->code_size);
+    WRITE1(il->n_per_block);
+    WRITE1(il->block_size);
 
     for (size_t i = 0; i < il->nlist; i++) {
         WRITEVECTOR(il->ids[i]);
@@ -126,13 +121,13 @@ void BlockInvertedListsIOHook::write(const InvertedLists *ils_in, IOWriter *f) c
     }
 }
 
-InvertedLists * BlockInvertedListsIOHook::read(IOReader *f, int /* io_flags */) const
-{
-    BlockInvertedLists *il = new BlockInvertedLists();
-    READ1 (il->nlist);
-    READ1 (il->code_size);
-    READ1 (il->n_per_block);
-    READ1 (il->block_size);
+InvertedLists* BlockInvertedListsIOHook::read(IOReader* f, int /* io_flags */)
+        const {
+    BlockInvertedLists* il = new BlockInvertedLists();
+    READ1(il->nlist);
+    READ1(il->code_size);
+    READ1(il->n_per_block);
+    READ1(il->block_size);
 
     il->ids.resize(il->nlist);
     il->codes.resize(il->nlist);
@@ -145,7 +140,4 @@ InvertedLists * BlockInvertedListsIOHook::read(IOReader *f, int /* io_flags */)
     return il;
 }
 
-
-
-
 } // namespace faiss

data/vendor/faiss/faiss/invlists/BlockInvertedLists.h

@@ -5,13 +5,12 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
+#include <faiss/index_io.h>
 #include <faiss/invlists/InvertedLists.h>
 #include <faiss/invlists/InvertedListsIOHook.h>
 #include <faiss/utils/AlignedTable.h>
-#include <faiss/index_io.h>
 
 namespace faiss {
 
@@ -28,49 +27,48 @@ namespace faiss {
  * The writing functions add_entries and update_entries operate on block-aligned
  * data.
  */
-struct BlockInvertedLists: InvertedLists {
-
-    size_t n_per_block;   // nb of vectors stored per block
-    size_t block_size;    // nb bytes per block
+struct BlockInvertedLists : InvertedLists {
+    size_t n_per_block; // nb of vectors stored per block
+    size_t block_size;  // nb bytes per block
 
     std::vector<AlignedTable<uint8_t>> codes;
     std::vector<std::vector<idx_t>> ids;
 
-
-    BlockInvertedLists (
-        size_t nlist, size_t vec_per_block,
-        size_t block_size
-    );
+    BlockInvertedLists(size_t nlist, size_t vec_per_block, size_t block_size);
 
     BlockInvertedLists();
 
     size_t list_size(size_t list_no) const override;
-    const uint8_t * get_codes (size_t list_no) const override;
-    const idx_t * get_ids (size_t list_no) const override;
+    const uint8_t* get_codes(size_t list_no) const override;
+    const idx_t* get_ids(size_t list_no) const override;
 
     // works only on empty BlockInvertedLists
     // the codes should be of size ceil(n_entry / n_per_block) * block_size
     // and padded with 0s
-    size_t add_entries (
-           size_t list_no, size_t n_entry,
-           const idx_t* ids, const uint8_t *code) override;
+    size_t add_entries(
+            size_t list_no,
+            size_t n_entry,
+            const idx_t* ids,
+            const uint8_t* code) override;
 
     /// not implemented
-    void update_entries (size_t list_no, size_t offset, size_t n_entry,
-                         const idx_t *ids, const uint8_t *code) override;
+    void update_entries(
+            size_t list_no,
+            size_t offset,
+            size_t n_entry,
+            const idx_t* ids,
+            const uint8_t* code) override;
 
     // also pads new data with 0s
-    void resize (size_t list_no, size_t new_size) override;
-
-    ~BlockInvertedLists () override;
+    void resize(size_t list_no, size_t new_size) override;
 
+    ~BlockInvertedLists() override;
 };
 
 struct BlockInvertedListsIOHook : InvertedListsIOHook {
     BlockInvertedListsIOHook();
-    void write(const InvertedLists *ils, IOWriter *f) const override;
-    InvertedLists * read(IOReader *f, int io_flags) const override;
+    void write(const InvertedLists* ils, IOWriter* f) const override;
+    InvertedLists* read(IOReader* f, int io_flags) const override;
 };
 
-
 } // namespace faiss