faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/impl/io.cpp

@@ -8,150 +8,141 @@
 // -*- c++ -*-
 
 #include <algorithm>
-#include <cstring>
 #include <cassert>
+#include <cstring>
 
-#include <faiss/impl/io.h>
 #include <faiss/impl/FaissAssert.h>
-
+#include <faiss/impl/io.h>
 
 namespace faiss {
 
-
 /***********************************************************************
  * IO functions
  ***********************************************************************/
 
-
-int IOReader::fileno ()
-{
-    FAISS_THROW_MSG ("IOReader does not support memory mapping");
+int IOReader::fileno() {
+    FAISS_THROW_MSG("IOReader does not support memory mapping");
 }
 
-int IOWriter::fileno ()
-{
-    FAISS_THROW_MSG ("IOWriter does not support memory mapping");
+int IOWriter::fileno() {
+    FAISS_THROW_MSG("IOWriter does not support memory mapping");
 }
 
 /***********************************************************************
  * IO Vector
  ***********************************************************************/
 
-
-size_t VectorIOWriter::operator()(
-                const void *ptr, size_t size, size_t nitems)
-{
+size_t VectorIOWriter::operator()(const void* ptr, size_t size, size_t nitems) {
     size_t bytes = size * nitems;
     if (bytes > 0) {
         size_t o = data.size();
         data.resize(o + bytes);
-        memcpy (&data[o], ptr, size * nitems);
+        memcpy(&data[o], ptr, size * nitems);
     }
     return nitems;
 }
 
-size_t VectorIOReader::operator()(
-                  void *ptr, size_t size, size_t nitems)
-{
-    if (rp >= data.size()) return 0;
+size_t VectorIOReader::operator()(void* ptr, size_t size, size_t nitems) {
+    if (rp >= data.size())
+        return 0;
     size_t nremain = (data.size() - rp) / size;
-    if (nremain < nitems) nitems = nremain;
+    if (nremain < nitems)
+        nitems = nremain;
     if (size * nitems > 0) {
-        memcpy (ptr, &data[rp], size * nitems);
+        memcpy(ptr, &data[rp], size * nitems);
         rp += size * nitems;
     }
     return nitems;
 }
 
-
-
-
 /***********************************************************************
  * IO File
  ***********************************************************************/
 
+FileIOReader::FileIOReader(FILE* rf) : f(rf) {}
 
-
-FileIOReader::FileIOReader(FILE *rf): f(rf) {}
-
-FileIOReader::FileIOReader(const char * fname)
-{
+FileIOReader::FileIOReader(const char* fname) {
     name = fname;
     f = fopen(fname, "rb");
-    FAISS_THROW_IF_NOT_FMT (f, "could not open %s for reading: %s",
-                            fname, strerror(errno));
+    FAISS_THROW_IF_NOT_FMT(
+            f, "could not open %s for reading: %s", fname, strerror(errno));
     need_close = true;
 }
 
-FileIOReader::~FileIOReader()  {
+FileIOReader::~FileIOReader() {
     if (need_close) {
         int ret = fclose(f);
-        if (ret != 0) {// we cannot raise and exception in the destructor
-            fprintf(stderr, "file %s close error: %s",
-                    name.c_str(), strerror(errno));
+        if (ret != 0) { // we cannot raise and exception in the destructor
+            fprintf(stderr,
+                    "file %s close error: %s",
+                    name.c_str(),
+                    strerror(errno));
         }
     }
 }
 
-size_t FileIOReader::operator()(void *ptr, size_t size, size_t nitems) {
+size_t FileIOReader::operator()(void* ptr, size_t size, size_t nitems) {
     return fread(ptr, size, nitems, f);
 }
 
-int FileIOReader::fileno()  {
-    return ::fileno (f);
+int FileIOReader::fileno() {
+    return ::fileno(f);
 }
 
+FileIOWriter::FileIOWriter(FILE* wf) : f(wf) {}
 
-FileIOWriter::FileIOWriter(FILE *wf): f(wf) {}
-
-FileIOWriter::FileIOWriter(const char * fname)
-{
+FileIOWriter::FileIOWriter(const char* fname) {
     name = fname;
     f = fopen(fname, "wb");
-    FAISS_THROW_IF_NOT_FMT (f, "could not open %s for writing: %s",
-                            fname, strerror(errno));
+    FAISS_THROW_IF_NOT_FMT(
+            f, "could not open %s for writing: %s", fname, strerror(errno));
     need_close = true;
 }
 
-FileIOWriter::~FileIOWriter()  {
+FileIOWriter::~FileIOWriter() {
     if (need_close) {
         int ret = fclose(f);
         if (ret != 0) {
             // we cannot raise and exception in the destructor
-            fprintf(stderr, "file %s close error: %s",
-                    name.c_str(), strerror(errno));
+            fprintf(stderr,
+                    "file %s close error: %s",
+                    name.c_str(),
+                    strerror(errno));
         }
     }
 }
 
-size_t FileIOWriter::operator()(const void *ptr, size_t size, size_t nitems) {
+size_t FileIOWriter::operator()(const void* ptr, size_t size, size_t nitems) {
     return fwrite(ptr, size, nitems, f);
 }
 
-int FileIOWriter::fileno()  {
-    return ::fileno (f);
+int FileIOWriter::fileno() {
+    return ::fileno(f);
 }
 
 /***********************************************************************
  * IO buffer
  ***********************************************************************/
 
-BufferedIOReader::BufferedIOReader(IOReader *reader, size_t bsz):
-    reader(reader), bsz(bsz), ofs(0), ofs2(0), b0(0), b1(0), buffer(bsz)
-{
-}
-
+BufferedIOReader::BufferedIOReader(IOReader* reader, size_t bsz)
+        : reader(reader),
+          bsz(bsz),
+          ofs(0),
+          ofs2(0),
+          b0(0),
+          b1(0),
+          buffer(bsz) {}
 
-size_t BufferedIOReader::operator()(void *ptr, size_t unitsize, size_t nitems)
-{
+size_t BufferedIOReader::operator()(void* ptr, size_t unitsize, size_t nitems) {
     size_t size = unitsize * nitems;
-    if (size == 0) return 0;
-    char * dst = (char*)ptr;
+    if (size == 0)
+        return 0;
+    char* dst = (char*)ptr;
     size_t nb;
 
     { // first copy available bytes
         nb = std::min(b1 - b0, size);
-        memcpy (dst, buffer.data() + b0, nb);
+        memcpy(dst, buffer.data() + b0, nb);
         b0 += nb;
         dst += nb;
         size -= nb;
@@ -159,7 +150,7 @@ size_t BufferedIOReader::operator()(void *ptr, size_t unitsize, size_t nitems)
 
     // while we would like to have more data
     while (size > 0) {
-        assert (b0 == b1); // buffer empty on input
+        assert(b0 == b1); // buffer empty on input
         // try to read from main reader
         b0 = 0;
         b1 = (*reader)(buffer.data(), 1, bsz);
@@ -172,7 +163,7 @@ size_t BufferedIOReader::operator()(void *ptr, size_t unitsize, size_t nitems)
 
         // copy remaining bytes
         size_t nb2 = std::min(b1, size);
-        memcpy (dst, buffer.data(), nb2);
+        memcpy(dst, buffer.data(), nb2);
         b0 = nb2;
         nb += nb2;
         dst += nb2;
@@ -182,22 +173,22 @@ size_t BufferedIOReader::operator()(void *ptr, size_t unitsize, size_t nitems)
     return nb / unitsize;
 }
 
+BufferedIOWriter::BufferedIOWriter(IOWriter* writer, size_t bsz)
+        : writer(writer), bsz(bsz), ofs2(0), b0(0), buffer(bsz) {}
 
-BufferedIOWriter::BufferedIOWriter(IOWriter *writer, size_t bsz):
-    writer(writer), bsz(bsz), ofs2(0), b0(0), buffer(bsz)
-{
-}
-
-size_t BufferedIOWriter::operator()(const void *ptr, size_t unitsize, size_t nitems)
-{
+size_t BufferedIOWriter::operator()(
+        const void* ptr,
+        size_t unitsize,
+        size_t nitems) {
     size_t size = unitsize * nitems;
-    if (size == 0) return 0;
-    const char * src = (const char*)ptr;
+    if (size == 0)
+        return 0;
+    const char* src = (const char*)ptr;
     size_t nb;
 
     { // copy as many bytes as possible to buffer
         nb = std::min(bsz - b0, size);
-        memcpy (buffer.data() + b0, src, nb);
+        memcpy(buffer.data() + b0, src, nb);
         b0 += nb;
         src += nb;
         size -= nb;
@@ -207,15 +198,15 @@ size_t BufferedIOWriter::operator()(const void *ptr, size_t unitsize, size_t nit
         // now we need to flush to add more bytes
         size_t ofs = 0;
         do {
-            assert (ofs < 10000000);
+            assert(ofs < 10000000);
             size_t written = (*writer)(buffer.data() + ofs, 1, bsz - ofs);
             FAISS_THROW_IF_NOT(written > 0);
             ofs += written;
-        } while(ofs != bsz);
+        } while (ofs != bsz);
 
         // copy src to buffer
         size_t nb1 = std::min(bsz, size);
-        memcpy (buffer.data(), src, nb1);
+        memcpy(buffer.data(), src, nb1);
         b0 = nb1;
         nb += nb1;
         src += nb1;
@@ -225,37 +216,31 @@ size_t BufferedIOWriter::operator()(const void *ptr, size_t unitsize, size_t nit
     return nb / unitsize;
 }
 
-BufferedIOWriter::~BufferedIOWriter()
-{
+BufferedIOWriter::~BufferedIOWriter() {
     size_t ofs = 0;
-    while(ofs != b0) {
+    while (ofs != b0) {
         // printf("Destructor write %zd \n", b0 - ofs);
         size_t written = (*writer)(buffer.data() + ofs, 1, b0 - ofs);
         FAISS_THROW_IF_NOT(written > 0);
         ofs += written;
     }
-
 }
 
-
-
-
-
-uint32_t fourcc (const  char sx[4]) {
-    FAISS_THROW_IF_NOT (4 == strlen(sx));
-    const unsigned char *x = (unsigned char*)sx;
+uint32_t fourcc(const char sx[4]) {
+    FAISS_THROW_IF_NOT(4 == strlen(sx));
+    const unsigned char* x = (unsigned char*)sx;
     return x[0] | x[1] << 8 | x[2] << 16 | x[3] << 24;
 }
 
-uint32_t fourcc (const std::string & sx) {
-    FAISS_THROW_IF_NOT (sx.length() == 4);
-    const unsigned char *x = (unsigned char*)sx.c_str();
+uint32_t fourcc(const std::string& sx) {
+    FAISS_THROW_IF_NOT(sx.length() == 4);
+    const unsigned char* x = (unsigned char*)sx.c_str();
     return x[0] | x[1] << 8 | x[2] << 16 | x[3] << 24;
 }
 
 void fourcc_inv(uint32_t x, char str[5]) {
     *(uint32_t*)str = x;
-    str[5] = 0;
+    str[4] = 0;
 }
 
 std::string fourcc_inv(uint32_t x) {
@@ -264,7 +249,6 @@ std::string fourcc_inv(uint32_t x) {
     return std::string(str);
 }
 
-
 std::string fourcc_inv_printable(uint32_t x) {
     char cstr[5];
     fourcc_inv(x, cstr);
@@ -282,7 +266,4 @@ std::string fourcc_inv_printable(uint32_t x) {
     return str;
 }
 
-
-
-
 } // namespace faiss

data/vendor/faiss/faiss/impl/io.h

@@ -16,25 +16,23 @@
 
 #pragma once
 
-#include <string>
 #include <cstdio>
+#include <string>
 #include <vector>
 
 #include <faiss/Index.h>
 
 namespace faiss {
 
-
 struct IOReader {
     // name that can be used in error messages
     std::string name;
 
     // fread. Returns number of items read or 0 in case of EOF.
-    virtual size_t operator()(
-         void *ptr, size_t size, size_t nitems) = 0;
+    virtual size_t operator()(void* ptr, size_t size, size_t nitems) = 0;
 
     // return a file number that can be memory-mapped
-    virtual int fileno ();
+    virtual int fileno();
 
     virtual ~IOReader() {}
 };
@@ -44,53 +42,51 @@ struct IOWriter {
     std::string name;
 
     // fwrite. Return number of items written
-    virtual size_t operator()(
-         const void *ptr, size_t size, size_t nitems) = 0;
+    virtual size_t operator()(const void* ptr, size_t size, size_t nitems) = 0;
 
     // return a file number that can be memory-mapped
-    virtual int fileno ();
+    virtual int fileno();
 
     virtual ~IOWriter() noexcept(false) {}
 };
 
-
-struct VectorIOReader:IOReader {
+struct VectorIOReader : IOReader {
     std::vector<uint8_t> data;
     size_t rp = 0;
-    size_t operator()(void *ptr, size_t size, size_t nitems) override;
+    size_t operator()(void* ptr, size_t size, size_t nitems) override;
 };
 
-struct VectorIOWriter:IOWriter {
+struct VectorIOWriter : IOWriter {
     std::vector<uint8_t> data;
-    size_t operator()(const void *ptr, size_t size, size_t nitems) override;
+    size_t operator()(const void* ptr, size_t size, size_t nitems) override;
 };
 
-struct FileIOReader: IOReader {
-    FILE *f = nullptr;
+struct FileIOReader : IOReader {
+    FILE* f = nullptr;
     bool need_close = false;
 
-    FileIOReader(FILE *rf);
+    FileIOReader(FILE* rf);
 
-    FileIOReader(const char * fname);
+    FileIOReader(const char* fname);
 
     ~FileIOReader() override;
 
-    size_t operator()(void *ptr, size_t size, size_t nitems) override;
+    size_t operator()(void* ptr, size_t size, size_t nitems) override;
 
     int fileno() override;
 };
 
-struct FileIOWriter: IOWriter {
-    FILE *f = nullptr;
+struct FileIOWriter : IOWriter {
+    FILE* f = nullptr;
     bool need_close = false;
 
-    FileIOWriter(FILE *wf);
+    FileIOWriter(FILE* wf);
 
-    FileIOWriter(const char * fname);
+    FileIOWriter(const char* fname);
 
     ~FileIOWriter() override;
 
-    size_t operator()(const void *ptr, size_t size, size_t nitems) override;
+    size_t operator()(const void* ptr, size_t size, size_t nitems) override;
 
     int fileno() override;
 };
@@ -103,12 +99,9 @@ struct FileIOWriter: IOWriter {
  * the read/write functions.
  *******************************************************/
 
-
-
 /** wraps an ioreader to make buffered reads to avoid too small reads */
-struct BufferedIOReader: IOReader {
-
-    IOReader *reader;
+struct BufferedIOReader : IOReader {
+    IOReader* reader;
     size_t bsz;
     size_t ofs;    ///< offset in input stream
     size_t ofs2;   ///< number of bytes returned to caller
@@ -119,37 +112,34 @@ struct BufferedIOReader: IOReader {
      * @param bsz    buffer size (bytes). Reads will be done by batched of
      *               this size
      */
-    explicit BufferedIOReader(IOReader *reader, size_t bsz = 1024 * 1024);
+    explicit BufferedIOReader(IOReader* reader, size_t bsz = 1024 * 1024);
 
-    size_t operator()(void *ptr, size_t size, size_t nitems) override;
+    size_t operator()(void* ptr, size_t size, size_t nitems) override;
 };
 
-
-struct BufferedIOWriter: IOWriter {
-
-    IOWriter *writer;
+struct BufferedIOWriter : IOWriter {
+    IOWriter* writer;
     size_t bsz;
     size_t ofs;
-    size_t ofs2;     ///< number of bytes received from caller
-    size_t b0;       ///< amount of data in buffer
+    size_t ofs2; ///< number of bytes received from caller
+    size_t b0;   ///< amount of data in buffer
     std::vector<char> buffer;
 
-    explicit BufferedIOWriter(IOWriter *writer, size_t bsz = 1024 * 1024);
+    explicit BufferedIOWriter(IOWriter* writer, size_t bsz = 1024 * 1024);
 
-    size_t operator()(const void *ptr, size_t size, size_t nitems) override;
+    size_t operator()(const void* ptr, size_t size, size_t nitems) override;
 
     // flushes
     ~BufferedIOWriter() override;
 };
 
 /// cast a 4-character string to a uint32_t that can be written and read easily
-uint32_t fourcc (const char sx[4]);
-uint32_t fourcc (const std::string & sx);
+uint32_t fourcc(const char sx[4]);
+uint32_t fourcc(const std::string& sx);
 
 // decoding of fourcc (int32 -> string)
 void fourcc_inv(uint32_t x, char str[5]);
 std::string fourcc_inv(uint32_t x);
 std::string fourcc_inv_printable(uint32_t x);
 
-
 } // namespace faiss

data/vendor/faiss/faiss/impl/io_macros.h

@@ -15,43 +15,74 @@
  * always called f and thus is not passed in as a macro parameter.
  **************************************************************/
 
-
-#define READANDCHECK(ptr, n) {                                  \
-        size_t ret = (*f)(ptr, sizeof(*(ptr)), n);              \
-        FAISS_THROW_IF_NOT_FMT(ret == (n),                      \
-            "read error in %s: %zd != %zd (%s)",                \
-            f->name.c_str(), ret, size_t(n), strerror(errno));  \
+#define READANDCHECK(ptr, n)                         \
+    {                                                \
+        size_t ret = (*f)(ptr, sizeof(*(ptr)), n);   \
+        FAISS_THROW_IF_NOT_FMT(                      \
+                ret == (n),                          \
+                "read error in %s: %zd != %zd (%s)", \
+                f->name.c_str(),                     \
+                ret,                                 \
+                size_t(n),                           \
+                strerror(errno));                    \
     }
 
-#define READ1(x)  READANDCHECK(&(x), 1)
+#define READ1(x) READANDCHECK(&(x), 1)
 
 // will fail if we write 256G of data at once...
-#define READVECTOR(vec)                                          \
-  {                                                              \
-    size_t size;                                                 \
-    READANDCHECK(&size, 1);                                      \
-    FAISS_THROW_IF_NOT(size >= 0 && size < (uint64_t{1} << 40)); \
-    (vec).resize(size);                                          \
-    READANDCHECK((vec).data(), size);                            \
-  }
-
-#define READSTRING(s) {                      \
-        size_t size = (s).size ();            \
-        WRITEANDCHECK (&size, 1);               \
-        WRITEANDCHECK ((s).c_str(), size);      \
+#define READVECTOR(vec)                                              \
+    {                                                                \
+        size_t size;                                                 \
+        READANDCHECK(&size, 1);                                      \
+        FAISS_THROW_IF_NOT(size >= 0 && size < (uint64_t{1} << 40)); \
+        (vec).resize(size);                                          \
+        READANDCHECK((vec).data(), size);                            \
+    }
+
+#define READSTRING(s)                     \
+    {                                     \
+        size_t size = (s).size();         \
+        WRITEANDCHECK(&size, 1);          \
+        WRITEANDCHECK((s).c_str(), size); \
     }
 
-#define WRITEANDCHECK(ptr, n) {                                 \
-        size_t ret = (*f)(ptr, sizeof(*(ptr)), n);              \
-        FAISS_THROW_IF_NOT_FMT(ret == (n),                      \
-            "write error in %s: %zd != %zd (%s)",               \
-            f->name.c_str(), ret, size_t(n), strerror(errno));  \
+#define WRITEANDCHECK(ptr, n)                         \
+    {                                                 \
+        size_t ret = (*f)(ptr, sizeof(*(ptr)), n);    \
+        FAISS_THROW_IF_NOT_FMT(                       \
+                ret == (n),                           \
+                "write error in %s: %zd != %zd (%s)", \
+                f->name.c_str(),                      \
+                ret,                                  \
+                size_t(n),                            \
+                strerror(errno));                     \
     }
 
 #define WRITE1(x) WRITEANDCHECK(&(x), 1)
 
-#define WRITEVECTOR(vec) {                      \
-        size_t size = (vec).size ();            \
-        WRITEANDCHECK (&size, 1);               \
-        WRITEANDCHECK ((vec).data (), size);    \
+#define WRITEVECTOR(vec)                   \
+    {                                      \
+        size_t size = (vec).size();        \
+        WRITEANDCHECK(&size, 1);           \
+        WRITEANDCHECK((vec).data(), size); \
+    }
+
+// read/write xb vector for backwards compatibility of IndexFlat
+
+#define WRITEXBVECTOR(vec)                         \
+    {                                              \
+        FAISS_THROW_IF_NOT((vec).size() % 4 == 0); \
+        size_t size = (vec).size() / 4;            \
+        WRITEANDCHECK(&size, 1);                   \
+        WRITEANDCHECK((vec).data(), size * 4);     \
+    }
+
+#define READXBVECTOR(vec)                                            \
+    {                                                                \
+        size_t size;                                                 \
+        READANDCHECK(&size, 1);                                      \
+        FAISS_THROW_IF_NOT(size >= 0 && size < (uint64_t{1} << 40)); \
+        size *= 4;                                                   \
+        (vec).resize(size);                                          \
+        READANDCHECK((vec).data(), size);                            \
     }