faiss 0.1.2 → 0.1.3

data/vendor/faiss/{impl → faiss/impl}/ScalarQuantizer.h

@@ -29,7 +29,7 @@ struct ScalarQuantizer {
         QT_8bit_uniform,     ///< same, shared range for all dimensions
         QT_4bit_uniform,
         QT_fp16,
-        QT_8bit_direct,      /// fast indexing of uint8s
+        QT_8bit_direct,      ///< fast indexing of uint8s
         QT_6bit,             ///< 6 bits per component
     };
 

data/vendor/faiss/{impl → faiss/impl}/index_read.cpp

@@ -12,13 +12,16 @@
 #include <cstdio>
 #include <cstdlib>
 
-#include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <unistd.h>
+
+#ifndef _MSC_VER
+#include <sys/mman.h>
+#endif // !_MSC_VER
 
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/io.h>
+#include <faiss/impl/io_macros.h>
 #include <faiss/utils/hamming.h>
 
 #include <faiss/IndexFlat.h>
@@ -36,46 +39,19 @@
 #include <faiss/IndexScalarQuantizer.h>
 #include <faiss/IndexHNSW.h>
 #include <faiss/IndexLattice.h>
-
-#include <faiss/OnDiskInvertedLists.h>
 #include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexBinaryFromFloat.h>
 #include <faiss/IndexBinaryHNSW.h>
 #include <faiss/IndexBinaryIVF.h>
 #include <faiss/IndexBinaryHash.h>
 
+#ifndef _MSC_VER
+#include <faiss/OnDiskInvertedLists.h>
+#endif // !_MSC_VER
 
 
 namespace faiss {
 
-/*************************************************************
- * I/O macros
- *
- * we use macros so that we have a line number to report in abort
- * (). This makes debugging a lot easier. The IOReader or IOWriter is
- * always called f and thus is not passed in as a macro parameter.
- **************************************************************/
-
-
-#define READANDCHECK(ptr, n) {                                  \
-        size_t ret = (*f)(ptr, sizeof(*(ptr)), n);              \
-        FAISS_THROW_IF_NOT_FMT(ret == (n),                      \
-            "read error in %s: %ld != %ld (%s)",                \
-            f->name.c_str(), ret, size_t(n), strerror(errno));  \
-    }
-
-#define READ1(x)  READANDCHECK(&(x), 1)
-
-// will fail if we write 256G of data at once...
-#define READVECTOR(vec) {                       \
-        long size;                            \
-        READANDCHECK (&size, 1);                \
-        FAISS_THROW_IF_NOT (size >= 0 && size < (1L << 40));  \
-        (vec).resize (size);                    \
-        READANDCHECK ((vec).data (), size);     \
-    }
-
-
 
 /*************************************************************
  * Read
@@ -202,7 +178,7 @@ InvertedLists *read_InvertedLists (IOReader *f, int io_flags) {
         fprintf(stderr, "read_InvertedLists:"
                 " WARN! inverted lists not stored with IVF object\n");
         return nullptr;
-    } else if (h == fourcc ("ilar") && !(io_flags & IO_FLAG_MMAP)) {
+    } else if (h == fourcc ("ilar") && !(io_flags & IO_FLAG_SKIP_IVF_DATA)) {
         auto ails = new ArrayInvertedLists (0, 0);
         READ1 (ails->nlist);
         READ1 (ails->code_size);
@@ -222,95 +198,31 @@ InvertedLists *read_InvertedLists (IOReader *f, int io_flags) {
             }
         }
         return ails;
-    } else if (h == fourcc ("ilar") && (io_flags & IO_FLAG_MMAP)) {
-        // then we load it as an OnDiskInvertedLists
 
-        FileIOReader *reader = dynamic_cast<FileIOReader*>(f);
-        FAISS_THROW_IF_NOT_MSG(reader, "mmap only supported for File objects");
-        FILE *fdesc = reader->f;
-
-        auto ails = new OnDiskInvertedLists ();
-        READ1 (ails->nlist);
-        READ1 (ails->code_size);
-        ails->read_only = true;
-        ails->lists.resize (ails->nlist);
-        std::vector<size_t> sizes (ails->nlist);
+#ifdef _MSC_VER
+    } else {
+        FAISS_THROW_MSG("Unsupported inverted list format for Windows");
+    }
+#else
+    } else if (h == fourcc ("ilar") && (io_flags & IO_FLAG_SKIP_IVF_DATA)) {
+        // code is always ilxx where xx is specific to the type of invlists we want
+        // so we get the 16 high bits from the io_flag and the 16 low bits as "il"
+        int h2 = (io_flags & 0xffff0000) | (fourcc("il__") & 0x0000ffff);
+        size_t nlist, code_size;
+        READ1 (nlist);
+        READ1 (code_size);
+        std::vector<size_t> sizes (nlist);
         read_ArrayInvertedLists_sizes (f, sizes);
-        size_t o0 = ftell(fdesc), o = o0;
-        { // do the mmap
-            struct stat buf;
-            int ret = fstat (fileno(fdesc), &buf);
-            FAISS_THROW_IF_NOT_FMT (ret == 0,
-                                    "fstat failed: %s", strerror(errno));
-            ails->totsize = buf.st_size;
-            ails->ptr = (uint8_t*)mmap (nullptr, ails->totsize,
-                                        PROT_READ, MAP_SHARED,
-                                        fileno(fdesc), 0);
-            FAISS_THROW_IF_NOT_FMT (ails->ptr != MAP_FAILED,
-                            "could not mmap: %s",
-                            strerror(errno));
-        }
-
-        for (size_t i = 0; i < ails->nlist; i++) {
-            OnDiskInvertedLists::List & l = ails->lists[i];
-            l.size = l.capacity = sizes[i];
-            l.offset = o;
-            o += l.size * (sizeof(OnDiskInvertedLists::idx_t) +
-                           ails->code_size);
-        }
-        FAISS_THROW_IF_NOT(o <= ails->totsize);
-        // resume normal reading of file
-        fseek (fdesc, o, SEEK_SET);
-        return ails;
-    } else if (h == fourcc ("ilod")) {
-        OnDiskInvertedLists *od = new OnDiskInvertedLists();
-        od->read_only = io_flags & IO_FLAG_READ_ONLY;
-        READ1 (od->nlist);
-        READ1 (od->code_size);
-        // this is a POD object
-        READVECTOR (od->lists);
-        {
-            std::vector<OnDiskInvertedLists::Slot> v;
-            READVECTOR(v);
-            od->slots.assign(v.begin(), v.end());
-        }
-        {
-            std::vector<char> x;
-            READVECTOR(x);
-            od->filename.assign(x.begin(), x.end());
-
-            if (io_flags & IO_FLAG_ONDISK_SAME_DIR) {
-                FileIOReader *reader = dynamic_cast<FileIOReader*>(f);
-                FAISS_THROW_IF_NOT_MSG (
-                    reader, "IO_FLAG_ONDISK_SAME_DIR only supported "
-                    "when reading from file");
-                std::string indexname = reader->name;
-                std::string dirname = "./";
-                size_t slash = indexname.find_last_of('/');
-                if (slash != std::string::npos) {
-                    dirname = indexname.substr(0, slash + 1);
-                }
-                std::string filename = od->filename;
-                slash = filename.find_last_of('/');
-                if (slash != std::string::npos) {
-                    filename = filename.substr(slash + 1);
-                }
-                filename = dirname + filename;
-                printf("IO_FLAG_ONDISK_SAME_DIR: "
-                       "updating ondisk filename from %s to %s\n",
-                       od->filename.c_str(), filename.c_str());
-                od->filename = filename;
-            }
-
-        }
-        READ1(od->totsize);
-        od->do_mmap();
-        return od;
+        return InvertedListsIOHook::lookup(h2)->read_ArrayInvertedLists(
+                f, io_flags, nlist, code_size, sizes);
     } else {
-        FAISS_THROW_MSG ("read_InvertedLists: unsupported invlist type");
+        return InvertedListsIOHook::lookup(h)->read(f, io_flags);
     }
+#endif // !_MSC_VER
+
 }
 
+
 static void read_InvertedLists (
         IndexIVF *ivf, IOReader *f, int io_flags) {
     InvertedLists *ils = read_InvertedLists (f, io_flags);
@@ -885,5 +797,76 @@ IndexBinary *read_index_binary (const char *fname, int io_flags) {
     return idx;
 }
 
+#ifndef _MSC_VER
+
+/**********************************************************
+ * InvertedListIOHook's
+ **********************************************************/
+
+InvertedListsIOHook::InvertedListsIOHook(
+        const std::string & key, const std::string & classname):
+    key(key), classname(classname)
+{}
+
+namespace {
+
+/// std::vector that deletes its contents
+struct IOHookTable: std::vector<InvertedListsIOHook*> {
+
+    IOHookTable() {
+        push_back(new OnDiskInvertedListsIOHook());
+    }
+
+    ~IOHookTable() {
+        for (auto x: *this) {
+            delete x;
+        }
+    }
+};
+
+static IOHookTable InvertedListsIOHook_table;
+
+} // anonymous namepsace
+
+InvertedListsIOHook* InvertedListsIOHook::lookup(int h)
+{
+    for(const auto & callback: InvertedListsIOHook_table) {
+        if (h == fourcc(callback->key)) {
+            return callback;
+        }
+    }
+    FAISS_THROW_FMT ("read_InvertedLists: could not load ArrayInvertedLists as %04x", h);
+}
+
+InvertedListsIOHook* InvertedListsIOHook::lookup_classname(const std::string & classname)
+{
+    for(const auto & callback: InvertedListsIOHook_table) {
+        if (callback->classname == classname) {
+            return callback;
+        }
+    }
+    FAISS_THROW_FMT ("read_InvertedLists: could not find classname %s", classname.c_str());
+}
+
+void InvertedListsIOHook::add_callback(InvertedListsIOHook *cb)
+{
+    InvertedListsIOHook_table.push_back(cb);
+}
+
+void InvertedListsIOHook::print_callbacks()
+{
+    printf("registered %zd InvertedListsIOHooks:\n",
+          InvertedListsIOHook_table.size());
+    for(const auto & cb: InvertedListsIOHook_table) {
+        printf("%08x %s %s\n",
+               fourcc(cb->key.c_str()),
+               cb->key.c_str(),
+               cb->classname.c_str());
+    }
+}
+
+#endif // !_MSC_VER
+
+
 
 } // namespace faiss

data/vendor/faiss/{impl → faiss/impl}/index_write.cpp

@@ -12,13 +12,16 @@
 #include <cstdio>
 #include <cstdlib>
 
-#include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <unistd.h>
+
+#ifndef _MSC_VER
+#include <sys/mman.h>
+#endif // !_MSC_VER
 
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/io.h>
+#include <faiss/impl/io_macros.h>
 #include <faiss/utils/hamming.h>
 
 #include <faiss/IndexFlat.h>
@@ -37,13 +40,15 @@
 #include <faiss/IndexHNSW.h>
 #include <faiss/IndexLattice.h>
 
-#include <faiss/OnDiskInvertedLists.h>
 #include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexBinaryFromFloat.h>
 #include <faiss/IndexBinaryHNSW.h>
 #include <faiss/IndexBinaryIVF.h>
 #include <faiss/IndexBinaryHash.h>
 
+#ifndef _MSC_VER
+#include <faiss/OnDiskInvertedLists.h>
+#endif // !_MSC_VER
 
 
 /*************************************************************
@@ -68,37 +73,9 @@
  * leak memory.
  **************************************************************/
 
-
-
 namespace faiss {
 
 
-/*************************************************************
- * I/O macros
- *
- * we use macros so that we have a line number to report in abort
- * (). This makes debugging a lot easier. The IOReader or IOWriter is
- * always called f and thus is not passed in as a macro parameter.
- **************************************************************/
-
-
-#define WRITEANDCHECK(ptr, n) {                                 \
-        size_t ret = (*f)(ptr, sizeof(*(ptr)), n);              \
-        FAISS_THROW_IF_NOT_FMT(ret == (n),                      \
-            "write error in %s: %ld != %ld (%s)",               \
-            f->name.c_str(), ret, size_t(n), strerror(errno));  \
-    }
-
-#define WRITE1(x) WRITEANDCHECK(&(x), 1)
-
-#define WRITEVECTOR(vec) {                      \
-        size_t size = (vec).size ();            \
-        WRITEANDCHECK (&size, 1);               \
-        WRITEANDCHECK ((vec).data (), size);    \
-    }
-
-
-
 /*************************************************************
  * Write
  **************************************************************/
@@ -239,31 +216,19 @@ void write_InvertedLists (const InvertedLists *ils, IOWriter *f) {
                 WRITEANDCHECK (ails->ids[i].data(), n);
             }
         }
-    } else if (const auto & od =
-               dynamic_cast<const OnDiskInvertedLists *>(ils)) {
-        uint32_t h = fourcc ("ilod");
-        WRITE1 (h);
-        WRITE1 (ils->nlist);
-        WRITE1 (ils->code_size);
-        // this is a POD object
-        WRITEVECTOR (od->lists);
+#ifndef _MSC_VER
+    } else {
 
-        {
-            std::vector<OnDiskInvertedLists::Slot> v(
-                      od->slots.begin(), od->slots.end());
-            WRITEVECTOR(v);
-        }
-        {
-            std::vector<char> x(od->filename.begin(), od->filename.end());
-            WRITEVECTOR(x);
-        }
-        WRITE1(od->totsize);
+        InvertedListsIOHook::lookup_classname(
+                typeid(*ils).name())->write(ils, f);
 
-    } else {
+        /*
         fprintf(stderr, "WARN! write_InvertedLists: unsupported invlist type, "
                 "saving null invlist\n");
         uint32_t h = fourcc ("il00");
         WRITE1 (h);
+        */
+#endif // !_MSC_VER
     }
 }
 

data/vendor/faiss/{impl → faiss/impl}/io.cpp

@@ -7,6 +7,7 @@
 
 // -*- c++ -*-
 
+#include <algorithm>
 #include <cstring>
 #include <cassert>
 
@@ -135,8 +136,8 @@ int FileIOWriter::fileno()  {
  * IO buffer
  ***********************************************************************/
 
-BufferedIOReader::BufferedIOReader(IOReader *reader, size_t bsz, size_t totsz):
-    reader(reader), bsz(bsz), totsz(totsz), ofs(0), b0(0), b1(0), buffer(bsz)
+BufferedIOReader::BufferedIOReader(IOReader *reader, size_t bsz):
+    reader(reader), bsz(bsz), ofs(0), ofs2(0), b0(0), b1(0), buffer(bsz)
 {
 }
 
@@ -156,15 +157,12 @@ size_t BufferedIOReader::operator()(void *ptr, size_t unitsize, size_t nitems)
         size -= nb;
     }
 
-    if (size > totsz - ofs) {
-        size = totsz - ofs;
-    }
     // while we would like to have more data
     while (size > 0) {
         assert (b0 == b1); // buffer empty on input
         // try to read from main reader
         b0 = 0;
-        b1 = (*reader)(buffer.data(), 1, std::min(bsz, size));
+        b1 = (*reader)(buffer.data(), 1, bsz);
 
         if (b1 == 0) {
             // no more bytes available
@@ -180,12 +178,13 @@ size_t BufferedIOReader::operator()(void *ptr, size_t unitsize, size_t nitems)
         dst += nb2;
         size -= nb2;
     }
+    ofs2 += nb;
     return nb / unitsize;
 }
 
 
 BufferedIOWriter::BufferedIOWriter(IOWriter *writer, size_t bsz):
-    writer(writer), bsz(bsz), b0(0), buffer(bsz)
+    writer(writer), bsz(bsz), ofs2(0), b0(0), buffer(bsz)
 {
 }
 
@@ -222,7 +221,7 @@ size_t BufferedIOWriter::operator()(const void *ptr, size_t unitsize, size_t nit
         src += nb1;
         size -= nb1;
     }
-
+    ofs2 += nb;
     return nb / unitsize;
 }
 
@@ -230,7 +229,7 @@ BufferedIOWriter::~BufferedIOWriter()
 {
     size_t ofs = 0;
     while(ofs != b0) {
-        printf("Destructor write %ld \n", b0 - ofs);
+        // printf("Destructor write %zd \n", b0 - ofs);
         size_t written = (*writer)(buffer.data() + ofs, 1, b0 - ofs);
         FAISS_THROW_IF_NOT(written > 0);
         ofs += written;
@@ -242,11 +241,17 @@ BufferedIOWriter::~BufferedIOWriter()
 
 
 
-uint32_t fourcc (const char sx[4]) {
+uint32_t fourcc (const  char sx[4]) {
     assert(4 == strlen(sx));
     const unsigned char *x = (unsigned char*)sx;
     return x[0] | x[1] << 8 | x[2] << 16 | x[3] << 24;
 }
 
+uint32_t fourcc (const std::string & sx) {
+    assert(sx.length() == 4);
+    const unsigned char *x = (unsigned char*)sx.c_str();
+    return x[0] | x[1] << 8 | x[2] << 16 | x[3] << 24;
+}
+
 
 } // namespace faiss