umappp 0.1.2

data/vendor/annoy/kissrandom.h

@@ -0,0 +1,120 @@
+#ifndef ANNOY_KISSRANDOM_H
+#define ANNOY_KISSRANDOM_H
+
+#if defined(_MSC_VER) && _MSC_VER == 1500
+typedef unsigned __int32    uint32_t;
+typedef unsigned __int64    uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+namespace Annoy {
+
+// KISS = "keep it simple, stupid", but high quality random number generator
+// http://www0.cs.ucl.ac.uk/staff/d.jones/GoodPracticeRNG.pdf -> "Use a good RNG and build it into your code"
+// http://mathforum.org/kb/message.jspa?messageID=6627731
+// https://de.wikipedia.org/wiki/KISS_(Zufallszahlengenerator)
+
+// 32 bit KISS
+struct Kiss32Random {
+  uint32_t x;
+  uint32_t y;
+  uint32_t z;
+  uint32_t c;
+
+  static const uint32_t default_seed = 123456789;
+#if __cplusplus < 201103L
+  typedef uint32_t seed_type;
+#endif
+
+  // seed must be != 0
+  Kiss32Random(uint32_t seed = default_seed) {
+    x = seed;
+    y = 362436000;
+    z = 521288629;
+    c = 7654321;
+  }
+
+  uint32_t kiss() {
+    // Linear congruence generator
+    x = 69069 * x + 12345;
+
+    // Xor shift
+    y ^= y << 13;
+    y ^= y >> 17;
+    y ^= y << 5;
+
+    // Multiply-with-carry
+    uint64_t t = 698769069ULL * z + c;
+    c = t >> 32;
+    z = (uint32_t) t;
+
+    return x + y + z;
+  }
+  inline int flip() {
+    // Draw random 0 or 1
+    return kiss() & 1;
+  }
+  inline size_t index(size_t n) {
+    // Draw random integer between 0 and n-1 where n is at most the number of data points you have
+    return kiss() % n;
+  }
+  inline void set_seed(uint32_t seed) {
+    x = seed;
+  }
+};
+
+// 64 bit KISS. Use this if you have more than about 2^24 data points ("big data" ;) )
+struct Kiss64Random {
+  uint64_t x;
+  uint64_t y;
+  uint64_t z;
+  uint64_t c;
+
+  static const uint64_t default_seed = 1234567890987654321ULL;
+#if __cplusplus < 201103L
+  typedef uint64_t seed_type;
+#endif
+
+  // seed must be != 0
+  Kiss64Random(uint64_t seed = default_seed) {
+    x = seed;
+    y = 362436362436362436ULL;
+    z = 1066149217761810ULL;
+    c = 123456123456123456ULL;
+  }
+
+  uint64_t kiss() {
+    // Linear congruence generator
+    z = 6906969069LL*z+1234567;
+
+    // Xor shift
+    y ^= (y<<13);
+    y ^= (y>>17);
+    y ^= (y<<43);
+
+    // Multiply-with-carry (uint128_t t = (2^58 + 1) * x + c; c = t >> 64; x = (uint64_t) t)
+    uint64_t t = (x<<58)+c;
+    c = (x>>6);
+    x += t;
+    c += (x<t);
+
+    return x + y + z;
+  }
+  inline int flip() {
+    // Draw random 0 or 1
+    return kiss() & 1;
+  }
+  inline size_t index(size_t n) {
+    // Draw random integer between 0 and n-1 where n is at most the number of data points you have
+    return kiss() % n;
+  }
+  inline void set_seed(uint64_t seed) {
+    x = seed;
+  }
+};
+
+}
+
+#endif
+// vim: tabstop=2 shiftwidth=2

data/vendor/annoy/mman.h

@@ -0,0 +1,242 @@
+
+// This is from https://code.google.com/p/mman-win32/
+// 
+// Licensed under MIT
+
+#ifndef _MMAN_WIN32_H
+#define _MMAN_WIN32_H
+
+#ifndef _WIN32_WINNT		// Allow use of features specific to Windows XP or later.                   
+#define _WIN32_WINNT 0x0501	// Change this to the appropriate value to target other versions of Windows.
+#endif						
+
+#include <sys/types.h>
+#include <windows.h>
+#include <errno.h>
+#include <io.h>
+
+#define PROT_NONE       0
+#define PROT_READ       1
+#define PROT_WRITE      2
+#define PROT_EXEC       4
+
+#define MAP_FILE        0
+#define MAP_SHARED      1
+#define MAP_PRIVATE     2
+#define MAP_TYPE        0xf
+#define MAP_FIXED       0x10
+#define MAP_ANONYMOUS   0x20
+#define MAP_ANON        MAP_ANONYMOUS
+
+#define MAP_FAILED      ((void *)-1)
+
+/* Flags for msync. */
+#define MS_ASYNC        1
+#define MS_SYNC         2
+#define MS_INVALIDATE   4
+
+#ifndef FILE_MAP_EXECUTE
+#define FILE_MAP_EXECUTE    0x0020
+#endif
+
+static int __map_mman_error(const DWORD err, const int deferr)
+{
+    if (err == 0)
+        return 0;
+    //TODO: implement
+    return err;
+}
+
+static DWORD __map_mmap_prot_page(const int prot)
+{
+    DWORD protect = 0;
+    
+    if (prot == PROT_NONE)
+        return protect;
+        
+    if ((prot & PROT_EXEC) != 0)
+    {
+        protect = ((prot & PROT_WRITE) != 0) ? 
+                    PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
+    }
+    else
+    {
+        protect = ((prot & PROT_WRITE) != 0) ?
+                    PAGE_READWRITE : PAGE_READONLY;
+    }
+    
+    return protect;
+}
+
+static DWORD __map_mmap_prot_file(const int prot)
+{
+    DWORD desiredAccess = 0;
+    
+    if (prot == PROT_NONE)
+        return desiredAccess;
+        
+    if ((prot & PROT_READ) != 0)
+        desiredAccess |= FILE_MAP_READ;
+    if ((prot & PROT_WRITE) != 0)
+        desiredAccess |= FILE_MAP_WRITE;
+    if ((prot & PROT_EXEC) != 0)
+        desiredAccess |= FILE_MAP_EXECUTE;
+    
+    return desiredAccess;
+}
+
+inline void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off)
+{
+    HANDLE fm, h;
+    
+    void * map = MAP_FAILED;
+    
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable: 4293)
+#endif
+
+    const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? 
+                    (DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
+    const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
+                    (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
+    const DWORD protect = __map_mmap_prot_page(prot);
+    const DWORD desiredAccess = __map_mmap_prot_file(prot);
+
+    const off_t maxSize = off + (off_t)len;
+
+    const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ? 
+                    (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
+    const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
+                    (DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+    errno = 0;
+    
+    if (len == 0 
+        /* Unsupported flag combinations */
+        || (flags & MAP_FIXED) != 0
+        /* Usupported protection combinations */
+        || prot == PROT_EXEC)
+    {
+        errno = EINVAL;
+        return MAP_FAILED;
+    }
+    
+    h = ((flags & MAP_ANONYMOUS) == 0) ? 
+                    (HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;
+
+    if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
+    {
+        errno = EBADF;
+        return MAP_FAILED;
+    }
+
+    fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
+
+    if (fm == NULL)
+    {
+        errno = __map_mman_error(GetLastError(), EPERM);
+        return MAP_FAILED;
+    }
+  
+    map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
+
+    CloseHandle(fm);
+  
+    if (map == NULL)
+    {
+        errno = __map_mman_error(GetLastError(), EPERM);
+        return MAP_FAILED;
+    }
+
+    return map;
+}
+
+inline int munmap(void *addr, size_t len)
+{
+    if (UnmapViewOfFile(addr))
+        return 0;
+        
+    errno =  __map_mman_error(GetLastError(), EPERM);
+    
+    return -1;
+}
+
+inline int mprotect(void *addr, size_t len, int prot)
+{
+    DWORD newProtect = __map_mmap_prot_page(prot);
+    DWORD oldProtect = 0;
+    
+    if (VirtualProtect(addr, len, newProtect, &oldProtect))
+        return 0;
+    
+    errno =  __map_mman_error(GetLastError(), EPERM);
+    
+    return -1;
+}
+
+inline int msync(void *addr, size_t len, int flags)
+{
+    if (FlushViewOfFile(addr, len))
+        return 0;
+    
+    errno =  __map_mman_error(GetLastError(), EPERM);
+    
+    return -1;
+}
+
+inline int mlock(const void *addr, size_t len)
+{
+    if (VirtualLock((LPVOID)addr, len))
+        return 0;
+        
+    errno =  __map_mman_error(GetLastError(), EPERM);
+    
+    return -1;
+}
+
+inline int munlock(const void *addr, size_t len)
+{
+    if (VirtualUnlock((LPVOID)addr, len))
+        return 0;
+        
+    errno =  __map_mman_error(GetLastError(), EPERM);
+    
+    return -1;
+}
+
+#if !defined(__MINGW32__)
+inline int ftruncate(const int fd, const int64_t size) {
+    if (fd < 0) {
+        errno = EBADF;
+        return -1;
+    }
+
+    HANDLE h = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
+    LARGE_INTEGER li_start, li_size;
+    li_start.QuadPart = static_cast<int64_t>(0);
+    li_size.QuadPart = size;
+    if (SetFilePointerEx(h, li_start, NULL, FILE_CURRENT) == ~0 ||
+        SetFilePointerEx(h, li_size, NULL, FILE_BEGIN) == ~0 ||
+        !SetEndOfFile(h)) {
+        unsigned long error = GetLastError();
+        fprintf(stderr, "I/O error while truncating: %lu\n", error);
+        switch (error) {
+            case ERROR_INVALID_HANDLE:
+                errno = EBADF;
+                break;
+            default:
+                errno = EIO;
+                break;
+        }
+        return -1;
+    }        
+    return 0;
+}
+#endif
+
+#endif 

data/vendor/hnswlib/bruteforce.h

@@ -0,0 +1,152 @@
+#pragma once
+#include <unordered_map>
+#include <fstream>
+#include <mutex>
+#include <algorithm>
+
+namespace hnswlib {
+    template<typename dist_t>
+    class BruteforceSearch : public AlgorithmInterface<dist_t> {
+    public:
+        BruteforceSearch(SpaceInterface <dist_t> *s) {
+
+        }
+        BruteforceSearch(SpaceInterface<dist_t> *s, const std::string &location) {
+            loadIndex(location, s);
+        }
+
+        BruteforceSearch(SpaceInterface <dist_t> *s, size_t maxElements) {
+            maxelements_ = maxElements;
+            data_size_ = s->get_data_size();
+            fstdistfunc_ = s->get_dist_func();
+            dist_func_param_ = s->get_dist_func_param();
+            size_per_element_ = data_size_ + sizeof(labeltype);
+            data_ = (char *) malloc(maxElements * size_per_element_);
+            if (data_ == nullptr)
+                std::runtime_error("Not enough memory: BruteforceSearch failed to allocate data");
+            cur_element_count = 0;
+        }
+
+        ~BruteforceSearch() {
+            free(data_);
+        }
+
+        char *data_;
+        size_t maxelements_;
+        size_t cur_element_count;
+        size_t size_per_element_;
+
+        size_t data_size_;
+        DISTFUNC <dist_t> fstdistfunc_;
+        void *dist_func_param_;
+        std::mutex index_lock;
+
+        std::unordered_map<labeltype,size_t > dict_external_to_internal;
+
+        void addPoint(const void *datapoint, labeltype label) {
+
+            int idx;
+            {
+                std::unique_lock<std::mutex> lock(index_lock);
+
+
+
+                auto search=dict_external_to_internal.find(label);
+                if (search != dict_external_to_internal.end()) {
+                    idx=search->second;
+                }
+                else{
+                    if (cur_element_count >= maxelements_) {
+                        throw std::runtime_error("The number of elements exceeds the specified limit\n");
+                    }
+                    idx=cur_element_count;
+                    dict_external_to_internal[label] = idx;
+                    cur_element_count++;
+                }
+            }
+            memcpy(data_ + size_per_element_ * idx + data_size_, &label, sizeof(labeltype));
+            memcpy(data_ + size_per_element_ * idx, datapoint, data_size_);
+
+
+
+
+        };
+
+        void removePoint(labeltype cur_external) {
+            size_t cur_c=dict_external_to_internal[cur_external];
+
+            dict_external_to_internal.erase(cur_external);
+
+            labeltype label=*((labeltype*)(data_ + size_per_element_ * (cur_element_count-1) + data_size_));
+            dict_external_to_internal[label]=cur_c;
+            memcpy(data_ + size_per_element_ * cur_c,
+                   data_ + size_per_element_ * (cur_element_count-1),
+                   data_size_+sizeof(labeltype));
+            cur_element_count--;
+
+        }
+
+
+        std::priority_queue<std::pair<dist_t, labeltype >>
+        searchKnn(const void *query_data, size_t k) const {
+            std::priority_queue<std::pair<dist_t, labeltype >> topResults;
+            if (cur_element_count == 0) return topResults;
+            for (int i = 0; i < k; i++) {
+                dist_t dist = fstdistfunc_(query_data, data_ + size_per_element_ * i, dist_func_param_);
+                topResults.push(std::pair<dist_t, labeltype>(dist, *((labeltype *) (data_ + size_per_element_ * i +
+                                                                                    data_size_))));
+            }
+            dist_t lastdist = topResults.top().first;
+            for (int i = k; i < cur_element_count; i++) {
+                dist_t dist = fstdistfunc_(query_data, data_ + size_per_element_ * i, dist_func_param_);
+                if (dist <= lastdist) {
+                    topResults.push(std::pair<dist_t, labeltype>(dist, *((labeltype *) (data_ + size_per_element_ * i +
+                                                                                        data_size_))));
+                    if (topResults.size() > k)
+                        topResults.pop();
+                    lastdist = topResults.top().first;
+                }
+
+            }
+            return topResults;
+        };
+
+        void saveIndex(const std::string &location) {
+            std::ofstream output(location, std::ios::binary);
+            std::streampos position;
+
+            writeBinaryPOD(output, maxelements_);
+            writeBinaryPOD(output, size_per_element_);
+            writeBinaryPOD(output, cur_element_count);
+
+            output.write(data_, maxelements_ * size_per_element_);
+
+            output.close();
+        }
+
+        void loadIndex(const std::string &location, SpaceInterface<dist_t> *s) {
+
+
+            std::ifstream input(location, std::ios::binary);
+            std::streampos position;
+
+            readBinaryPOD(input, maxelements_);
+            readBinaryPOD(input, size_per_element_);
+            readBinaryPOD(input, cur_element_count);
+
+            data_size_ = s->get_data_size();
+            fstdistfunc_ = s->get_dist_func();
+            dist_func_param_ = s->get_dist_func_param();
+            size_per_element_ = data_size_ + sizeof(labeltype);
+            data_ = (char *) malloc(maxelements_ * size_per_element_);
+            if (data_ == nullptr)
+                std::runtime_error("Not enough memory: loadIndex failed to allocate data");
+
+            input.read(data_, maxelements_ * size_per_element_);
+
+            input.close();
+
+        }
+
+    };
+}