logosdb 0.7.7 → 0.7.10

package/deps/core/src/wal.h

@@ -0,0 +1,99 @@
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace logosdb
+{
+namespace internal
+{
+
+// Write-Ahead Log for atomic Put operations.
+//
+// The WAL records a Put intent before any store is modified. On crash,
+// incomplete entries are replayed on the next open() to ensure consistency
+// across vector storage, metadata, and HNSW index.
+//
+// File format (append-only binary):
+//   [magic "LOGW" (4 bytes)]
+//   [version uint32 (4 bytes)]
+//   [sequence of WALEntry records]
+//
+// Each WALEntry:
+//   [state uint8: 0=pending, 1=committed]
+//   [dim uint32 (4 bytes)]
+//   [vector_bytes uint32 (4 bytes)]
+//   [vector data (dim*4 bytes)]
+//   [text_len uint32 (4 bytes)]
+//   [text (text_len bytes)]
+//   [ts_len uint32 (4 bytes)]
+//   [timestamp (ts_len bytes)]
+//   [expected_id uint64 (8 bytes)]
+
+enum class WALState : uint8_t
+{
+    PENDING = 0,
+    COMMITTED = 1,
+    ABORTED = 2
+};
+
+struct WALEntry
+{
+    WALState state = WALState::PENDING;
+    uint32_t dim = 0;
+    std::vector<float> vector;
+    std::string text;
+    std::string timestamp;
+    uint64_t expected_id = 0;  // Expected row id (for validation)
+};
+
+class WriteAheadLog
+{
+  public:
+    WriteAheadLog() = default;
+    ~WriteAheadLog();
+
+    WriteAheadLog(const WriteAheadLog&) = delete;
+    WriteAheadLog& operator=(const WriteAheadLog&) = delete;
+
+    // Open or create WAL file at the given path.
+    bool open(const std::string& path, std::string& err);
+    void close();
+
+    // Append a new pending entry. Returns the entry offset in the file
+    // (needed to mark committed later), or -1 on error.
+    int64_t append_pending(const float* vec,
+                           int dim,
+                           const char* text,
+                           const char* timestamp,
+                           uint64_t expected_id,
+                           std::string& err);
+
+    // Mark an entry as committed by its offset.
+    bool mark_committed(int64_t offset, std::string& err);
+
+    // Replay all pending entries, calling the provided function for each.
+    // Entries are marked committed after successful replay.
+    // Returns number of entries replayed, or -1 on error.
+    int replay_pending(std::function<bool(const WALEntry&, std::string&)> replay_fn,
+                       std::string& err);
+
+    // Sync WAL to disk.
+    bool sync(std::string& err);
+
+    // Get count of pending entries (for debugging/metrics).
+    size_t pending_count() const { return pending_count_; }
+
+  private:
+    bool read_entry_at(int64_t offset, WALEntry& entry, std::string& err);
+    bool write_state_at(int64_t offset, WALState state, std::string& err);
+
+    std::string path_;
+    int fd_ = -1;
+    size_t pending_count_ = 0;
+};
+
+}  // namespace internal
+}  // namespace logosdb

package/deps/core/third_party/hnswlib/hnswlib/bruteforce.h

@@ -0,0 +1,163 @@
+#pragma once
+#include <unordered_map>
+#include <fstream>
+#include <mutex>
+#include <algorithm>
+#include <assert.h>
+
+namespace hnswlib {
+template<typename dist_t>
+class BruteforceSearch : public AlgorithmInterface<dist_t> {
+ public:
+    char *data_;
+    size_t maxelements_;
+    size_t cur_element_count;
+    size_t size_per_element_;
+
+    size_t data_size_;
+    DISTFUNC <dist_t> fstdistfunc_;
+    void *dist_func_param_;
+    std::mutex index_lock;
+
+    std::unordered_map<labeltype, size_t > dict_external_to_internal;
+
+
+    BruteforceSearch(SpaceInterface <dist_t> *s)
+        : data_(nullptr),
+            maxelements_(0),
+            cur_element_count(0),
+            size_per_element_(0),
+            data_size_(0),
+            dist_func_param_(nullptr) {
+    }
+
+
+    BruteforceSearch(SpaceInterface<dist_t> *s, const std::string &location)
+        : data_(nullptr),
+            maxelements_(0),
+            cur_element_count(0),
+            size_per_element_(0),
+            data_size_(0),
+            dist_func_param_(nullptr) {
+        loadIndex(location, s);
+    }
+
+
+    BruteforceSearch(SpaceInterface <dist_t> *s, size_t maxElements) {
+        maxelements_ = maxElements;
+        data_size_ = s->get_data_size();
+        fstdistfunc_ = s->get_dist_func();
+        dist_func_param_ = s->get_dist_func_param();
+        size_per_element_ = data_size_ + sizeof(labeltype);
+        data_ = (char *) malloc(maxElements * size_per_element_);
+        if (data_ == nullptr)
+            throw std::runtime_error("Not enough memory: BruteforceSearch failed to allocate data");
+        cur_element_count = 0;
+    }
+
+
+    ~BruteforceSearch() {
+        free(data_);
+    }
+
+
+    void addPoint(const void *datapoint, labeltype label, bool replace_deleted = false) {
+        int idx;
+        {
+            std::unique_lock<std::mutex> lock(index_lock);
+
+            auto search = dict_external_to_internal.find(label);
+            if (search != dict_external_to_internal.end()) {
+                idx = search->second;
+            } else {
+                if (cur_element_count >= maxelements_) {
+                    throw std::runtime_error("The number of elements exceeds the specified limit\n");
+                }
+                idx = cur_element_count;
+                dict_external_to_internal[label] = idx;
+                cur_element_count++;
+            }
+        }
+        memcpy(data_ + size_per_element_ * idx + data_size_, &label, sizeof(labeltype));
+        memcpy(data_ + size_per_element_ * idx, datapoint, data_size_);
+    }
+
+
+    void removePoint(labeltype cur_external) {
+        std::unique_lock<std::mutex> lock(index_lock);
+
+        auto found = dict_external_to_internal.find(cur_external);
+        if (found == dict_external_to_internal.end()) {
+            return;
+        }
+
+        dict_external_to_internal.erase(found);
+
+        size_t cur_c = found->second;
+        labeltype label = *((labeltype*)(data_ + size_per_element_ * (cur_element_count-1) + data_size_));
+        dict_external_to_internal[label] = cur_c;
+        memcpy(data_ + size_per_element_ * cur_c,
+                data_ + size_per_element_ * (cur_element_count-1),
+                data_size_+sizeof(labeltype));
+        cur_element_count--;
+    }
+
+
+    std::priority_queue<std::pair<dist_t, labeltype >>
+    searchKnn(const void *query_data, size_t k, BaseFilterFunctor* isIdAllowed = nullptr) const {
+        assert(k <= cur_element_count);
+        std::priority_queue<std::pair<dist_t, labeltype >> topResults;
+        dist_t lastdist = std::numeric_limits<dist_t>::max();
+        for (int i = 0; i < cur_element_count; i++) {
+            dist_t dist = fstdistfunc_(query_data, data_ + size_per_element_ * i, dist_func_param_);
+            if (dist <= lastdist || topResults.size() < k) {
+                labeltype label = *((labeltype *) (data_ + size_per_element_ * i + data_size_));
+                if ((!isIdAllowed) || (*isIdAllowed)(label)) {
+                    topResults.emplace(dist, label);
+                    if (topResults.size() > k)
+                        topResults.pop();
+                    if (!topResults.empty())
+                        lastdist = topResults.top().first;
+                }
+            }
+        }
+        return topResults;
+    }
+
+
+    void saveIndex(const std::string &location) {
+        std::ofstream output(location, std::ios::binary);
+        std::streampos position;
+
+        writeBinaryPOD(output, maxelements_);
+        writeBinaryPOD(output, size_per_element_);
+        writeBinaryPOD(output, cur_element_count);
+
+        output.write(data_, maxelements_ * size_per_element_);
+
+        output.close();
+    }
+
+
+    void loadIndex(const std::string &location, SpaceInterface<dist_t> *s) {
+        std::ifstream input(location, std::ios::binary);
+        std::streampos position;
+
+        readBinaryPOD(input, maxelements_);
+        readBinaryPOD(input, size_per_element_);
+        readBinaryPOD(input, cur_element_count);
+
+        data_size_ = s->get_data_size();
+        fstdistfunc_ = s->get_dist_func();
+        dist_func_param_ = s->get_dist_func_param();
+        size_per_element_ = data_size_ + sizeof(labeltype);
+        data_ = (char *) malloc(maxelements_ * size_per_element_);
+        if (data_ == nullptr)
+            throw std::runtime_error("Not enough memory: loadIndex failed to allocate data");
+
+        input.read(data_, maxelements_ * size_per_element_);
+
+        input.close();
+    }
+};
+}  // namespace hnswlib