logosdb 0.7.8 → 0.7.11

package/deps/core/src/metadata.cpp

@@ -0,0 +1,266 @@
+#include "metadata.h"
+
+#include <fcntl.h>
+#include <sys/types.h>
+
+#include <cerrno>
+#include <cstring>
+#include <fstream>
+#include <nlohmann/json.hpp>
+
+#ifdef _WIN32
+#include <io.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace logosdb
+{
+namespace internal
+{
+
+using json = nlohmann::json;
+
+MetadataStore::~MetadataStore()
+{
+    close();
+}
+
+bool MetadataStore::open(const std::string& path, std::string& err)
+{
+    close();
+    path_ = path;
+
+#ifdef _WIN32
+    int flags = O_RDWR | O_CREAT | O_APPEND | O_BINARY;
+#else
+    int flags = O_RDWR | O_CREAT | O_APPEND;
+#endif
+    fd_ = ::open(path.c_str(), flags, 0644);
+    if (fd_ < 0)
+    {
+        err = std::string("open meta: ") + strerror(errno);
+        return false;
+    }
+
+    std::ifstream in(path);
+    if (in.good())
+    {
+        std::string line;
+        while (std::getline(in, line))
+        {
+            if (line.empty())
+                continue;
+
+            // Try to parse as JSON
+            json j;
+            try
+            {
+                j = json::parse(line);
+            }
+            catch (const json::exception& e)
+            {
+                // Invalid JSON line - skip but don't fail
+                continue;
+            }
+
+            // Tombstone record: {"op":"del","id":N}
+            if (j.contains("op") && j["op"] == "del" && j.contains("id"))
+            {
+                uint64_t id = j["id"].get<uint64_t>();
+                if (id < rows_.size() && !rows_[id].deleted)
+                {
+                    rows_[id].deleted = true;
+                    ++num_deleted_;
+                }
+                continue;
+            }
+
+            // Data row: {"text":"...","ts":"..."}
+            MetaRow r;
+            if (j.contains("text"))
+            {
+                r.text = j["text"].get<std::string>();
+            }
+            if (j.contains("ts"))
+            {
+                r.timestamp = j["ts"].get<std::string>();
+            }
+            rows_.push_back(std::move(r));
+        }
+    }
+    return true;
+}
+
+void MetadataStore::close()
+{
+    if (fd_ >= 0)
+    {
+#ifdef _WIN32
+        _close(fd_);
+#else
+        ::close(fd_);
+#endif
+        fd_ = -1;
+    }
+    rows_.clear();
+    num_deleted_ = 0;
+    path_.clear();
+}
+
+uint64_t MetadataStore::append(const char* text, const char* timestamp, std::string& err)
+{
+    if (fd_ < 0)
+    {
+        err = "meta not open";
+        return UINT64_MAX;
+    }
+
+    json j;
+    j["text"] = text ? text : "";
+    j["ts"] = timestamp ? timestamp : "";
+    std::string line = j.dump() + "\n";
+
+#ifdef _WIN32
+    int written = _write(fd_, line.data(), (int)line.size());
+    if (written != (int)line.size())
+    {
+#else
+    ssize_t written = ::write(fd_, line.data(), line.size());
+    if (written != (ssize_t)line.size())
+    {
+#endif
+        err = std::string("write meta: ") + strerror(errno);
+        return UINT64_MAX;
+    }
+
+    uint64_t id = rows_.size();
+    rows_.push_back({text ? text : "", timestamp ? timestamp : ""});
+    return id;
+}
+
+uint64_t MetadataStore::append_batch(const char* const* texts,
+                                     const char* const* timestamps,
+                                     int n,
+                                     std::string& err)
+{
+    if (fd_ < 0)
+    {
+        err = "meta not open";
+        return UINT64_MAX;
+    }
+    if (n <= 0)
+    {
+        return rows_.size();
+    }
+
+    // Build all JSON lines and write in a single batch
+    std::string batch;
+    batch.reserve(n * 64);  // rough estimate
+
+    for (int i = 0; i < n; ++i)
+    {
+        json j;
+        j["text"] = texts && texts[i] ? texts[i] : "";
+        j["ts"] = timestamps && timestamps[i] ? timestamps[i] : "";
+        batch += j.dump();
+        batch += "\n";
+    }
+
+#ifdef _WIN32
+    int written = _write(fd_, batch.data(), (int)batch.size());
+    if (written != (int)batch.size())
+    {
+#else
+    ssize_t written = ::write(fd_, batch.data(), batch.size());
+    if (written != (ssize_t)batch.size())
+    {
+#endif
+        err = std::string("write meta batch: ") + strerror(errno);
+        return UINT64_MAX;
+    }
+
+    uint64_t start_id = rows_.size();
+    for (int i = 0; i < n; ++i)
+    {
+        rows_.push_back(
+            {texts && texts[i] ? texts[i] : "", timestamps && timestamps[i] ? timestamps[i] : ""});
+    }
+    return start_id;
+}
+
+bool MetadataStore::mark_deleted(uint64_t id, std::string& err)
+{
+    if (fd_ < 0)
+    {
+        err = "meta not open";
+        return false;
+    }
+    if (id >= rows_.size())
+    {
+        err = "delete: id out of range";
+        return false;
+    }
+    if (rows_[id].deleted)
+    {
+        err = "delete: id already deleted";
+        return false;
+    }
+
+    json j;
+    j["op"] = "del";
+    j["id"] = id;
+    std::string line = j.dump() + "\n";
+
+#ifdef _WIN32
+    int written = _write(fd_, line.data(), (int)line.size());
+    if (written != (int)line.size())
+    {
+#else
+    ssize_t written = ::write(fd_, line.data(), line.size());
+    if (written != (ssize_t)line.size())
+    {
+#endif
+        err = std::string("write tombstone: ") + strerror(errno);
+        return false;
+    }
+
+    rows_[id].deleted = true;
+    ++num_deleted_;
+    return true;
+}
+
+std::vector<uint64_t> MetadataStore::deleted_ids() const
+{
+    std::vector<uint64_t> out;
+    out.reserve(num_deleted_);
+    for (size_t i = 0; i < rows_.size(); ++i)
+    {
+        if (rows_[i].deleted)
+            out.push_back((uint64_t)i);
+    }
+    return out;
+}
+
+bool MetadataStore::sync(std::string& err)
+{
+    if (fd_ < 0)
+    {
+        err = "meta not open";
+        return false;
+    }
+#ifdef _WIN32
+    if (_commit(fd_) != 0)
+    {
+#else
+    if (::fsync(fd_) != 0)
+    {
+#endif
+        err = std::string("fsync meta: ") + strerror(errno);
+        return false;
+    }
+    return true;
+}
+
+}  // namespace internal
+}  // namespace logosdb

package/deps/core/src/metadata.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace logosdb
+{
+namespace internal
+{
+
+struct MetaRow
+{
+    std::string text;
+    std::string timestamp;
+    bool deleted = false;
+};
+
+// Append-only JSONL metadata store.
+//
+// Data rows:      {"text":"...","ts":"..."}
+// Tombstone rows: {"op":"del","id":N}
+//
+// Tombstones mark an earlier data row as logically deleted. They do not
+// occupy a row index themselves.
+class MetadataStore
+{
+  public:
+    MetadataStore() = default;
+    ~MetadataStore();
+
+    MetadataStore(const MetadataStore&) = delete;
+    MetadataStore& operator=(const MetadataStore&) = delete;
+
+    bool open(const std::string& path, std::string& err);
+    void close();
+
+    uint64_t append(const char* text, const char* timestamp, std::string& err);
+
+    /* Append n metadata rows efficiently. Returns the starting id, or UINT64_MAX on error. */
+    uint64_t
+    append_batch(const char* const* texts, const char* const* timestamps, int n, std::string& err);
+
+    // Append a tombstone for `id`. Returns false if the id is out of range
+    // or already deleted. `err` is set on failure.
+    bool mark_deleted(uint64_t id, std::string& err);
+
+    size_t count() const { return rows_.size(); }
+    size_t deleted_count() const { return num_deleted_; }
+
+    bool is_deleted(uint64_t id) const { return id < rows_.size() && rows_[id].deleted; }
+
+    const MetaRow* row(uint64_t idx) const { return idx < rows_.size() ? &rows_[idx] : nullptr; }
+
+    // Iterate over all currently-tombstoned ids. Used by the DB on open to
+    // re-apply deletion marks to a freshly-rebuilt HNSW index.
+    std::vector<uint64_t> deleted_ids() const;
+
+    bool sync(std::string& err);
+
+  private:
+    std::string path_;
+    int fd_ = -1;
+    std::vector<MetaRow> rows_;
+    size_t num_deleted_ = 0;
+};
+
+}  // namespace internal
+}  // namespace logosdb

package/deps/core/src/platform.cpp

@@ -0,0 +1,342 @@
+#include "platform.h"
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <cerrno>
+#include <cstring>
+
+#ifdef _WIN32
+#include <io.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace logosdb
+{
+namespace internal
+{
+namespace platform
+{
+
+bool file_exists(const std::string& path)
+{
+#ifdef _WIN32
+    struct _stat st;
+    return _stat(path.c_str(), &st) == 0;
+#else
+    struct stat st;
+    return stat(path.c_str(), &st) == 0;
+#endif
+}
+
+#ifdef _WIN32
+// Windows memory mapping implementation
+
+bool mmap_open(const std::string& path, size_t& out_size, MappedFile& out_map, std::string& err)
+{
+    out_map = {};  // Clear
+
+    HANDLE file_handle = CreateFileA(path.c_str(),
+                                     GENERIC_READ,
+                                     FILE_SHARE_READ,
+                                     nullptr,
+                                     OPEN_EXISTING,
+                                     FILE_ATTRIBUTE_NORMAL,
+                                     nullptr);
+
+    if (file_handle == INVALID_HANDLE_VALUE)
+    {
+        err = "CreateFileA failed: " + std::to_string(GetLastError());
+        return false;
+    }
+
+    LARGE_INTEGER file_size;
+    if (!GetFileSizeEx(file_handle, &file_size))
+    {
+        err = "GetFileSizeEx failed: " + std::to_string(GetLastError());
+        CloseHandle(file_handle);
+        return false;
+    }
+
+    if (file_size.QuadPart == 0)
+    {
+        // Empty file - no mapping needed
+        out_map.file_handle = file_handle;
+        out_map.data = nullptr;
+        out_map.size = 0;
+        out_size = 0;
+        return true;
+    }
+
+    HANDLE map_handle = CreateFileMapping(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr);
+
+    if (map_handle == INVALID_HANDLE_VALUE)
+    {
+        err = "CreateFileMapping failed: " + std::to_string(GetLastError());
+        CloseHandle(file_handle);
+        return false;
+    }
+
+    void* data = MapViewOfFile(map_handle, FILE_MAP_READ, 0, 0, 0);
+
+    if (!data)
+    {
+        err = "MapViewOfFile failed: " + std::to_string(GetLastError());
+        CloseHandle(map_handle);
+        CloseHandle(file_handle);
+        return false;
+    }
+
+    out_map.file_handle = file_handle;
+    out_map.map_handle = map_handle;
+    out_map.data = static_cast<uint8_t*>(data);
+    out_map.size = static_cast<size_t>(file_size.QuadPart);
+    out_size = out_map.size;
+    return true;
+}
+
+void mmap_close(MappedFile& map)
+{
+    if (map.data)
+    {
+        UnmapViewOfFile(map.data);
+        map.data = nullptr;
+    }
+    if (map.map_handle != INVALID_HANDLE_VALUE)
+    {
+        CloseHandle(map.map_handle);
+        map.map_handle = INVALID_HANDLE_VALUE;
+    }
+    if (map.file_handle != INVALID_HANDLE_VALUE)
+    {
+        CloseHandle(map.file_handle);
+        map.file_handle = INVALID_HANDLE_VALUE;
+    }
+    map.size = 0;
+}
+
+bool mmap_resize(MappedFile& map, size_t new_size, std::string& err)
+{
+    // On Windows, we need to close and reopen the mapping
+    // This is used when the file grows
+    mmap_close(map);
+
+    // For simplicity on Windows, we don't remap here - the caller
+    // should use mmap_open again after closing the file
+    err = "mmap_resize not implemented for Windows - use close/reopen pattern";
+    return false;
+}
+
+bool mmap_reserve(const std::string& path,
+                  size_t reserve_size,
+                  MappedFile& out_map,
+                  std::string& err)
+{
+    out_map = {};  // Clear
+
+    // First, get the current file size
+    HANDLE file_handle =
+        CreateFileA(path.c_str(),
+                    GENERIC_READ,
+                    FILE_SHARE_READ | FILE_SHARE_WRITE,  // Allow others to write (file growth)
+                    nullptr,
+                    OPEN_EXISTING,
+                    FILE_ATTRIBUTE_NORMAL,
+                    nullptr);
+
+    if (file_handle == INVALID_HANDLE_VALUE)
+    {
+        err = "CreateFileA failed: " + std::to_string(GetLastError());
+        return false;
+    }
+
+    LARGE_INTEGER file_size;
+    if (!GetFileSizeEx(file_handle, &file_size))
+    {
+        err = "GetFileSizeEx failed: " + std::to_string(GetLastError());
+        CloseHandle(file_handle);
+        return false;
+    }
+
+    // Create a file mapping that reserves virtual address space
+    // We use a large maximum size for reservation, but only commit what's needed
+    HANDLE map_handle =
+        CreateFileMapping(file_handle,
+                          nullptr,
+                          PAGE_READONLY,
+                          0,
+                          static_cast<DWORD>(reserve_size),  // Maximum size for reservation
+                          nullptr);
+
+    if (map_handle == INVALID_HANDLE_VALUE)
+    {
+        err = "CreateFileMapping failed: " + std::to_string(GetLastError());
+        CloseHandle(file_handle);
+        return false;
+    }
+
+    // Map only the current file size initially
+    void* data = MapViewOfFile(map_handle,
+                               FILE_MAP_READ,
+                               0,
+                               0,
+                               static_cast<size_t>(file_size.QuadPart)  // Initial view size
+    );
+
+    if (!data)
+    {
+        err = "MapViewOfFile failed: " + std::to_string(GetLastError());
+        CloseHandle(map_handle);
+        CloseHandle(file_handle);
+        return false;
+    }
+
+    out_map.file_handle = file_handle;
+    out_map.map_handle = map_handle;
+    out_map.data = static_cast<uint8_t*>(data);
+    out_map.size = static_cast<size_t>(file_size.QuadPart);
+    return true;
+}
+
+size_t mmap_commit(MappedFile& map, size_t file_size)
+{
+#ifdef _WIN32
+    // On Windows with file mapping, the view automatically extends
+    // when the file grows (as long as we're within the mapping's max size)
+    // We just need to unmap and remap to the new size
+    if (map.data)
+    {
+        UnmapViewOfFile(map.data);
+    }
+
+    void* data = MapViewOfFile(map.map_handle,
+                               FILE_MAP_READ,
+                               0,
+                               0,
+                               file_size  // New view size
+    );
+
+    if (!data)
+    {
+        return map.size;  // Failed, keep old size
+    }
+
+    map.data = static_cast<uint8_t*>(data);
+    map.size = file_size;
+    return map.size;
+#else
+    (void)map;
+    return file_size;
+#endif
+}
+
+#else
+// POSIX memory mapping implementation (Linux/macOS)
+
+bool mmap_open(const std::string& path, size_t& out_size, MappedFile& out_map, std::string& err)
+{
+    out_map = {};  // Clear
+
+    int fd = ::open(path.c_str(), O_RDONLY);
+    if (fd < 0)
+    {
+        err = std::string("open: ") + strerror(errno);
+        return false;
+    }
+
+    struct stat st;
+    if (fstat(fd, &st) != 0)
+    {
+        err = std::string("fstat: ") + strerror(errno);
+        ::close(fd);
+        return false;
+    }
+
+    size_t size = static_cast<size_t>(st.st_size);
+
+    if (size == 0)
+    {
+        // Empty file - no mapping needed
+        out_map.fd = fd;
+        out_map.data = nullptr;
+        out_map.size = 0;
+        out_size = 0;
+        return true;
+    }
+
+    void* data = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);
+    if (data == MAP_FAILED)
+    {
+        err = std::string("mmap: ") + strerror(errno);
+        ::close(fd);
+        return false;
+    }
+
+    out_map.fd = fd;
+    out_map.data = static_cast<uint8_t*>(data);
+    out_map.size = size;
+    out_size = size;
+    return true;
+}
+
+void mmap_close(MappedFile& map)
+{
+    if (map.data && map.size > 0)
+    {
+        munmap(map.data, map.size);
+    }
+    if (map.fd >= 0)
+    {
+        ::close(map.fd);
+    }
+    map.data = nullptr;
+    map.fd = -1;
+    map.size = 0;
+}
+
+bool mmap_resize(MappedFile& map, size_t new_size, std::string& err)
+{
+    // On POSIX, unmap and remap
+    if (map.data && map.size > 0)
+    {
+        munmap(map.data, map.size);
+    }
+
+    void* data = mmap(nullptr, new_size, PROT_READ, MAP_SHARED, map.fd, 0);
+    if (data == MAP_FAILED)
+    {
+        err = std::string("mmap resize: ") + strerror(errno);
+        return false;
+    }
+
+    map.data = static_cast<uint8_t*>(data);
+    map.size = new_size;
+    return true;
+}
+
+// Stub implementations for POSIX - reservation is handled directly in storage.cpp
+bool mmap_reserve(const std::string& path,
+                  size_t reserve_size,
+                  MappedFile& out_map,
+                  std::string& err)
+{
+    (void)path;
+    (void)reserve_size;
+    (void)out_map;
+    err = "mmap_reserve not implemented for POSIX - use storage.cpp implementation";
+    return false;
+}
+
+size_t mmap_commit(MappedFile& map, size_t file_size)
+{
+    (void)map;
+    return file_size;
+}
+
+#endif
+
+}  // namespace platform
+}  // namespace internal
+}  // namespace logosdb