kenaz 0.3.2 → 0.3.3

package/core/.kenaz/knowledge/ladybug_2c51da1e_shadow_file_frame_group.patch

@@ -1,163 +1,163 @@
-diff --git a/src/include/storage/file_handle.h b/src/include/storage/file_handle.h
-index bff124907..ec9cf3e6a 100644
---- a/src/include/storage/file_handle.h
-+++ b/src/include/storage/file_handle.h
-@@ -104,6 +104,9 @@ class FileHandle {
-     common::page_idx_t getNumPages() const { return numPages; }
-     common::FileInfo* getFileInfo() const { return fileInfo.get(); }
-     void resetFileInfo() { fileInfo.reset(); }
-+    void setFileInfo(std::unique_ptr<common::FileInfo> fileInfo_) {
-+        fileInfo = std::move(fileInfo_);
-+    }
- 
-     uint64_t getPageSize() const {
-         return isLargePaged() ? common::TEMP_PAGE_SIZE : common::LBUG_PAGE_SIZE;
-diff --git a/src/storage/file_handle.cpp b/src/storage/file_handle.cpp
-index 5f141f805..ae4f4f327 100644
---- a/src/storage/file_handle.cpp
-+++ b/src/storage/file_handle.cpp
-@@ -72,6 +72,10 @@ page_idx_t FileHandle::addNewPages(page_idx_t numNewPages) {
- page_idx_t FileHandle::addNewPageWithoutLock() {
-     if (numPages == pageCapacity) {
-         addNewPageGroupWithoutLock();
-+    } else if (numPages >= static_cast<page_idx_t>(pageStates.size())) {
-+        // Reusing retained capacity after a truncate: re-expose the already-allocated slots.
-+        // ConcurrentVector::resize never deallocates, so this is cheap and leak-free.
-+        pageStates.resize(pageCapacity);
-     }
-     pageStates[numPages].resetToEvicted();
-     const auto pageIdx = numPages++;
-@@ -133,14 +137,13 @@ void FileHandle::removePageIdxAndTruncateIfNecessary(page_idx_t pageIdx) {
-         return;
-     }
-     numPages = pageIdx;
--    pageStates.resize(numPages);
--    const auto numPageGroups = getNumPageGroups();
--    if (numPageGroups == frameGroupIdxes.size()) {
--        return;
-+    // Keep frameGroupIdxes and pageCapacity at their high-water mark so truncated page groups
-+    // reuse their already-allocated VM frame groups when the file grows again. Shrinking them
-+    // here permanently leaks a VMRegion frame group per truncated page group
-+    // (VMRegion::addNewFrameGroup only grows up to max_db_size).
-+    if (pageStates.size() > numPages) {
-+        pageStates.resize(numPages);
-     }
--    DASSERT(numPageGroups < frameGroupIdxes.size());
--    frameGroupIdxes.resize(numPageGroups);
--    pageCapacity = numPageGroups * StorageConstants::PAGE_GROUP_SIZE;
- }
- 
- void FileHandle::removePageFromFrameIfNecessary(page_idx_t pageIdx) {
-diff --git a/src/storage/shadow_file.cpp b/src/storage/shadow_file.cpp
-index eaac734ab..9811166c6 100644
---- a/src/storage/shadow_file.cpp
-+++ b/src/storage/shadow_file.cpp
-@@ -191,31 +191,62 @@ void ShadowFile::flushAll(main::ClientContext& context) const {
- 
- void ShadowFile::clear(BufferManager& bm) {
-     DASSERT(shadowingFH);
--    // TODO(Guodong): We should remove shadow file here. This requires changes:
--    // 1. We need to make shadow file not going through BM.
--    // 2. We need to remove fileHandles held in BM, so that BM only keeps FH for the data file.
-+    // Evict pages, truncate to zero pages (which also truncates the on-disk file), then unlink
-+    // the path and drop the stale fd so no leftover .shadow file exists at rest (a present
-+    // .shadow is treated as "checkpoint in progress" by recovery/startup probes). The
-+    // retained FileHandle keeps its VM frame groups (see reset()); the file is re-created
-+    // lazily by getOrCreateShadowingFH on the next checkpoint, which also re-reserves the
-+    // header page.
-     bm.removeFilePagesFromFrames(*shadowingFH);
-     shadowingFH->resetToZeroPagesAndPageCapacity();
-+    shadowingFH->resetFileInfo();
-+    vfs->removeFileIfExists(shadowFilePath);
-     shadowPagesMap.clear();
-     shadowPageRecords.clear();
--    // Reserve header page.
--    shadowingFH->addNewPage();
- }
- 
- void ShadowFile::reset() {
--    shadowingFH->resetFileInfo();
--    shadowingFH = nullptr;
--    vfs->removeFileIfExists(shadowFilePath);
-+    // Keep the shadowing FileHandle (and its VM frame groups) alive across checkpoints:
-+    // each FileHandle allocates VMRegion frame groups that count against max_db_size and
-+    // are never reclaimed, so orphaning the handle here exhausted max_db_size after a
-+    // handful of checkpoints (see #924). Evict pages, truncate to zero pages (which also
-+    // truncates the on-disk file), drop the stale fd, and unlink the path so
-+    // recovery/startup probes — which treat a present .shadow file as "checkpoint in
-+    // progress" — don't trip on it. getOrCreateShadowingFH re-opens the path into the
-+    // same retained handle on the next checkpoint, so subsequent shadow pages land in a
-+    // real on-disk file visible to recovery (which replays by path).
-+    if (shadowingFH == nullptr) {
-+        return;
-+    }
-+    // If clear() already ran (the normal checkpoint flow calls both), the file was truncated,
-+    // unlinked and the fd dropped; just make sure the in-memory maps are empty.
-+    if (shadowingFH->getFileInfo() != nullptr) {
-+        bm.removeFilePagesFromFrames(*shadowingFH);
-+        shadowingFH->resetToZeroPagesAndPageCapacity();
-+        shadowingFH->resetFileInfo();
-+        vfs->removeFileIfExists(shadowFilePath);
-+    }
-+    shadowPagesMap.clear();
-+    shadowPageRecords.clear();
- }
- 
- FileHandle* ShadowFile::getOrCreateShadowingFH() {
-     if (!shadowingFH) {
-         shadowingFH = bm.getFileHandle(shadowFilePath,
-             FileHandle::O_PERSISTENT_FILE_CREATE_NOT_EXISTS, vfs, nullptr);
--        if (shadowingFH->getNumPages() == 0) {
--            // Reserve the first page for the header.
--            shadowingFH->addNewPage();
--        }
-+    } else if (shadowingFH->getFileInfo() == nullptr) {
-+        // reset() unlinked the on-disk file and dropped the stale fd (which pointed at the
-+        // unlinked inode): re-open the same retained FileHandle in place so its identity
-+        // (fileIndex, frame groups) is preserved and no new VM accounting is consumed.
-+        // Matches constructPersistentFileHandle's flags for O_PERSISTENT_FILE_CREATE_NOT_EXISTS.
-+        shadowingFH->setFileInfo(vfs->openFile(shadowFilePath,
-+            FileOpenFlags(
-+                FileFlags::WRITE | FileFlags::READ_ONLY | FileFlags::CREATE_IF_NOT_EXISTS),
-+            nullptr));
-+    }
-+    if (shadowingFH->getNumPages() == 0) {
-+        // Reserve the first page for the header.
-+        shadowingFH->addNewPage();
-     }
-     return shadowingFH;
- }
-diff --git a/test/api/system_config_test.cpp b/test/api/system_config_test.cpp
-index 638b0a685..e398ddc4b 100644
---- a/test/api/system_config_test.cpp
-+++ b/test/api/system_config_test.cpp
-@@ -101,6 +101,29 @@ TEST_F(SystemConfigTest, testMaxDBSize) {
-     }
- }
- 
-+TEST_F(SystemConfigTest, testRepeatedCheckpointsDoNotExhaustMaxDBSize) {
-+    // Regression test for https://github.com/LadybugDB/ladybug/issues/924: each CHECKPOINT
-+    // used to permanently consume ~1 VM frame group of max_db_size accounting (the shadowing
-+    // FileHandle was orphaned on every checkpoint), so ~7 checkpoints exhausted a 64 MiB cap.
-+    if (databasePath == "" || databasePath == ":memory:") {
-+        GTEST_SKIP();
-+    }
-+    systemConfig->maxDBSize = 64 * 1024 * 1024;
-+    auto db = std::make_unique<Database>(databasePath, *systemConfig);
-+    auto con = std::make_unique<Connection>(db.get());
-+    assertQuery(*con->query("CREATE NODE TABLE Node(id STRING, payload STRING, PRIMARY KEY(id))"));
-+    for (auto i = 0; i < 50; ++i) {
-+        auto res = con->query("MERGE (n:Node {id: 'node-" + std::to_string(i % 10) +
-+                              "'}) ON CREATE SET n.payload = 'x' ON MATCH SET n.payload = 'x'");
-+        ASSERT_TRUE(res->isSuccess()) << res->toString();
-+        auto checkpoint = con->query("CHECKPOINT");
-+        ASSERT_TRUE(checkpoint->isSuccess()) << checkpoint->toString();
-+    }
-+    auto count = con->query("MATCH (n:Node) RETURN COUNT(*)");
-+    ASSERT_TRUE(count->isSuccess()) << count->toString();
-+    ASSERT_EQ(TestHelper::convertResultToString(*count), std::vector<std::string>{"10"});
-+}
-+
- TEST_F(SystemConfigTest, testBufferPoolSize) {
-     systemConfig->bufferPoolSize = 1024;
-     try {
+diff --git a/src/include/storage/file_handle.h b/src/include/storage/file_handle.h
+index bff124907..ec9cf3e6a 100644
+--- a/src/include/storage/file_handle.h
++++ b/src/include/storage/file_handle.h
+@@ -104,6 +104,9 @@ class FileHandle {
+     common::page_idx_t getNumPages() const { return numPages; }
+     common::FileInfo* getFileInfo() const { return fileInfo.get(); }
+     void resetFileInfo() { fileInfo.reset(); }
++    void setFileInfo(std::unique_ptr<common::FileInfo> fileInfo_) {
++        fileInfo = std::move(fileInfo_);
++    }
+ 
+     uint64_t getPageSize() const {
+         return isLargePaged() ? common::TEMP_PAGE_SIZE : common::LBUG_PAGE_SIZE;
+diff --git a/src/storage/file_handle.cpp b/src/storage/file_handle.cpp
+index 5f141f805..ae4f4f327 100644
+--- a/src/storage/file_handle.cpp
++++ b/src/storage/file_handle.cpp
+@@ -72,6 +72,10 @@ page_idx_t FileHandle::addNewPages(page_idx_t numNewPages) {
+ page_idx_t FileHandle::addNewPageWithoutLock() {
+     if (numPages == pageCapacity) {
+         addNewPageGroupWithoutLock();
++    } else if (numPages >= static_cast<page_idx_t>(pageStates.size())) {
++        // Reusing retained capacity after a truncate: re-expose the already-allocated slots.
++        // ConcurrentVector::resize never deallocates, so this is cheap and leak-free.
++        pageStates.resize(pageCapacity);
+     }
+     pageStates[numPages].resetToEvicted();
+     const auto pageIdx = numPages++;
+@@ -133,14 +137,13 @@ void FileHandle::removePageIdxAndTruncateIfNecessary(page_idx_t pageIdx) {
+         return;
+     }
+     numPages = pageIdx;
+-    pageStates.resize(numPages);
+-    const auto numPageGroups = getNumPageGroups();
+-    if (numPageGroups == frameGroupIdxes.size()) {
+-        return;
++    // Keep frameGroupIdxes and pageCapacity at their high-water mark so truncated page groups
++    // reuse their already-allocated VM frame groups when the file grows again. Shrinking them
++    // here permanently leaks a VMRegion frame group per truncated page group
++    // (VMRegion::addNewFrameGroup only grows up to max_db_size).
++    if (pageStates.size() > numPages) {
++        pageStates.resize(numPages);
+     }
+-    DASSERT(numPageGroups < frameGroupIdxes.size());
+-    frameGroupIdxes.resize(numPageGroups);
+-    pageCapacity = numPageGroups * StorageConstants::PAGE_GROUP_SIZE;
+ }
+ 
+ void FileHandle::removePageFromFrameIfNecessary(page_idx_t pageIdx) {
+diff --git a/src/storage/shadow_file.cpp b/src/storage/shadow_file.cpp
+index eaac734ab..9811166c6 100644
+--- a/src/storage/shadow_file.cpp
++++ b/src/storage/shadow_file.cpp
+@@ -191,31 +191,62 @@ void ShadowFile::flushAll(main::ClientContext& context) const {
+ 
+ void ShadowFile::clear(BufferManager& bm) {
+     DASSERT(shadowingFH);
+-    // TODO(Guodong): We should remove shadow file here. This requires changes:
+-    // 1. We need to make shadow file not going through BM.
+-    // 2. We need to remove fileHandles held in BM, so that BM only keeps FH for the data file.
++    // Evict pages, truncate to zero pages (which also truncates the on-disk file), then unlink
++    // the path and drop the stale fd so no leftover .shadow file exists at rest (a present
++    // .shadow is treated as "checkpoint in progress" by recovery/startup probes). The
++    // retained FileHandle keeps its VM frame groups (see reset()); the file is re-created
++    // lazily by getOrCreateShadowingFH on the next checkpoint, which also re-reserves the
++    // header page.
+     bm.removeFilePagesFromFrames(*shadowingFH);
+     shadowingFH->resetToZeroPagesAndPageCapacity();
++    shadowingFH->resetFileInfo();
++    vfs->removeFileIfExists(shadowFilePath);
+     shadowPagesMap.clear();
+     shadowPageRecords.clear();
+-    // Reserve header page.
+-    shadowingFH->addNewPage();
+ }
+ 
+ void ShadowFile::reset() {
+-    shadowingFH->resetFileInfo();
+-    shadowingFH = nullptr;
+-    vfs->removeFileIfExists(shadowFilePath);
++    // Keep the shadowing FileHandle (and its VM frame groups) alive across checkpoints:
++    // each FileHandle allocates VMRegion frame groups that count against max_db_size and
++    // are never reclaimed, so orphaning the handle here exhausted max_db_size after a
++    // handful of checkpoints (see #924). Evict pages, truncate to zero pages (which also
++    // truncates the on-disk file), drop the stale fd, and unlink the path so
++    // recovery/startup probes — which treat a present .shadow file as "checkpoint in
++    // progress" — don't trip on it. getOrCreateShadowingFH re-opens the path into the
++    // same retained handle on the next checkpoint, so subsequent shadow pages land in a
++    // real on-disk file visible to recovery (which replays by path).
++    if (shadowingFH == nullptr) {
++        return;
++    }
++    // If clear() already ran (the normal checkpoint flow calls both), the file was truncated,
++    // unlinked and the fd dropped; just make sure the in-memory maps are empty.
++    if (shadowingFH->getFileInfo() != nullptr) {
++        bm.removeFilePagesFromFrames(*shadowingFH);
++        shadowingFH->resetToZeroPagesAndPageCapacity();
++        shadowingFH->resetFileInfo();
++        vfs->removeFileIfExists(shadowFilePath);
++    }
++    shadowPagesMap.clear();
++    shadowPageRecords.clear();
+ }
+ 
+ FileHandle* ShadowFile::getOrCreateShadowingFH() {
+     if (!shadowingFH) {
+         shadowingFH = bm.getFileHandle(shadowFilePath,
+             FileHandle::O_PERSISTENT_FILE_CREATE_NOT_EXISTS, vfs, nullptr);
+-        if (shadowingFH->getNumPages() == 0) {
+-            // Reserve the first page for the header.
+-            shadowingFH->addNewPage();
+-        }
++    } else if (shadowingFH->getFileInfo() == nullptr) {
++        // reset() unlinked the on-disk file and dropped the stale fd (which pointed at the
++        // unlinked inode): re-open the same retained FileHandle in place so its identity
++        // (fileIndex, frame groups) is preserved and no new VM accounting is consumed.
++        // Matches constructPersistentFileHandle's flags for O_PERSISTENT_FILE_CREATE_NOT_EXISTS.
++        shadowingFH->setFileInfo(vfs->openFile(shadowFilePath,
++            FileOpenFlags(
++                FileFlags::WRITE | FileFlags::READ_ONLY | FileFlags::CREATE_IF_NOT_EXISTS),
++            nullptr));
++    }
++    if (shadowingFH->getNumPages() == 0) {
++        // Reserve the first page for the header.
++        shadowingFH->addNewPage();
+     }
+     return shadowingFH;
+ }
+diff --git a/test/api/system_config_test.cpp b/test/api/system_config_test.cpp
+index 638b0a685..e398ddc4b 100644
+--- a/test/api/system_config_test.cpp
++++ b/test/api/system_config_test.cpp
+@@ -101,6 +101,29 @@ TEST_F(SystemConfigTest, testMaxDBSize) {
+     }
+ }
+ 
++TEST_F(SystemConfigTest, testRepeatedCheckpointsDoNotExhaustMaxDBSize) {
++    // Regression test for https://github.com/LadybugDB/ladybug/issues/924: each CHECKPOINT
++    // used to permanently consume ~1 VM frame group of max_db_size accounting (the shadowing
++    // FileHandle was orphaned on every checkpoint), so ~7 checkpoints exhausted a 64 MiB cap.
++    if (databasePath == "" || databasePath == ":memory:") {
++        GTEST_SKIP();
++    }
++    systemConfig->maxDBSize = 64 * 1024 * 1024;
++    auto db = std::make_unique<Database>(databasePath, *systemConfig);
++    auto con = std::make_unique<Connection>(db.get());
++    assertQuery(*con->query("CREATE NODE TABLE Node(id STRING, payload STRING, PRIMARY KEY(id))"));
++    for (auto i = 0; i < 50; ++i) {
++        auto res = con->query("MERGE (n:Node {id: 'node-" + std::to_string(i % 10) +
++                              "'}) ON CREATE SET n.payload = 'x' ON MATCH SET n.payload = 'x'");
++        ASSERT_TRUE(res->isSuccess()) << res->toString();
++        auto checkpoint = con->query("CHECKPOINT");
++        ASSERT_TRUE(checkpoint->isSuccess()) << checkpoint->toString();
++    }
++    auto count = con->query("MATCH (n:Node) RETURN COUNT(*)");
++    ASSERT_TRUE(count->isSuccess()) << count->toString();
++    ASSERT_EQ(TestHelper::convertResultToString(*count), std::vector<std::string>{"10"});
++}
++
+ TEST_F(SystemConfigTest, testBufferPoolSize) {
+     systemConfig->bufferPoolSize = 1024;
+     try {