kenaz 0.0.0-stage → 0.3.2

package/core/.kenaz/knowledge/ladybug_926a0ba1_column_checkpoint.patch

@@ -0,0 +1,412 @@
+From 926a0ba10d221eb437add0885ff492e7576bb7c9 Mon Sep 17 00:00:00 2001
+From: kory-io <kory-io@users.noreply.github.com>
+Date: Thu, 24 Sep 2026 18:56:55 -0600
+Subject: [PATCH] fix(storage): keep segment on disk when an out-of-place
+ checkpoint fails
+
+Column::checkpointColumnChunkOutOfPlace rewrote a segment by switching
+the persistent ColumnChunkData itself to IN_MEMORY (setToInMemory) and
+then resizing, scanning, writing, finalizing and flushing it. Any of
+those steps can throw, for example "Unable to allocate memory! The
+buffer pool is full" from the resize or the scan. The checkpoint is
+then rolled back, but Checkpointer::rollback only discards pending page
+frees and rolls back node-table indexes; it does not touch node group
+state. The node group keeps the half-rewritten segment, which stays
+IN_MEMORY, and with runtime checks enabled (Debug, or RUNTIME_CHECKS=1
+as in CI) every later checkpoint of that table fails the
+`segment->getResidencyState() == ResidencyState::ON_DISK` assertion in
+ColumnChunk::checkpoint, so the database can no longer checkpoint.
+
+Rewrite the segment into a new in-memory chunk of the same kind
+instead, and return it once it has been flushed, the same way split
+segments are already returned. The persistent segment object is not
+touched until the rewrite has succeeded, and its pages are only freed
+after that, so a failure leaves it on disk and unchanged and the next
+checkpoint starts from it again. Segments rewritten earlier in the
+same failed checkpoint stay replaced; their old pages are only
+reclaimed after the database is reopened.
+
+ColumnChunk::checkpoint already installs returned segments. The nested
+null, list offset and list size chunks, which are checkpointed without
+splitting, now install the returned chunk into their parent.
+StringColumn and StructColumn also call syncNumValues on the returned
+segments, as they already did on the persistent segment.
+
+Add FlakyCheckpointerTest.RecoverFromFailureDuringOutOfPlaceCheckpoint.
+It fails a checkpoint at the flush of an out-of-place rewrite, then
+checks that the data is intact and that the next checkpoint succeeds.
+Add an e2e case that checkpoints a list column in place while its
+offsets and nulls are rewritten out of place.
+---
+ src/include/storage/table/column.h            |   4 +-
+ src/storage/table/column.cpp                  |  58 +++++++---
+ src/storage/table/column_chunk.cpp            |   4 +-
+ src/storage/table/list_column.cpp             |  15 ++-
+ src/storage/table/string_column.cpp           |   3 +
+ src/storage/table/struct_column.cpp           |   3 +
+ .../transaction/set_node/set_empty.test       |  41 +++++++
+ test/transaction/checkpoint_test.cpp          | 106 ++++++++++++++++++
+ 8 files changed, 209 insertions(+), 25 deletions(-)
+
+diff --git a/src/include/storage/table/column.h b/src/include/storage/table/column.h
+index 16584cb027..d56fefbf5f 100644
+--- a/src/include/storage/table/column.h
++++ b/src/include/storage/table/column.h
+@@ -95,8 +95,8 @@ class Column {
+         return common::dynamic_cast_checked<TARGET&>(*this);
+     }
+ 
+-    // Return value is the new segments if segment splitting occurs during an out of place
+-    // checkpoint
++    // Return value is the new segment(s) replacing the persistent segment if it was checkpointed
++    // out of place (more than one if it was split), or empty if it was checkpointed in place
+     virtual std::vector<std::unique_ptr<ColumnChunkData>> checkpointSegment(
+         ColumnCheckpointState&& checkpointState, PageAllocator& pageAllocator,
+         bool canSplitSegment = true) const;
+diff --git a/src/storage/table/column.cpp b/src/storage/table/column.cpp
+index 72b9a2f206..9d23776188 100644
+--- a/src/storage/table/column.cpp
++++ b/src/storage/table/column.cpp
+@@ -3,6 +3,7 @@
+ #include <algorithm>
+ #include <cstdint>
+ #include <memory>
++#include <tuple>
+ 
+ #include "common/assert.h"
+ #include "common/data_chunk/sel_vector.h"
+@@ -439,42 +440,63 @@ void Column::checkpointNullData(const ColumnCheckpointState& checkpointState,
+             segmentCheckpointState.numRows);
+     }
+     DASSERT(checkpointState.persistentData.hasNullData());
+-    nullColumn->checkpointSegment(
++    auto newNullData = nullColumn->checkpointSegment(
+         ColumnCheckpointState(*checkpointState.persistentData.getNullData(),
+             std::move(nullSegmentCheckpointStates)),
+         pageAllocator, false);
++    if (!newNullData.empty()) {
++        DASSERT(newNullData.size() == 1);
++        auto* nullChunk = &newNullData[0]->cast<NullChunkData>();
++        std::ignore = newNullData[0].release();
++        checkpointState.persistentData.setNullData(std::unique_ptr<NullChunkData>(nullChunk));
++    }
++}
++
++static std::unique_ptr<ColumnChunkData> createInMemoryChunkLike(const ColumnChunkData& chunk,
++    uint64_t capacity) {
++    if (dynamic_cast<const NullChunkData*>(&chunk) != nullptr) {
++        return ColumnChunkFactory::createNullChunkData(chunk.getMemoryManager(),
++            chunk.isCompressionEnabled(), capacity, ResidencyState::IN_MEMORY);
++    }
++    return ColumnChunkFactory::createColumnChunkData(chunk.getMemoryManager(),
++        chunk.getDataType().copy(), chunk.isCompressionEnabled(), capacity,
++        ResidencyState::IN_MEMORY, chunk.hasNullData());
+ }
+ 
+ std::vector<std::unique_ptr<ColumnChunkData>> Column::checkpointColumnChunkOutOfPlace(
+     const SegmentState& state, const ColumnCheckpointState& checkpointState,
+     PageAllocator& pageAllocator, bool canSplitSegment) const {
+     const auto numRows = std::max(checkpointState.endRowIdxToWrite, state.metadata.numValues);
+-    checkpointState.persistentData.setToInMemory();
+-    checkpointState.persistentData.resize(numRows);
+-    DASSERT(checkpointState.persistentData.getNumValues() == 0);
+-    scanSegment(state, &checkpointState.persistentData, 0, state.metadata.numValues);
+-    state.reclaimAllocatedPages(pageAllocator);
++    // Rewrite the segment into a new chunk instead of converting persistentData to an in-memory
++    // chunk in place. If the rewrite fails part way (e.g. the buffer pool is full), the checkpoint
++    // is rolled back but the segment is kept, so it must still describe its on-disk pages for the
++    // data to stay readable and for the next checkpoint to be able to rewrite it.
++    auto newData = createInMemoryChunkLike(checkpointState.persistentData, numRows);
++    DASSERT(newData->getNumValues() == 0);
++    scanSegment(state, newData.get(), 0, state.metadata.numValues);
+     // TODO(bmwinger): for simple compression types, we can predict whether or not we will need to
+     // split the segment and avoid having to re-write it multiple times
+     for (auto& segmentCheckpointState : checkpointState.segmentCheckpointStates) {
+-        checkpointState.persistentData.write(&segmentCheckpointState.chunkData,
+-            segmentCheckpointState.startRowInData, segmentCheckpointState.offsetInSegment,
+-            segmentCheckpointState.numRows);
++        newData->write(&segmentCheckpointState.chunkData, segmentCheckpointState.startRowInData,
++            segmentCheckpointState.offsetInSegment, segmentCheckpointState.numRows);
+     }
+     // Finalize is necessary prior to splitting for strings and lists so that pruned values don't
+     // have an impact on the number/size of segments It should not be necessary after splitting
+     // since the function is used to prune unused values (or duplicated dictionary entries in the
+     // case of strings) and those will never be introduced when splitting.
+-    checkpointState.persistentData.finalize();
+-    if (canSplitSegment && checkpointState.persistentData.shouldSplit()) {
+-        auto newSegments = checkpointState.persistentData.split();
+-        for (auto& segment : newSegments) {
+-            segment->flush(pageAllocator);
+-        }
+-        return newSegments;
++    newData->finalize();
++    std::vector<std::unique_ptr<ColumnChunkData>> newSegments;
++    if (canSplitSegment && newData->shouldSplit()) {
++        newSegments = newData->split();
++    } else {
++        newSegments.push_back(std::move(newData));
+     }
+-    checkpointState.persistentData.flush(pageAllocator);
+-    return {};
++    for (auto& segment : newSegments) {
++        segment->flush(pageAllocator);
++    }
++    // Only free the old pages once the replacement is on disk.
++    state.reclaimAllocatedPages(pageAllocator);
++    return newSegments;
+ }
+ 
+ bool Column::canCheckpointInPlace(const SegmentState& state,
+diff --git a/src/storage/table/column_chunk.cpp b/src/storage/table/column_chunk.cpp
+index 9adf90426d..4c0f277344 100644
+--- a/src/storage/table/column_chunk.cpp
++++ b/src/storage/table/column_chunk.cpp
+@@ -311,8 +311,8 @@ void ColumnChunk::checkpoint(Column& column,
+             }
+         }
+         auto segmentEnd = segmentStart + segment->getNumValues();
+-        // If the segment was split during checkpointing we need to insert the new segments into the
+-        // ColumnChunk
++        // If the segment was rewritten out of place (and possibly split) during checkpointing we
++        // need to replace it with the new segments in the ColumnChunk
+         auto newSegments = column.checkpointSegment(
+             ColumnCheckpointState(*segment, std::move(segmentCheckpointStates)), pageAllocator);
+         if (!newSegments.empty()) {
+diff --git a/src/storage/table/list_column.cpp b/src/storage/table/list_column.cpp
+index 0ac0465896..e4a1ece4e2 100644
+--- a/src/storage/table/list_column.cpp
++++ b/src/storage/table/list_column.cpp
+@@ -403,10 +403,14 @@ std::vector<std::unique_ptr<ColumnChunkData>> ListColumn::checkpointSegment(
+     }
+ 
+     // We do not allow nested splitting of offset/size segments
+-    offsetColumn->checkpointSegment(
++    auto newOffsetChunk = offsetColumn->checkpointSegment(
+         ColumnCheckpointState(*persistentListChunk.getOffsetColumnChunk(),
+             std::move(offsetChunkCheckpointStates)),
+         pageAllocator, false);
++    if (!newOffsetChunk.empty()) {
++        DASSERT(newOffsetChunk.size() == 1);
++        persistentListChunk.setOffsetColumnChunk(std::move(newOffsetChunk[0]));
++    }
+ 
+     // Checkpoint size data.
+     std::vector<SegmentCheckpointState> sizeChunkCheckpointStates;
+@@ -416,9 +420,14 @@ std::vector<std::unique_ptr<ColumnChunkData>> ListColumn::checkpointSegment(
+             segmentCheckpointState.startRowInData, segmentCheckpointState.offsetInSegment,
+             segmentCheckpointState.numRows});
+     }
+-    sizeColumn->checkpointSegment(ColumnCheckpointState(*persistentListChunk.getSizeColumnChunk(),
+-                                      std::move(sizeChunkCheckpointStates)),
++    auto newSizeChunk = sizeColumn->checkpointSegment(
++        ColumnCheckpointState(*persistentListChunk.getSizeColumnChunk(),
++            std::move(sizeChunkCheckpointStates)),
+         pageAllocator, false);
++    if (!newSizeChunk.empty()) {
++        DASSERT(newSizeChunk.size() == 1);
++        persistentListChunk.setSizeColumnChunk(std::move(newSizeChunk[0]));
++    }
+     // Checkpoint null data.
+     Column::checkpointNullData(checkpointState, pageAllocator);
+ 
+diff --git a/src/storage/table/string_column.cpp b/src/storage/table/string_column.cpp
+index b7f94dc3ea..38b508765a 100644
+--- a/src/storage/table/string_column.cpp
++++ b/src/storage/table/string_column.cpp
+@@ -116,6 +116,9 @@ std::vector<std::unique_ptr<ColumnChunkData>> StringColumn::checkpointSegment(
+     auto result =
+         Column::checkpointSegment(std::move(checkpointState), pageAllocator, canSplitSegment);
+     persistentData.syncNumValues();
++    for (auto& segment : result) {
++        segment->syncNumValues();
++    }
+     return result;
+ }
+ 
+diff --git a/src/storage/table/struct_column.cpp b/src/storage/table/struct_column.cpp
+index b16e3d83ff..673cb6c468 100644
+--- a/src/storage/table/struct_column.cpp
++++ b/src/storage/table/struct_column.cpp
+@@ -114,6 +114,9 @@ std::vector<std::unique_ptr<ColumnChunkData>> StructColumn::checkpointSegment(
+     auto result =
+         Column::checkpointSegment(std::move(checkpointState), pageAllocator, canSplitSegment);
+     persistentStructChunk.syncNumValues();
++    for (auto& segment : result) {
++        segment->syncNumValues();
++    }
+     return result;
+ }
+ 
+diff --git a/test/test_files/transaction/set_node/set_empty.test b/test/test_files/transaction/set_node/set_empty.test
+index 72dbeefd09..b6e1495e07 100644
+--- a/test/test_files/transaction/set_node/set_empty.test
++++ b/test/test_files/transaction/set_node/set_empty.test
+@@ -33,3 +33,44 @@
+ ---- 2
+ 0|1010101010.300000
+ 1341|1010101010.200000
++
++-CASE ListInPlaceWithOutOfPlaceOffsetsAndNulls
++-STATEMENT CALL auto_checkpoint=false;
++---- ok
++-STATEMENT CREATE NODE TABLE t2(id INT64 PRIMARY KEY, l INT64[]);
++---- ok
++-STATEMENT UNWIND range(0, 4094) AS i CREATE (:t2 {id: i, l: [0, 1]});
++---- ok
++-STATEMENT CHECKPOINT;
++---- ok
++-STATEMENT MATCH (a:t2) WHERE a.id = 5 SET a.l = [1,0,1,0,1,0,1,0,1,0];
++---- ok
++-STATEMENT MATCH (a:t2) WHERE a.id = 7 SET a.l = NULL;
++---- ok
++-STATEMENT CHECKPOINT;
++---- ok
++-STATEMENT MATCH (a:t2) RETURN COUNT(a.l);
++---- 1
++4094
++-STATEMENT MATCH (a:t2) UNWIND a.l AS x RETURN COUNT(x), SUM(x);
++---- 1
++8196|4098
++-STATEMENT MATCH (a:t2) WHERE a.id = 5 RETURN SIZE(a.l), a.l[1], a.l[2];
++---- 1
++10|1|0
++-STATEMENT MATCH (a:t2) WHERE a.id = 6 RETURN SIZE(a.l), a.l[1], a.l[2];
++---- 1
++2|0|1
++-RELOADDB
++-STATEMENT MATCH (a:t2) RETURN COUNT(a.l);
++---- 1
++4094
++-STATEMENT MATCH (a:t2) UNWIND a.l AS x RETURN COUNT(x), SUM(x);
++---- 1
++8196|4098
++-STATEMENT MATCH (a:t2) WHERE a.id = 5 RETURN SIZE(a.l), a.l[1], a.l[2];
++---- 1
++10|1|0
++-STATEMENT MATCH (a:t2) WHERE a.id = 4094 RETURN SIZE(a.l), a.l[1], a.l[2];
++---- 1
++2|0|1
+diff --git a/test/transaction/checkpoint_test.cpp b/test/transaction/checkpoint_test.cpp
+index 2777560616..2049d21d19 100644
+--- a/test/transaction/checkpoint_test.cpp
++++ b/test/transaction/checkpoint_test.cpp
+@@ -13,6 +13,8 @@
+ #include "catalog/catalog_entry/table_catalog_entry.h"
+ #include "common/exception/runtime.h"
+ #include "storage/checkpointer.h"
++#include "storage/page_allocator.h"
++#include "storage/page_manager.h"
+ #include "storage/storage_manager.h"
+ #include "storage/table/node_table.h"
+ #include "storage/table/string_chunk_data.h"
+@@ -235,6 +237,110 @@ TEST_F(FlakyCheckpointerTest, RecoverFromCheckpointClearingFilesFailure) {
+     runTest(flakyCheckpointer);
+ }
+ 
++// Page allocator that fails the first page allocation. In-place checkpoints do not allocate
++// pages, so during the storage phase of a checkpoint that only updates a node group with
++// persistent data, the first allocation is the flush of an out-of-place column chunk rewrite and
++// the failure lands in the middle of that rewrite.
++class FailFirstAllocationPageAllocator final : public PageAllocator {
++public:
++    FailFirstAllocationPageAllocator(PageAllocator& inner, bool& failed)
++        : PageAllocator(inner.getDataFH()), inner{inner}, failed{failed} {}
++
++    PageRange allocatePageRange(page_idx_t numPages) override {
++        if (!failed) {
++            failed = true;
++            throw RuntimeException("checkpoint failed.");
++        }
++        return inner.allocatePageRange(numPages);
++    }
++    void freePageRange(PageRange block) override { inner.freePageRange(block); }
++
++private:
++    PageAllocator& inner;
++    bool& failed;
++};
++
++class FlakyCheckpointerFailsDuringOutOfPlaceRewrite final : public Checkpointer {
++public:
++    FlakyCheckpointerFailsDuringOutOfPlaceRewrite(main::ClientContext& context, bool& failed)
++        : Checkpointer(context), failed{failed} {}
++
++    bool checkpointStorage() override {
++        for (const auto& target : checkpointTargets) {
++            FailFirstAllocationPageAllocator pageAllocator(
++                *target.storageManager->getDataFH()->getPageManager(), failed);
++            const Transaction snapshotTxn(TransactionType::CHECKPOINT,
++                Transaction::DUMMY_TRANSACTION_ID, snapshotTS);
++            target.storageManager->checkpoint(&clientContext, *target.catalog, snapshotTxn,
++                pageAllocator, tableEpochWatermarksByManager.at(target.storageManager));
++        }
++        throw RuntimeException("expected the injected page allocation failure to fire.");
++    }
++
++private:
++    bool& failed;
++};
++
++// A checkpoint that fails while a column chunk segment is being rewritten out of place must
++// leave that segment as it was, so that the data stays readable and later checkpoints succeed.
++TEST_F(FlakyCheckpointerTest, RecoverFromFailureDuringOutOfPlaceCheckpoint) {
++    if (inMemMode || systemConfig->checkpointThreshold == 0) {
++        GTEST_SKIP();
++    }
++    conn->query("CALL force_checkpoint_on_close=false;");
++    conn->query("CALL auto_checkpoint=false;");
++    ASSERT_TRUE(
++        conn->query("CREATE NODE TABLE test(id INT64 PRIMARY KEY, name STRING);")->isSuccess());
++    constexpr int64_t numInitialRows = 3000;
++    constexpr int64_t numRows = 6000;
++    constexpr std::string_view namePrefix = "a longer name so the pages fill up ";
++    auto nameOf = [&](int64_t i) { return std::format("{}{}", namePrefix, i); };
++    auto insertRows = [&](int64_t start, int64_t end) {
++        auto res = conn->query(std::format("UNWIND range({}, {}) AS i CREATE (a:test {{id: i, "
++                                           "name: concat('{}', CAST(i AS STRING))}});",
++            start, end - 1, namePrefix));
++        ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
++    };
++    auto checkRows = [&]() {
++        auto res = conn->query("MATCH (a:test) RETURN COUNT(a), CAST(SUM(a.id) AS INT64), "
++                               "CAST(SUM(SIZE(a.name)) AS INT64), MIN(a.name), MAX(a.name);");
++        ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
++        auto row = res->getNext();
++        ASSERT_EQ(row->getValue(0)->getValue<int64_t>(), numRows);
++        ASSERT_EQ(row->getValue(1)->getValue<int64_t>(), numRows * (numRows - 1) / 2);
++        int64_t totalNameSize = 0;
++        for (auto i = 0; i < numRows; i++) {
++            totalNameSize += nameOf(i).size();
++        }
++        ASSERT_EQ(row->getValue(2)->getValue<int64_t>(), totalNameSize);
++        ASSERT_EQ(row->getValue(3)->getValue<std::string>(), nameOf(0));
++        ASSERT_EQ(row->getValue(4)->getValue<std::string>(), nameOf(999));
++    };
++    insertRows(0, numInitialRows);
++    ASSERT_TRUE(conn->query("CHECKPOINT;")->isSuccess());
++    // Appending to the persistent node group outgrows the pages of its column chunks, so the
++    // next checkpoint has to rewrite them out of place.
++    insertRows(numInitialRows, numRows);
++
++    auto context = getClientContext(*conn);
++    bool failed = false;
++    FlakyCheckpointer flakyCheckpointer([&failed](main::ClientContext& ctx) {
++        return std::make_unique<FlakyCheckpointerFailsDuringOutOfPlaceRewrite>(ctx, failed);
++    });
++    flakyCheckpointer.setCheckpointer(*context);
++    auto res = conn->query("CHECKPOINT;");
++    ASSERT_FALSE(res->isSuccess());
++    ASSERT_TRUE(failed);
++    checkRows();
++
++    FlakyCheckpointer defaultCheckpointer(
++        [](main::ClientContext& ctx) { return std::make_unique<Checkpointer>(ctx); });
++    defaultCheckpointer.setCheckpointer(*context);
++    res = conn->query("CHECKPOINT;");
++    ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
++    checkRows();
++}
++
+ // Simulates a situation where a database attempts to replay a shadow file from an older database
+ // with the same path
+ TEST_F(FlakyCheckpointerTest, ShadowFileDatabaseIDMismatchExistingDB) {

package/core/.kenaz/knowledge/ladybug_a346c4c4b6_pk_index_header_pages.patch

@@ -0,0 +1,144 @@
+diff --git a/src/storage/index/hash_index.cpp b/src/storage/index/hash_index.cpp
+index b65b3da86..2e952966e 100644
+--- a/src/storage/index/hash_index.cpp
++++ b/src/storage/index/hash_index.cpp
+@@ -770,13 +770,15 @@ void PrimaryKeyIndex::checkpoint(main::ClientContext*, storage::PageAllocator& p
+             indexChanged = true;
+         }
+     }
++    // Assign both header pages under one condition: the constructor rejects an overflow
++    // header page without index header pages.
+     if (indexChanged) {
+         writeHeaders(pageAllocator);
+         hashIndexDiskArrays->checkpoint(getDiskArrayFirstHeaderPage(), pageAllocator);
+-    }
+-    if (overflowFile) {
+-        overflowFile->checkpoint(pageAllocator);
+-        updateOverflowHeaderPageIfNeeded(storageInfo.get(), overflowFile.get());
++        if (overflowFile) {
++            overflowFile->checkpoint(pageAllocator);
++            updateOverflowHeaderPageIfNeeded(storageInfo.get(), overflowFile.get());
++        }
+     }
+     // Make sure that changes which bypassed the WAL are written.
+     // There is no other mechanism for enforcing that they are flushed
+diff --git a/test/storage/CMakeLists.txt b/test/storage/CMakeLists.txt
+index 581979387..3cabd5c2b 100644
+--- a/test/storage/CMakeLists.txt
++++ b/test/storage/CMakeLists.txt
+@@ -3,6 +3,7 @@ add_lbug_test(compression_test compression_test.cpp compress_chunk_test.cpp)
+ add_lbug_test(alp_exception_chunk_test alp_exception_chunk_test.cpp)
+ add_lbug_test(column_chunk_metadata_test column_chunk_metadata_test.cpp)
+ add_lbug_test(local_hash_index_test local_hash_index_test.cpp)
++add_lbug_test(pk_index_checkpoint_test pk_index_checkpoint_test.cpp)
+ add_lbug_test(disk_array_collection_test disk_array_collection_test.cpp)
+ add_lbug_test(buffer_manager_test buffer_manager_test.cpp)
+ add_lbug_test(rel_tests rel_scan_test.cpp rel_delete_test.cpp)
+diff --git a/test/storage/pk_index_checkpoint_test.cpp b/test/storage/pk_index_checkpoint_test.cpp
+new file mode 100644
+index 000000000..1e577c45f
+--- /dev/null
++++ b/test/storage/pk_index_checkpoint_test.cpp
+@@ -0,0 +1,102 @@
++#include "catalog/catalog.h"
++#include "catalog/catalog_entry/node_table_catalog_entry.h"
++#include "graph_test/private_graph_test.h"
++#include "storage/index/hash_index.h"
++#include "storage/storage_manager.h"
++#include "storage/table/node_table.h"
++#include "test_helper/test_helper.h"
++#include "transaction/transaction.h"
++
++using namespace lbug::common;
++using namespace lbug::catalog;
++using namespace lbug::storage;
++using namespace lbug::transaction;
++
++namespace lbug {
++namespace testing {
++
++// Regression tests for #464: checkpointing an empty STRING primary-key index must not persist an
++// overflow header page without the index header pages, which the constructor rejects on reopen.
++class PKIndexCheckpointTest : public EmptyDBTest {
++protected:
++    void SetUp() override {
++        EmptyDBTest::SetUp();
++        createDBAndConn();
++    }
++
++    void assertQuery(const std::string& query) const {
++        auto result = conn->query(query);
++        ASSERT_TRUE(result->isSuccess()) << query << ": " << result->toString();
++    }
++
++    const PrimaryKeyIndexStorageInfo& getPKStorageInfo(const std::string& tableName) const {
++        auto* entry = database->getCatalog()
++                          ->getTableCatalogEntry(&DUMMY_CHECKPOINT_TRANSACTION, tableName)
++                          ->ptrCast<NodeTableCatalogEntry>();
++        auto& nodeTable =
++            database->getStorageManager()->getTable(entry->getTableID())->cast<NodeTable>();
++        return nodeTable.getPKIndex()->getStorageInfo().constCast<PrimaryKeyIndexStorageInfo>();
++    }
++
++    // Both header pointers are assigned together, so an unset index header page implies an unset
++    // overflow header page.
++    void assertHeaderPagesConsistent(const std::string& tableName) const {
++        const auto& info = getPKStorageInfo(tableName);
++        if (info.firstHeaderPage == INVALID_PAGE_IDX) {
++            EXPECT_EQ(info.overflowHeaderPage, INVALID_PAGE_IDX)
++                << "overflow header page persisted without index header pages";
++        }
++    }
++
++    void reopen() {
++        conn.reset();
++        createDBAndConn();
++    }
++};
++
++TEST_F(PKIndexCheckpointTest, EmptyStringPKCheckpointAfterAlter) {
++    if (inMemMode) {
++        GTEST_SKIP();
++    }
++    assertQuery("CREATE NODE TABLE t(id STRING PRIMARY KEY);");
++    assertQuery("ALTER TABLE t ADD v INT64;");
++    assertQuery("CHECKPOINT;");
++    assertHeaderPagesConsistent("t");
++
++    ASSERT_NO_THROW(reopen());
++    auto result = conn->query("MATCH (n:t) RETURN count(*);");
++    ASSERT_TRUE(result->isSuccess()) << result->toString();
++    EXPECT_EQ(TestHelper::convertResultToString(*result), std::vector<std::string>{"0"});
++    assertHeaderPagesConsistent("t");
++}
++
++TEST_F(PKIndexCheckpointTest, InsertAfterEmptyStringPKCheckpoint) {
++    if (inMemMode) {
++        GTEST_SKIP();
++    }
++    assertQuery("CREATE NODE TABLE t(id STRING PRIMARY KEY);");
++    assertQuery("ALTER TABLE t ADD v INT64;");
++    assertQuery("CHECKPOINT;");
++    assertQuery("CREATE (:t {id: 'a-key-longer-than-twelve-bytes', v: 1});");
++    assertQuery("CREATE (:t {id: 'short', v: 2});");
++    assertQuery("CHECKPOINT;");
++    assertHeaderPagesConsistent("t");
++
++    ASSERT_NO_THROW(reopen());
++    auto result = conn->query("MATCH (n:t) WHERE n.id = 'a-key-longer-than-twelve-bytes' "
++                              "RETURN n.v;");
++    ASSERT_TRUE(result->isSuccess()) << result->toString();
++    EXPECT_EQ(TestHelper::convertResultToString(*result), std::vector<std::string>{"1"});
++    result = conn->query("MATCH (n:t) WHERE n.id = 'short' RETURN n.v;");
++    ASSERT_TRUE(result->isSuccess()) << result->toString();
++    EXPECT_EQ(TestHelper::convertResultToString(*result), std::vector<std::string>{"2"});
++    // The primary key must still reject duplicates after reopen.
++    result = conn->query("CREATE (:t {id: 'a-key-longer-than-twelve-bytes', v: 3});");
++    EXPECT_FALSE(result->isSuccess());
++    const auto& info = getPKStorageInfo("t");
++    EXPECT_NE(info.firstHeaderPage, INVALID_PAGE_IDX);
++    EXPECT_NE(info.overflowHeaderPage, INVALID_PAGE_IDX);
++}
++
++} // namespace testing
++} // namespace lbug

package/core/.kenaz/knowledge/ladybug_ba5f38815b_csr_node_group.patch

@@ -0,0 +1,86 @@
+@@ -290,6 +290,12 @@ NodeGroupScanResult CSRNodeGroup::scanCommittedInMemSequential(const Transaction
+     const ChunkedNodeGroup* chunkedGroup = nullptr;
+     {
+         const auto lock = chunkedGroups.lock();
++        // A stale CSR index entry (e.g. an INVALID_ROW_IDX sentinel) can yield an
++        // out-of-range chunk group index. Guard against it instead of dereferencing
++        // a null/out-of-bounds group. See LadybugDB/ladybug#611.
++        if (chunkIdx >= chunkedGroups.getNumGroups(lock)) {
++            return NODE_GROUP_SCAN_EMPTY_RESULT;
++        }
+         chunkedGroup = chunkedGroups.getGroup(lock, chunkIdx);
+     }
+     chunkedGroup->scan(transaction, tableState, nodeGroupScanState, startRowInChunk, numRows);
+@@ -315,8 +321,15 @@ NodeGroupScanResult CSRNodeGroup::scanCommittedInMemRandom(const Transaction* tr
+         auto [chunkIdx, rowInChunk] =
+             StorageUtils::getQuotientRemainder(rowIdx, StorageConfig::CHUNKED_NODE_GROUP_CAPACITY);
+         if (chunkIdx != currentChunkIdx) {
+-            currentChunkIdx = chunkIdx;
+             const auto lock = chunkedGroups.lock();
++            // A stale CSR index entry (e.g. an INVALID_ROW_IDX sentinel) can yield an
++            // out-of-range chunk group index. Skip such rows instead of dereferencing
++            // a null/out-of-bounds group. See LadybugDB/ladybug#611.
++            if (chunkIdx >= chunkedGroups.getNumGroups(lock)) {
++                nextRow++;
++                continue;
++            }
++            currentChunkIdx = chunkIdx;
+             chunkedGroup = chunkedGroups.getGroup(lock, chunkIdx);
+         }
+         DASSERT(chunkedGroup);
+@@ -409,6 +422,11 @@ void CSRNodeGroup::update(const Transaction* transaction, CSRNodeGroupScanSource
+         auto [chunkIdx, rowInChunk] = StorageUtils::getQuotientRemainder(rowIdxInGroup,
+             StorageConfig::CHUNKED_NODE_GROUP_CAPACITY);
+         const auto lock = chunkedGroups.lock();
++        // A stale CSR index entry can yield an out-of-range chunk group index; treat
++        // the update as a no-op in that case. See LadybugDB/ladybug#611.
++        if (chunkIdx >= chunkedGroups.getNumGroups(lock)) {
++            return;
++        }
+         const auto chunkedGroup = chunkedGroups.getGroup(lock, chunkIdx);
+         return chunkedGroup->update(transaction, rowInChunk, columnID, propertyVector);
+     }
+@@ -431,6 +449,11 @@ bool CSRNodeGroup::delete_(const Transaction* transaction, CSRNodeGroupScanSourc
+         auto [chunkIdx, rowInChunk] = StorageUtils::getQuotientRemainder(rowIdxInGroup,
+             StorageConfig::CHUNKED_NODE_GROUP_CAPACITY);
+         const auto lock = chunkedGroups.lock();
++        // A stale CSR index entry can yield an out-of-range chunk group index; treat
++        // the delete as a no-op in that case. See LadybugDB/ladybug#611.
++        if (chunkIdx >= chunkedGroups.getNumGroups(lock)) {
++            return false;
++        }
+         const auto chunkedGroup = chunkedGroups.getGroup(lock, chunkIdx);
+         return chunkedGroup->delete_(transaction, rowInChunk);
+     }
+@@ -913,6 +936,15 @@ void CSRNodeGroup::collectInMemRegionChangesAndUpdateHeaderLength(const UniqLock
+                 const auto row = rows[i];
+                 auto [chunkIdx, rowInChunk] = StorageUtils::getQuotientRemainder(row,
+                     StorageConfig::CHUNKED_NODE_GROUP_CAPACITY);
++                // A stale CSR index entry (e.g. an INVALID_ROW_IDX sentinel from a
++                // prior checkpoint pass) can yield an out-of-range chunk group index.
++                // Treat it as already deleted and skip. See LadybugDB/ladybug#611.
++                if (chunkIdx >= chunkedGroups.getNumGroups(lock)) {
++                    csrIndex->indices[nodeOffset].turnToNonSequential();
++                    csrIndex->indices[nodeOffset].setInvalid(i);
++                    numInMemDeletionsInCSR++;
++                    continue;
++                }
+                 const auto chunkedGroup = chunkedGroups.getGroup(lock, chunkIdx);
+                 if (chunkedGroup->isDeleted(txn, rowInChunk)) {
+                     csrIndex->indices[nodeOffset].turnToNonSequential();
+@@ -1117,6 +1149,15 @@ void CSRNodeGroup::populateCSRLengthInMemOnly(const UniqLock& lock, offset_t num
+             const auto row = rows[i];
+             auto [chunkIdx, rowInChunk] =
+                 StorageUtils::getQuotientRemainder(row, StorageConfig::CHUNKED_NODE_GROUP_CAPACITY);
++            // A stale CSR index entry (e.g. an INVALID_ROW_IDX sentinel from a prior
++            // checkpoint pass) can yield an out-of-range chunk group index. Treat it
++            // as already deleted. See LadybugDB/ladybug#611.
++            if (chunkIdx >= chunkedGroups.getNumGroups(lock)) {
++                csrIndex->indices[offset].turnToNonSequential();
++                csrIndex->indices[offset].setInvalid(i);
++                lengthAfterDelete--;
++                continue;
++            }
+             const auto chunkedGroup = chunkedGroups.getGroup(lock, chunkIdx);
+             const auto isDeleted = chunkedGroup->isDeleted(txn, rowInChunk);
+             if (isDeleted) {