kenaz 0.3.3 → 0.3.6

package/core/.kenaz/knowledge/ladybug_30cf85690e_shadow_page_read_during_checkpoint.patch

@@ -1,393 +1,393 @@
-diff --git a/src/include/storage/shadow_file.h b/src/include/storage/shadow_file.h
-index 6b186b167..a968b3af6 100644
---- a/src/include/storage/shadow_file.h
-+++ b/src/include/storage/shadow_file.h
-@@ -1,5 +1,9 @@
- #pragma once
- 
-+#include <atomic>
-+#include <functional>
-+#include <shared_mutex>
-+
- #include "common/types/uuid.h"
- #include "storage/file_handle.h"
- 
-@@ -24,11 +28,17 @@ struct ShadowFileHeader {
- static_assert(std::is_trivially_copyable_v<ShadowFileHeader>);
- 
- class BufferManager;
--// NOTE: This class is NOT thread-safe for now, as we are not checkpointing in parallel yet.
-+// NOTE: Only the checkpointing thread may create, publish or clear shadow pages. Read
-+// transactions running concurrently with a checkpoint may only use readShadowVersionIfExists.
- class ShadowFile {
- public:
-     ShadowFile(BufferManager& bm, common::VirtualFileSystem* vfs, const std::string& databasePath);
- 
-+    // Mutex protocol: hasShadowPage, getShadowPage, clearShadowPage, createShadowPage and
-+    // publishShadowPage are for the checkpointing thread only. It is the only writer of
-+    // shadowPagesMap, so it may read the map without taking mtx; the functions that modify the
-+    // map take mtx exclusively. Read transactions running concurrently with a checkpoint must not
-+    // call these and must use readShadowVersionIfExists, which takes mtx shared.
-     // TODO(Guodong): Remove originalFile param.
-     bool hasShadowPage(common::file_idx_t originalFile, common::page_idx_t originalPage) const {
-         return shadowPagesMap.contains(originalFile) &&
-@@ -37,8 +47,21 @@ class ShadowFile {
-     void clearShadowPage(common::file_idx_t originalFile, common::page_idx_t originalPage);
-     common::page_idx_t getShadowPage(common::file_idx_t originalFile,
-         common::page_idx_t originalPage) const;
--    common::page_idx_t getOrCreateShadowPage(common::file_idx_t originalFile,
-+    // Adds a new shadow page for originalPage. The page is not visible to
-+    // readShadowVersionIfExists (nor hasShadowPage) until publishShadowPage is called, so the
-+    // caller can fill it with the contents of the original page first. Must not be called while
-+    // a published shadow page of a column data page is pinned (see mtx).
-+    common::page_idx_t createShadowPage(common::file_idx_t originalFile,
-         common::page_idx_t originalPage);
-+    void publishShadowPage(common::file_idx_t originalFile, common::page_idx_t originalPage,
-+        common::page_idx_t shadowPageIdx);
-+    // Reads the shadow version of a page if the current checkpoint has one, and returns false
-+    // without calling readOp otherwise. Unlike the functions above, this is safe to call from
-+    // read transactions running concurrently with a checkpoint: a checkpoint publishes the new
-+    // metadata of in-place updated column chunks before the shadow pages are applied to the data
-+    // file, so those readers must see the shadow version of such pages.
-+    bool readShadowVersionIfExists(common::file_idx_t originalFile, common::page_idx_t originalPage,
-+        const std::function<void(uint8_t*)>& readOp) const;
- 
-     FileHandle& getShadowingFH() const { return *shadowingFH; }
- 
-@@ -83,6 +106,15 @@ class ShadowFile {
-         std::unordered_map<common::page_idx_t, common::page_idx_t>>
-         shadowPagesMap;
-     std::vector<ShadowPageRecord> shadowPageRecords;
-+    // Protects shadowPagesMap against concurrent readers (see readShadowVersionIfExists). Only
-+    // the checkpointing thread modifies the map.
-+    // Readers hold mtx shared while spinning in optimisticRead on a locked shadow page of a page
-+    // they reach through ColumnReadWriter::readFromPage (column data pages). The checkpointing
-+    // thread therefore must not hold such a published shadow page pinned while it takes mtx
-+    // exclusively (create, publish, clear or reset), or it would deadlock with those readers.
-+    mutable std::shared_mutex mtx;
-+    // Fast path for readers: false when there is no shadow page.
-+    std::atomic<bool> hasShadowPages{false};
- };
- 
- } // namespace storage
-diff --git a/src/storage/shadow_file.cpp b/src/storage/shadow_file.cpp
-index 9811166c6..8984d1eb8 100644
---- a/src/storage/shadow_file.cpp
-+++ b/src/storage/shadow_file.cpp
-@@ -45,6 +45,7 @@ void ShadowFile::setDatabasePath(const std::string& databasePath_) {
- }
- 
- void ShadowFile::clearShadowPage(file_idx_t originalFile, page_idx_t originalPage) {
-+    std::unique_lock lck{mtx};
-     if (hasShadowPage(originalFile, originalPage)) {
-         shadowPagesMap.at(originalFile).erase(originalPage);
-         if (shadowPagesMap.at(originalFile).empty()) {
-@@ -53,20 +54,44 @@ void ShadowFile::clearShadowPage(file_idx_t originalFile, page_idx_t originalPag
-     }
- }
- 
--page_idx_t ShadowFile::getOrCreateShadowPage(file_idx_t originalFile, page_idx_t originalPage) {
--    if (hasShadowPage(originalFile, originalPage)) {
--        return shadowPagesMap[originalFile][originalPage];
--    }
-+page_idx_t ShadowFile::createShadowPage(file_idx_t originalFile, page_idx_t originalPage) {
-+    DASSERT(!hasShadowPage(originalFile, originalPage));
-+    std::unique_lock lck{mtx};
-     const auto shadowPageIdx = getOrCreateShadowingFH()->addNewPage();
--    shadowPagesMap[originalFile][originalPage] = shadowPageIdx;
-+    // Records must stay in the same order as the shadow pages (see applyShadowPages). Readers
-+    // only use shadowPagesMap, so the page is not visible to them until it is published.
-     shadowPageRecords.push_back({originalFile, originalPage});
-     return shadowPageIdx;
- }
- 
-+void ShadowFile::publishShadowPage(file_idx_t originalFile, page_idx_t originalPage,
-+    page_idx_t shadowPageIdx) {
-+    std::unique_lock lck{mtx};
-+    shadowPagesMap[originalFile][originalPage] = shadowPageIdx;
-+    // Pairs with the acquire load in readShadowVersionIfExists.
-+    hasShadowPages.store(true, std::memory_order_release);
-+}
-+
- page_idx_t ShadowFile::getShadowPage(file_idx_t originalFile, page_idx_t originalPage) const {
-     DASSERT(hasShadowPage(originalFile, originalPage));
-     return shadowPagesMap.at(originalFile).at(originalPage);
- }
-+
-+bool ShadowFile::readShadowVersionIfExists(file_idx_t originalFile, page_idx_t originalPage,
-+    const std::function<void(uint8_t*)>& readOp) const {
-+    if (!hasShadowPages.load(std::memory_order_acquire)) {
-+        return false;
-+    }
-+    std::shared_lock lck{mtx};
-+    if (!hasShadowPage(originalFile, originalPage)) {
-+        return false;
-+    }
-+    // Optimistic reads let concurrent readers share the page and retry if the checkpointer is
-+    // updating it at the same time.
-+    shadowingFH->optimisticReadPage(shadowPagesMap.at(originalFile).at(originalPage), readOp);
-+    return true;
-+}
-+
- void ShadowFile::applyShadowPages(StorageManager& storageManager, ClientContext& context) const {
-     const auto pageBuffer = std::make_unique<uint8_t[]>(LBUG_PAGE_SIZE);
-     page_idx_t shadowPageIdx = 1; // Skip header page.
-@@ -191,6 +216,8 @@ void ShadowFile::flushAll(main::ClientContext& context) const {
- 
- void ShadowFile::clear(BufferManager& bm) {
-     DASSERT(shadowingFH);
-+    // Concurrent readers may be reading shadow pages (see readShadowVersionIfExists).
-+    std::unique_lock lck{mtx};
-     // Evict pages, truncate to zero pages (which also truncates the on-disk file), then unlink
-     // the path and drop the stale fd so no leftover .shadow file exists at rest (a present
-     // .shadow is treated as "checkpoint in progress" by recovery/startup probes). The
-@@ -203,6 +230,7 @@ void ShadowFile::clear(BufferManager& bm) {
-     vfs->removeFileIfExists(shadowFilePath);
-     shadowPagesMap.clear();
-     shadowPageRecords.clear();
-+    hasShadowPages.store(false, std::memory_order_relaxed);
- }
- 
- void ShadowFile::reset() {
-@@ -218,6 +246,7 @@ void ShadowFile::reset() {
-     if (shadowingFH == nullptr) {
-         return;
-     }
-+    std::unique_lock lck{mtx};
-     // If clear() already ran (the normal checkpoint flow calls both), the file was truncated,
-     // unlinked and the fd dropped; just make sure the in-memory maps are empty.
-     if (shadowingFH->getFileInfo() != nullptr) {
-@@ -228,6 +257,7 @@ void ShadowFile::reset() {
-     }
-     shadowPagesMap.clear();
-     shadowPageRecords.clear();
-+    hasShadowPages.store(false, std::memory_order_relaxed);
- }
- 
- FileHandle* ShadowFile::getOrCreateShadowingFH() {
-diff --git a/src/storage/shadow_utils.cpp b/src/storage/shadow_utils.cpp
-index 572990231..57538f3ff 100644
---- a/src/storage/shadow_utils.cpp
-+++ b/src/storage/shadow_utils.cpp
-@@ -12,14 +12,16 @@ namespace storage {
- ShadowPageAndFrame ShadowUtils::createShadowVersionIfNecessaryAndPinPage(page_idx_t originalPage,
-     bool skipReadingOriginalPage, FileHandle& fileHandle, ShadowFile& shadowFile) {
-     DASSERT(!fileHandle.isInMemoryMode());
--    const auto hasShadowPage = shadowFile.hasShadowPage(fileHandle.getFileIndex(), originalPage);
--    auto shadowPage = shadowFile.getOrCreateShadowPage(fileHandle.getFileIndex(), originalPage);
-+    const auto fileIdx = fileHandle.getFileIndex();
-+    page_idx_t shadowPage = INVALID_PAGE_IDX;
-     uint8_t* shadowFrame = nullptr;
-     try {
--        if (hasShadowPage) {
-+        if (shadowFile.hasShadowPage(fileIdx, originalPage)) {
-+            shadowPage = shadowFile.getShadowPage(fileIdx, originalPage);
-             shadowFrame =
-                 shadowFile.getShadowingFH().pinPage(shadowPage, PageReadPolicy::READ_PAGE);
-         } else {
-+            shadowPage = shadowFile.createShadowPage(fileIdx, originalPage);
-             shadowFrame =
-                 shadowFile.getShadowingFH().pinPage(shadowPage, PageReadPolicy::DONT_READ_PAGE);
-             if (!skipReadingOriginalPage) {
-@@ -27,6 +29,11 @@ ShadowPageAndFrame ShadowUtils::createShadowVersionIfNecessaryAndPinPage(page_id
-                     memcpy(shadowFrame, frame, LBUG_PAGE_SIZE);
-                 });
-             }
-+            // Publish the shadow page only once it holds the original contents, since read
-+            // transactions may read shadow pages concurrently (see
-+            // ShadowFile::readShadowVersionIfExists). The page stays locked until it is unpinned,
-+            // so concurrent optimistic readers wait for the caller's update to finish.
-+            shadowFile.publishShadowPage(fileIdx, originalPage, shadowPage);
-         }
-         // The shadow page existing already does not mean that it's already dirty
-         // It may have been flushed to disk to free memory and then read again
-diff --git a/src/storage/table/column_reader_writer.cpp b/src/storage/table/column_reader_writer.cpp
-index fb71533d3..3e768de2f 100644
---- a/src/storage/table/column_reader_writer.cpp
-+++ b/src/storage/table/column_reader_writer.cpp
-@@ -8,6 +8,7 @@
- #include "common/vector/value_vector.h"
- #include "storage/compression/float_compression.h"
- #include "storage/file_handle.h"
-+#include "storage/shadow_file.h"
- #include "storage/shadow_utils.h"
- #include "storage/storage_utils.h"
- #include "storage/table/column_chunk_data.h"
-@@ -462,6 +463,12 @@ void ColumnReadWriter::readFromPage(page_idx_t pageIdx,
-     if (pageIdx == INVALID_PAGE_IDX) {
-         return readFunc(nullptr);
-     }
-+    // A checkpoint in progress may already have published metadata which refers to in-place
-+    // updates that so far only exist in shadow pages.
-+    if (shadowFile &&
-+        shadowFile->readShadowVersionIfExists(dataFH->getFileIndex(), pageIdx, readFunc)) {
-+        return;
-+    }
-     dataFH->optimisticReadPage(pageIdx, readFunc);
- }
- 
-diff --git a/src/transaction/transaction_manager.cpp b/src/transaction/transaction_manager.cpp
-index 32e155213..48cbd8a2a 100644
---- a/src/transaction/transaction_manager.cpp
-+++ b/src/transaction/transaction_manager.cpp
-@@ -303,8 +303,10 @@ void TransactionManager::tryCheckpoint(main::ClientContext& clientContext) {
- void TransactionManager::checkpointNoLock(main::ClientContext& clientContext) {
-     QueryProgressScope progress{clientContext, 0.01};
-     // We only need to wait for active write transactions to leave the system before
--    // checkpointing. Read transactions can continue safely because they use MVCC snapshot
--    // isolation and shadow pages are applied with per-page locking.
-+    // checkpointing. Read transactions can continue because they use MVCC snapshot isolation.
-+    // Column chunk metadata of in-place checkpointed data is published before the shadow pages
-+    // are applied, so column reads resolve such in-flight pages through the shadow file (see
-+    // ColumnReadWriter::readFromPage); applying shadow pages uses per-page locking.
-     UniqLock writeGate;
-     try {
-         writeGate = stopNewWriteTransactionsAndWaitUntilAllWriteTransactionsLeave();
-diff --git a/test/transaction/checkpoint_test.cpp b/test/transaction/checkpoint_test.cpp
-index 0a9899dca..277756061 100644
---- a/test/transaction/checkpoint_test.cpp
-+++ b/test/transaction/checkpoint_test.cpp
-@@ -2,14 +2,20 @@
- #include <chrono>
- #include <cstdlib>
- #include <fstream>
-+#include <functional>
- #include <future>
- #include <mutex>
- #include <thread>
-+#include <unordered_set>
- 
- #include "api_test/private_api_test.h"
-+#include "catalog/catalog.h"
-+#include "catalog/catalog_entry/table_catalog_entry.h"
- #include "common/exception/runtime.h"
- #include "storage/checkpointer.h"
- #include "storage/storage_manager.h"
-+#include "storage/table/node_table.h"
-+#include "storage/table/string_chunk_data.h"
- #include "storage/wal/wal.h"
- #include "test_env.h"
- #include "transaction/transaction_manager.h"
-@@ -308,6 +314,122 @@ TEST_F(FlakyCheckpointerTest, ShadowFileDatabaseIDMismatchCorruptedDB) {
-     EXPECT_THROW(createDBAndConn(), InternalException);
- }
- 
-+// A checkpoint publishes the new chunk metadata of a node group during its storage phase, but
-+// in-place updates to existing pages are only written to the shadow file until the shadow pages
-+// are applied at the end of the checkpoint. Read-only transactions are not blocked by a
-+// checkpoint, so a scan that starts in between must not observe the new metadata together with
-+// the old page contents (e.g. appended string dictionary offsets that are not on disk yet).
-+class CheckpointerWithReadBeforeApplyingShadowPages final : public Checkpointer {
-+public:
-+    CheckpointerWithReadBeforeApplyingShadowPages(main::ClientContext& clientContext,
-+        std::function<void()> readFunc)
-+        : Checkpointer(clientContext), readFunc(std::move(readFunc)) {}
-+
-+    void logCheckpointAndApplyShadowPages(bool walRotated) override {
-+        readFunc();
-+        Checkpointer::logCheckpointAndApplyShadowPages(walRotated);
-+    }
-+
-+private:
-+    std::function<void()> readFunc;
-+};
-+
-+#ifndef __SINGLE_THREADED__
-+TEST_F(FlakyCheckpointerTest, ReadBeforeShadowPagesAreAppliedSeesCommittedStrings) {
-+    if (inMemMode || systemConfig->checkpointThreshold == 0) {
-+        GTEST_SKIP();
-+    }
-+    conn->query("CALL force_checkpoint_on_close=false;");
-+    conn->query("CALL auto_checkpoint=false;");
-+    ASSERT_TRUE(
-+        conn->query("CREATE NODE TABLE test(id INT64 PRIMARY KEY, name STRING);")->isSuccess());
-+    // Strings longer than the inline limit, few enough that the appended values below fit in
-+    // the pages already allocated to the dictionary, so the checkpoint updates them in place.
-+    auto getName = [](uint64_t i) { return std::format("persistent string value {:04}", i); };
-+    auto insertRows = [&](uint64_t start, uint64_t end) {
-+        for (auto i = start; i < end; i++) {
-+            auto res =
-+                conn->query(std::format("CREATE (:test {{id: {}, name: '{}'}});", i, getName(i)));
-+            ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
-+        }
-+    };
-+    constexpr uint64_t numInitialRows = 20;
-+    constexpr uint64_t numRows = 25;
-+    insertRows(0, numInitialRows);
-+    ASSERT_TRUE(conn->query("CHECKPOINT;")->isSuccess());
-+    insertRows(numInitialRows, numRows);
-+
-+    // Pages of the persistent `name` column chunk (dictionary offsets, string data and indices).
-+    // An in-place checkpoint of the column updates these pages through the shadow file, while an
-+    // out-of-place checkpoint writes new pages and leaves these untouched.
-+    auto context = getClientContext(*conn);
-+    auto storageManager = StorageManager::Get(*context);
-+    const auto tableEntry = catalog::Catalog::Get(*context)->getTableCatalogEntry(
-+        &DUMMY_CHECKPOINT_TRANSACTION, "test");
-+    auto& nodeTable = storageManager->getTable(tableEntry->getTableID())->cast<NodeTable>();
-+    const auto* persistentGroup = nodeTable.getNodeGroup(0)->getChunkedNodeGroup(0);
-+    ASSERT_EQ(persistentGroup->getResidencyState(), ResidencyState::ON_DISK);
-+    std::unordered_set<page_idx_t> namePages;
-+    for (const auto* segment :
-+        persistentGroup->getColumnChunk(tableEntry->getColumnID("name")).getSegments()) {
-+        const auto& stringChunk = segment->cast<StringChunkData>();
-+        for (const auto* chunk :
-+            std::initializer_list<const ColumnChunkData*>{stringChunk.getIndexColumnChunk(),
-+                stringChunk.getDictionaryChunk().getOffsetChunk(),
-+                stringChunk.getDictionaryChunk().getStringDataChunk()}) {
-+            const auto& metadata = chunk->getMetadata();
-+            for (auto i = 0u; i < metadata.getNumPages(); i++) {
-+                namePages.insert(metadata.getStartPageIdx() + i);
-+            }
-+        }
-+    }
-+    ASSERT_FALSE(namePages.empty());
-+
-+    bool readRan = false;
-+    uint64_t numShadowedNamePages = 0;
-+    std::string readError;
-+    std::vector<std::string> readNames;
-+    auto readFunc = [&]() {
-+        // Make sure the `name` column was checkpointed in place, i.e. its updated pages are still
-+        // pending in the shadow file; otherwise the read below would not read shadow pages.
-+        auto& shadowFile = storageManager->getShadowFile();
-+        for (const auto pageIdx : namePages) {
-+            if (shadowFile.hasShadowPage(storageManager->getDataFH()->getFileIndex(), pageIdx)) {
-+                numShadowedNamePages++;
-+            }
-+        }
-+        // Run on a separate thread with its own connection, like a concurrent reader would.
-+        std::thread reader([&]() {
-+            auto readConn = std::make_unique<main::Connection>(database.get());
-+            auto res = readConn->query("MATCH (t:test) RETURN t.name ORDER BY t.id;");
-+            readRan = true;
-+            if (!res->isSuccess()) {
-+                readError = res->getErrorMessage();
-+                return;
-+            }
-+            while (res->hasNext()) {
-+                readNames.push_back(res->getNext()->getValue(0)->getValue<std::string>());
-+            }
-+        });
-+        reader.join();
-+    };
-+    FlakyCheckpointer checkpointer([&](main::ClientContext& clientContext) {
-+        return std::make_unique<CheckpointerWithReadBeforeApplyingShadowPages>(clientContext,
-+            readFunc);
-+    });
-+    checkpointer.setCheckpointer(*context);
-+    auto res = conn->query("CHECKPOINT;");
-+    ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
-+    ASSERT_TRUE(readRan);
-+    ASSERT_GT(numShadowedNamePages, 0u);
-+    ASSERT_TRUE(readError.empty()) << readError;
-+    ASSERT_EQ(readNames.size(), numRows);
-+    for (auto i = 0u; i < numRows; i++) {
-+        EXPECT_EQ(readNames[i], getName(i));
-+    }
-+}
-+#endif // __SINGLE_THREADED__
-+
- // ─────────────────────────────────────────────────────────────────────────────
- // ReviewFixesTest
- // Targeted regression tests for the three fixes made in response to adsharma's
+diff --git a/src/include/storage/shadow_file.h b/src/include/storage/shadow_file.h
+index 6b186b167..a968b3af6 100644
+--- a/src/include/storage/shadow_file.h
++++ b/src/include/storage/shadow_file.h
+@@ -1,5 +1,9 @@
+ #pragma once
+ 
++#include <atomic>
++#include <functional>
++#include <shared_mutex>
++
+ #include "common/types/uuid.h"
+ #include "storage/file_handle.h"
+ 
+@@ -24,11 +28,17 @@ struct ShadowFileHeader {
+ static_assert(std::is_trivially_copyable_v<ShadowFileHeader>);
+ 
+ class BufferManager;
+-// NOTE: This class is NOT thread-safe for now, as we are not checkpointing in parallel yet.
++// NOTE: Only the checkpointing thread may create, publish or clear shadow pages. Read
++// transactions running concurrently with a checkpoint may only use readShadowVersionIfExists.
+ class ShadowFile {
+ public:
+     ShadowFile(BufferManager& bm, common::VirtualFileSystem* vfs, const std::string& databasePath);
+ 
++    // Mutex protocol: hasShadowPage, getShadowPage, clearShadowPage, createShadowPage and
++    // publishShadowPage are for the checkpointing thread only. It is the only writer of
++    // shadowPagesMap, so it may read the map without taking mtx; the functions that modify the
++    // map take mtx exclusively. Read transactions running concurrently with a checkpoint must not
++    // call these and must use readShadowVersionIfExists, which takes mtx shared.
+     // TODO(Guodong): Remove originalFile param.
+     bool hasShadowPage(common::file_idx_t originalFile, common::page_idx_t originalPage) const {
+         return shadowPagesMap.contains(originalFile) &&
+@@ -37,8 +47,21 @@ class ShadowFile {
+     void clearShadowPage(common::file_idx_t originalFile, common::page_idx_t originalPage);
+     common::page_idx_t getShadowPage(common::file_idx_t originalFile,
+         common::page_idx_t originalPage) const;
+-    common::page_idx_t getOrCreateShadowPage(common::file_idx_t originalFile,
++    // Adds a new shadow page for originalPage. The page is not visible to
++    // readShadowVersionIfExists (nor hasShadowPage) until publishShadowPage is called, so the
++    // caller can fill it with the contents of the original page first. Must not be called while
++    // a published shadow page of a column data page is pinned (see mtx).
++    common::page_idx_t createShadowPage(common::file_idx_t originalFile,
+         common::page_idx_t originalPage);
++    void publishShadowPage(common::file_idx_t originalFile, common::page_idx_t originalPage,
++        common::page_idx_t shadowPageIdx);
++    // Reads the shadow version of a page if the current checkpoint has one, and returns false
++    // without calling readOp otherwise. Unlike the functions above, this is safe to call from
++    // read transactions running concurrently with a checkpoint: a checkpoint publishes the new
++    // metadata of in-place updated column chunks before the shadow pages are applied to the data
++    // file, so those readers must see the shadow version of such pages.
++    bool readShadowVersionIfExists(common::file_idx_t originalFile, common::page_idx_t originalPage,
++        const std::function<void(uint8_t*)>& readOp) const;
+ 
+     FileHandle& getShadowingFH() const { return *shadowingFH; }
+ 
+@@ -83,6 +106,15 @@ class ShadowFile {
+         std::unordered_map<common::page_idx_t, common::page_idx_t>>
+         shadowPagesMap;
+     std::vector<ShadowPageRecord> shadowPageRecords;
++    // Protects shadowPagesMap against concurrent readers (see readShadowVersionIfExists). Only
++    // the checkpointing thread modifies the map.
++    // Readers hold mtx shared while spinning in optimisticRead on a locked shadow page of a page
++    // they reach through ColumnReadWriter::readFromPage (column data pages). The checkpointing
++    // thread therefore must not hold such a published shadow page pinned while it takes mtx
++    // exclusively (create, publish, clear or reset), or it would deadlock with those readers.
++    mutable std::shared_mutex mtx;
++    // Fast path for readers: false when there is no shadow page.
++    std::atomic<bool> hasShadowPages{false};
+ };
+ 
+ } // namespace storage
+diff --git a/src/storage/shadow_file.cpp b/src/storage/shadow_file.cpp
+index 9811166c6..8984d1eb8 100644
+--- a/src/storage/shadow_file.cpp
++++ b/src/storage/shadow_file.cpp
+@@ -45,6 +45,7 @@ void ShadowFile::setDatabasePath(const std::string& databasePath_) {
+ }
+ 
+ void ShadowFile::clearShadowPage(file_idx_t originalFile, page_idx_t originalPage) {
++    std::unique_lock lck{mtx};
+     if (hasShadowPage(originalFile, originalPage)) {
+         shadowPagesMap.at(originalFile).erase(originalPage);
+         if (shadowPagesMap.at(originalFile).empty()) {
+@@ -53,20 +54,44 @@ void ShadowFile::clearShadowPage(file_idx_t originalFile, page_idx_t originalPag
+     }
+ }
+ 
+-page_idx_t ShadowFile::getOrCreateShadowPage(file_idx_t originalFile, page_idx_t originalPage) {
+-    if (hasShadowPage(originalFile, originalPage)) {
+-        return shadowPagesMap[originalFile][originalPage];
+-    }
++page_idx_t ShadowFile::createShadowPage(file_idx_t originalFile, page_idx_t originalPage) {
++    DASSERT(!hasShadowPage(originalFile, originalPage));
++    std::unique_lock lck{mtx};
+     const auto shadowPageIdx = getOrCreateShadowingFH()->addNewPage();
+-    shadowPagesMap[originalFile][originalPage] = shadowPageIdx;
++    // Records must stay in the same order as the shadow pages (see applyShadowPages). Readers
++    // only use shadowPagesMap, so the page is not visible to them until it is published.
+     shadowPageRecords.push_back({originalFile, originalPage});
+     return shadowPageIdx;
+ }
+ 
++void ShadowFile::publishShadowPage(file_idx_t originalFile, page_idx_t originalPage,
++    page_idx_t shadowPageIdx) {
++    std::unique_lock lck{mtx};
++    shadowPagesMap[originalFile][originalPage] = shadowPageIdx;
++    // Pairs with the acquire load in readShadowVersionIfExists.
++    hasShadowPages.store(true, std::memory_order_release);
++}
++
+ page_idx_t ShadowFile::getShadowPage(file_idx_t originalFile, page_idx_t originalPage) const {
+     DASSERT(hasShadowPage(originalFile, originalPage));
+     return shadowPagesMap.at(originalFile).at(originalPage);
+ }
++
++bool ShadowFile::readShadowVersionIfExists(file_idx_t originalFile, page_idx_t originalPage,
++    const std::function<void(uint8_t*)>& readOp) const {
++    if (!hasShadowPages.load(std::memory_order_acquire)) {
++        return false;
++    }
++    std::shared_lock lck{mtx};
++    if (!hasShadowPage(originalFile, originalPage)) {
++        return false;
++    }
++    // Optimistic reads let concurrent readers share the page and retry if the checkpointer is
++    // updating it at the same time.
++    shadowingFH->optimisticReadPage(shadowPagesMap.at(originalFile).at(originalPage), readOp);
++    return true;
++}
++
+ void ShadowFile::applyShadowPages(StorageManager& storageManager, ClientContext& context) const {
+     const auto pageBuffer = std::make_unique<uint8_t[]>(LBUG_PAGE_SIZE);
+     page_idx_t shadowPageIdx = 1; // Skip header page.
+@@ -191,6 +216,8 @@ void ShadowFile::flushAll(main::ClientContext& context) const {
+ 
+ void ShadowFile::clear(BufferManager& bm) {
+     DASSERT(shadowingFH);
++    // Concurrent readers may be reading shadow pages (see readShadowVersionIfExists).
++    std::unique_lock lck{mtx};
+     // Evict pages, truncate to zero pages (which also truncates the on-disk file), then unlink
+     // the path and drop the stale fd so no leftover .shadow file exists at rest (a present
+     // .shadow is treated as "checkpoint in progress" by recovery/startup probes). The
+@@ -203,6 +230,7 @@ void ShadowFile::clear(BufferManager& bm) {
+     vfs->removeFileIfExists(shadowFilePath);
+     shadowPagesMap.clear();
+     shadowPageRecords.clear();
++    hasShadowPages.store(false, std::memory_order_relaxed);
+ }
+ 
+ void ShadowFile::reset() {
+@@ -218,6 +246,7 @@ void ShadowFile::reset() {
+     if (shadowingFH == nullptr) {
+         return;
+     }
++    std::unique_lock lck{mtx};
+     // If clear() already ran (the normal checkpoint flow calls both), the file was truncated,
+     // unlinked and the fd dropped; just make sure the in-memory maps are empty.
+     if (shadowingFH->getFileInfo() != nullptr) {
+@@ -228,6 +257,7 @@ void ShadowFile::reset() {
+     }
+     shadowPagesMap.clear();
+     shadowPageRecords.clear();
++    hasShadowPages.store(false, std::memory_order_relaxed);
+ }
+ 
+ FileHandle* ShadowFile::getOrCreateShadowingFH() {
+diff --git a/src/storage/shadow_utils.cpp b/src/storage/shadow_utils.cpp
+index 572990231..57538f3ff 100644
+--- a/src/storage/shadow_utils.cpp
++++ b/src/storage/shadow_utils.cpp
+@@ -12,14 +12,16 @@ namespace storage {
+ ShadowPageAndFrame ShadowUtils::createShadowVersionIfNecessaryAndPinPage(page_idx_t originalPage,
+     bool skipReadingOriginalPage, FileHandle& fileHandle, ShadowFile& shadowFile) {
+     DASSERT(!fileHandle.isInMemoryMode());
+-    const auto hasShadowPage = shadowFile.hasShadowPage(fileHandle.getFileIndex(), originalPage);
+-    auto shadowPage = shadowFile.getOrCreateShadowPage(fileHandle.getFileIndex(), originalPage);
++    const auto fileIdx = fileHandle.getFileIndex();
++    page_idx_t shadowPage = INVALID_PAGE_IDX;
+     uint8_t* shadowFrame = nullptr;
+     try {
+-        if (hasShadowPage) {
++        if (shadowFile.hasShadowPage(fileIdx, originalPage)) {
++            shadowPage = shadowFile.getShadowPage(fileIdx, originalPage);
+             shadowFrame =
+                 shadowFile.getShadowingFH().pinPage(shadowPage, PageReadPolicy::READ_PAGE);
+         } else {
++            shadowPage = shadowFile.createShadowPage(fileIdx, originalPage);
+             shadowFrame =
+                 shadowFile.getShadowingFH().pinPage(shadowPage, PageReadPolicy::DONT_READ_PAGE);
+             if (!skipReadingOriginalPage) {
+@@ -27,6 +29,11 @@ ShadowPageAndFrame ShadowUtils::createShadowVersionIfNecessaryAndPinPage(page_id
+                     memcpy(shadowFrame, frame, LBUG_PAGE_SIZE);
+                 });
+             }
++            // Publish the shadow page only once it holds the original contents, since read
++            // transactions may read shadow pages concurrently (see
++            // ShadowFile::readShadowVersionIfExists). The page stays locked until it is unpinned,
++            // so concurrent optimistic readers wait for the caller's update to finish.
++            shadowFile.publishShadowPage(fileIdx, originalPage, shadowPage);
+         }
+         // The shadow page existing already does not mean that it's already dirty
+         // It may have been flushed to disk to free memory and then read again
+diff --git a/src/storage/table/column_reader_writer.cpp b/src/storage/table/column_reader_writer.cpp
+index fb71533d3..3e768de2f 100644
+--- a/src/storage/table/column_reader_writer.cpp
++++ b/src/storage/table/column_reader_writer.cpp
+@@ -8,6 +8,7 @@
+ #include "common/vector/value_vector.h"
+ #include "storage/compression/float_compression.h"
+ #include "storage/file_handle.h"
++#include "storage/shadow_file.h"
+ #include "storage/shadow_utils.h"
+ #include "storage/storage_utils.h"
+ #include "storage/table/column_chunk_data.h"
+@@ -462,6 +463,12 @@ void ColumnReadWriter::readFromPage(page_idx_t pageIdx,
+     if (pageIdx == INVALID_PAGE_IDX) {
+         return readFunc(nullptr);
+     }
++    // A checkpoint in progress may already have published metadata which refers to in-place
++    // updates that so far only exist in shadow pages.
++    if (shadowFile &&
++        shadowFile->readShadowVersionIfExists(dataFH->getFileIndex(), pageIdx, readFunc)) {
++        return;
++    }
+     dataFH->optimisticReadPage(pageIdx, readFunc);
+ }
+ 
+diff --git a/src/transaction/transaction_manager.cpp b/src/transaction/transaction_manager.cpp
+index 32e155213..48cbd8a2a 100644
+--- a/src/transaction/transaction_manager.cpp
++++ b/src/transaction/transaction_manager.cpp
+@@ -303,8 +303,10 @@ void TransactionManager::tryCheckpoint(main::ClientContext& clientContext) {
+ void TransactionManager::checkpointNoLock(main::ClientContext& clientContext) {
+     QueryProgressScope progress{clientContext, 0.01};
+     // We only need to wait for active write transactions to leave the system before
+-    // checkpointing. Read transactions can continue safely because they use MVCC snapshot
+-    // isolation and shadow pages are applied with per-page locking.
++    // checkpointing. Read transactions can continue because they use MVCC snapshot isolation.
++    // Column chunk metadata of in-place checkpointed data is published before the shadow pages
++    // are applied, so column reads resolve such in-flight pages through the shadow file (see
++    // ColumnReadWriter::readFromPage); applying shadow pages uses per-page locking.
+     UniqLock writeGate;
+     try {
+         writeGate = stopNewWriteTransactionsAndWaitUntilAllWriteTransactionsLeave();
+diff --git a/test/transaction/checkpoint_test.cpp b/test/transaction/checkpoint_test.cpp
+index 0a9899dca..277756061 100644
+--- a/test/transaction/checkpoint_test.cpp
++++ b/test/transaction/checkpoint_test.cpp
+@@ -2,14 +2,20 @@
+ #include <chrono>
+ #include <cstdlib>
+ #include <fstream>
++#include <functional>
+ #include <future>
+ #include <mutex>
+ #include <thread>
++#include <unordered_set>
+ 
+ #include "api_test/private_api_test.h"
++#include "catalog/catalog.h"
++#include "catalog/catalog_entry/table_catalog_entry.h"
+ #include "common/exception/runtime.h"
+ #include "storage/checkpointer.h"
+ #include "storage/storage_manager.h"
++#include "storage/table/node_table.h"
++#include "storage/table/string_chunk_data.h"
+ #include "storage/wal/wal.h"
+ #include "test_env.h"
+ #include "transaction/transaction_manager.h"
+@@ -308,6 +314,122 @@ TEST_F(FlakyCheckpointerTest, ShadowFileDatabaseIDMismatchCorruptedDB) {
+     EXPECT_THROW(createDBAndConn(), InternalException);
+ }
+ 
++// A checkpoint publishes the new chunk metadata of a node group during its storage phase, but
++// in-place updates to existing pages are only written to the shadow file until the shadow pages
++// are applied at the end of the checkpoint. Read-only transactions are not blocked by a
++// checkpoint, so a scan that starts in between must not observe the new metadata together with
++// the old page contents (e.g. appended string dictionary offsets that are not on disk yet).
++class CheckpointerWithReadBeforeApplyingShadowPages final : public Checkpointer {
++public:
++    CheckpointerWithReadBeforeApplyingShadowPages(main::ClientContext& clientContext,
++        std::function<void()> readFunc)
++        : Checkpointer(clientContext), readFunc(std::move(readFunc)) {}
++
++    void logCheckpointAndApplyShadowPages(bool walRotated) override {
++        readFunc();
++        Checkpointer::logCheckpointAndApplyShadowPages(walRotated);
++    }
++
++private:
++    std::function<void()> readFunc;
++};
++
++#ifndef __SINGLE_THREADED__
++TEST_F(FlakyCheckpointerTest, ReadBeforeShadowPagesAreAppliedSeesCommittedStrings) {
++    if (inMemMode || systemConfig->checkpointThreshold == 0) {
++        GTEST_SKIP();
++    }
++    conn->query("CALL force_checkpoint_on_close=false;");
++    conn->query("CALL auto_checkpoint=false;");
++    ASSERT_TRUE(
++        conn->query("CREATE NODE TABLE test(id INT64 PRIMARY KEY, name STRING);")->isSuccess());
++    // Strings longer than the inline limit, few enough that the appended values below fit in
++    // the pages already allocated to the dictionary, so the checkpoint updates them in place.
++    auto getName = [](uint64_t i) { return std::format("persistent string value {:04}", i); };
++    auto insertRows = [&](uint64_t start, uint64_t end) {
++        for (auto i = start; i < end; i++) {
++            auto res =
++                conn->query(std::format("CREATE (:test {{id: {}, name: '{}'}});", i, getName(i)));
++            ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
++        }
++    };
++    constexpr uint64_t numInitialRows = 20;
++    constexpr uint64_t numRows = 25;
++    insertRows(0, numInitialRows);
++    ASSERT_TRUE(conn->query("CHECKPOINT;")->isSuccess());
++    insertRows(numInitialRows, numRows);
++
++    // Pages of the persistent `name` column chunk (dictionary offsets, string data and indices).
++    // An in-place checkpoint of the column updates these pages through the shadow file, while an
++    // out-of-place checkpoint writes new pages and leaves these untouched.
++    auto context = getClientContext(*conn);
++    auto storageManager = StorageManager::Get(*context);
++    const auto tableEntry = catalog::Catalog::Get(*context)->getTableCatalogEntry(
++        &DUMMY_CHECKPOINT_TRANSACTION, "test");
++    auto& nodeTable = storageManager->getTable(tableEntry->getTableID())->cast<NodeTable>();
++    const auto* persistentGroup = nodeTable.getNodeGroup(0)->getChunkedNodeGroup(0);
++    ASSERT_EQ(persistentGroup->getResidencyState(), ResidencyState::ON_DISK);
++    std::unordered_set<page_idx_t> namePages;
++    for (const auto* segment :
++        persistentGroup->getColumnChunk(tableEntry->getColumnID("name")).getSegments()) {
++        const auto& stringChunk = segment->cast<StringChunkData>();
++        for (const auto* chunk :
++            std::initializer_list<const ColumnChunkData*>{stringChunk.getIndexColumnChunk(),
++                stringChunk.getDictionaryChunk().getOffsetChunk(),
++                stringChunk.getDictionaryChunk().getStringDataChunk()}) {
++            const auto& metadata = chunk->getMetadata();
++            for (auto i = 0u; i < metadata.getNumPages(); i++) {
++                namePages.insert(metadata.getStartPageIdx() + i);
++            }
++        }
++    }
++    ASSERT_FALSE(namePages.empty());
++
++    bool readRan = false;
++    uint64_t numShadowedNamePages = 0;
++    std::string readError;
++    std::vector<std::string> readNames;
++    auto readFunc = [&]() {
++        // Make sure the `name` column was checkpointed in place, i.e. its updated pages are still
++        // pending in the shadow file; otherwise the read below would not read shadow pages.
++        auto& shadowFile = storageManager->getShadowFile();
++        for (const auto pageIdx : namePages) {
++            if (shadowFile.hasShadowPage(storageManager->getDataFH()->getFileIndex(), pageIdx)) {
++                numShadowedNamePages++;
++            }
++        }
++        // Run on a separate thread with its own connection, like a concurrent reader would.
++        std::thread reader([&]() {
++            auto readConn = std::make_unique<main::Connection>(database.get());
++            auto res = readConn->query("MATCH (t:test) RETURN t.name ORDER BY t.id;");
++            readRan = true;
++            if (!res->isSuccess()) {
++                readError = res->getErrorMessage();
++                return;
++            }
++            while (res->hasNext()) {
++                readNames.push_back(res->getNext()->getValue(0)->getValue<std::string>());
++            }
++        });
++        reader.join();
++    };
++    FlakyCheckpointer checkpointer([&](main::ClientContext& clientContext) {
++        return std::make_unique<CheckpointerWithReadBeforeApplyingShadowPages>(clientContext,
++            readFunc);
++    });
++    checkpointer.setCheckpointer(*context);
++    auto res = conn->query("CHECKPOINT;");
++    ASSERT_TRUE(res->isSuccess()) << res->getErrorMessage();
++    ASSERT_TRUE(readRan);
++    ASSERT_GT(numShadowedNamePages, 0u);
++    ASSERT_TRUE(readError.empty()) << readError;
++    ASSERT_EQ(readNames.size(), numRows);
++    for (auto i = 0u; i < numRows; i++) {
++        EXPECT_EQ(readNames[i], getName(i));
++    }
++}
++#endif // __SINGLE_THREADED__
++
+ // ─────────────────────────────────────────────────────────────────────────────
+ // ReviewFixesTest
+ // Targeted regression tests for the three fixes made in response to adsharma's