@silexlabs/silex 3.3.1-3 → 3.3.1-5

package/src/ts/plugins/server/GitlabConnector.ts

@@ -18,7 +18,7 @@
 import { API_CONNECTOR_LOGIN_CALLBACK, API_CONNECTOR_PATH, API_PATH, WEBSITE_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
 import { ServerConfig } from '../../server/config'
 import { ConnectorFile, ConnectorFileContent, StatusCallback, StorageConnector, contentToBuffer, contentToString, toConnectorData } from '../../server/connectors/connectors'
-import { ApiError, ConnectorType, ConnectorUser, WebsiteData, WebsiteId, WebsiteMeta, WebsiteMetaFileContent, JobStatus, Page } from '../../types'
+import { ApiError, ConnectorType, ConnectorUser, WebsiteData, WebsiteId, WebsiteMeta, WebsiteMetaFileContent, JobStatus } from '../../types'
 import fetch from 'node-fetch'
 import crypto, { createHash } from 'crypto'
 import { join } from 'path'
@@ -26,6 +26,8 @@ import { Agent } from 'https'
 import { getPageSlug } from '../../page'
 import e from 'express'
 import { fork } from 'child_process'
+import { Page } from 'grapesjs'
+import { stringify, split, merge } from '../../server/utils/websiteDataSerialization'
 
 /**
  * Gitlab connector
@@ -120,16 +122,6 @@ interface GitlabFetchCommits {
   since: string
 }
 
-interface GitlabPage {
-  id: string
-  name: string
-  isFile: true
-}
-
-interface GitlabWebsiteData {
-  fileFormatVersion: string
-  pages: GitlabPage[]
-}
 
 // interface MetaRepoFileContent {
 //   websites: {
@@ -220,6 +212,7 @@ function sanitizeGitlabPath(name: string): string {
     .toLowerCase()
 }
 
+
 export default class GitlabConnector implements StorageConnector {
   connectorId = 'gitlab'
   connectorType = ConnectorType.STORAGE
@@ -327,15 +320,21 @@ export default class GitlabConnector implements StorageConnector {
       'Content-Type': 'application/json',
     }
     if (method === 'GET' && requestBody) {
-      console.error('Gitlab API error (4) - GET request with body', { url, method, body: requestBody, params })
+      console.error('[GitlabConnector] Invalid GET request: GET requests should not have a body', { url, method, body: requestBody, params })
     }
     // With or without body
-    let response
+    let response: Response
     const body = requestBody ? JSON.stringify(requestBody) : undefined
     try {
-      if(body && Buffer.byteLength(body) > MAX_BODY_SIZE_KB * 1024) {
-        // TODO: warn the end user
-        console.warn('Gitlab API warning - body too big', Buffer.byteLength(body), 'bytes', { url, method, params })
+      if (body && Buffer.byteLength(body) > MAX_BODY_SIZE_KB * 1024) {
+        console.warn('[GitlabConnector] Request body size exceeds Gitlab API limit', {
+          size: Buffer.byteLength(body),
+          maxAllowed: MAX_BODY_SIZE_KB * 1024,
+          url,
+          method,
+          params,
+          session,
+        })
       }
       response = await fetch(url, requestBody && method !== 'GET' ? {
         agent: this.getAgent(),
@@ -348,8 +347,16 @@ export default class GitlabConnector implements StorageConnector {
         headers,
       })
     } catch (e) {
-      console.error('Gitlab API error (0)', e)
-      throw new ApiError(`Gitlab API error (0): ${e.message} ${e.text} ${e.code} ${e.name} ${e.type}`, 500)
+      console.error('[GitlabConnector] Failed to reach Gitlab API endpoint', {
+        error: e,
+        url,
+        method,
+        body: requestBody,
+        params,
+        session,
+        stack: e?.stack || new Error().stack,
+      })
+      throw new ApiError(`Could not reach Gitlab API: ${e.message || e}`, 500)
     }
     // Pass the response headers to the caller
     response.headers.forEach((value, name) => responseHeaders[name] = value)
@@ -358,14 +365,30 @@ export default class GitlabConnector implements StorageConnector {
       try {
         return await response.text()
       } catch (e) {
-        console.error('Gitlab API error (6) - could not parse response', response.status, response.statusText, { url, method, body: requestBody, params }, e)
-        throw new ApiError(`Gitlab API error (6): response body not available. ${e.message}`, 500)
+        console.error('[GitlabConnector] Failed to read response body from Gitlab API', {
+          status: response.status,
+          statusText: response.statusText,
+          url,
+          method,
+          body: requestBody,
+          params,
+          error: e
+        })
+        throw new ApiError(`Gitlab API: Could not read response body (${e.message})`, 500)
       }
     }()
     if (!response.ok) {
-      console.error('Gitlab API error (7) - response not ok', response.status, response.statusText, { url, method, body: requestBody, params, text: text })
+      console.error('[GitlabConnector] Gitlab API responded with error status', {
+        status: response.status,
+        statusText: response.statusText,
+        url,
+        method,
+        body: requestBody,
+        params,
+        responseText: text
+      })
       if (text.includes('A file with this name doesn\'t exist')) {
-        throw new ApiError('Gitlab API error (5): Not Found', 404)
+        throw new ApiError('Gitlab API: File not found', 404)
       } else if (response.status === 401 && this.getSessionToken(session).token?.refresh_token) {
         // Refresh the token
         const token = this.getSessionToken(session).token
@@ -401,17 +424,28 @@ export default class GitlabConnector implements StorageConnector {
           })
         } else {
           const message = typeof refreshJson?.message === 'object' ? Object.entries(refreshJson.message).map(entry => entry.join(' ')).join(' ') : refreshJson?.message ?? refreshJson?.error ?? response.statusText
-          console.error('Gitlab API error (2) - could not refresh token', response.status, response.statusText, { message }, 'refresh_token:', token?.refresh_token)
-          // Workaround for when the token is invalid
-          // It happens often which is not normal (refresh token should last 6 months)
+          console.error('[GitlabConnector] Failed to refresh Gitlab OAuth token', {
+            status: response.status,
+            statusText: response.statusText,
+            message,
+            refresh_token: token?.refresh_token
+          })
           this.logout(session)
-          // Notify the user
-          throw new ApiError(`Gitlab API error (2): ${message}`, response.status)
+          throw new ApiError(`Gitlab API: Could not refresh token (${message})`, response.status)
         }
       } else {
         const message = response.statusText
-        console.error('Gitlab API error (1)', response.status, response.statusText, { url, method, body: requestBody, params, text: text, message })
-        throw new ApiError(`Gitlab API error (1): ${message} (${text})`, response.status)
+        console.error('[GitlabConnector] Unhandled Gitlab API error response', {
+          status: response.status,
+          statusText: response.statusText,
+          url,
+          method,
+          body: requestBody,
+          params,
+          responseText: text,
+          message
+        })
+        throw new ApiError(`Gitlab API error: ${message} (${text})`, response.status)
       }
     }
     let json: { message: string, error: string } | any
@@ -419,11 +453,19 @@ export default class GitlabConnector implements StorageConnector {
       json = JSON.parse(text)
     } catch (e) {
       if (!response.ok) {
-        // A real error
         throw e
       } else {
-        // Useless error linked to the fact that the response is not JSON
-        console.error('Gitlab API error (3) - could not parse response', response.status, response.statusText, { url, method, body: requestBody, params, text: text })
+        console.error('[GitlabConnector] Response from Gitlab API is not valid JSON', {
+          statusText: response.statusText,
+          url,
+          method,
+          body: requestBody,
+          params,
+          responseText: text,
+          error: e,
+          session,
+          stack: e?.stack || new Error().stack,
+        })
         return text
       }
     }
@@ -451,18 +493,30 @@ export default class GitlabConnector implements StorageConnector {
     if (!fileRes.ok) {
       const errText = await fileRes.text()
       if (errText.includes('not found') || fileRes.status === 404) {
-        throw new ApiError('GitLab raw error (5): Not Found', 404)
+        console.error('[GitlabConnector] GitLab raw file not found', {
+          status: fileRes.status,
+          statusText: fileRes.statusText,
+          url: rawUrl,
+          responseText: errText
+        })
+        throw new ApiError(`GitLab raw file not found: filePath="${filePath}" (status ${fileRes.status})`, 404)
       }
-      console.error('GitLab raw error (1)', fileRes.status, fileRes.statusText, { rawUrl, errText })
-      throw new ApiError(`GitLab raw error (1): ${fileRes.statusText} (${errText})`, fileRes.status)
+      console.error('[GitlabConnector] Failed to fetch raw file from GitLab', {
+        filePath,
+        url: rawUrl,
+        status: fileRes.status,
+        statusText: fileRes.statusText,
+        responseText: errText
+      })
+      throw new ApiError(`Failed to fetch raw file from GitLab: filePath="${filePath}" - ${fileRes.statusText} (${errText})`, fileRes.status)
     }
 
     try {
       const buffer = await fileRes.buffer()
       return buffer
     } catch (e) {
-      console.error('GitLab raw error (3): could not read buffer', e)
-      throw new ApiError('GitLab raw error (3): failed to read binary content', 500)
+      console.error('[GitlabConnector] Error reading binary content from GitLab raw file', e)
+      throw new ApiError('Failed to read binary content from GitLab raw file', 500)
     }
   }
 
@@ -673,39 +727,15 @@ export default class GitlabConnector implements StorageConnector {
    */
   async readWebsite(session: GitlabSession, websiteId: string): Promise<WebsiteData> {
     const websiteDataBuf = await this.downloadRawFile(session, websiteId, WEBSITE_DATA_FILE)
-    const websiteDataOb = JSON.parse(websiteDataBuf.toString('utf8')) as GitlabWebsiteData & WebsiteData
-    const { fileFormatVersion, ...websiteData } = websiteDataOb
-
-    // Check the file format version
-    if (fileFormatVersion !== WEBSITE_DATA_FILE_FORMAT_VERSION) {
-      // This should be handled by a migration mechanism
-      console.warn('Gitlab connector: website data file format version mismatch', fileFormatVersion, '!=', WEBSITE_DATA_FILE_FORMAT_VERSION)
-    }
+    const websiteDataContent = websiteDataBuf.toString('utf8')
 
-    // This happens when the website was just created
-    // Let grapesjs create the pages in the frontend
-    if (!websiteData.pages) {
-      return websiteData as WebsiteData
+    // Use the common merge function to reconstruct website data
+    const pageLoader = async (pagePath: string): Promise<string> => {
+      const pageBuffer = await this.downloadRawFile(session, websiteId, pagePath)
+      return pageBuffer.toString('utf8')
     }
 
-    // Load each page in parallel
-    const pages = await Promise.all(websiteData.pages.map(async (page: GitlabPage | Page) => {
-      if ((page as GitlabPage).isFile) {
-        const name = getPageSlug(page.name)
-        const fileName = (`${name}-${page.id}`)
-        const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}.json`
-        const pageContent = await this.downloadRawFile(session, websiteId, filePath)
-        const res = JSON.parse(pageContent.toString('utf8')) as Page
-        return res
-      }
-      return page as Page
-    }))
-
-    // Read each page file if needed
-    return {
-      ...websiteData,
-      pages,
-    } as WebsiteData
+    return await merge(websiteDataContent, pageLoader)
   }
 
   /**
@@ -720,7 +750,7 @@ export default class GitlabConnector implements StorageConnector {
         name: this.options.repoPrefix + websiteMeta.name,
       }
     }) as any
-    await this.createFile(session, project.id, WEBSITE_DATA_FILE, JSON.stringify({} as WebsiteData))
+    await this.createFile(session, project.id, WEBSITE_DATA_FILE, stringify({} as WebsiteData))
     //await this.createFile(session, project.id, WEBSITE_META_DATA_FILE, JSON.stringify(websiteMeta))
     //await this.updateWebsite(session, project.id, {} as WebsiteData)
     //await this.setWebsiteMeta(session, project.id, websiteMeta)
@@ -745,16 +775,19 @@ export default class GitlabConnector implements StorageConnector {
     })
 
     // **
-    // Create the pages for batch upload
-    const pages = websiteData.pages.map((page: Page) => {
-      const fileName = encodeURIComponent(`${getPageSlug(page.name)}-${page.id}`)
-      const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}.json`
-      const content = JSON.stringify(page)
+    // Use the common split function to create files
+    const filesToWrite = split(websiteData)
+
+    // **
+    // Process each file to create/update
+    for (const file of filesToWrite) {
+      const filePath = file.path
+      const content = file.content
       const newSha = computeGitBlobSha(content, false)
 
-      const existingSha = existingFiles.get(filePath)
+      const existingSha = existingFiles.get(filePath) || (filePath === WEBSITE_DATA_FILE ? 'always-update' : undefined)
 
-      if (existingSha) {
+      if (existingSha && existingSha !== 'always-update') {
         if (existingSha !== newSha) {
           batchActions.push({
             action: 'update',
@@ -764,26 +797,18 @@ export default class GitlabConnector implements StorageConnector {
         } // else: skip unchanged file
       } else {
         batchActions.push({
-          action: 'create',
+          action: file.path === WEBSITE_DATA_FILE ? 'update' : 'create',
           file_path: filePath,
           content,
         })
       }
-
-      return {
-        name: page.name,
-        id: page.id,
-        isFile: true,
-      }
-    })
+    }
 
     // **
     // Delete pages that are not in the new website data
+    const newPageFiles = new Set(filesToWrite.filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER)).map(f => f.path))
     for (const filePath of existingFiles.keys()) {
-      const pageName = filePath.replace(/.*\//, '').replace(/\.json$/, '')
-      const pageId = pageName.split('-').pop()
-      const page = websiteData.pages.find((p: Page) => p.id === pageId)
-      if (!page) {
+      if (!newPageFiles.has(filePath)) {
         batchActions.push({
           action: 'delete',
           file_path: filePath,
@@ -791,21 +816,6 @@ export default class GitlabConnector implements StorageConnector {
       }
     }
 
-    // **
-    // Add the main website data file
-    const websiteDataWithGitlabPages = {
-      ...websiteData,
-      fileFormatVersion: WEBSITE_DATA_FILE_FORMAT_VERSION,
-      pages,
-    } as GitlabWebsiteData
-
-    const websiteJsonContent = JSON.stringify(websiteDataWithGitlabPages)
-    batchActions.push({
-      action: 'update',
-      file_path: WEBSITE_DATA_FILE,
-      content: websiteJsonContent,
-    })
-
     // **
     // Perform a batch commit
     return this.callApi({

package/src/ts/server/connectors/FsStorage.test.ts

@@ -3,6 +3,7 @@ import { ServerConfig } from '../config'
 import { FsStorage } from './FsStorage'
 import { readFileSync, rmdirSync, statSync } from 'fs'
 import { join } from 'path'
+import { WEBSITE_DATA_FILE } from '../../constants'
 
 const storageRootPath = '/tmp/silex-tests'
 const assetsFolder = 'assets'
@@ -45,7 +46,10 @@ describe('FsStorage website', () => {
     })
     const id = await connector.createWebsite(dummySession, { name: 'dummy name', connectorUserSettings: {} })
     expect(id).toBeDefined()
-    expect(() => statSync(join(storageRootPath, id, 'website.json'))).not.toThrow()
-    expect(readFileSync(join(storageRootPath, id, 'website.json'), 'utf8')).toBe(JSON.stringify({}))
+    expect(() => statSync(join(storageRootPath, id, WEBSITE_DATA_FILE))).not.toThrow()
+    const content = readFileSync(join(storageRootPath, id, WEBSITE_DATA_FILE), 'utf8')
+    const parsed = JSON.parse(content)
+    expect(parsed).toHaveProperty('fileFormatVersion', '1.0.0')
+    expect(parsed).toHaveProperty('pages', [])
   })
 })

package/src/ts/server/connectors/FsStorage.ts

@@ -23,10 +23,11 @@ import { ConnectorUser, WebsiteMeta, JobStatus, WebsiteId, ConnectorType, Websit
 import { userInfo } from 'os'
 import { requiredParam } from '../utils/validation'
 import { ServerConfig } from '../config'
-import { DEFAULT_WEBSITE_ID, WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE } from '../../constants'
+import { DEFAULT_WEBSITE_ID, WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
 import { Readable } from 'stream'
 import { v4 as uuid } from 'uuid'
 import { fileURLToPath } from 'url'
+import { stringify, split, merge } from '../utils/websiteDataSerialization'
 
 // Variables needed for jest tests
 if(!globalThis.__dirname) {
@@ -145,7 +146,7 @@ export class FsStorage implements StorageConnector<FsSession> {
 
   async setWebsiteMeta(session: any, id: string, data: WebsiteMetaFileContent): Promise<void> {
     const websiteId = requiredParam<WebsiteId>(id, 'website id')
-    const content = JSON.stringify(data)
+    const content = stringify(data)
     const path = join(this.options.path, id, WEBSITE_META_DATA_FILE)
     await fs.writeFile(path, content)
   }
@@ -183,14 +184,68 @@ export class FsStorage implements StorageConnector<FsSession> {
   async readWebsite(session: FsSession, websiteId: WebsiteId): Promise<WebsiteData> {
     const id = requiredParam<WebsiteId>(websiteId, 'website id')
     const path = join(this.options.path, id, WEBSITE_DATA_FILE)
+
     const content = await fs.readFile(path)
-    return JSON.parse(content.toString())
+    const websiteDataContent = content.toString()
+
+    // Check if this is the new split format
+    const parsedData = JSON.parse(websiteDataContent)
+    if (parsedData.fileFormatVersion === '1.0.0') {
+      // Use the merge function to reconstruct website data
+      const pageLoader = async (pagePath: string): Promise<string> => {
+        const fullPath = join(this.options.path, id, pagePath)
+        const pageContent = await fs.readFile(fullPath)
+        return pageContent.toString()
+      }
+
+      return await merge(websiteDataContent, pageLoader)
+    } else {
+      // Legacy format, return as is
+      return parsedData
+    }
   }
 
   async updateWebsite(session: FsSession, websiteId: WebsiteId, data: WebsiteData): Promise<void> {
     const id = requiredParam<WebsiteId>(websiteId, 'website id')
-    const path = join(this.options.path, id, WEBSITE_DATA_FILE)
-    await fs.writeFile(path, JSON.stringify(data))
+    const websitePath = join(this.options.path, id)
+
+    // Use the split function to create separate files for pages
+    const filesToWrite = split(data)
+
+    // Ensure the website directory exists
+    await fs.mkdir(websitePath, { recursive: true })
+
+    // Ensure the pages directory exists if we have page files
+    const hasPageFiles = filesToWrite.some(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
+    if (hasPageFiles) {
+      await fs.mkdir(join(websitePath, WEBSITE_PAGES_FOLDER), { recursive: true })
+    }
+
+    // **
+    // Delete pages that are not in the new website data
+    const pagesPath = join(websitePath, WEBSITE_PAGES_FOLDER)
+    try {
+      const existingPageFiles = await fs.readdir(pagesPath)
+      const newPageFiles = new Set(
+        filesToWrite
+          .filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
+          .map(f => f.path.replace(`${WEBSITE_PAGES_FOLDER}/`, ''))
+      )
+      
+      for (const existingFile of existingPageFiles) {
+        if (existingFile.endsWith('.json') && !newPageFiles.has(existingFile)) {
+          await fs.unlink(join(pagesPath, existingFile))
+        }
+      }
+    } catch (error) {
+      // Ignore error if pages directory doesn't exist yet
+    }
+
+    // Write all files
+    for (const file of filesToWrite) {
+      const filePath = join(websitePath, file.path)
+      await fs.writeFile(filePath, file.content)
+    }
   }
 
   async deleteWebsite(session: FsSession, websiteId: WebsiteId): Promise<void> {

package/src/ts/server/utils/websiteDataSerialization.test.ts

@@ -0,0 +1,59 @@
+import { stringify, split, merge } from './websiteDataSerialization'
+import { expect, it, describe } from '@jest/globals'
+import { WebsiteData, defaultWebsiteData } from '../../types'
+import { WEBSITE_DATA_FILE } from '../../constants'
+
+describe('websiteDataSerialization', () => {
+  describe('stringify', () => {
+    it('should format JSON with sorted keys', () => {
+      const data = { z: 1, a: 2, m: { z: 3, a: 4 } }
+      const result = stringify(data)
+      expect(result).toBe(`{
+  "a": 2,
+  "m": {
+    "a": 4,
+    "z": 3
+  },
+  "z": 1
+}`)
+    })
+  })
+
+  describe('split', () => {
+    it('should split website data into separate files', () => {
+      const websiteData: WebsiteData = {
+        ...defaultWebsiteData,
+        pages: [
+          { id: 'page1', getName: () => 'Home' } as any,
+          { id: 'page2', getName: () => 'About' } as any
+        ]
+      }
+      
+      const files = split(websiteData)
+      
+      expect(files).toHaveLength(3)
+      expect(files[0].path).toBe('src/home-page1.json')
+      expect(files[1].path).toBe('src/about-page2.json')
+      expect(files[2].path).toBe(WEBSITE_DATA_FILE)
+      
+      const mainFile = JSON.parse(files[2].content)
+      expect(mainFile.fileFormatVersion).toBe('1.0.0')
+      expect(mainFile.pages[0].isFile).toBe(true)
+    })
+  })
+
+  describe('merge', () => {
+    it('should handle legacy format', async () => {
+      const legacyContent = `{
+        "settings": {},
+        "pages": [{ "id": "page1" }]
+      }`
+      
+      const pageLoader = async () => { throw new Error('Should not be called') }
+      const result = await merge(legacyContent, pageLoader)
+      
+      expect(result.pages).toHaveLength(1)
+      expect(result.pages[0].id).toBe('page1')
+    })
+  })
+})

package/src/ts/server/utils/websiteDataSerialization.ts

@@ -0,0 +1,143 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { WebsiteData } from '../../types'
+import { Page } from 'grapesjs'
+import { getPageSlug } from '../../page'
+import { WEBSITE_PAGES_FOLDER, WEBSITE_DATA_FILE } from '../../constants'
+
+/**
+ * Serialize JSON with stable formatting for git
+ * - Uses 2-space indentation for readability
+ * - Sorts object keys deterministically to avoid random diffs
+ */
+export function stringify(data: any): string {
+  return JSON.stringify(data, sortKeys, 2)
+}
+
+/**
+ * Sort object keys for consistent output across serializations
+ */
+function sortKeys(key: string, value: any): any {
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    const sorted: any = {}
+    Object.keys(value).sort().forEach(k => {
+      sorted[k] = value[k]
+    })
+    return sorted
+  }
+  return value
+}
+
+/**
+ * Split website data into multiple files
+ * Returns array of files to create/update
+ */
+export function split(websiteData: WebsiteData): Array<{path: string, content: string}> {
+  const files: Array<{path: string, content: string}> = []
+
+  // Split pages into separate files
+  const pages = websiteData.pages?.map((page: Page) => {
+    const pageName = getPageName(page)
+    const slug = getPageSlug(pageName)
+    const fileName = `${slug}-${page.id}.json`
+    const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}`
+
+    files.push({
+      path: filePath,
+      content: stringify(page)
+    })
+
+    return {
+      name: pageName,
+      id: page.id,
+      isFile: true,
+    }
+  }) || []
+
+  // Create main website data file without pages content
+  const websiteDataWithoutPages = {
+    ...websiteData,
+    fileFormatVersion: '1.0.0',
+    pages,
+  }
+
+  files.push({
+    path: WEBSITE_DATA_FILE,
+    content: stringify(websiteDataWithoutPages)
+  })
+
+  return files
+}
+
+/**
+ * Reconstruct website data from main file and page loader function
+ * The pageLoader function will be called for each page that needs to be loaded
+ */
+export async function merge(
+  websiteDataContent: string,
+  pageLoader: (pagePath: string) => Promise<string>
+): Promise<WebsiteData> {
+  const websiteDataOb = JSON.parse(websiteDataContent) as any
+  const { fileFormatVersion, ...websiteData } = websiteDataOb
+
+  // Check file format version
+  if (fileFormatVersion && fileFormatVersion !== '1.0.0') {
+    console.warn('Website data file format version mismatch', fileFormatVersion, '!=', '1.0.0')
+  }
+
+  // Handle legacy format (no split pages) or empty pages
+  if (!websiteData.pages || websiteData.pages.length === 0) {
+    return websiteData as WebsiteData
+  }
+
+  // Check if pages are already embedded (legacy format)
+  if (websiteData.pages[0] && !websiteData.pages[0].isFile) {
+    return websiteData as WebsiteData
+  }
+
+  // Load pages from separate files
+  const pages = await Promise.all(websiteData.pages.map(async (pageRef: any) => {
+    if (pageRef.isFile) {
+      const pageName = pageRef.name
+      const slug = getPageSlug(pageName)
+      const fileName = `${slug}-${pageRef.id}.json`
+      const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}`
+
+      try {
+        const pageContent = await pageLoader(filePath)
+        return JSON.parse(pageContent) as Page
+      } catch (error) {
+        console.warn(`Could not load page file: ${filePath}`, error)
+        return pageRef
+      }
+    }
+    return pageRef as Page
+  }))
+
+  return {
+    ...websiteData,
+    pages,
+  } as WebsiteData
+}
+
+/**
+ * Get page name from page object
+ */
+function getPageName(page: Page | any): string {
+  return (page as any).getName ? (page as Page).getName() : (page as any).name
+}