@silexlabs/silex 3.3.1-4 → 3.3.1-5

package/src/ts/plugins/server/GitlabConnector.ts

@@ -27,6 +27,7 @@ import { getPageSlug } from '../../page'
 import e from 'express'
 import { fork } from 'child_process'
 import { Page } from 'grapesjs'
+import { stringify, split, merge } from '../../server/utils/websiteDataSerialization'
 
 /**
  * Gitlab connector
@@ -121,16 +122,6 @@ interface GitlabFetchCommits {
   since: string
 }
 
-interface GitlabPage {
-  id: string
-  name: string
-  isFile: true
-}
-
-interface GitlabWebsiteData {
-  fileFormatVersion: string
-  pages: GitlabPage[]
-}
 
 // interface MetaRepoFileContent {
 //   websites: {
@@ -221,9 +212,6 @@ function sanitizeGitlabPath(name: string): string {
     .toLowerCase()
 }
 
-function getPageName(page: Page | GitlabPage) {
-  return (page as any).getName ? (page as Page).getName() : (page as GitlabPage).name
-}
 
 export default class GitlabConnector implements StorageConnector {
   connectorId = 'gitlab'
@@ -739,40 +727,15 @@ export default class GitlabConnector implements StorageConnector {
    */
   async readWebsite(session: GitlabSession, websiteId: string): Promise<WebsiteData> {
     const websiteDataBuf = await this.downloadRawFile(session, websiteId, WEBSITE_DATA_FILE)
-    const websiteDataOb = JSON.parse(websiteDataBuf.toString('utf8')) as GitlabWebsiteData & WebsiteData
-    const { fileFormatVersion, ...websiteData } = websiteDataOb
+    const websiteDataContent = websiteDataBuf.toString('utf8')
 
-    // Check the file format version
-    if (fileFormatVersion !== WEBSITE_DATA_FILE_FORMAT_VERSION) {
-      // This should be handled by a migration mechanism
-      console.warn('Gitlab connector: website data file format version mismatch', fileFormatVersion, '!=', WEBSITE_DATA_FILE_FORMAT_VERSION)
+    // Use the common merge function to reconstruct website data
+    const pageLoader = async (pagePath: string): Promise<string> => {
+      const pageBuffer = await this.downloadRawFile(session, websiteId, pagePath)
+      return pageBuffer.toString('utf8')
     }
 
-    // This happens when the website was just created
-    // Let grapesjs create the pages in the frontend
-    if (!websiteData.pages) {
-      return websiteData as WebsiteData
-    }
-
-    // Load each page in parallel
-    const pages = await Promise.all(websiteData.pages.map(async (page: GitlabPage | Page) => {
-      if ((page as GitlabPage).isFile) {
-        const name = getPageName(page)
-        const slug = getPageSlug(name)
-        const fileName = (`${slug}-${page.id}`)
-        const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}.json`
-        const pageContent = await this.downloadRawFile(session, websiteId, filePath)
-        const res = JSON.parse(pageContent.toString('utf8')) as Page
-        return res
-      }
-      return page as Page
-    }))
-
-    // Read each page file if needed
-    return {
-      ...websiteData,
-      pages,
-    } as WebsiteData
+    return await merge(websiteDataContent, pageLoader)
   }
 
   /**
@@ -787,7 +750,7 @@ export default class GitlabConnector implements StorageConnector {
         name: this.options.repoPrefix + websiteMeta.name,
       }
     }) as any
-    await this.createFile(session, project.id, WEBSITE_DATA_FILE, JSON.stringify({} as WebsiteData))
+    await this.createFile(session, project.id, WEBSITE_DATA_FILE, stringify({} as WebsiteData))
     //await this.createFile(session, project.id, WEBSITE_META_DATA_FILE, JSON.stringify(websiteMeta))
     //await this.updateWebsite(session, project.id, {} as WebsiteData)
     //await this.setWebsiteMeta(session, project.id, websiteMeta)
@@ -812,17 +775,19 @@ export default class GitlabConnector implements StorageConnector {
     })
 
     // **
-    // Create the pages for batch upload
-    const pages = websiteData.pages.map((page: Page) => {
-      /* @ts-ignore */
-      const fileName = encodeURIComponent(`${getPageSlug(getPageName(page))}-${page.id}`)
-      const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}.json`
-      const content = JSON.stringify(page)
+    // Use the common split function to create files
+    const filesToWrite = split(websiteData)
+
+    // **
+    // Process each file to create/update
+    for (const file of filesToWrite) {
+      const filePath = file.path
+      const content = file.content
       const newSha = computeGitBlobSha(content, false)
 
-      const existingSha = existingFiles.get(filePath)
+      const existingSha = existingFiles.get(filePath) || (filePath === WEBSITE_DATA_FILE ? 'always-update' : undefined)
 
-      if (existingSha) {
+      if (existingSha && existingSha !== 'always-update') {
         if (existingSha !== newSha) {
           batchActions.push({
             action: 'update',
@@ -832,26 +797,18 @@ export default class GitlabConnector implements StorageConnector {
         } // else: skip unchanged file
       } else {
         batchActions.push({
-          action: 'create',
+          action: file.path === WEBSITE_DATA_FILE ? 'update' : 'create',
           file_path: filePath,
           content,
         })
       }
-
-      return {
-        name: getPageName(page),
-        id: page.id,
-        isFile: true,
-      }
-    })
+    }
 
     // **
     // Delete pages that are not in the new website data
+    const newPageFiles = new Set(filesToWrite.filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER)).map(f => f.path))
     for (const filePath of existingFiles.keys()) {
-      const pageName = filePath.replace(/.*\//, '').replace(/\.json$/, '')
-      const pageId = pageName.split('-').pop()
-      const page = websiteData.pages.find((p: Page) => p.id === pageId)
-      if (!page) {
+      if (!newPageFiles.has(filePath)) {
         batchActions.push({
           action: 'delete',
           file_path: filePath,
@@ -859,21 +816,6 @@ export default class GitlabConnector implements StorageConnector {
       }
     }
 
-    // **
-    // Add the main website data file
-    const websiteDataWithGitlabPages = {
-      ...websiteData,
-      fileFormatVersion: WEBSITE_DATA_FILE_FORMAT_VERSION,
-      pages,
-    } as GitlabWebsiteData
-
-    const websiteJsonContent = JSON.stringify(websiteDataWithGitlabPages)
-    batchActions.push({
-      action: 'update',
-      file_path: WEBSITE_DATA_FILE,
-      content: websiteJsonContent,
-    })
-
     // **
     // Perform a batch commit
     return this.callApi({

package/src/ts/server/connectors/FsStorage.test.ts

@@ -3,6 +3,7 @@ import { ServerConfig } from '../config'
 import { FsStorage } from './FsStorage'
 import { readFileSync, rmdirSync, statSync } from 'fs'
 import { join } from 'path'
+import { WEBSITE_DATA_FILE } from '../../constants'
 
 const storageRootPath = '/tmp/silex-tests'
 const assetsFolder = 'assets'
@@ -45,7 +46,10 @@ describe('FsStorage website', () => {
     })
     const id = await connector.createWebsite(dummySession, { name: 'dummy name', connectorUserSettings: {} })
     expect(id).toBeDefined()
-    expect(() => statSync(join(storageRootPath, id, 'website.json'))).not.toThrow()
-    expect(readFileSync(join(storageRootPath, id, 'website.json'), 'utf8')).toBe(JSON.stringify({}))
+    expect(() => statSync(join(storageRootPath, id, WEBSITE_DATA_FILE))).not.toThrow()
+    const content = readFileSync(join(storageRootPath, id, WEBSITE_DATA_FILE), 'utf8')
+    const parsed = JSON.parse(content)
+    expect(parsed).toHaveProperty('fileFormatVersion', '1.0.0')
+    expect(parsed).toHaveProperty('pages', [])
   })
 })

package/src/ts/server/connectors/FsStorage.ts

@@ -23,10 +23,11 @@ import { ConnectorUser, WebsiteMeta, JobStatus, WebsiteId, ConnectorType, Websit
 import { userInfo } from 'os'
 import { requiredParam } from '../utils/validation'
 import { ServerConfig } from '../config'
-import { DEFAULT_WEBSITE_ID, WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE } from '../../constants'
+import { DEFAULT_WEBSITE_ID, WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
 import { Readable } from 'stream'
 import { v4 as uuid } from 'uuid'
 import { fileURLToPath } from 'url'
+import { stringify, split, merge } from '../utils/websiteDataSerialization'
 
 // Variables needed for jest tests
 if(!globalThis.__dirname) {
@@ -145,7 +146,7 @@ export class FsStorage implements StorageConnector<FsSession> {
 
   async setWebsiteMeta(session: any, id: string, data: WebsiteMetaFileContent): Promise<void> {
     const websiteId = requiredParam<WebsiteId>(id, 'website id')
-    const content = JSON.stringify(data)
+    const content = stringify(data)
     const path = join(this.options.path, id, WEBSITE_META_DATA_FILE)
     await fs.writeFile(path, content)
   }
@@ -183,14 +184,68 @@ export class FsStorage implements StorageConnector<FsSession> {
   async readWebsite(session: FsSession, websiteId: WebsiteId): Promise<WebsiteData> {
     const id = requiredParam<WebsiteId>(websiteId, 'website id')
     const path = join(this.options.path, id, WEBSITE_DATA_FILE)
+
     const content = await fs.readFile(path)
-    return JSON.parse(content.toString())
+    const websiteDataContent = content.toString()
+
+    // Check if this is the new split format
+    const parsedData = JSON.parse(websiteDataContent)
+    if (parsedData.fileFormatVersion === '1.0.0') {
+      // Use the merge function to reconstruct website data
+      const pageLoader = async (pagePath: string): Promise<string> => {
+        const fullPath = join(this.options.path, id, pagePath)
+        const pageContent = await fs.readFile(fullPath)
+        return pageContent.toString()
+      }
+
+      return await merge(websiteDataContent, pageLoader)
+    } else {
+      // Legacy format, return as is
+      return parsedData
+    }
   }
 
   async updateWebsite(session: FsSession, websiteId: WebsiteId, data: WebsiteData): Promise<void> {
     const id = requiredParam<WebsiteId>(websiteId, 'website id')
-    const path = join(this.options.path, id, WEBSITE_DATA_FILE)
-    await fs.writeFile(path, JSON.stringify(data))
+    const websitePath = join(this.options.path, id)
+
+    // Use the split function to create separate files for pages
+    const filesToWrite = split(data)
+
+    // Ensure the website directory exists
+    await fs.mkdir(websitePath, { recursive: true })
+
+    // Ensure the pages directory exists if we have page files
+    const hasPageFiles = filesToWrite.some(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
+    if (hasPageFiles) {
+      await fs.mkdir(join(websitePath, WEBSITE_PAGES_FOLDER), { recursive: true })
+    }
+
+    // **
+    // Delete pages that are not in the new website data
+    const pagesPath = join(websitePath, WEBSITE_PAGES_FOLDER)
+    try {
+      const existingPageFiles = await fs.readdir(pagesPath)
+      const newPageFiles = new Set(
+        filesToWrite
+          .filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
+          .map(f => f.path.replace(`${WEBSITE_PAGES_FOLDER}/`, ''))
+      )
+      
+      for (const existingFile of existingPageFiles) {
+        if (existingFile.endsWith('.json') && !newPageFiles.has(existingFile)) {
+          await fs.unlink(join(pagesPath, existingFile))
+        }
+      }
+    } catch (error) {
+      // Ignore error if pages directory doesn't exist yet
+    }
+
+    // Write all files
+    for (const file of filesToWrite) {
+      const filePath = join(websitePath, file.path)
+      await fs.writeFile(filePath, file.content)
+    }
   }
 
   async deleteWebsite(session: FsSession, websiteId: WebsiteId): Promise<void> {

package/src/ts/server/utils/websiteDataSerialization.test.ts

@@ -0,0 +1,59 @@
+import { stringify, split, merge } from './websiteDataSerialization'
+import { expect, it, describe } from '@jest/globals'
+import { WebsiteData, defaultWebsiteData } from '../../types'
+import { WEBSITE_DATA_FILE } from '../../constants'
+
+describe('websiteDataSerialization', () => {
+  describe('stringify', () => {
+    it('should format JSON with sorted keys', () => {
+      const data = { z: 1, a: 2, m: { z: 3, a: 4 } }
+      const result = stringify(data)
+      expect(result).toBe(`{
+  "a": 2,
+  "m": {
+    "a": 4,
+    "z": 3
+  },
+  "z": 1
+}`)
+    })
+  })
+
+  describe('split', () => {
+    it('should split website data into separate files', () => {
+      const websiteData: WebsiteData = {
+        ...defaultWebsiteData,
+        pages: [
+          { id: 'page1', getName: () => 'Home' } as any,
+          { id: 'page2', getName: () => 'About' } as any
+        ]
+      }
+      
+      const files = split(websiteData)
+      
+      expect(files).toHaveLength(3)
+      expect(files[0].path).toBe('src/home-page1.json')
+      expect(files[1].path).toBe('src/about-page2.json')
+      expect(files[2].path).toBe(WEBSITE_DATA_FILE)
+      
+      const mainFile = JSON.parse(files[2].content)
+      expect(mainFile.fileFormatVersion).toBe('1.0.0')
+      expect(mainFile.pages[0].isFile).toBe(true)
+    })
+  })
+
+  describe('merge', () => {
+    it('should handle legacy format', async () => {
+      const legacyContent = `{
+        "settings": {},
+        "pages": [{ "id": "page1" }]
+      }`
+      
+      const pageLoader = async () => { throw new Error('Should not be called') }
+      const result = await merge(legacyContent, pageLoader)
+      
+      expect(result.pages).toHaveLength(1)
+      expect(result.pages[0].id).toBe('page1')
+    })
+  })
+})

package/src/ts/server/utils/websiteDataSerialization.ts

@@ -0,0 +1,143 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { WebsiteData } from '../../types'
+import { Page } from 'grapesjs'
+import { getPageSlug } from '../../page'
+import { WEBSITE_PAGES_FOLDER, WEBSITE_DATA_FILE } from '../../constants'
+
+/**
+ * Serialize JSON with stable formatting for git
+ * - Uses 2-space indentation for readability
+ * - Sorts object keys deterministically to avoid random diffs
+ */
+export function stringify(data: any): string {
+  return JSON.stringify(data, sortKeys, 2)
+}
+
+/**
+ * Sort object keys for consistent output across serializations
+ */
+function sortKeys(key: string, value: any): any {
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    const sorted: any = {}
+    Object.keys(value).sort().forEach(k => {
+      sorted[k] = value[k]
+    })
+    return sorted
+  }
+  return value
+}
+
+/**
+ * Split website data into multiple files
+ * Returns array of files to create/update
+ */
+export function split(websiteData: WebsiteData): Array<{path: string, content: string}> {
+  const files: Array<{path: string, content: string}> = []
+
+  // Split pages into separate files
+  const pages = websiteData.pages?.map((page: Page) => {
+    const pageName = getPageName(page)
+    const slug = getPageSlug(pageName)
+    const fileName = `${slug}-${page.id}.json`
+    const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}`
+
+    files.push({
+      path: filePath,
+      content: stringify(page)
+    })
+
+    return {
+      name: pageName,
+      id: page.id,
+      isFile: true,
+    }
+  }) || []
+
+  // Create main website data file without pages content
+  const websiteDataWithoutPages = {
+    ...websiteData,
+    fileFormatVersion: '1.0.0',
+    pages,
+  }
+
+  files.push({
+    path: WEBSITE_DATA_FILE,
+    content: stringify(websiteDataWithoutPages)
+  })
+
+  return files
+}
+
+/**
+ * Reconstruct website data from main file and page loader function
+ * The pageLoader function will be called for each page that needs to be loaded
+ */
+export async function merge(
+  websiteDataContent: string,
+  pageLoader: (pagePath: string) => Promise<string>
+): Promise<WebsiteData> {
+  const websiteDataOb = JSON.parse(websiteDataContent) as any
+  const { fileFormatVersion, ...websiteData } = websiteDataOb
+
+  // Check file format version
+  if (fileFormatVersion && fileFormatVersion !== '1.0.0') {
+    console.warn('Website data file format version mismatch', fileFormatVersion, '!=', '1.0.0')
+  }
+
+  // Handle legacy format (no split pages) or empty pages
+  if (!websiteData.pages || websiteData.pages.length === 0) {
+    return websiteData as WebsiteData
+  }
+
+  // Check if pages are already embedded (legacy format)
+  if (websiteData.pages[0] && !websiteData.pages[0].isFile) {
+    return websiteData as WebsiteData
+  }
+
+  // Load pages from separate files
+  const pages = await Promise.all(websiteData.pages.map(async (pageRef: any) => {
+    if (pageRef.isFile) {
+      const pageName = pageRef.name
+      const slug = getPageSlug(pageName)
+      const fileName = `${slug}-${pageRef.id}.json`
+      const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}`
+
+      try {
+        const pageContent = await pageLoader(filePath)
+        return JSON.parse(pageContent) as Page
+      } catch (error) {
+        console.warn(`Could not load page file: ${filePath}`, error)
+        return pageRef
+      }
+    }
+    return pageRef as Page
+  }))
+
+  return {
+    ...websiteData,
+    pages,
+  } as WebsiteData
+}
+
+/**
+ * Get page name from page object
+ */
+function getPageName(page: Page | any): string {
+  return (page as any).getName ? (page as Page).getName() : (page as any).name
+}

package/src/ts/client/grapesjs/dynamic-plugins.ts

@@ -1,63 +0,0 @@
-/**
- * Dynamic plugin loader for external GrapesJS plugins
- * These plugins are loaded at runtime to reduce initial bundle size
- */
-
-export interface PluginDefinition {
-  name: string
-  value: any
-}
-
-/**
- * Load all external plugins dynamically
- */
-export async function loadDynamicPlugins(): Promise<PluginDefinition[]> {
-  const [
-    blocksBasicPlugin,
-    styleFilterPlugin,
-    formPlugin,
-    codePlugin,
-    filterStyles,
-    symbolsPlugin,
-    loadingPlugin,
-    fontsDialogPlugin,
-    selectorPlugin,
-    rateLimitPlugin,
-    borderPlugin,
-    backgroundPlugin,
-    notificationsPlugin,
-    keymapsDialogPlugin,
-  ] = await Promise.all([
-    import('grapesjs-blocks-basic').then(m => m.default),
-    import('grapesjs-style-filter').then(m => m.default),
-    import('grapesjs-plugin-forms').then(m => m.default),
-    import('grapesjs-custom-code').then(m => m.default),
-    import('@silexlabs/grapesjs-filter-styles').then(m => m.default),
-    import('@silexlabs/grapesjs-symbols').then(m => m.default),
-    import('@silexlabs/grapesjs-loading').then(m => m.default),
-    import('@silexlabs/grapesjs-fonts').then(m => m.default),
-    import('@silexlabs/grapesjs-advanced-selector').then(m => m.default),
-    import('@silexlabs/grapesjs-storage-rate-limit').then(m => m.default),
-    import('grapesjs-style-border').then(m => m.default),
-    import('grapesjs-style-bg').then(m => m.default),
-    import('@silexlabs/grapesjs-notifications').then(m => m.default),
-    import('@silexlabs/grapesjs-keymaps-dialog').then(m => m.default),
-  ])
-
-  return [
-    { name: 'grapesjs-blocks-basic', value: blocksBasicPlugin },
-    { name: 'grapesjs-style-filter', value: styleFilterPlugin },
-    { name: 'grapesjs-plugin-forms', value: formPlugin },
-    { name: 'grapesjs-custom-code', value: codePlugin },
-    { name: '@silexlabs/grapesjs-filter-styles', value: filterStyles },
-    { name: '@silexlabs/grapesjs-symbols', value: symbolsPlugin },
-    { name: '@silexlabs/grapesjs-loading', value: loadingPlugin },
-    { name: '@silexlabs/grapesjs-fonts', value: fontsDialogPlugin },
-    { name: '@silexlabs/grapesjs-advanced-selector', value: selectorPlugin },
-    { name: '@silexlabs/grapesjs-storage-rate-limit', value: rateLimitPlugin },
-    { name: 'grapesjs-style-border', value: borderPlugin },
-    { name: 'grapesjs-style-bg', value: backgroundPlugin },
-    { name: '@silexlabs/grapesjs-notifications', value: notificationsPlugin },
-    { name: '@silexlabs/grapesjs-keymaps-dialog', value: keymapsDialogPlugin },
-  ]
-}