@silexlabs/silex 3.4.0 → 3.5.1-1

package/src/ts/plugins/server/FtpConnector.ts

@@ -19,8 +19,8 @@ import { Client } from 'basic-ftp'
 import { Readable } from 'stream'
 import { ConnectorFile, ConnectorFileContent, HostingConnector, StatusCallback, StorageConnector, contentToReadable, contentToString, toConnectorData, toConnectorEnum} from '../../server/connectors/connectors'
 import { requiredParam } from '../../server/utils/validation'
-import { WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
-import { ConnectorType, ConnectorUser, WebsiteMeta, FileMeta, JobData, JobStatus, WebsiteId, PublicationJobData, WebsiteMetaFileContent, defaultWebsiteData, WebsiteData, ConnectorOptions } from '../../types'
+import { WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE, LEGACY_WEBSITE_PAGES_FOLDER } from '../../constants'
+import { ConnectorType, ConnectorUser, WebsiteMeta, FileMeta, JobData, JobStatus, WebsiteId, PublicationJobData, WebsiteMetaFileContent, EMPTY_WEBSITE, WebsiteData, ConnectorOptions } from '../../types'
 import { ServerConfig } from '../../server/config'
 import { join } from 'path'
 import { tmpdir } from 'os'
@@ -28,7 +28,7 @@ import { v4 as uuid } from 'uuid'
 import { JobManager } from '../../server/jobs'
 import { mkdtemp, rm, rmdir } from 'fs/promises'
 import { createReadStream } from 'fs'
-import { stringify, split, merge } from '../../server/utils/websiteDataSerialization'
+import { stringify, split, merge, getPagesFolder } from '../../server/utils/websiteDataSerialization'
 
 /**
  * @fileoverview FTP connector for Silex
@@ -443,7 +443,7 @@ export default class FtpConnector implements StorageConnector<FtpSession> {
     await this.mkdir(ftp, assetsPath)
     // create website data file
     const websiteDataPath = join(this.rootPath(session), id, WEBSITE_DATA_FILE)
-    await this.write(ftp, websiteDataPath, stringify(defaultWebsiteData))
+    await this.write(ftp, websiteDataPath, stringify(EMPTY_WEBSITE))
     // create website meta data file
     const websiteMetaDataPath = join(this.rootPath(session), id, WEBSITE_META_DATA_FILE)
     await this.write(ftp, websiteMetaDataPath, stringify({}))
@@ -477,22 +477,16 @@ export default class FtpConnector implements StorageConnector<FtpSession> {
 
       // Check if this is the new split format by trying to parse
       const parsedData = JSON.parse(websiteDataContent)
-      if (parsedData.fileFormatVersion === '1.0.0') {
-        // Use the merge function to reconstruct website data
-        const pageLoader = async (pagePath: string): Promise<string> => {
-          const fullPath = join(storageRootPath, websiteId, pagePath)
-          const pageStream = await this.read(ftp, fullPath)
-          return await contentToString(pageStream)
-        }
-
-        const result = await merge(websiteDataContent, pageLoader)
-        this.closeClient(ftp)
-        return result
-      } else {
-        // Legacy format, return as is
-        this.closeClient(ftp)
-        return parsedData
+      // Use the merge function to reconstruct website data
+      const pageLoader = async (pagePath: string): Promise<string> => {
+        const fullPath = join(storageRootPath, websiteId, pagePath)
+        const pageStream = await this.read(ftp, fullPath)
+        return await contentToString(pageStream)
       }
+
+      const result = await merge(websiteDataContent, pageLoader)
+      this.closeClient(ftp)
+      return result
     } catch (error) {
       this.closeClient(ftp)
       throw error
@@ -506,17 +500,20 @@ export default class FtpConnector implements StorageConnector<FtpSession> {
     // Use the split function to create separate files for pages
     const filesToWrite = split(data)
 
+    // Get the pages folder path from website data
+    const pagesFolder = getPagesFolder(data)
+
     // **
     // Delete pages that are not in the new website data
-    const pagesPath = join(storageRootPath, websiteId, WEBSITE_PAGES_FOLDER)
+    const pagesPath = join(storageRootPath, websiteId, pagesFolder)
     try {
       const existingPageFiles = await this.readdir(ftp, pagesPath)
       const newPageFiles = new Set(
         filesToWrite
-          .filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
-          .map(f => f.path.replace(`${WEBSITE_PAGES_FOLDER}/`, ''))
+          .filter(f => f.path.startsWith(pagesFolder))
+          .map(f => f.path.replace(`${pagesFolder}/`, ''))
       )
-      
+
       for (const existingFile of existingPageFiles) {
         if (existingFile.name.endsWith('.json') && !newPageFiles.has(existingFile.name)) {
           await this.unlink(ftp, join(pagesPath, existingFile.name))

package/src/ts/plugins/server/GitlabConnector.ts

@@ -15,10 +15,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import { API_CONNECTOR_LOGIN_CALLBACK, API_CONNECTOR_PATH, API_PATH, WEBSITE_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
+import { API_CONNECTOR_LOGIN_CALLBACK, API_CONNECTOR_PATH, API_PATH, WEBSITE_DATA_FILE, LEGACY_WEBSITE_PAGES_FOLDER, WEBSITE_PAGES_FOLDER } from '../../constants'
 import { ServerConfig } from '../../server/config'
 import { ConnectorFile, ConnectorFileContent, StatusCallback, StorageConnector, contentToBuffer, contentToString, toConnectorData } from '../../server/connectors/connectors'
-import { ApiError, ConnectorType, ConnectorUser, WebsiteData, WebsiteId, WebsiteMeta, WebsiteMetaFileContent, JobStatus } from '../../types'
+import { ApiError, ConnectorType, ConnectorUser, WebsiteData, WebsiteId, WebsiteMeta, WebsiteMetaFileContent, JobStatus, EMPTY_WEBSITE } from '../../types'
 import fetch from 'node-fetch'
 import crypto, { createHash } from 'crypto'
 import { join } from 'path'
@@ -27,7 +27,7 @@ import { getPageSlug } from '../../page'
 import e from 'express'
 import { fork } from 'child_process'
 import { Page } from 'grapesjs'
-import { stringify, split, merge } from '../../server/utils/websiteDataSerialization'
+import { stringify, split, merge, getPagesFolder } from '../../server/utils/websiteDataSerialization'
 
 /**
  * Gitlab connector
@@ -750,7 +750,7 @@ export default class GitlabConnector implements StorageConnector {
         name: this.options.repoPrefix + websiteMeta.name,
       }
     }) as any
-    await this.createFile(session, project.id, WEBSITE_DATA_FILE, stringify({} as WebsiteData))
+    await this.createFile(session, project.id, WEBSITE_DATA_FILE, stringify(EMPTY_WEBSITE))
     //await this.createFile(session, project.id, WEBSITE_META_DATA_FILE, JSON.stringify(websiteMeta))
     //await this.updateWebsite(session, project.id, {} as WebsiteData)
     //await this.setWebsiteMeta(session, project.id, websiteMeta)
@@ -766,14 +766,44 @@ export default class GitlabConnector implements StorageConnector {
     const batchActions: GitlabAction[] = []
 
     // **
-    // List existing files in the pages folder
+    // Handle the legacy sites that have no pagesFolder
+    // We want them to use "pages/" by default despite their files being in "src/" for now
+    let isLegacySite = !websiteData.pagesFolder
+
+    // **
+    // Backward compatibility case
+    // The second time a legacy site is saved, the new `pages/` folder has been created
+    // but the front end still doesn't have the `pagesFolder` key
+    // This will be the case until the front end reloads
+    if (isLegacySite) {
+      const rootFiles = await this.ls({
+        session,
+        websiteId,
+        path: WEBSITE_PAGES_FOLDER,
+        recursive: false,
+      })
+      if (rootFiles.size) {
+        // Here the new `pages/` folder has been created already
+        isLegacySite = false
+        websiteData.pagesFolder = WEBSITE_PAGES_FOLDER
+      }
+    }
+
+    // **
+    // List existing files in the OLD pages folder (to detect files to delete)
     const existingFiles = await this.ls({
       session,
       websiteId,
-      path: WEBSITE_PAGES_FOLDER,
+      path: getPagesFolder(websiteData),
       recursive: false,
     })
 
+    // **
+    // Force pagesFolder to 'pages' for writing if not defined
+    if (isLegacySite) {
+      websiteData.pagesFolder = WEBSITE_PAGES_FOLDER
+    }
+
     // **
     // Use the common split function to create files
     const filesToWrite = split(websiteData)
@@ -787,7 +817,7 @@ export default class GitlabConnector implements StorageConnector {
 
       const existingSha = existingFiles.get(filePath) || (filePath === WEBSITE_DATA_FILE ? 'always-update' : undefined)
 
-      if (existingSha && existingSha !== 'always-update') {
+      if (existingSha) {
         if (existingSha !== newSha) {
           batchActions.push({
             action: 'update',
@@ -797,7 +827,7 @@ export default class GitlabConnector implements StorageConnector {
         } // else: skip unchanged file
       } else {
         batchActions.push({
-          action: file.path === WEBSITE_DATA_FILE ? 'update' : 'create',
+          action: 'create',
           file_path: filePath,
           content,
         })
@@ -806,9 +836,9 @@ export default class GitlabConnector implements StorageConnector {
 
     // **
     // Delete pages that are not in the new website data
-    const newPageFiles = new Set(filesToWrite.filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER)).map(f => f.path))
+    const pathsToWrite = filesToWrite.map(f => f.path)
     for (const filePath of existingFiles.keys()) {
-      if (!newPageFiles.has(filePath)) {
+      if (!pathsToWrite.includes(filePath)) {
         batchActions.push({
           action: 'delete',
           file_path: filePath,

package/src/ts/server/connectors/FsStorage.test.ts

@@ -4,6 +4,7 @@ import { FsStorage } from './FsStorage'
 import { readFileSync, rmdirSync, statSync } from 'fs'
 import { join } from 'path'
 import { WEBSITE_DATA_FILE } from '../../constants'
+import { EMPTY_WEBSITE } from '../../types'
 
 const storageRootPath = '/tmp/silex-tests'
 const assetsFolder = 'assets'
@@ -49,7 +50,7 @@ describe('FsStorage website', () => {
     expect(() => statSync(join(storageRootPath, id, WEBSITE_DATA_FILE))).not.toThrow()
     const content = readFileSync(join(storageRootPath, id, WEBSITE_DATA_FILE), 'utf8')
     const parsed = JSON.parse(content)
-    expect(parsed).toHaveProperty('fileFormatVersion', '1.0.0')
-    expect(parsed).toHaveProperty('pages', [])
+    console.log({parsed, content, EMPTY_WEBSITE})
+    expect(parsed).toEqual(EMPTY_WEBSITE)
   })
 })

package/src/ts/server/connectors/FsStorage.ts

@@ -19,15 +19,15 @@ import fs from 'fs/promises'
 import { createWriteStream } from 'fs'
 import { ConnectorFile, StorageConnector, StatusCallback, ConnectorSession, toConnectorData, ConnectorFileContent} from './connectors'
 import { dirname, join } from 'path'
-import { ConnectorUser, WebsiteMeta, JobStatus, WebsiteId, ConnectorType, WebsiteMetaFileContent, WebsiteData, defaultWebsiteData, ConnectorOptions } from '../../types'
+import { ConnectorUser, WebsiteMeta, JobStatus, WebsiteId, ConnectorType, WebsiteMetaFileContent, WebsiteData, EMPTY_WEBSITE, ConnectorOptions } from '../../types'
 import { userInfo } from 'os'
 import { requiredParam } from '../utils/validation'
 import { ServerConfig } from '../config'
-import { DEFAULT_WEBSITE_ID, WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
+import { DEFAULT_WEBSITE_ID, WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE, LEGACY_WEBSITE_PAGES_FOLDER } from '../../constants'
 import { Readable } from 'stream'
 import { v4 as uuid } from 'uuid'
 import { fileURLToPath } from 'url'
-import { stringify, split, merge } from '../utils/websiteDataSerialization'
+import { stringify, split, merge, getPagesFolder } from '../utils/websiteDataSerialization'
 
 // Variables needed for jest tests
 if(!globalThis.__dirname) {
@@ -88,7 +88,7 @@ export class FsStorage implements StorageConnector<FsSession> {
       const id = DEFAULT_WEBSITE_ID
       await fs.mkdir(join(this.options.path, id, this.options.assetsFolder), { recursive: true })
       await this.setWebsiteMeta({}, id, { name: 'Default website', connectorUserSettings: {} })
-      await this.updateWebsite({}, id, defaultWebsiteData)
+      await this.updateWebsite({}, id, EMPTY_WEBSITE)
       console.info(`> [FsStorage] Created ${id} in ${this.options.path}`)
     }
   }
@@ -177,7 +177,7 @@ export class FsStorage implements StorageConnector<FsSession> {
     const id = uuid()
     await fs.mkdir(join(this.options.path, id, this.options.assetsFolder), { recursive: true })
     await this.setWebsiteMeta(session, id, meta)
-    await this.updateWebsite(session, id, defaultWebsiteData)
+    await this.updateWebsite(session, id, EMPTY_WEBSITE)
     return id
   }
 
@@ -190,19 +190,14 @@ export class FsStorage implements StorageConnector<FsSession> {
 
     // Check if this is the new split format
     const parsedData = JSON.parse(websiteDataContent)
-    if (parsedData.fileFormatVersion === '1.0.0') {
-      // Use the merge function to reconstruct website data
-      const pageLoader = async (pagePath: string): Promise<string> => {
-        const fullPath = join(this.options.path, id, pagePath)
-        const pageContent = await fs.readFile(fullPath)
-        return pageContent.toString()
-      }
-
-      return await merge(websiteDataContent, pageLoader)
-    } else {
-      // Legacy format, return as is
-      return parsedData
+    // Use the merge function to reconstruct website data
+    const pageLoader = async (pagePath: string): Promise<string> => {
+      const fullPath = join(this.options.path, id, pagePath)
+      const pageContent = await fs.readFile(fullPath)
+      return pageContent.toString()
     }
+
+    return await merge(websiteDataContent, pageLoader)
   }
 
   async updateWebsite(session: FsSession, websiteId: WebsiteId, data: WebsiteData): Promise<void> {
@@ -212,26 +207,29 @@ export class FsStorage implements StorageConnector<FsSession> {
     // Use the split function to create separate files for pages
     const filesToWrite = split(data)
 
+    // Get the pages folder path from website data
+    const pagesFolder = getPagesFolder(data)
+
     // Ensure the website directory exists
     await fs.mkdir(websitePath, { recursive: true })
 
     // Ensure the pages directory exists if we have page files
-    const hasPageFiles = filesToWrite.some(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
+    const hasPageFiles = filesToWrite.some(f => f.path.startsWith(pagesFolder))
     if (hasPageFiles) {
-      await fs.mkdir(join(websitePath, WEBSITE_PAGES_FOLDER), { recursive: true })
+      await fs.mkdir(join(websitePath, pagesFolder), { recursive: true })
     }
 
     // **
     // Delete pages that are not in the new website data
-    const pagesPath = join(websitePath, WEBSITE_PAGES_FOLDER)
+    const pagesPath = join(websitePath, pagesFolder)
     try {
       const existingPageFiles = await fs.readdir(pagesPath)
       const newPageFiles = new Set(
         filesToWrite
-          .filter(f => f.path.startsWith(WEBSITE_PAGES_FOLDER))
-          .map(f => f.path.replace(`${WEBSITE_PAGES_FOLDER}/`, ''))
+          .filter(f => f.path.startsWith(pagesFolder))
+          .map(f => f.path.replace(`${pagesFolder}/`, ''))
       )
-      
+
       for (const existingFile of existingPageFiles) {
         if (existingFile.endsWith('.json') && !newPageFiles.has(existingFile)) {
           await fs.unlink(join(pagesPath, existingFile))

package/src/ts/server/utils/websiteDataSerialization.test.ts

@@ -1,7 +1,7 @@
-import { stringify, split, merge } from './websiteDataSerialization'
+import { stringify, split, merge, getPagesFolder } from './websiteDataSerialization'
 import { expect, it, describe } from '@jest/globals'
-import { WebsiteData, defaultWebsiteData } from '../../types'
-import { WEBSITE_DATA_FILE } from '../../constants'
+import { WebsiteData, EMPTY_WEBSITE } from '../../types'
+import { WEBSITE_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants'
 
 describe('websiteDataSerialization', () => {
   describe('stringify', () => {
@@ -22,22 +22,21 @@ describe('websiteDataSerialization', () => {
   describe('split', () => {
     it('should split website data into separate files', () => {
       const websiteData: WebsiteData = {
-        ...defaultWebsiteData,
+        ...EMPTY_WEBSITE,
         pages: [
           { id: 'page1', getName: () => 'Home' } as any,
           { id: 'page2', getName: () => 'About' } as any
         ]
       }
-      
+
       const files = split(websiteData)
-      
+
       expect(files).toHaveLength(3)
-      expect(files[0].path).toBe('src/home-page1.json')
-      expect(files[1].path).toBe('src/about-page2.json')
+      expect(files[0].path).toBe('pages/home-page1.json')
+      expect(files[1].path).toBe('pages/about-page2.json')
       expect(files[2].path).toBe(WEBSITE_DATA_FILE)
-      
+
       const mainFile = JSON.parse(files[2].content)
-      expect(mainFile.fileFormatVersion).toBe('1.0.0')
       expect(mainFile.pages[0].isFile).toBe(true)
     })
   })
@@ -48,12 +47,45 @@ describe('websiteDataSerialization', () => {
         "settings": {},
         "pages": [{ "id": "page1" }]
       }`
-      
+
       const pageLoader = async () => { throw new Error('Should not be called') }
       const result = await merge(legacyContent, pageLoader)
-      
+
       expect(result.pages).toHaveLength(1)
       expect(result.pages[0].id).toBe('page1')
     })
   })
-})
+
+  describe('getPagesFolder', () => {
+    it('should return default folder when pagesFolder is not set', () => {
+      const websiteData = { ...EMPTY_WEBSITE }
+      const result = getPagesFolder(websiteData)
+      expect(result).toBe(WEBSITE_PAGES_FOLDER)
+    })
+
+    it('should return custom folder when pagesFolder is set', () => {
+      const websiteData = { ...EMPTY_WEBSITE, pagesFolder: 'custom-pages' }
+      const result = getPagesFolder(websiteData)
+      expect(result).toBe('custom-pages')
+    })
+  })
+
+  describe('split with custom pages folder', () => {
+    it('should use custom pages folder in file paths', () => {
+      const websiteData = {
+        ...EMPTY_WEBSITE,
+        pagesFolder: 'custom-pages',
+        pages: [{
+          id: 'page1',
+          getName: () => 'Home'
+        } as any]
+      }
+
+      const files = split(websiteData)
+      const pageFile = files.find(f => f.path.includes('page1'))
+
+      expect(pageFile).toBeDefined()
+      expect(pageFile!.path).toMatch(/^custom-pages\/.*\.json$/)
+    })
+  })
+})

package/src/ts/server/utils/websiteDataSerialization.ts

@@ -15,10 +15,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import { WebsiteData } from '../../types'
+import { EMPTY_PAGES, WebsiteData } from '../../types'
 import { Page } from 'grapesjs'
 import { getPageSlug } from '../../page'
-import { WEBSITE_PAGES_FOLDER, WEBSITE_DATA_FILE } from '../../constants'
+import { LEGACY_WEBSITE_PAGES_FOLDER, WEBSITE_DATA_FILE } from '../../constants'
 
 /**
  * Serialize JSON with stable formatting for git
@@ -50,12 +50,17 @@ function sortKeys(key: string, value: any): any {
 export function split(websiteData: WebsiteData): Array<{path: string, content: string}> {
   const files: Array<{path: string, content: string}> = []
 
+  // Use custom pages folder if specified, otherwise use default
+  const pagesFolder = websiteData.pagesFolder || LEGACY_WEBSITE_PAGES_FOLDER
+
   // Split pages into separate files
   const pages = websiteData.pages?.map((page: Page) => {
     const pageName = getPageName(page)
     const slug = getPageSlug(pageName)
     const fileName = `${slug}-${page.id}.json`
-    const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}`
+    const filePath = `${pagesFolder}/${fileName}`
+
+    if(!page.id) return page // It is the {} from EMPTY_PAGES, this happens only with unit tests
 
     files.push({
       path: filePath,
@@ -67,18 +72,18 @@ export function split(websiteData: WebsiteData): Array<{path: string, content: s
       id: page.id,
       isFile: true,
     }
-  }) || []
+  }) || EMPTY_PAGES
 
   // Create main website data file without pages content
-  const websiteDataWithoutPages = {
+  const websiteDataLinkedPages = {
     ...websiteData,
-    fileFormatVersion: '1.0.0',
+    pagesFolder,
     pages,
   }
 
   files.push({
     path: WEBSITE_DATA_FILE,
-    content: stringify(websiteDataWithoutPages)
+    content: stringify(websiteDataLinkedPages)
   })
 
   return files
@@ -92,13 +97,10 @@ export async function merge(
   websiteDataContent: string,
   pageLoader: (pagePath: string) => Promise<string>
 ): Promise<WebsiteData> {
-  const websiteDataOb = JSON.parse(websiteDataContent) as any
-  const { fileFormatVersion, ...websiteData } = websiteDataOb
+  const websiteData = JSON.parse(websiteDataContent) as any
 
-  // Check file format version
-  if (fileFormatVersion && fileFormatVersion !== '1.0.0') {
-    console.warn('Website data file format version mismatch', fileFormatVersion, '!=', '1.0.0')
-  }
+  // Use custom pages folder if specified, otherwise use default
+  const pagesFolder = websiteData.pagesFolder || LEGACY_WEBSITE_PAGES_FOLDER
 
   // Handle legacy format (no split pages) or empty pages
   if (!websiteData.pages || websiteData.pages.length === 0) {
@@ -116,7 +118,7 @@ export async function merge(
       const pageName = pageRef.name
       const slug = getPageSlug(pageName)
       const fileName = `${slug}-${pageRef.id}.json`
-      const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}`
+      const filePath = `${pagesFolder}/${fileName}`
 
       try {
         const pageContent = await pageLoader(filePath)
@@ -135,6 +137,13 @@ export async function merge(
   } as WebsiteData
 }
 
+/**
+ * Get the pages folder path for a website, using custom path or default
+ */
+export function getPagesFolder(websiteData: WebsiteData): string {
+  return websiteData.pagesFolder || LEGACY_WEBSITE_PAGES_FOLDER
+}
+
 /**
  * Get page name from page object
  */

package/src/ts/types.ts

@@ -89,7 +89,9 @@ export type ApiConnectorUserResponse = ConnectorUser
 
 // **
 // Website API
-export const defaultWebsiteData: WebsiteData = {
+export const EMPTY_PAGES = [{}] as Page[] // This is what grapesjs understands, it will create an empty page
+
+export const EMPTY_WEBSITE: WebsiteData = {
   //pages: [],
   //assets: [],
   //styles: [],
@@ -97,15 +99,17 @@ export const defaultWebsiteData: WebsiteData = {
   //fonts: [],
   //symbols: [],
   //publication: {},
+  pages: EMPTY_PAGES,
+  pagesFolder: 'pages',
 } as WebsiteData
 
-
 export interface WebsiteData {
   pages: Page[],
   assets: Asset[],
   styles: Style[],
   //name: string,
   settings: WebsiteSettings,
+  pagesFolder: string,
   fonts: Font[],
   symbols: symbol[],
   publication: PublicationSettings,