@silexlabs/silex 3.0.1-alpha.0 → 3.0.1

package/src/ts/server/connectors/connectors.ts

@@ -0,0 +1,203 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { Readable } from 'stream'
+import { ServerConfig } from '../config'
+import { JobStatus, JobData, ConnectorData, ConnectorId, WebsiteId, ConnectorType, ConnectorUser, WebsiteMeta, WebsiteMetaFileContent, WebsiteData, ConnectorOptions } from '../../types'
+import { JobManager } from '../jobs'
+
+/**
+ * @fileoverview define types for Silex connectors
+ * Bakends can provide storage for website data and assets, and/or hosting to publish the website online
+ */
+
+/**
+ * Connector data stored in the website meta file
+ */
+export type ConnectorFileContent = string | Buffer | Readable
+
+/**
+ * Files are stored in connector as a File object
+ * @see types/ClientSideFile
+ * @see contentToString()
+ */
+export interface ConnectorFile {
+  path: string,
+  content: ConnectorFileContent,
+}
+
+/**
+ * Callback to update the publication status
+ */
+export type StatusCallback = ({message, status}: {message: string, status: JobStatus}) => Promise<void>
+
+export type ConnectorSession = object
+/**
+ * Connectors are the base interface for Storage and Hosting connectors
+ */
+export interface Connector<Session extends ConnectorSession = ConnectorSession> {
+  connectorId: ConnectorId
+  connectorType: ConnectorType
+  displayName: string
+  icon: string
+  disableLogout?: boolean
+  color: string
+  background: string
+  // Get the URL to start the authentication process with OAuth (not used for basic auth)
+  getOAuthUrl(session: Session): Promise<string | null>
+  // Get the form to display to the user to authenticate (not used for OAuth)
+  getLoginForm(session: Session, redirectTo: string): Promise<string | null>
+  // Get the form to display to the user to set the connector settings for a given website
+  getSettingsForm(session: Session, redirectTo: string): Promise<string | null>
+  // Auth and user management
+  isLoggedIn(session: Session): Promise<boolean>
+  setToken(session: Session, token: object): Promise<void>
+  logout(session: Session): Promise<void>
+  getUser(session: Session): Promise<ConnectorUser | null>
+  // Get the connector options from login form
+  // They are stored in the website meta file for hosting connectors
+  // And in the user session for storage connectors
+  getOptions(formData: object): ConnectorOptions
+}
+
+/**
+ * Storage are used to store the website data and assets
+ * And possibly rename files and directories, and get the URL of a file
+ *
+ */
+export interface StorageConnector<Session extends ConnectorSession = ConnectorSession> extends Connector<Session> {
+  // CRUD on websites
+  listWebsites(session: Session): Promise<WebsiteMeta[]>
+  readWebsite(session: Session, websiteId: WebsiteId): Promise<WebsiteData | Readable>
+  createWebsite(session: Session, data: WebsiteMetaFileContent): Promise<WebsiteId>
+  updateWebsite(session: Session, websiteId: WebsiteId, data: WebsiteData): Promise<void>
+  deleteWebsite(session: Session, websiteId: WebsiteId): Promise<void>
+  duplicateWebsite(session: Session, websiteId: WebsiteId): Promise<void>
+  // CRUD on assets
+  writeAssets(session: Session, websiteId: WebsiteId, files: ConnectorFile[], status?: StatusCallback): Promise<string[] | void>
+  readAsset(session: Session, websiteId: WebsiteId, fileName: string): Promise<ConnectorFileContent>
+  deleteAssets(session: Session, websiteId: WebsiteId, fileNames: string[]): Promise<void>
+  //getFileUrl(session: Session, websiteId: WebsiteId, path: string): Promise<string>
+  // Handle website meta file
+  getWebsiteMeta(session: Session, websiteId: WebsiteId): Promise<WebsiteMeta>
+  setWebsiteMeta(session: Session, websiteId: WebsiteId, data: WebsiteMetaFileContent): Promise<void>
+}
+
+/**
+ * Hosting connectors are used to publish the website
+ */
+export interface HostingConnector<Session extends ConnectorSession = ConnectorSession> extends Connector<Session> {
+  publish(session: Session, websiteId: WebsiteId, files: ConnectorFile[], jobManager: JobManager): Promise<JobData> // Pass the jobManager as plugins do not neccessarily share the same module instance
+  getUrl(session: Session, websiteId: WebsiteId): Promise<string>
+}
+
+export function toConnectorEnum(type: string | ConnectorType): ConnectorType {
+  const result = ConnectorType[type.toString().toUpperCase() as keyof typeof ConnectorType]
+  if(!result) throw new Error(`Unknown connector type ${type}. It should be one of ${Object.keys(ConnectorType).join(', ')}`)
+  return result
+}
+
+/**
+ * Get a connector by id or by type
+ */
+export async function getConnector<T extends Connector>(config: ServerConfig, session: any, type: ConnectorType, connectorId?: ConnectorId): Promise<T | undefined> {
+  // Get the connectors for this type
+  const connectors = config.getConnectors(type) as T[]
+
+  // Find the connector by id
+  if (connectorId) return connectors.find(s => s.connectorId === connectorId && s.connectorType === type)
+  // Find the first logged in connector
+  for (const connector of connectors) {
+    if (await connector.isLoggedIn(session)) {
+      return connector
+    }
+  }
+  // Defaults to the first connector
+  return connectors[0]
+}
+
+/**
+ * Convert a connector to a ConnectorData object to be sent to the frontend
+ */
+export async function toConnectorData(session: any, connector: Connector): Promise<ConnectorData> {
+  return {
+    connectorId: connector.connectorId,
+    type: connector.connectorType,
+    displayName: connector.displayName,
+    icon: connector.icon,
+    disableLogout: !!connector.disableLogout,
+    isLoggedIn: await connector.isLoggedIn(session),
+    oauthUrl: await connector.getOAuthUrl(session),
+    color: connector.color,
+    background: connector.background,
+  }
+}
+
+export async function contentToString(content: ConnectorFileContent): Promise<string> {
+  // String
+  if (typeof content === 'string') return content
+  // Buffer
+  if (Buffer.isBuffer(content)) return content.toString('utf8')
+  // Stream
+  return new Promise((resolve, reject) => {
+    let result = ''
+    content.on('data', (chunk: Buffer) => {
+      result += chunk.toString('utf8')
+    })
+    content.on('error', (err: Error) => {
+      reject(err)
+    })
+    content.on('end', () => {
+      resolve(result)
+    })
+  })
+}
+
+export async function contentToBuffer(content: ConnectorFileContent): Promise<Buffer> {
+  // String
+  if (typeof content === 'string') return Buffer.from(content, 'utf8')
+  // Buffer
+  if (Buffer.isBuffer(content)) return content
+  // Stream
+  if(content instanceof Readable) {
+    return new Promise((resolve, reject) => {
+      const chunks: Buffer[] = []
+      content.on('data', (chunk: Buffer) => {
+        chunks.push(chunk)
+      })
+      content.on('error', (err: Error) => {
+        reject(err)
+      })
+      content.on('end', () => {
+        resolve(Buffer.concat(chunks))
+      })
+    })
+  }
+  // Unknown
+  throw new Error('Unknown content type')
+}
+
+export function contentToReadable(content: ConnectorFileContent): Readable {
+  // String
+  if (typeof content === 'string') return Readable.from([content])
+  // Buffer
+  if (Buffer.isBuffer(content)) return Readable.from([content])
+  // Stream
+  if (content instanceof Readable) return content
+  // Unknown
+  throw new Error('Unknown content type')
+}

package/src/ts/server/events.ts

@@ -0,0 +1,37 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { ConnectorId, PublicationData, WebsiteData, WebsiteId } from '../types'
+import { ConnectorFile } from './connectors/connectors'
+
+export enum ServerEvent {
+  STARTUP_START = 'startup-start',
+  STARTUP_END = 'startup-end',
+  PUBLISH_START = 'publish-start',
+  PUBLISH_END = 'publish-end',
+  WEBSITE_STORE_START = 'store-start',
+  WEBSITE_STORE_END = 'store-end',
+  WEBSITE_ASSET_STORE_START = 'asset-store-start',
+  WEBSITE_ASSET_STORE_END = 'asset-store-end',
+}
+
+export type PublishStartEventType = PublicationData
+export type PublishEndEventType = Error | null
+export type WebsiteStoreStartEventType = { websiteId: WebsiteId, websiteData: WebsiteData, connectorId: ConnectorId}
+export type WebsiteStoreEndEventType = Error | null
+export type WebsiteAssetStoreStartEventType = { files: ConnectorFile[], websiteId: WebsiteId, connectorId: ConnectorId}
+export type WebsiteAssetStoreEndEventType = Error | null

package/src/ts/server/expose.ts

@@ -0,0 +1,19 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+
+export * as events from './events'

package/src/ts/server/express.ts

@@ -1,17 +1,69 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 import express, { Application } from 'express'
+import bodyParser from 'body-parser'
+import compression from 'compression'
+import cookieParser from 'cookie-parser'
+import session from 'cookie-session'
+import cors from 'cors'
 
 import { ServerConfig } from './config'
-import { EVENT_STARTUP_START, EVENT_STARTUP_END } from '../events'
+import { ServerEvent } from './events'
 
-export default async function(config: ServerConfig): Promise<Application> {
-  // Plugins hook to create API routes
+export function create(config: ServerConfig): Application {
+  // Express app
   const app = express()
-  await config.emit(EVENT_STARTUP_START, { app })
+  app.set('config', config)
+
+  // CORS
+  const options = config.expressOptions
+  if (options.cors) {
+    console.info('> CORS are ENABLED:', options.cors)
+    app.use(cors({
+      origin: options.cors,
+    }))
+  }
+  // compress gzip when possible
+  app.use(compression() as any)
+
+  // cookie & session
+  app.use(bodyParser.json({ limit: options.jsonLimit }))
+  app.use(bodyParser.text({ limit: options.textLimit }))
+  app.use(bodyParser.urlencoded({ limit: options.urlencodedLimit }))
+  console.info('> Session name:', options.sessionName)
+  app.use(cookieParser() as any)
+  app.use(session({
+    name: options.sessionName,
+    secret: options.sessionSecret,
+  }) as any)
+  return app
+}
+
+export async function start(app: Application): Promise<Application> {
+  const config = app.get('config') as ServerConfig
+
+  // Plugins hook to create API routes
+  config.emit(ServerEvent.STARTUP_START, { app })
 
   // Start server
   return new Promise((resolve, reject) => {
     app.listen(config.port, () => {
-      config.emit(EVENT_STARTUP_END, { app })
+      config.emit(ServerEvent.STARTUP_END, { app })
       resolve(app)
     })
   })

package/src/ts/server/index.ts

@@ -1,24 +1,84 @@
-import getConfig from './config'
-import start  from './express'
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 
-import * as config from './config'
-import * as types from '../types'
-import * as events from '../events'
-import * as constants from '../constants'
-import * as page from '../page'
-export default { config, types, events, constants, page }
+import * as silexApp from './express'
+import api from './api/api'
+import { ServerConfig } from './config'
+import dotenv from 'dotenv'
+import { join } from 'path'
+import { DEV_MESSAGE } from '../constants'
+
+// Expose API to calling app as function silex()
+export * from './expose'
 
 // Main app
-async function main() {
-  // Get the default config object
-  const config = await getConfig()
+export default async function silex() {
+  // Log Silex version
+  console.info(DEV_MESSAGE)
+  console.info(`Node version: ${process.version}`)
+  console.info(`CWD: ${process.cwd()}`)
+
+  // Load env vars from .env file if any
+  const curDirEnv = dotenv.config()
+  if (curDirEnv.error) {
+    if(curDirEnv.error['code'] === 'ENOENT') {
+      console.info('> No .env file found in current directory')
+    } else {
+      throw curDirEnv.error
+    }
+  } else {
+    console.info('> Env vars loaded from .env file')
+  }
 
-  // Here one can mutate the config object
-  // Check the docs, look for the "add silex to your project as an npm dependency"
+  // Load default env vars
+  // This will not override existing env vars
+  const DEFAULT_ENV_FILE = join(__dirname, '../../../.env.default')
+  const rootDirEnv = dotenv.config({ path: DEFAULT_ENV_FILE })
+  if (rootDirEnv.error) {
+    if(rootDirEnv.error['code'] === 'ENOENT') {
+      throw new Error(`\n\nFailed to load default env vars. File not found ${DEFAULT_ENV_FILE}\n\n`)
+    } else {
+      throw new Error(`\n\nFailed to load default env vars. Error in ${DEFAULT_ENV_FILE}: ${ rootDirEnv.error.message }\n\n`)
+    }
+  }
+  console.info('> Default env vars loaded')
+
+  // Get the default config object
+  const config = new ServerConfig()
 
   // start silex
-  const app = await start(config)
+  const app = silexApp.create(config)
 
+  // Serve the client config file
+  await config.addRoutes(app)
+
+  // Load the config files
+  await config.loadConfigFiles()
+
+  // Init the connectors in case no plugin adds them
+  await config.initDefaultConnectors()
+
+  // APIs
+  app.use('/api', api(config))
+
+  // Start the server
+  await silexApp.start(app)
+
+  // All good, server is ready
   console.info(`
 I'm ready, listening to port ${config.port}
   `)
@@ -26,14 +86,13 @@ I'm ready, listening to port ${config.port}
     startLiverReload()
   }
 }
-main()
 
 // livereload
 async function startLiverReload() {
   // Load modules only when needed (they may not even be installed)
   const { createServer } = await import('livereload')
   const { resolve } = await import('path')
-  const dist = resolve(__dirname, '../client')
+  const dist = resolve(__dirname, '../../client')
   const server = createServer({
     delay: 0,
   }, () => {
@@ -41,3 +100,7 @@ async function startLiverReload() {
   })
   server.watch(dist)
 }
+
+if (require.main === module) {
+  silex()
+}

package/src/ts/server/jobs.test.ts

@@ -0,0 +1,75 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { JobData, JobStatus } from '../types'
+import { startJob, jobSuccess, jobError, getJob, killJob } from './jobs'
+import { expect, afterEach, it, describe } from '@jest/globals'
+
+let job: JobData | undefined
+afterEach(() => {
+  // Avoid memory leak
+  if(job) killJob(job)
+})
+
+describe('Job Module', () => {
+  it('should start a job', () => {
+    const {jobId} = startJob('Test job')
+    job = getJob(jobId)
+    expect(job).not.toBeNull()
+    if (job) {
+      expect(job.jobId).toEqual(jobId)
+      expect(job.status).toEqual(JobStatus.IN_PROGRESS)
+      expect(job.message).toEqual('Test job')
+      killJob(job)
+      expect(getJob(jobId)).toBeUndefined()
+    }
+  })
+
+  it('should end a job with success', () => {
+    const {jobId} = startJob('Test job')
+    const result = jobSuccess(jobId, 'Job completed successfully')
+    expect(result).toBeTruthy()
+    job = getJob(jobId)
+    expect(job).not.toBeNull()
+    if (job) {
+      expect(job.status).toEqual(JobStatus.SUCCESS)
+      expect(job.message).toEqual('Job completed successfully')
+    }
+  })
+
+  it('should end a job with error', () => {
+    const {jobId} = startJob('Test job')
+    const result = jobError(jobId, 'Job ended with error')
+    expect(result).toBeTruthy()
+    job = getJob(jobId)
+    expect(job).not.toBeNull()
+    if (job) {
+      expect(job.status).toEqual(JobStatus.ERROR)
+      expect(job.message).toEqual('Job ended with error')
+    }
+  })
+
+  it('should not find a job after ending', done => {
+    const {jobId} = startJob('Test job')
+    jobSuccess(jobId, 'Job completed successfully')
+    setTimeout(() => {
+      job = getJob(jobId)
+      expect(job).not.toBeUndefined()
+      done()
+    }, 1000)
+  })
+})

package/src/ts/server/jobs.ts

@@ -0,0 +1,89 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { JobData, JobStatus, JobId } from '../types'
+
+// Delay after which the job is deleted from memory
+const TIME_TO_KEEP_AFTER_STOP = 60*10*1000 // 10 min
+
+// Store the jobs in memory
+const jobs = new Array<JobData>()
+const timers = new WeakMap<JobData, NodeJS.Timeout>()
+
+// Key which will change on server restart
+const processKey = Math.ceil(Math.random() * 1000000)
+
+// Next id for the next job
+let nextJobId = 0
+
+// Create a new job
+export function startJob(message = ''): JobData {
+  const jobId = `${processKey}_${++nextJobId}`
+  const job: JobData = {
+    jobId,
+    status: JobStatus.IN_PROGRESS,
+    message: message
+  }
+  jobs.push(job)
+  return job
+}
+
+// Retriev a job if it exists
+export function getJob(id: JobId): JobData | undefined {
+  return jobs.find(job => job.jobId === id)
+}
+
+
+// End the job with success
+export function jobSuccess(id: JobId, message = ''): boolean {
+  return end(id, JobStatus.SUCCESS, message)
+}
+
+// End the job with error
+export function jobError(id: JobId, message = ''): boolean {
+  return end(id, JobStatus.ERROR, message)
+}
+
+// Remove a job from memory
+// This is also used in unit tests to avoid memory lea
+export function killJob(job: JobData) {
+  clearTimeout(timers.get(job))
+  timers.delete(job)
+  jobs.splice(jobs.findIndex(j => job.jobId === j.jobId), 1)
+}
+
+// End the job
+function end(id: JobId, status: JobStatus, message = ''): boolean {
+  const job = getJob(id)
+  if(job) {
+    job.status = status
+    if(message) job.message = message
+    timers.set(job, setTimeout(() => killJob(job), TIME_TO_KEEP_AFTER_STOP))
+    return true
+  }
+  return false
+}
+
+// Type definition of the job manager
+export interface JobManager {
+  startJob: typeof startJob
+  jobSuccess: typeof jobSuccess
+  jobError: typeof jobError
+}
+
+// Export the job manager
+export const jobManager: JobManager = { startJob, jobSuccess, jobError }