@silexlabs/silex 3.0.1-alpha.0 → 3.0.2

package/src/ts/plugins/server/FtpConnector.ts

@@ -0,0 +1,621 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { Client } from 'basic-ftp'
+import { Readable } from 'stream'
+import { ConnectorFile, ConnectorFileContent, HostingConnector, StatusCallback, StorageConnector, contentToReadable, contentToString, toConnectorData, toConnectorEnum} from '../../server/connectors/connectors'
+import { requiredParam } from '../../server/utils/validation'
+import { WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE } from '../../constants'
+import { ConnectorType, ConnectorUser, WebsiteMeta, FileMeta, JobData, JobStatus, WebsiteId, PublicationJobData, WebsiteMetaFileContent, defaultWebsiteData, WebsiteData, ConnectorOptions } from '../../types'
+import { ServerConfig } from '../../server/config'
+import { join } from 'path'
+import { tmpdir } from 'os'
+import { v4 as uuid } from 'uuid'
+import { JobManager } from '../../server/jobs'
+import { mkdtemp, rm, rmdir } from 'fs/promises'
+import { createReadStream } from 'fs'
+
+/**
+ * @fileoverview FTP connector for Silex
+ * This is a connector for Silex, which allows to store the website on a FTP server
+ *   and to publish the website on a FTP server
+ * FIXME: As a hosting connector, this connector should only store the password in the session and the rest in the website data
+ *   to prevent publishing a site on the wrong server + multiple sites on 1 server
+ * FIXME: Allow reuse of FTP session between calls
+ */
+
+export interface FtpSessionData {
+  host: string
+  user: string
+  pass: string
+  port: number
+  secure: boolean
+  // Storage options
+  storageRootPath?: string
+  // Hosting options
+  publicationPath?: string
+  websiteUrl?: string
+}
+
+export interface FtpSession {
+  [ConnectorType.STORAGE]?: FtpSessionData
+  [ConnectorType.HOSTING]?: FtpSessionData
+}
+
+interface FileStatus {
+  file: ConnectorFile
+  message: string
+  status: JobStatus
+}
+
+export interface FtpOptions {
+  type: ConnectorType
+  path: string
+  assetsFolder: string
+  cssFolder: string // For publication only
+  authorizeUrl: string
+  authorizePath: string
+}
+
+// **
+// Utils methos
+function formHtml(redirectTo: string, type: ConnectorType, { host, user, pass, port, secure, storageRootPath, publicationPath, websiteUrl}: FtpSessionData, err = '') {
+  return `
+    ${ err && `<div class="error">${err || ''}</div>` }
+    <form method="POST" action="${redirectTo}">
+      <label for="host">Host</label>
+      <input placeholder="ftp.example.com" type="text" name="host" value="${host || ''}" />
+      <label for="user">User</label>
+      <input placeholder="user" type="text" name="user" value="${user || ''}" />
+      <label for="pass">Pass</label>
+      <input placeholder="****" type="password" name="pass" value="${pass || ''}" />
+      <label for="port">Port</label>
+      <input placeholder="21" type="number" name="port" value="${port || '21'}" />
+      <div class="checkbox-container">
+        <input class="checkbox" type="checkbox" name="secure" value="true" ${secure ? 'checked' : ''} />
+        <label for="secure">Secure</label>
+      </div>
+      ${ type === ConnectorType.STORAGE ? `
+      <details>
+        <summary>Storage options</summary>
+        <p>If you are not sure, don't change this</p>
+        <label for="storageRootPath">Root path where to store the website files</label>
+        <input placeholder="/silex/" type="text" name="storageRootPath" value="${storageRootPath || '/silex/'}" />
+      </details>
+      ` : ''}
+      ${ type === ConnectorType.HOSTING ? `
+      <fieldset>
+        <legend>Publication options</legend>
+        <label for="publicationPath">Path where to publish</label>
+        <input placeholder="/public_html/" type="text" name="publicationPath" value="${publicationPath || ''}" />
+        <label for="websiteUrl">URL where to the site will be accessible</label>
+        <input placeholder="https://mysite.com/" type="text" name="websiteUrl" value="${websiteUrl || ''}" />
+      </fieldset>
+      ` : ''}
+      <div class="button-container">
+        <button type="submit" class="primary-button">Login</button>
+        <button type="button" class="secondary-button">Cancel</button>
+      </div>
+    </form>
+  `
+}
+
+const formCss = `
+/* Reset default form styles */
+form {
+  margin: 0;
+  padding: 0;
+}
+
+/* Center the form */
+body {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100vh;
+  background-color: #f5f5f5;
+}
+
+/* Style form container */
+form {
+  width: 400px;
+  padding: 20px;
+  border-radius: 4px;
+  background-color: white;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}
+
+/* Style form labels */
+label {
+  font-weight: bold;
+  margin-bottom: 5px;
+  color: #333;
+}
+
+/* Style form inputs */
+input {
+  width: 100%;
+  padding: 8px;
+  margin-bottom: 10px;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+}
+input.checkbox {
+  width: 20px;
+  height: 20px;
+}
+fieldset {
+  padding: 20px;
+}
+
+/* Style form buttons */
+.button-container {
+  display: flex;
+  justify-content: space-between;
+  margin-top: 20px;
+}
+
+.primary-button,
+.secondary-button {
+  width: 48%;
+  padding: 10px;
+  border: none;
+  border-radius: 4px;
+  cursor: pointer;
+  transition: background-color 0.3s ease;
+}
+
+.primary-button {
+  background-color: #4caf50;
+  color: white;
+}
+
+.secondary-button {
+  background-color: #e0e0e0;
+  color: #333;
+}
+
+.primary-button:hover,
+.secondary-button:hover {
+  background-color: #388e3c;
+}
+
+/* Style form checkbox */
+.checkbox-container {
+  display: flex;
+  align-items: center;
+  margin-bottom: 10px;
+  margin: 20px 0;
+}
+
+.checkbox-container label {
+  margin-left: 5px;
+  color: #333;
+}
+
+/* Style details and summary elements */
+details {
+  margin-top: 20px;
+  color: #333;
+}
+
+summary {
+  font-weight: bold;
+  cursor: pointer;
+}
+
+details > p {
+  margin-top: 10px;
+}
+`
+
+/**
+ * @class FtpConnector
+ * @implements {HostingConnector}
+ * @implements {StorageConnector}
+ */
+export default class FtpConnector implements StorageConnector<FtpSession> {
+  connectorId = 'ftp'
+  displayName = 'Ftp'
+  icon = '/assets/ftp.png'
+  options: FtpOptions
+  connectorType: ConnectorType
+  color = '#ffffff'
+  background = '#0066CC'
+
+  constructor(config: ServerConfig, opts: Partial<FtpOptions>) {
+    this.options = {
+      path: '',
+      assetsFolder: 'assets',
+      cssFolder: 'css',
+      authorizeUrl: './api/authorize/ftp/',
+      authorizePath: './api/authorize/ftp/',
+      ...opts,
+    } as FtpOptions
+    if(!this.options.type) throw new Error('missing type in option of FtpConnector')
+    this.connectorType = toConnectorEnum(this.options.type)
+  }
+
+  // **
+  // Utils
+  sessionData(session: FtpSession): FtpSessionData {
+    return session[`ftp-${this.options.type}`] ?? {} as FtpSessionData
+  }
+
+  rootPath(session: FtpSession): string {
+    return this.connectorType === ConnectorType.STORAGE ?
+      requiredParam<string>(this.sessionData(session).storageRootPath, 'storage root path') :
+      requiredParam<string>(this.sessionData(session).publicationPath, 'publication path')
+  }
+
+  // **
+  // FTP methods
+  private async write(ftp: Client, path: string, content: ConnectorFileContent, progress?: (message: string) => void): Promise<void> {
+    ftp.trackProgress(info => {
+      progress && progress(`Uploading ${info.bytes / 1000} KB}`)
+    })
+    progress && progress('Upload started')
+    await ftp.uploadFrom(contentToReadable(content), path)
+    progress && progress('Upload complete')
+  }
+
+  private async read(ftp: Client, path: string): Promise<Readable> {
+    const tempDir = await mkdtemp(join(tmpdir(), 'silex-tmp'))
+    const tempPath = join(tempDir, 'silex-ftp.tmp')
+    await ftp.downloadTo(tempPath, path)
+    const rStream = createReadStream(tempPath)
+    await rm(tempPath)
+    await rmdir(tempDir)
+    return rStream
+  }
+
+  private async readdir(ftp: Client, path: string): Promise<FileMeta[]> {
+    const list = await ftp.list(path)
+    return list.map((file) => ({
+      name: file.name,
+      isDir: file.isDirectory,
+      size: file.size,
+      createdAt: file.modifiedAt,
+      updatedAt: file.modifiedAt,
+      metaData: file,
+    } as FileMeta))
+  }
+
+  private async mkdir(ftp: Client, path: string) {
+    return ftp.ensureDir(path)
+  }
+
+  private async rmdir(ftp: Client, path: string) {
+    return ftp.removeDir(path)
+  }
+
+  private async unlink(ftp: Client, path: string) {
+    return ftp.remove(path)
+  }
+
+  private async getClient({host, user, pass, port, secure}): Promise<Client> {
+    const ftp = new Client()
+    await ftp.access({
+      host,
+      port,
+      user,
+      password: pass,
+      secure,
+    })
+    return ftp
+  }
+
+  private closeClient(ftp: Client) {
+    if(ftp && !ftp.closed) {
+      ftp.close()
+    } else {
+      console.warn('ftp already closed')
+    }
+  }
+
+  // **
+  // Connector interface
+  getOptions(formData: object): ConnectorOptions {
+    return {
+      // Storage
+      storageRootPath: formData['storageRootPath'],
+      // Hosting
+      publicationPath: formData['publicationPath'],
+      websiteUrl: formData['websiteUrl'],
+    }
+  }
+
+  async getOAuthUrl(session: FtpSession): Promise<null> { return null }
+
+  async getLoginForm(session: FtpSession, redirectTo: string): Promise<string | null> {
+    const { host, user, pass, port, secure, publicationPath, storageRootPath, websiteUrl } = this.sessionData(session)
+    return `
+        <style>
+          ${formCss}
+        </style>
+        ${formHtml(redirectTo, this.connectorType, { host, user, pass, port, secure, publicationPath, storageRootPath, websiteUrl })}
+      `
+  }
+
+  async getSettingsForm(session: FtpSession, redirectTo: string): Promise<string | null> { return null }
+
+  async setToken(session: FtpSession, token: object): Promise<void> {
+    // Check all required params are present
+    const { host, user, pass, port, secure = false, publicationPath, storageRootPath, websiteUrl } = token as FtpSessionData
+    requiredParam(session, 'session')
+    requiredParam(host, 'host')
+    requiredParam(user, 'user')
+    requiredParam(pass, 'pass')
+    requiredParam(port, 'port')
+
+    // Check if the connection is valid
+    const ftp = await this.getClient({ host, user, pass, port, secure })
+    // Save the token
+    session[`ftp-${this.connectorType}`] = { host, user, pass, port, secure, publicationPath, storageRootPath, websiteUrl }
+    // Clean up
+    this.closeClient(ftp)
+  }
+
+  async logout(session: FtpSession) {
+    delete session[`ftp-${this.options.type}`]
+  }
+
+  async getUser(session: FtpSession): Promise<ConnectorUser> {
+    return {
+      name: this.sessionData(session).user,
+      picture: 'data:image/svg+xml,%3Csvg xmlns=\'http://www.w3.org/2000/svg\' height=\'1em\' viewBox=\'0 0 448 512\'%3E%3C!--! Font Awesome Free 6.4.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license (Commercial License) Copyright 2023 Fonticons, Inc. --%3E%3Cpath d=\'M224 256A128 128 0 1 0 224 0a128 128 0 1 0 0 256zm-45.7 48C79.8 304 0 383.8 0 482.3C0 498.7 13.3 512 29.7 512H418.3c16.4 0 29.7-13.3 29.7-29.7C448 383.8 368.2 304 269.7 304H178.3z\'/%3E%3C/svg%3E',
+      storage: await toConnectorData(session, this)
+    }
+  }
+
+  async isLoggedIn(session: FtpSession) {
+    try {
+      if (!session[`ftp-${this.options.type}`]) {
+        return false
+      }
+      const ftp = await this.getClient(this.sessionData(session))
+      this.closeClient(ftp)
+      return true
+    } catch(err) {
+      return false
+    }
+  }
+
+  async setWebsiteMeta(session: FtpSession, id: string, data: WebsiteMetaFileContent): Promise<void> {
+    const websiteId = requiredParam<WebsiteId>(id, 'website id')
+    const path = join(this.rootPath(session), websiteId, WEBSITE_META_DATA_FILE)
+    const ftp = await this.getClient(this.sessionData(session))
+    await this.write(ftp, path, JSON.stringify(data))
+    this.closeClient(ftp)
+  }
+
+  async getWebsiteMeta(session: FtpSession, id: WebsiteId): Promise<WebsiteMeta> {
+    try {
+      const websiteId = requiredParam<WebsiteId>(id, 'website id')
+      const ftp = await this.getClient(this.sessionData(session))
+      // Get stats for the website folder
+      const folder = join(this.rootPath(session), websiteId)
+      const path = join(this.rootPath(session), websiteId, WEBSITE_META_DATA_FILE)
+      const readable = await this.read(ftp, path)
+      const meta = JSON.parse(await contentToString(readable)) as WebsiteMetaFileContent
+      this.closeClient(ftp)
+      // Return all meta
+      return {
+        websiteId,
+        ...meta,
+      }
+    } catch(err) {
+      console.error(err)
+      throw err
+    }
+  }
+
+  // **
+  // Storage interface
+  /**
+   * Create necessary folders
+   * Assets and root folders
+   */
+  async createWebsite(session: FtpSession): Promise<WebsiteId> {
+    const ftp = await this.getClient(this.sessionData(session))
+    // Generate a random id
+    const id = uuid()
+    // // create root folder
+    // const rootPath = join(this.storageRootPath(session), id)
+    // await this.mkdir(ftp, rootPath)
+    // create assets folder
+    const assetsPath = join(this.rootPath(session), id, '/assets')
+    await this.mkdir(ftp, assetsPath)
+    // create website data file
+    const websiteDataPath = join(this.rootPath(session), id, WEBSITE_DATA_FILE)
+    await this.write(ftp, websiteDataPath, JSON.stringify(defaultWebsiteData))
+    // create website meta data file
+    const websiteMetaDataPath = join(this.rootPath(session), id, WEBSITE_META_DATA_FILE)
+    await this.write(ftp, websiteMetaDataPath, JSON.stringify({}))
+    // Clean up
+    this.closeClient(ftp)
+    // All good
+    return id
+  }
+
+  async listWebsites(session: FtpSession): Promise<WebsiteMeta[]> {
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    const files = await this.readdir(ftp, storageRootPath)
+    const list = await Promise.all(files.map(async file => {
+      const websiteId = file.name
+      const websiteMeta = await this.getWebsiteMeta(session, websiteId)
+      return websiteMeta
+    }))
+    this.closeClient(ftp)
+    return list
+  }
+
+  async readWebsite(session: FtpSession, websiteId: string): Promise<WebsiteData | Readable> {
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    const websiteDataPath = join(storageRootPath, websiteId, WEBSITE_DATA_FILE)
+    const data = await this.read(ftp, websiteDataPath)
+    this.closeClient(ftp)
+    return data
+  }
+
+  async updateWebsite(session: FtpSession, websiteId: string, data: WebsiteData): Promise<void> {
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    const websiteDataPath = join(storageRootPath, websiteId, WEBSITE_DATA_FILE)
+    await this.write(ftp, websiteDataPath, JSON.stringify(data))
+    this.closeClient(ftp)
+  }
+
+  async deleteWebsite(session: FtpSession, websiteId: string): Promise<void> {
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    const websitePath = join(storageRootPath, websiteId)
+    await this.rmdir(ftp, websitePath)
+    this.closeClient(ftp)
+  }
+
+  async duplicateWebsite(session: FtpSession, websiteId: string): Promise<void> {
+    const newWebsiteId = uuid()
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    const websitePath = join(storageRootPath, websiteId)
+    const newWebsitePath = join(storageRootPath, newWebsiteId)
+    const tempDir = await mkdtemp(tmpdir())
+    await ftp.downloadToDir(websitePath, tempDir)
+    await ftp.uploadFromDir(tempDir, newWebsitePath)
+    this.closeClient(ftp)
+  }
+
+  async writeAssets(
+    session: any,
+    id: WebsiteId,
+    files: ConnectorFile[],
+    statusCbk?: StatusCallback,
+  ): Promise<void> {
+    return this.writeFile(session, id, files, this.options.assetsFolder, statusCbk)
+  }
+
+  async writeFile(
+    session: any,
+    id: WebsiteId,
+    files: ConnectorFile[],
+    relativePath: string,
+    statusCbk?: StatusCallback,
+  ): Promise<void> {
+    // Connect to FTP server
+    statusCbk && statusCbk({
+      message: 'Connecting to FTP server',
+      status: JobStatus.IN_PROGRESS,
+    })
+    const ftp = await this.getClient(this.sessionData(session))
+    const rootPath = this.rootPath(session)
+    // Make sure that root folder exists
+    statusCbk && statusCbk({
+      message: 'Making sure that root folder exists',
+      status: JobStatus.IN_PROGRESS,
+    })
+    // Useless as ftp write will create the folder
+    // await this.mkdir(ftp, rootPath)
+    // Write files
+    let lastFile: ConnectorFile | undefined
+    try {
+      // Sequentially write files
+      for(const file of files) {
+        statusCbk && statusCbk({
+          message: `Writing file ${file.path}`,
+          status: JobStatus.IN_PROGRESS,
+        })
+        const dstPath = join(this.options.path, rootPath, id, relativePath, file.path)
+        lastFile = file
+        const result = await this.write(ftp, dstPath, file.content, message => {
+          statusCbk && statusCbk({
+            message: `Writing file ${file.path.split('/').pop()} to ${dstPath.split('/').slice(0, -1).join('/')}: ${message}`,
+            status: JobStatus.IN_PROGRESS,
+          })
+        })
+      }
+      this.closeClient(ftp)
+      statusCbk && statusCbk({
+        message: `Finished writing ${files.length} files to ${rootPath}`,
+        status: JobStatus.SUCCESS,
+      })
+    } catch(err) {
+      // Not sure why it never gets here
+      statusCbk && statusCbk({
+        message: `Error writing file ${lastFile?.path}: ${err.message}`,
+        status: JobStatus.ERROR,
+      })
+      this.closeClient(ftp)
+    }
+  }
+
+  async readAsset(session: FtpSession, id: string, path: string): Promise<ConnectorFileContent> {
+    if (!this.sessionData(session)) throw new Error('Not logged in')
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    const dirPath = join(this.options.path, storageRootPath, id, this.options.assetsFolder, path)
+    const asset = await this.read(ftp, dirPath)
+    this.closeClient(ftp)
+    return asset
+  }
+
+  async deleteAssets(session: FtpSession, id: WebsiteId, paths: string[]): Promise<void> {
+    const storageRootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    await Promise.all(
+      paths.map((path) => this.unlink(ftp, join(this.options.path, storageRootPath, id, this.options.assetsFolder, path)))
+    )
+    this.closeClient(ftp)
+  }
+
+  // **
+  // Hosting interface
+  async getUrl(session: FtpSession, id: WebsiteId): Promise<string> {
+    // FIXME: do not store websiteUrl in the session, but in the website data
+    console.warn('FIXME: do not store websiteUrl in the session, but in the website data')
+    return this.sessionData(session).websiteUrl ?? ''
+  }
+
+  async publish(session: FtpSession, id: WebsiteId, files: ConnectorFile[], {startJob, jobSuccess, jobError}: JobManager): Promise<JobData> {
+    const job = startJob(`Publishing to ${this.displayName}`) as PublicationJobData
+    job.logs = [[`Publishing to ${this.displayName}`]]
+    job.errors = [[]]
+    // Create folders
+    const rootPath = this.rootPath(session)
+    const ftp = await this.getClient(this.sessionData(session))
+    await this.mkdir(ftp, rootPath)
+    await this.mkdir(ftp, join(rootPath, this.options.assetsFolder))
+    await this.mkdir(ftp, join(rootPath, this.options.cssFolder))
+    // Write files
+    // Do not await for the result, return the job and continue the publication in the background
+    this.writeFile(session, '', files, '', async ({status, message}) => {
+      // Update the job status
+      job.status = status
+      job.message = message
+      job.logs[0].push(message)
+      if(status === JobStatus.SUCCESS) {
+        jobSuccess(job.jobId, message)
+      } else if(status === JobStatus.ERROR) {
+        job.errors[0].push(message)
+        jobError(job.jobId, message)
+      }
+    })
+    return job
+  }
+}