@silexlabs/silex 3.0.1-alpha.0 → 3.0.1

package/src/ts/plugins/server/GitlabHostingConnector.ts

@@ -0,0 +1,215 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Gitlab hosting connector
+ * @fileoverview Gitlab hosting connector for Silex, connect to the user's Gitlab account to host websites with .gitlab-ci.yml file template for plain html by default
+ * @see https://docs.gitlab.com/ee/api/oauth2.html
+ * @see https://docs.gitlab.com/ee/user/project/pages/getting_started/pages_ci_cd_template.html
+ */
+
+import dedent from 'dedent'
+import GitlabConnector, { GitlabOptions, GitlabSession} from './GitlabConnector'
+import { HostingConnector, ConnectorFile } from '../../server/connectors/connectors'
+import { ConnectorType, WebsiteId, JobData, JobStatus, PublicationJobData } from '../../types'
+import { JobManager } from '../../server/jobs'
+import { ServerConfig } from '../../server/config'
+import { setTimeout } from 'timers/promises'
+
+const waitTimeOut = 5000 /* for wait loop in job pages getting */
+const SILEX_OVERWRITE_NOTICE = '# silexOverwrite: true'
+
+export default class GitlabHostingConnector extends GitlabConnector implements HostingConnector {
+
+  displayName = 'Gitlab hosting'
+  connectorType = ConnectorType.HOSTING
+
+  constructor( config: ServerConfig, opts: Partial<GitlabOptions>) {
+    super (config, opts)
+    // public directory for standard gitlab pages
+    this.options.assetsFolder= 'public'
+  }
+
+  async publish(session: GitlabSession, websiteId: WebsiteId, files: ConnectorFile[], {startJob, jobSuccess, jobError}: JobManager): Promise<JobData> {
+    const job = startJob(`Publishing to ${this.displayName}`) as PublicationJobData
+    job.logs = [[`Publishing to ${this.displayName}`]]
+    job.errors = [[]]
+    /* Configuration file .gitlab-ci.yml contains template for plain html Gitlab pages*/
+    const pathYml = '.gitlab-ci.yml'
+    const contentYml = dedent`
+      ${SILEX_OVERWRITE_NOTICE}
+      # You will want to remove these lines if you customize your build process
+      # Silex will overwrite this file unless you remove these lines
+      # This is the default build process for plain eleventy sites
+      image: node:20
+      pages:
+        stage: deploy
+        environment: production
+        script:${files.find(file => file.path.includes('.11tydata.')) ? `
+          - npx @11ty/eleventy@v3.0.0-alpha.20 --input=public --output=_site
+          - mkdir -p public/css public/assets && cp -R public/css public/assets _site/
+          - rm -rf public && mv _site public`
+    : ''}
+          - echo "The site will be deployed to $CI_PAGES_URL"
+        artifacts:
+          paths:
+            - public
+        rules:
+          - if: '$CI_COMMIT_TAG'
+          - if: '$CI_PIPELINE_SOURCE == "trigger"'
+    `
+    try {
+      const originalCi = await this.readFile(session, websiteId, pathYml)
+      if(originalCi.toString().startsWith(SILEX_OVERWRITE_NOTICE)) {
+        await this.updateFile(session, websiteId, pathYml, contentYml)
+      }
+    } catch (e) {
+      // If the file .gitlab-ci.yml does not exist, create it, otherwise do nothing (do not overwriting existing one)
+      if (e.statusCode === 404 || e.httpStatusCode === 404 || e.message.endsWith('A file with this name doesn\'t exist')) {
+        await this.createFile(session, websiteId, pathYml, contentYml)
+      } else {
+        jobError(job.jobId, e.message)
+      }
+    }
+
+    // publishing all files for website
+    // Do not await for the result, return the job and continue the publication in the background
+    this.writeAssets(session, websiteId, files, async ({status, message}) => {
+      // Update the job status
+      if(status === JobStatus.SUCCESS) {
+        /* Squash and tag the commits */
+        const successTag = await this.createTag(session, websiteId, job, { startJob, jobSuccess, jobError })
+        if(!successTag) {
+          // jobError will have been called in createTag
+          return
+        }
+        try {
+          job.message = 'Getting the website URL...'
+          job.logs[0].push(job.message)
+          const gitlabUrl = await this.getUrl(session, websiteId)
+          job.logs[0].push(`Website URL: ${gitlabUrl}`)
+          job.message = 'Getting the admin URL...'
+          job.logs[0].push(job.message)
+          const adminUrl = await this.getAdminUrl(session, websiteId)
+          job.logs[0].push(`Admin URL: ${adminUrl}`)
+          job.message = 'Getting the page URL...'
+          job.logs[0].push(job.message)
+          const pageUrl = await this.getPageUrl(session, websiteId, adminUrl)
+          job.logs[0].push(`Page URL: ${pageUrl}`)
+          job.message = 'Getting the deployment logs URL...'
+          job.logs[0].push(job.message)
+          const gitlabJobLogsUrl = await this.getGitlabJobLogsUrl(session, websiteId, job, { startJob, jobSuccess, jobError }, adminUrl, successTag)
+          // Because of the GitLab policy, this can be null (and we suggest the user to verify their account)
+          if (!gitlabJobLogsUrl) {
+            let errorMessage = 'Could not retrieve the deployment logs URL.'
+
+            if (this.isUsingOfficialInstance()) {
+              const verifyURL = 'https://gitlab.com/-/identity_verification'
+              errorMessage +=
+                `<div class="notice">
+                    If your GitLab account is recent, you may need to verify it <a href="${verifyURL}" target="_blank">here</a>
+                    in order to be able to use pipelines (this is GitLab's policy, not Silex's).
+                 </div>`
+            }
+
+            throw new Error(errorMessage)
+          }
+          job.logs[0].push(`Deployment logs URL: ${gitlabJobLogsUrl}`)
+          const message = `
+            <p><a href="${gitlabUrl}" target="_blank">Your website will soon be live here</a>.</p>
+            <p>Changes may take a few minutes to appear, <a href="${gitlabJobLogsUrl}" target="_blank">follow deployment here</a>.</p>
+            <p>Manage <a href="${pageUrl}" target="_blank">GitLab Pages settings</a>.</p>
+          `
+          job.logs[0].push(message)
+          jobSuccess(job.jobId, message)
+        } catch (error) {
+          console.error('Error during getting the website URLs:', error.message)
+          jobError(job.jobId, `Failed to get the website URLs: ${error.message}`)
+        }
+      } else if (status === JobStatus.ERROR) {
+        job.errors[0].push(message)
+        jobError(job.jobId, message)
+      } else {
+        // Update the job status while uploading files
+        job.status = status
+        job.message = message
+        job.logs[0].push(message)
+      }
+    })
+      .catch(e => {
+        console.error('Error uploading files to gitlab:', e.message)
+        jobError(job.jobId, `Failed to upload files: ${e.message}`)
+      })
+
+    return job
+  }
+
+  /* Get and return Url Gitlab Pages */
+  async getUrl(session: GitlabSession, websiteId: WebsiteId): Promise<string> {
+    const response = await this.callApi(session, `api/v4/projects/${websiteId}/pages`, 'GET')
+    return response.url
+  }
+
+  async getAdminUrl(session: GitlabSession, websiteId: WebsiteId): Promise<string> {
+    const projectInfo = await this.callApi(session, `api/v4/projects/${websiteId}`, 'GET')
+    return projectInfo.web_url
+  }
+
+  async getPageUrl(session: GitlabSession, websiteId: WebsiteId, projectUrl: string): Promise<string> {
+    return `${projectUrl}/pages`
+  }
+
+  // waiting for the job corresponding to the current tag
+  async getGitlabJobLogsUrl(session: GitlabSession, websiteId: WebsiteId, job: PublicationJobData, { startJob, jobSuccess, jobError }: JobManager, projectUrl: string, tag): Promise<string | null> {
+    const t0 = Date.now()
+    do {
+      const jobs = await this.callApi(session, `api/v4/projects/${websiteId}/jobs`, 'GET')
+      if (!jobs.length) return null
+      if (jobs[0].ref === tag) {return `${projectUrl}/-/jobs/${jobs[0].id}`}
+      await setTimeout(waitTimeOut)
+    } while ((Date.now() - t0) < this.options.timeOut)
+      
+    // failed in timelaps allowed (avoiding infinite loop)
+    jobError(job.jobId, 'Failed to get job id')
+    job.message = 'Unable to get job id'
+    job.logs[0].push(job.message)
+    return `${projectUrl}/-/jobs/`
+  }
+
+  async createTag(session: GitlabSession, websiteId: WebsiteId, job: PublicationJobData, { startJob, jobSuccess, jobError }: JobManager): Promise<string | null> {
+    const projectId = websiteId // Assuming websiteId corresponds to GitLab project ID
+    const newTag = '_silex_' + Date.now()
+    // Create a new tag
+    try {
+      job.message = `Creating new tag ${newTag}...`
+      job.logs[0].push(job.message)
+      await this.callApi(session, `api/v4/projects/${projectId}/repository/tags`, 'POST', {
+        tag_name: newTag,
+        ref: 'main',
+        message: 'Publication from Silex',
+      })
+    } catch (error) {
+      console.error('Error during creating new tag:', error.message)
+      jobError(job.jobId, `Failed to create new tag: ${error.message}`)
+      return null
+    }
+    
+    // Return new tag
+    return newTag
+  }
+
+}

package/src/ts/plugins/server/PublicationOptimizer.ts

@@ -0,0 +1,14 @@
+
+//import { minify } from 'html-minifier'
+//const optim = files.map(file => ({
+//  ...file,
+//  html: minify(file.html, {
+//    continueOnParseError: true,
+//    collapseInlineTagWhitespace: true,
+//    collapseWhitespace: true,
+//    minifyCSS: true,
+//    minifyJS: true,
+//    removeScriptTypeAttributes: true,
+//    removeStyleLinkTypeAttributes: true,
+//  }).trim(),
+//}))

package/src/ts/plugins/server/SslPlugin.ts

@@ -0,0 +1,84 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import express from 'express'
+import { readFileSync } from 'fs'
+import { createServer } from 'https'
+import forceSSL from 'express-force-ssl'
+import { ServerEvent } from '../../server/events'
+
+// interface SslOptions {
+//   forceHttps?: boolean
+//   trustXFPHeader?: boolean
+//   privateKey?: string
+//   certificate?: string
+//   sslPort?: string
+// }
+
+export default async function(config, opts = {}) {
+  // Options with defaults
+  const options = {
+    forceHttps: process.env.SILEX_FORCE_HTTPS === 'true',
+    trustXFPHeader: process.env.SILEX_FORCE_HTTPS_TRUST_XFP_HEADER === 'true',
+    privateKey: process.env.SILEX_SSL_PRIVATE_KEY,
+    certificate: process.env.SILEX_SSL_CERTIFICATE,
+    sslPort: process.env.SSL_PORT || '443',
+    ...opts,
+  }
+
+  // Add routes on silex startup
+  config.on(ServerEvent.STARTUP_START, ({app}) => {
+    const router = express.Router()
+
+    // SSL
+    // force ssl if the env var SILEX_FORCE_HTTPS is set
+    if (options.forceHttps) {
+      console.info('> [SslPlugin] Force SSL option is enabled')
+      app.set('forceSSLOptions', {
+        trustXFPHeader: !!options.trustXFPHeader,
+      })
+      router.use(forceSSL)
+    } else {
+      console.info('> [SslPlugin] Force SSL option is disabled, env var SILEX_FORCE_HTTPS not set')
+    }
+
+    // SSL certificate
+    if (options.privateKey && options.certificate) {
+      console.info('> [SslPlugin] SSL certificate is enabled, found certificate:', options.certificate)
+      try {
+        const privateKey = readFileSync(options.privateKey).toString()
+        const certificate = readFileSync(options.certificate).toString()
+
+        const sslServerOptions = {
+          key: privateKey,
+          cert: certificate,
+          requestCert: true,
+          rejectUnauthorized: false,
+        }
+
+        createServer(sslServerOptions, this).listen(options.sslPort, () => {
+          console.info('> [SslPlugin] Listening on port ', options.sslPort)
+        })
+      } catch (e) {
+        console.error('> [SslPlugin] Load certificate failed.', e)
+      }
+    } else {
+      console.info('> [SslPlugin] Certificate disabled, env vars SILEX_SSL_CERTIFICATE and SILEX_SSL_PRIVATE_KEY not set')
+    }
+    app.use(router)
+  })
+}

package/src/ts/plugins/server/StaticPlugin.ts

@@ -0,0 +1,45 @@
+/*
+ * Silex website builder, free/libre no-code tool for makers.
+ * Copyright (c) 2023 lexoyo and Silex Labs foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import express from 'express'
+import serveStatic from 'serve-static'
+import { withCache } from './Cache'
+import { ServerEvent } from '../../server/events'
+
+type StaticOptions = {
+  routes: {
+    path: string
+    route: string
+  }[]
+}
+
+export default async function(config, options: StaticOptions = { routes: [] }) {
+  if(!options.routes) throw new Error('The config for static module has no `routes` attribute')
+  console.info(`> [StaticPlugin] Serving ${options.routes.length} static files`)
+
+  config.on(ServerEvent.STARTUP_START, ({app}) => {
+    const router = express.Router()
+    options.routes
+      .forEach(folder => {
+        if (!folder.route) throw new Error('The config for static module has no `route` attribute')
+        if (!folder.path) throw new Error('The config for static module has no `path` attribute')
+        console.info(`> [StaticPlugin] Serving static files from ${folder.path} on ${folder.route}`)
+        router.use(folder.route, serveStatic(folder.path))
+      })
+    app.use(withCache,  router)
+  })
+}

package/src/ts/server/api/api.ts

@@ -0,0 +1,19 @@
+import { Router } from 'express'
+import { ServerConfig } from '../config'
+import { noCache } from '../../plugins/server/Cache'
+
+import connectorApi from './connectorApi'
+import websiteApi from './websiteApi'
+import publicationApi from './publicationApi'
+import { API_CONNECTOR_PATH, API_PUBLICATION_PATH, API_WEBSITE_PATH } from '../../constants'
+
+export default function(config: ServerConfig): Router {
+  const router = Router()
+  router.use(noCache)
+
+  router.use(API_CONNECTOR_PATH, connectorApi(config))
+  router.use(API_WEBSITE_PATH, websiteApi(config))
+  router.use(API_PUBLICATION_PATH, publicationApi(config))
+
+  return router
+}