@saltcorn/plugins-loader 0.9.5-beta.10

package/download_utils.js

@@ -0,0 +1,146 @@
+const fs = require("fs");
+const { rm } = require("fs").promises;
+const { mkdir, pathExists } = require("fs-extra");
+const { tmpName } = require("tmp-promise");
+const { execSync } = require("child_process");
+const { extract } = require("tar");
+const { join } = require("path");
+const { createWriteStream, unlink } = require("fs");
+const { get } = require("https");
+const npmFetch = require("npm-registry-fetch");
+
+const rootFolder = process.cwd();
+
+const downloadFromGithub = async (plugin, pluginDir) => {
+  const tarballUrl = `https://api.github.com/repos/${plugin.location}/tarball`;
+  const fileName = plugin.name.split("/").pop();
+  const filePath = await loadTarball(tarballUrl, fileName);
+  await mkdir(pluginDir, { recursive: true });
+  await extractTarball(filePath, pluginDir);
+};
+
+const downloadFromNpm = async (plugin, pluginDir, pckJson) => {
+  const pkgInfo = await npmFetch.json(
+    `https://registry.npmjs.org/${plugin.location}`
+  );
+  const keys = Object.keys(pkgInfo.versions);
+  const latest = keys[keys.length - 1];
+  const vToInstall =
+    plugin.version && plugin.version !== "latest" ? plugin.version : latest;
+
+  if (pckJson && pckJson.version === vToInstall) return false;
+  else {
+    const tarballUrl = pkgInfo.versions[vToInstall].dist.tarball;
+    const fileName = plugin.name.split("/").pop();
+    const filePath = await loadTarball(tarballUrl, fileName);
+    await mkdir(pluginDir, { recursive: true });
+    await extractTarball(filePath, pluginDir);
+    return true;
+  }
+};
+
+const loadTarball = (url, name) => {
+  const options = {
+    headers: {
+      "User-Agent": "request",
+    },
+  };
+  const writeTarball = async (res) => {
+    const filePath = join(rootFolder, "plugins_folder", `${name}.tar.gz`);
+    const stream = createWriteStream(filePath);
+    res.pipe(stream);
+    return new Promise((resolve, reject) => {
+      stream.on("finish", () => {
+        stream.close();
+        resolve(filePath);
+      });
+      stream.on("error", (err) => {
+        stream.close();
+        reject(err);
+      });
+    });
+  };
+
+  return new Promise((resolve, reject) => {
+    get(url, options, async (res) => {
+      if (res.statusCode === 302) {
+        get(res.headers.location, options, async (redirect) => {
+          if (redirect.statusCode === 200) {
+            const filePath = await writeTarball(redirect);
+            resolve(filePath);
+          } else
+            reject(
+              new Error(
+                `Error downloading tarball: http code ${redirect.statusCode}`
+              )
+            );
+        });
+      } else if (res.statusCode !== 200)
+        reject(
+          new Error(`Error downloading tarball: http code ${res.statusCode}`)
+        );
+      else {
+        const filePath = await writeTarball(res);
+        resolve(filePath);
+      }
+    }).on("error", (err) => {
+      reject(err);
+    });
+  });
+};
+
+/**
+ * Git pull or clone
+ * @param plugin
+ */
+const gitPullOrClone = async (plugin, pluginDir) => {
+  await fs.promises.mkdir("git_plugins", { recursive: true });
+  let keyfnm,
+    setKey = `-c core.sshCommand="ssh -oBatchMode=yes -o 'StrictHostKeyChecking no'" `;
+  if (plugin.deploy_private_key) {
+    keyfnm = await tmpName();
+    await fs.promises.writeFile(
+      keyfnm,
+      plugin.deploy_private_key.replace(/[\r]+/g, "") + "\n",
+      {
+        mode: 0o600,
+        encoding: "ascii",
+      }
+    );
+    setKey = `-c core.sshCommand="ssh -oBatchMode=yes -o 'StrictHostKeyChecking no' -i ${keyfnm}" `;
+  }
+  if (fs.existsSync(pluginDir)) {
+    execSync(`git ${setKey} -C ${pluginDir} pull`);
+  } else {
+    execSync(`git ${setKey} clone ${plugin.location} ${pluginDir}`);
+  }
+  if (plugin.deploy_private_key && keyfnm) await fs.promises.unlink(keyfnm);
+};
+
+const extractTarball = async (tarFile, destination) => {
+  await extract({
+    file: tarFile,
+    cwd: destination,
+    strip: 1,
+  });
+};
+
+const tarballExists = async (plugin) => {
+  const fileName = `${plugin.name.split("/").pop()}.tar.gz`;
+  return await pathExists(join(rootFolder, "plugins_folder", fileName));
+};
+
+const removeTarball = async (plugin) => {
+  const fileName = `${plugin.name.split("/").pop()}.tar.gz`;
+  await rm(join(rootFolder, "plugins_folder", fileName));
+};
+
+module.exports = {
+  downloadFromGithub,
+  downloadFromNpm,
+  gitPullOrClone,
+  loadTarball,
+  extractTarball,
+  tarballExists,
+  removeTarball,
+};

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@saltcorn/plugins-loader",
+  "version": "0.9.5-beta.10",
+  "description": "Saltcorn plugin loader",
+  "homepage": "https://saltcorn.com",
+  "scripts": {
+    "test": "echo 'No tests'",
+    "tsc": "echo 'No TypeScript build'",
+    "clean": "echo 'No TypeScript build'"
+  },
+  "dependencies": {
+    "env-paths": "^2.2.0",
+    "@saltcorn/data": "0.9.5-beta.10"
+  },
+  "author": "Christian Hugo",
+  "license": "MIT",
+  "main": "index.js",
+  "repository": "github:saltcorn/saltcorn",
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/plugin_installer.js

@@ -0,0 +1,234 @@
+const { join, normalize, dirname } = require("path");
+const { writeFile, mkdir, pathExists, copy, symlink } = require("fs-extra");
+const { spawn } = require("child_process");
+const {
+  downloadFromNpm,
+  downloadFromGithub,
+  gitPullOrClone,
+  tarballExists,
+  removeTarball,
+} = require("./download_utils");
+const { getState } = require("@saltcorn/data/db/state");
+const { rm, rename, cp, readFile } = require("fs").promises;
+const envPaths = require("env-paths");
+
+const staticDeps = ["@saltcorn/markup", "@saltcorn/data", "jest"];
+const fixedPlugins = ["@saltcorn/base-plugin", "@saltcorn/sbadmin2"];
+
+const isGitCheckout = async () => {
+  const gitPath = join(__dirname, "..", "..", "Dockerfile.release");
+  return await pathExists(gitPath);
+};
+
+const readPackageJson = async (filePath) => {
+  if (await pathExists(filePath)) return JSON.parse(await readFile(filePath));
+  else return null;
+};
+
+class PluginInstaller {
+  constructor(plugin, opts = {}) {
+    this.plugin = plugin;
+    this.rootFolder = opts.rootFolder || process.cwd();
+    this.tempRootFolder =
+      opts.tempRootFolder || envPaths("saltcorn", { suffix: "tmp" }).temp;
+    const tokens =
+      plugin.source === "npm"
+        ? plugin.location.split("/")
+        : plugin.name.split("/");
+    this.name = tokens[tokens.length - 1];
+    this.pluginDir = join(
+      this.rootFolder,
+      plugin.source === "git" ? "git_plugins" : "plugins_folder",
+      ...tokens
+    );
+    this.pckJsonPath = join(this.pluginDir, "package.json");
+    this.tempDir = join(this.tempRootFolder, "temp_install", ...tokens);
+    this.tempPckJsonPath = join(this.tempDir, "package.json");
+  }
+
+  async install(force) {
+    await this.ensurePluginsRootFolders();
+    if (fixedPlugins.includes(this.plugin.location))
+      return { plugin_module: require(this.plugin.location) };
+
+    let pckJSON = await readPackageJson(this.pckJsonPath);
+    const installer = async () => {
+      if (await this.prepPluginsFolder(force, pckJSON)) {
+        const tmpPckJSON = await this.removeDependencies(
+          await readPackageJson(this.tempPckJsonPath)
+        );
+        await this.npmInstall(tmpPckJSON);
+        await this.movePlugin();
+        if (await tarballExists(this.plugin)) await removeTarball(this.plugin);
+      }
+      pckJSON = await readPackageJson(this.pckJsonPath);
+    };
+    await installer();
+    let module = null;
+    let loadedWithReload = false;
+    try {
+      // try importing it and if it fails, remove and try again
+      // could happen when there is a directory with a valid package.json
+      // but without a valid node modules folder
+      module = await this.loadMainFile(pckJSON);
+    } catch (e) {
+      getState().log(
+        2,
+        `Error loading plugin ${this.plugin.name}. Removing and trying again.`
+      );
+      if (force) {
+        await this.remove();
+        pckJSON = null;
+        await installer();
+      }
+      module = await this.loadMainFile(pckJSON, true);
+      loadedWithReload = true;
+    }
+    return {
+      version: pckJSON.version,
+      plugin_module: module,
+      location: this.pluginDir,
+      name: this.name,
+      loadedWithReload,
+    };
+  }
+
+  async remove() {
+    if (await pathExists(this.pluginDir))
+      await rm(this.pluginDir, { recursive: true });
+  }
+
+  async prepPluginsFolder(force, pckJSON) {
+    let wasLoaded = false;
+    switch (this.plugin.source) {
+      case "npm":
+        if (
+          (force && !(await this.versionIsInstalled(pckJSON))) ||
+          !(await pathExists(this.pluginDir))
+        ) {
+          wasLoaded = await downloadFromNpm(this.plugin, this.tempDir, pckJSON);
+        }
+        break;
+      case "github":
+        if (force || !(await pathExists(this.pluginDir))) {
+          await downloadFromGithub(this.plugin, this.tempDir);
+          wasLoaded = true;
+        }
+        break;
+      case "local":
+        if (force || !(await pathExists(this.pluginDir))) {
+          await copy(this.plugin.location, this.tempDir);
+          wasLoaded = true;
+        }
+        break;
+      case "git":
+        if (force || !(await pathExists(this.pluginDir))) {
+          await gitPullOrClone(this.plugin, this.tempDir);
+          this.pckJsonPath = join(this.pluginDir, "package.json");
+          wasLoaded = true;
+        }
+        break;
+    }
+    return wasLoaded;
+  }
+
+  async ensurePluginsRootFolders() {
+    const isWindows = process.platform === "win32";
+    const ensureFn = async (folder) => {
+      const pluginsFolder = join(this.rootFolder, folder);
+      if (!(await pathExists(pluginsFolder))) await mkdir(pluginsFolder);
+      const symLinkDst = join(pluginsFolder, "node_modules");
+      const symLinkSrc = (await isGitCheckout())
+        ? join(__dirname, "..", "..", "node_modules")
+        : join(dirname(require.resolve("@saltcorn/cli")), "..", "node_modules");
+      if (!(await pathExists(symLinkDst)))
+        await symlink(symLinkSrc, symLinkDst, !isWindows ? "dir" : "junction");
+    };
+    for (const folder of ["plugins_folder", "git_plugins"])
+      await ensureFn(folder);
+  }
+
+  isFixedVersion() {
+    return !!this.plugin.version && this.plugin.version !== "latest";
+  }
+
+  async versionIsInstalled(pckJSON) {
+    if (!pckJSON || !this.isFixedVersion()) return false;
+    else {
+      const vInstalled = pckJSON.version;
+      if (vInstalled === this.plugin.version) return true;
+      else return false;
+    }
+  }
+
+  async loadMainFile(pckJSON, reload) {
+    const isWindows = process.platform === "win32";
+    if (process.env.NODE_ENV === "test") {
+      // in jest, downgrad to require
+      return require(normalize(join(this.pluginDir, pckJSON.main)));
+    } else {
+      const res = await import(
+        `${isWindows ? `file://` : ""}${normalize(
+          join(this.pluginDir, pckJSON.main + (reload ? "?reload=true" : ""))
+        )}`
+      );
+      return res.default;
+    }
+  }
+
+  async removeDependencies(tmpPckJSON) {
+    const pckJSON = { ...tmpPckJSON };
+    const oldDepsLength = Object.keys(pckJSON.dependencies || {}).length;
+    const oldDevDepsLength = Object.keys(pckJSON.devDependencies || {}).length;
+    const staticsRemover = (deps) => {
+      for (const staticDep of staticDeps) {
+        if (deps[staticDep]) delete deps[staticDep];
+      }
+    };
+    if (pckJSON.dependencies) staticsRemover(pckJSON.dependencies);
+    if (pckJSON.devDependencies) staticsRemover(pckJSON.devDependencies);
+    if (
+      Object.keys(pckJSON.dependencies || {}).length !== oldDepsLength ||
+      Object.keys(pckJSON.devDependencies || {}).length !== oldDevDepsLength
+    )
+      await writeFile(
+        join(this.tempDir, "package.json"),
+        JSON.stringify(pckJSON, null, 2)
+      );
+    return pckJSON;
+  }
+
+  async npmInstall(pckJSON) {
+    const isWindows = process.platform === "win32";
+    if (
+      Object.keys(pckJSON.dependencies || {}).length > 0 ||
+      Object.keys(pckJSON.devDependencies || {}).length > 0
+    ) {
+      const child = spawn("npm", ["install"], {
+        cwd: this.tempDir,
+        ...(isWindows ? { shell: true } : {}),
+      });
+      return new Promise((resolve, reject) => {
+        child.on("exit", (exitCode, signal) => {
+          resolve({ success: exitCode === 0 });
+        });
+
+        child.on("error", (msg) => {
+          reject(msg);
+        });
+      });
+    }
+  }
+
+  async movePlugin() {
+    const isWindows = process.platform === "win32";
+    if (await pathExists(this.pluginDir))
+      await rm(this.pluginDir, { recursive: true });
+    await mkdir(this.pluginDir, { recursive: true });
+    if (!isWindows) await rename(this.tempDir, this.pluginDir);
+    else
+      await cp(this.tempDir, this.pluginDir, { recursive: true, force: true });
+  }
+}
+
+module.exports = PluginInstaller;