@lazycatcloud/lzc-cli 1.1.5 → 1.1.8

package/lib/app/lpk_devshell.js

@@ -0,0 +1,621 @@
+// lzc-cli app devshell
+import path from "node:path";
+import fs from "node:fs";
+import logger from "loglevel";
+import { spawnSync, execSync } from "child_process";
+import { LpkBuild } from "./lpk_build.js";
+import { LpkInstaller } from "./lpk_installer.js";
+import {
+  mergeYamlInMemory,
+  contextDirname,
+  ensureDir,
+  isFileExist,
+  GitIgnore,
+  sleep,
+  envTemplateFile,
+  md5String,
+  createTemplateFileCommon,
+} from "../utils.js";
+import yaml from "js-yaml";
+import Key from "../key.js";
+import os from "node:os";
+import { sdkEnv } from "../env.js";
+import commandExists from "command-exists";
+import chokidar from "chokidar";
+import execa from "execa";
+
+function sdkSSHAddress() {
+  const sdkUrl = sdkEnv.sdkUrl;
+  let url = new URL(sdkUrl);
+  return `box@${url.hostname}:2222`;
+}
+
+export class AppDevShell {
+  constructor(cwd, builder) {
+    this.cwd = cwd ? path.resolve(cwd) : process.cwd();
+    this.lpkBuild = builder;
+  }
+
+  async init() {
+    if (!this.lpkBuild) {
+      this.lpkBuild = new LpkBuild(this.cwd);
+      await this.lpkBuild.init();
+    }
+  }
+
+  async shellWithBuild() {
+    // 确保 sdk key ，并上传到 sdk
+    const k = new Key();
+    await k.ensure(sdkEnv.sdkUrl);
+    const pairs = await k.getKeyPair();
+
+    // devshell 不需要执行 buildscript 和 contentdir 也不需要
+    this.lpkBuild.onBeforeBuildPackage(async (options) => {
+      logger.debug("devshell delete 'buildscript' field");
+      logger.debug("devshell delete 'contentdir' field");
+      delete options["buildscript"];
+      delete options["contentdir"];
+      return options;
+    });
+
+    // 用 sdk ssh key 并复制到 contentdir 目录
+    // docker 中 sshd 的配置中已改成 /lzcapp/pkg/content/devshell/authorized_keys
+    this.lpkBuild.onBeforeTarContent(async (contentdir) => {
+      let publicKey = pairs["public"];
+      let dest = path.join(contentdir, "devshell", "authorized_keys");
+      ensureDir(dest);
+      fs.copyFileSync(publicKey, dest);
+    });
+
+    // 在生成 manifest.yml 之前合并 sync 模板字段
+    this.lpkBuild.onBeforeDumpYaml(async (manifest, options) => {
+      const services = manifest["services"];
+      if (services && Object.keys(services).some((key) => key === "sync")) {
+        throw "自定义的 services 中与 devshell 中的 sync 名称冲突";
+      }
+
+      const syncManifestPath = path.join(
+        contextDirname(import.meta.url),
+        "../../template/_lpk/sync.manifest.yml.in"
+      );
+      const syncManifest = yaml.load(
+        await envTemplateFile(syncManifestPath, manifest)
+      );
+      return mergeYamlInMemory([manifest, syncManifest]);
+    });
+
+    // 在生成 manifest.yml 之前合并 devshell manifest 模板字段
+    this.lpkBuild.onBeforeDumpYaml(async (manifest, options) => {
+      logger.debug("merge lzc-build.yml devshell routes field");
+      const devshell = options["devshell"];
+      if (!devshell) {
+        throw "devshell 模式下，devshell 字段必须要指定";
+      }
+
+      const routes = devshell["routes"];
+      if (!routes || routes.length == 0) {
+        throw "devshell 模式下，必须要指定 routes 内容";
+      }
+
+      logger.debug("options devshell delete 'routes' field");
+      delete options["devshell"]["routes"];
+      // 如果 devshell 中的 router 和 manifest 中的 prefix 出现冲突
+      // 优先使用 devshell 中的。
+      routes.forEach((r) => {
+        if (!r) {
+          return;
+        }
+
+        let prefix = r.split("=")[0];
+        let index = manifest["application"]["routes"].findIndex((mr) => {
+          if (!mr) {
+            return false;
+          }
+          return mr.split("=")[0] == prefix;
+        });
+        if (index > -1) {
+          manifest["application"]["routes"].splice(index, 1);
+        }
+      });
+      const application = { routes };
+      return mergeYamlInMemory([manifest, { application }]);
+    });
+
+    // 在生成 manifest.yml 之前合并 lzc-build.yml devshell 字段的值
+    this.lpkBuild.onBeforeDumpYaml(async (manifest, options) => {
+      logger.debug("merge lzc-build.yml devshell services\n", options);
+      const devshell = {
+        application: {
+          devshell: options["devshell"],
+        },
+      };
+      return mergeYamlInMemory([manifest, devshell]);
+    });
+
+    // 如果 services/devshell 中有 dependencies 字段，优先使用
+    this.lpkBuild.onBeforeDumpYaml(async (manifest) => {
+      const config = manifest["application"]["devshell"];
+      if (!config || !config["dependencies"]) {
+        return manifest;
+      }
+
+      const deps = config["dependencies"];
+      if (deps.length == 0) {
+        logger.warn("dependencies 内容为空，跳过 dependencies");
+        delete manifest["application"]["devshell"]["dependencies"];
+        return manifest;
+      }
+
+      const depsStr = deps.sort().join(" ");
+      logger.debug("开始创建 Dockerfile 文件");
+      const tempDir = fs.mkdtempSync(".lzc-cli-build-dependencies");
+      const dockerfilePath = path.join(
+        contextDirname(import.meta.url),
+        "../../template/_lpk/Dockerfile.in"
+      );
+      await createTemplateFileCommon(
+        dockerfilePath,
+        path.join(tempDir, "Dockerfile"),
+        { dependencies: depsStr }
+      );
+
+      const tag = `${await md5String(depsStr)}:latest`;
+      logger.debug(`开始在盒子中构建 ${tag} 镜像 from ${tempDir}`);
+      const sdk = new sdkDocker();
+      await sdk.buildImage(tag, tempDir, tempDir);
+
+      fs.rmSync(tempDir, { recursive: true });
+
+      delete manifest["application"]["devshell"];
+      manifest["application"]["image"] = tag;
+      return manifest;
+    });
+
+    // 如果 services 中有 devshell 的字段，需要检测是否需要提前构建
+    this.lpkBuild.onBeforeDumpYaml(async (manifest) => {
+      const application = manifest["application"];
+      if (!application || !application["devshell"]) {
+        return manifest;
+      }
+
+      const config = manifest["application"]["devshell"];
+      if (!config || !config["build"]) {
+        return manifest;
+      }
+
+      const tag = `${manifest["package"]}-devshell:${manifest["version"]}`;
+
+      logger.debug(`开始在盒子中构建 ${tag} 镜像`);
+
+      const sdk = new sdkDocker();
+      await sdk.buildImage(tag, config["build"], process.cwd());
+
+      delete manifest["application"]["devshell"];
+      manifest["application"]["image"] = tag;
+      return manifest;
+    });
+
+    // 如果没有找到 devshell 中没有指定 image 不存在，将默认使用的 lzc-cli/devshell 容器
+    this.lpkBuild.onBeforeDumpYaml(async (manifest) => {
+      const config = manifest["application"];
+      if (config["image"]) {
+        return manifest;
+      }
+      manifest["application"][
+        "image"
+      ] = `registry.lazycat.cloud/lzc-cli/devshell:latest`;
+      return manifest;
+    });
+
+    // 在构建生成 lpk 包后，调用 deploy 进行部署
+    let installer = new LpkInstaller();
+    await installer.init();
+    await installer.deploy(this.lpkBuild);
+
+    await sleep(2000);
+
+    // 通过 rsync 同步源码到应用容器中
+    // TODO: 改成用 sshfs 的方式
+    await this.shell(true);
+  }
+
+  async shell(build = false) {
+    const k = new Key();
+    const pairs = await k.getKeyPair();
+    const manifest = await this.lpkBuild.getManifest();
+    const devshell = new DevShell(pairs["private"], manifest["package"]);
+    if (build || !devshell.isRsyncWatch()) {
+      await devshell.shell();
+    } else {
+      await devshell.directShell();
+    }
+    logger.debug("exit shell");
+    // TODO: shell 在正常情况下，按 Ctrl-D 就会退出，回到原来的本地的 shell ，但
+    // 现在会一直卡在退出状态后，必须要另外手动的指定 pkill node
+    process.exit(0);
+  }
+}
+
+class DevShell {
+  constructor(keyFile, appId) {
+    logger.debug("keyFile", keyFile);
+    logger.debug("appid", appId);
+
+    this.tempDir = path.join(os.tmpdir(), "lzc-cli-devshell", appId);
+    if (!fs.existsSync(this.tmpdir)) {
+      fs.mkdirSync(this.tempDir, { recursive: true });
+    }
+    this.keyFile = keyFile;
+    this.appId = appId;
+  }
+
+  async syncProject(keyFile, appAddr, appId) {
+    let jump = sdkSSHAddress();
+    // prettier-ignore
+    let rsh = [
+      "ssh",
+      "-J", jump,
+      "-o", `"StrictHostKeyChecking=no"`,
+      "-o", `"UserKnownHostsFile=/dev/null"`,
+      "-o", `"ConnectionAttempts=3"`,
+      "-o", `"ConnectTimeout=30"`,
+      "-o", `"LogLevel=ERROR"`,
+      "-i", keyFile,
+    ].join(" ");
+    // 检查rsync工具是否存在:提示用户
+    const rsyncExisted = commandExists.sync("rsync");
+    if (!rsyncExisted) {
+      console.log(chalk.red("请检查 rsync 是否安装，路径是否正确!"));
+      process.exit(1);
+    }
+
+    // FIXME: 下方执行命令不确定是否有兼容性问题
+    execSync(
+      `rsync --rsh='${rsh}' -czrPR . root@${appAddr}:/lzcapp/cache/${appId} --filter=':- .gitignore' --exclude='node_modules' --exclude='.git' --delete`,
+      { stdio: ["ignore", "ignore", "inherit"] }
+    );
+  }
+
+  // fallback fs.watch on not darwin and windows platform
+  // FILEPATH directory or file path
+  // CALLBACK => function(eventType, filename)
+  //  fs.watch 虽然不支持递归，但可以直接监听整个文件夹的变动
+  async fallbackWatch(filepath, gitignore, callback) {
+    if (gitignore.contain(filepath)) {
+      return Promise.resolve();
+    }
+
+    if (filepath.endsWith(".git") || filepath.endsWith(".lazycat")) {
+      return Promise.resolve();
+    }
+
+    fs.watch(filepath, callback(filepath));
+
+    // 如果为一个文件夹，则扫描当中是否含有子文件夹
+    if (isDirSync(filepath)) {
+      return gitignore.readdir(filepath, (err, files) => {
+        if (err) {
+          throw err;
+        }
+
+        if (files.length <= 0) {
+          return;
+        }
+
+        files.forEach((f) => {
+          if (f.isDirectory()) {
+            this.fallbackWatch(
+              path.join(filepath, f.name),
+              gitignore,
+              callback
+            );
+          }
+        });
+      });
+    }
+  }
+
+  // 监听非.gitignore文件
+  // TODO： 目前仅仅监听process.cwd()以下的文件
+  async watchFile(keyFile, appAddr, appId) {
+    const ignore = new GitIgnore(process.cwd());
+    await ignore.collect();
+    chokidar
+      .watch(".", {
+        ignored: (path) => {
+          if ([".git", ".lazycat"].some((p) => path.startsWith(p))) return true;
+
+          return ignore.contain(path);
+        },
+        ignoreInitial: true,
+      })
+      .on("all", (event, path) => {
+        // console.log(event, path);
+        this.syncProject(keyFile, appAddr, appId);
+      });
+  }
+
+  // 保存一个dev shell状态的文件,如果重复执行dev shell
+  // 会先判断这个临时文件是否存在，如果不存在重新部署，更新流程
+  // 否则直接ssh连接
+  async storeShellStatus(isWatch) {
+    fs.writeFileSync(
+      path.join(this.tempDir, ".shellStatus"),
+      JSON.stringify({
+        isWatch,
+      })
+    );
+  }
+
+  isRsyncWatch() {
+    try {
+      let obj = this.readShellStatus();
+      return obj.isWatch;
+    } catch {
+      return false;
+    }
+  }
+
+  async readShellStatus() {
+    return JSON.parse(
+      fs.readFileSync(path.join(this.tempDir, ".shellStatus"), "utf-8")
+    );
+  }
+
+  async directShell() {
+    const keyFile = this.keyFile;
+    const appAddr = `sync.${this.appId}.lzcapp`;
+
+    let isWatch;
+    try {
+      ({ isWatch } = await this.readShellStatus());
+
+      if (!isWatch) {
+        await this.syncProject(keyFile, appAddr, this.appId);
+        // 注册watch函数
+        await this.watchFile(keyFile, appAddr, this.appId);
+
+        this.storeShellStatus(true);
+      }
+
+      await this.connectShell(this.appId);
+    } catch (e) {
+      return Promise.reject(e);
+    } finally {
+      if (!isWatch) {
+        this.storeShellStatus(false);
+      }
+    }
+  }
+
+  async shell() {
+    let keyFile = this.keyFile;
+    const appAddr = `sync.${this.appId}.lzcapp`;
+
+    try {
+      // 当进入shell的时候，都同步一次
+      await this.syncProject(keyFile, appAddr, this.appId);
+      // 注册watch函数
+      await this.watchFile(keyFile, appAddr, this.appId);
+
+      this.storeShellStatus(true);
+
+      await this.connectShell(this.appId);
+    } catch (e) {
+      console.log(e);
+      // this.reset();
+      return Promise.reject(e);
+    } finally {
+      // 当shell退出后，更新isWatch状态
+      this.storeShellStatus(false);
+    }
+  }
+
+  async connectShell(appId) {
+    const sdk = new sdkDocker();
+    const replacedAppId = appId.replaceAll(".", "_").replaceAll("-", "__");
+    await sdk.interactiveShell(`lzc--${replacedAppId}-app-1`);
+  }
+}
+
+// build strategies based on lzc-build.yml devshell field
+import { DockerClient, dockerPullLzcAppsImage } from "../sdk.js";
+import { DockerfileParser } from "dockerfile-ast";
+import glob from "fast-glob";
+import ignore from "@balena/dockerignore";
+import { createLogUpdate } from "log-update";
+
+class sdkDocker {
+  constructor() {}
+
+  // builder 需要在app 的场景下使用, 不过有两个路径要注意
+  // 一个是 docker build 时的context. 这个一般是运行cli时候的路径
+  async buildImage(tag, buildDir, contextDir) {
+    const dockerfilePath = path.join(buildDir, "Dockerfile");
+    try {
+      await this.dockerRemoteBuildV2(contextDir, dockerfilePath, tag);
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  async collectContextFromDockerFile(contextDir, dockerfilePath) {
+    if (!fs.existsSync(dockerfilePath)) {
+      throw "未发现 Dockerfile";
+    }
+
+    let src = [path.relative(contextDir, dockerfilePath)];
+
+    // 通过 COPY 和 ADD 获取所有context中的文件
+    const ast = DockerfileParser.parse(fs.readFileSync(dockerfilePath, "utf8"));
+    for (let a of ast.getInstructions()) {
+      if (["COPY", "ADD"].includes(a.getInstruction())) {
+        const from = a.getArguments()[0].getValue().replace(/^\//, "");
+
+        const fromFullPath = path.join(contextDir, from);
+
+        if (fs.existsSync(fromFullPath)) {
+          const stat = fs.statSync(path.join(contextDir, from));
+          if (stat.isDirectory()) {
+            let files = await glob(path.join(from, "**"), {
+              cwd: contextDir,
+            });
+            src = src.concat(files);
+          } else if (stat.isFile()) {
+            src.push(from);
+          }
+        } else {
+          // try use glob
+          let files = await glob(from, {
+            cwd: contextDir,
+          });
+          src = src.concat(files);
+        }
+      }
+    }
+    // filter by dockerignore
+    const dockerIgnoreFile = path.join(contextDir, ".dockerignore");
+    //
+    if (fs.existsSync(dockerIgnoreFile)) {
+      let ig = ignore();
+      let data = fs.readFileSync(dockerIgnoreFile, "utf8");
+      ig.add(data.split("\n"));
+      src = ig.filter(src);
+    }
+    return src;
+  }
+
+  /**
+   * v2 版本会收集所有需要的context 文件,将其传入buildImage 方法中
+   * @param contextDir
+   * @param dockerfile
+   * @param tag
+   */
+  async dockerRemoteBuildV2(contextDir, dockerfile, tag) {
+    const src = await this.collectContextFromDockerFile(contextDir, dockerfile);
+    const host = new URL(sdkEnv.sdkUrl).host;
+    console.log(host);
+    const docker = await new DockerClient(host).init();
+    try {
+      await dockerPullLzcAppsImage(host);
+
+      return new Promise(async (resolve, reject) => {
+        let refresher = new StatusRefresher();
+
+        logger.info("开始在设备中构建镜像");
+        const stream = await docker.buildImage(
+          {
+            context: contextDir,
+            src: src,
+          },
+          {
+            dockerfile: src[0],
+            t: tag,
+          }
+        );
+        docker.modem.followProgress(
+          stream,
+          (err, res) => {
+            err ? reject(err) : resolve(res);
+          },
+          (res) => {
+            if (res.status) {
+              if (res.id) {
+                if (["Downloading", "Extracting"].includes(res.status)) {
+                  refresher.updateStatus({
+                    id: res.id,
+                    content: `${res.status} ${res.progress}`,
+                  });
+                } else {
+                  refresher.updateStatus({
+                    id: res.id,
+                    content: res.status,
+                  });
+                }
+              } else {
+                process.stdout.write(res.status);
+                process.stdout.write("\n");
+              }
+            }
+
+            if (res.stream) {
+              if (res.stream.startsWith("Step")) {
+                refresher.reset();
+              }
+              process.stdout.write(res.stream);
+            }
+            if (res.error) {
+              reject(res.error);
+            }
+          }
+        );
+      });
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  async interactiveShell(containerName, shell = "sh") {
+    const host = new URL(sdkEnv.sdkUrl).hostname;
+    const docker = await new DockerClient(host).init();
+    const container = docker.getContainer(containerName);
+    const exec = await container.exec({
+      Cmd: [shell],
+      AttachStderr: true,
+      AttachStdout: true,
+      AttachStdin: true,
+      Tty: true,
+    });
+
+    const stream = await exec.start({ stdin: true });
+
+    process.stdin.setRawMode(true);
+    stream.pipe(process.stdout);
+    process.stdin.pipe(stream);
+
+    // prepare command
+    stream.write("cd /lzcapp/cache\n");
+    stream.write("ls\n");
+
+    await new Promise((resolve) => {
+      stream.on("end", () => {
+        logger.info("exit");
+        resolve();
+      });
+    });
+
+    process.stdin.removeAllListeners();
+    process.stdin.setRawMode(false);
+    process.stdin.resume();
+  }
+}
+
+class StatusRefresher {
+  constructor() {
+    this.reset();
+  }
+  refresh() {
+    this.logUpdate(this.pulls.map((p) => `${p.id}: ${p.content}`).join("\n"));
+  }
+
+  reset() {
+    this.pulls = [];
+    this.logUpdate = createLogUpdate(process.stdout);
+  }
+
+  updateStatus({ id, content }) {
+    let index = this.pulls.findIndex((p) => p.id == id);
+    if (index > -1) {
+      this.pulls[index] = {
+        id: id,
+        content: content,
+      };
+    } else {
+      this.pulls.push({
+        id: id,
+        content: content,
+      });
+    }
+    this.refresh();
+  }
+}

package/lib/app/lpk_installer.js

@@ -0,0 +1,67 @@
+import BoxAPI from "../api.js";
+import { sdkEnv } from "../env.js";
+import logger from "loglevel";
+import fs from "node:fs";
+import path from "node:path";
+import { dockerPullLzcAppsImage } from "../sdk.js";
+import Key from "../key.js";
+
+export class LpkInstaller {
+  constructor() {}
+
+  async init() {
+    // 1. 确保 sdk 已经安装
+    await sdkEnv.ensure();
+
+    // 2. pull 镜像需要ssh key
+    const k = new Key();
+    await k.ensure(sdkEnv.sdkUrl);
+
+    const host = new URL(sdkEnv.sdkUrl).hostname;
+    await dockerPullLzcAppsImage(host);
+  }
+
+  async deploy(builder) {
+    if (!builder) {
+      throw "deploy 必须传递一个 builder";
+    }
+
+    let manifest = await builder.getManifest();
+    let api = new BoxAPI(manifest["package"], sdkEnv.sdkUrl);
+
+    try {
+      let pkgPath = await builder.exec("");
+      logger.info("开始部署应用");
+      await api.install(pkgPath);
+      await api.checkStatus();
+      logger.info(
+        `请在浏览器中访问 ${sdkEnv.sdkUrl.replace(
+          /sdk/,
+          manifest["application"]["subdomain"]
+        )}`
+      );
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  async install(pkgPath) {
+    if (!pkgPath) {
+      throw "install 必须指定一个 pkg 路径";
+    }
+    let pkgNameParts = path.basename(pkgPath).split("-v");
+    if (pkgNameParts.length < 2) {
+      throw "lpk 文件名不合法：应该使用 $appid-$version.lpk 的格式（如 xxx-v0.0.1.lpk）"
+    }
+    let pkgName = pkgNameParts[0]
+    let api = new BoxAPI(pkgName, sdkEnv.sdkUrl);
+    try {
+      logger.info("开始安装应用");
+      await api.install(pkgPath);
+      await api.checkStatus();
+      logger.info(`安装成功！`);
+    } catch (e) {
+      throw e;
+    }
+  }
+}