@blocklet/cli 1.16.33 → 1.16.34-beta-20241120-080738-bbbe036c

package/lib/commands/blocklet/bundle/bundlers/markdown.js

@@ -0,0 +1,241 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { BLOCKLET_BUNDLE_FOLDER } = require('@blocklet/constant');
+const fromMarkdown = require('mdast-util-from-markdown');
+const toMarkdown = require('mdast-util-to-markdown');
+const crypto = require('crypto');
+const { parser } = require('posthtml-parser');
+const { render } = require('posthtml-render');
+const isEmpty = require('lodash/isEmpty');
+const isArray = require('lodash/isArray');
+const chalk = require('chalk');
+const { isLocalUri } = require('../../../../util/what-uri');
+const { printError } = require('../../../../util');
+
+class MarkdownBundler {
+  /**
+   * Creates an instance of MarkdownBundler.
+   * @param {{
+   *  blockletDir: string
+   * }} { blockletDir }
+   * @memberof MarkdownBundler
+   */
+  constructor({ blockletDir }) {
+    this.blockletDir = blockletDir;
+    this.bundleDir = path.join(this.blockletDir, BLOCKLET_BUNDLE_FOLDER);
+    this.mediaBundleDir = path.join(this.bundleDir, 'media');
+    this.mediaBaseUrl = '/assets/media';
+  }
+
+  /**
+   *
+   * @return {string[]} blocklet.md files absolute path list
+   * @memberof MarkdownBundler
+   */
+  _find() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   *
+   *
+   * @return {Promise<void>}
+   * @memberof MarkdownBundler
+   */
+  bundle() {
+    return this._bundle();
+  }
+
+  /**
+   *
+   * @description 从本地所有的blocklet.md提取本地静态文件到当前目录的media文件夹下面
+   * @param {*} blockletDir
+   */
+  async _bundle() {
+    const markdownAbsolutePaths = await this._find();
+    const cache = new Map();
+
+    while (!isEmpty(markdownAbsolutePaths)) {
+      const markdownAbsolutePath = markdownAbsolutePaths.shift();
+      const mdAstNode = fromMarkdown(fs.readFileSync(markdownAbsolutePath));
+
+      const done = this._handleMdAstNode(mdAstNode, (astNode) => {
+        const referenceFile = astNode?.url || astNode?.attrs?.src || astNode?.attrs?.href;
+        const localUrl = this._getLocalUrl(referenceFile);
+
+        // FIXME: 后续jest mock做一个单测吧
+        if (!fs.existsSync(localUrl)) {
+          printError(
+            `Referenced file ${chalk.red(referenceFile)} not found when bundling ${chalk.cyan(
+              path.basename(markdownAbsolutePath)
+            )}`
+          );
+          process.exit(1);
+        }
+
+        // 如果路径指向了一个文件夹,应该通过报错告知用户,文件夹不能存在于markdown文件当中
+        if (fs.statSync(localUrl).isDirectory()) {
+          printError(
+            `Only files links are allowed in ${chalk.cyan(
+              path.basename(markdownAbsolutePath)
+            )}, found a folder ${chalk.red(referenceFile)}`
+          );
+          process.exit(1);
+        }
+
+        if (!cache.get(localUrl)) {
+          cache.set(localUrl, null);
+          return false;
+        }
+
+        const { storeStaticUrl } = cache.get(localUrl);
+
+        if (astNode?.url) {
+          astNode.url = storeStaticUrl;
+        }
+
+        if (astNode?.attrs?.src) {
+          astNode.attrs.src = storeStaticUrl;
+        }
+
+        if (astNode?.attrs?.href) {
+          astNode.attrs.href = storeStaticUrl;
+        }
+
+        return true;
+      });
+
+      if (done) {
+        // eslint-disable-next-line no-await-in-loop
+        await this.writeFile(path.join(this.bundleDir, path.basename(markdownAbsolutePath)), toMarkdown(mdAstNode));
+      } else {
+        // 说明有些资源还没有计算好hash值和复制到bundle下,blocklet.md没处理完,当然也就不能直接修改blocklet.md里面的引用了
+        // eslint-disable-next-line no-await-in-loop
+        await this._batchHash(cache);
+        // eslint-disable-next-line no-await-in-loop
+        await this._batchCopy(cache);
+        // 还没处理完成,等到下一轮再处理即可
+        markdownAbsolutePaths.push(markdownAbsolutePath);
+      }
+    }
+  }
+
+  /**
+   *
+   *
+   * @param {string} dest
+   * @param {string | NodeJS.ArrayBufferView} data
+   * @memberof MarkdownBundler
+   */
+  writeFile(dest, data) {
+    fs.writeFileSync(dest, data);
+  }
+
+  _batchHash(cache) {
+    return Promise.all(
+      [...cache.keys()].map((localUrl) => {
+        return this._getTargetUrls(localUrl).then((targetUrls) => {
+          cache.set(localUrl, targetUrls);
+        });
+      })
+    );
+  }
+
+  _batchCopy(cache) {
+    return Promise.all(
+      [...cache.keys()].map((localUrl) => {
+        const { mediaBundleUrl } = cache.get(localUrl);
+        return fs.ensureDir(path.dirname(mediaBundleUrl)).then(() => fs.copy(localUrl, mediaBundleUrl));
+      })
+    );
+  }
+
+  _handleMdAstNode(mdAstNode, callback) {
+    let done = true;
+
+    if (this._isLocalUrl(mdAstNode?.url)) {
+      done = done && callback(mdAstNode);
+    }
+
+    if (this._isHtmlNode(mdAstNode)) {
+      const hastNode = parser(mdAstNode?.value);
+      done = done && this._handleHastNode(hastNode, callback);
+      mdAstNode.value = render(hastNode);
+    }
+
+    if (mdAstNode?.children) {
+      done = mdAstNode.children.reduce(
+        (preValue, astNode) => this._handleMdAstNode(astNode, callback) && preValue,
+        done
+      );
+    }
+
+    return done;
+  }
+
+  _handleHastNode(root, callback) {
+    let done = true;
+
+    const currentRoot = isArray(root) ? root : [root];
+
+    currentRoot.forEach((children) => {
+      if (this._isLocalUrl(children?.attrs?.src || children?.attrs?.href)) {
+        done = callback(children) && done;
+      }
+
+      if (isArray(children?.content)) {
+        done = children.content.reduce((preValue, _root) => this._handleHastNode(_root, callback) && preValue, done);
+      }
+    });
+
+    return done;
+  }
+
+  /**
+   *
+   *
+   * @param {string} url
+   * @return {boolean}
+   * @memberof BundleMarkdown
+   */
+  _isLocalUrl(url) {
+    return isLocalUri(url);
+  }
+
+  _isHtmlNode(node) {
+    return node?.type === 'html';
+  }
+
+  _getLocalUrl(url) {
+    return path.join(this.blockletDir, url);
+  }
+
+  async _getTargetUrls(localUrl) {
+    const fileHash = await this._getFileHash(localUrl);
+
+    const fileDirNameAndFileName = path.join(fileHash.slice(0, 2), `${fileHash.slice(2)}${path.extname(localUrl)}`);
+
+    return {
+      mediaBundleUrl: path.join(this.mediaBundleDir, fileDirNameAndFileName),
+      storeStaticUrl: path.join(this.mediaBaseUrl, fileDirNameAndFileName),
+    };
+  }
+
+  _getFileHash(url) {
+    return new Promise((resolve, reject) => {
+      const hash = crypto.createHash('md5');
+      const stream = fs.createReadStream(url);
+      stream.on('error', (err) => {
+        stream?.destroy();
+        reject(err);
+      });
+      stream.on('data', (chunk) => hash.update(chunk));
+      stream.on('end', () => {
+        stream?.destroy();
+        resolve(hash.digest('hex'));
+      });
+    });
+  }
+}
+
+module.exports = MarkdownBundler;

package/lib/commands/blocklet/bundle/bundlers/preference.js

@@ -0,0 +1,50 @@
+const path = require('path');
+const fs = require('fs-extra');
+const { BLOCKLET_BUNDLE_FOLDER, BLOCKLET_PREFERENCE_FILE } = require('@blocklet/constant');
+const fg = require('fast-glob');
+const chalk = require('chalk');
+const { printError } = require('../../../../util');
+
+class PreferenceBundler {
+  constructor({ blockletDir }) {
+    this.blockletDir = blockletDir;
+    this.bundlerDir = path.join(this.blockletDir, BLOCKLET_BUNDLE_FOLDER);
+    this.fileName = BLOCKLET_PREFERENCE_FILE;
+  }
+
+  _find() {
+    const found = fg.sync(this.fileName, {
+      cwd: this.blockletDir,
+      deep: 1,
+      caseSensitiveMatch: false,
+      onlyFiles: true,
+      absolute: true,
+    });
+
+    if (found.length) {
+      const [filePath] = found;
+      try {
+        fs.readJsonSync(filePath);
+        return filePath;
+      } catch (err) {
+        printError(`Invalid ${chalk.red(this.fileName)} find ${chalk.cyan(this.blockletDir)}`);
+        process.exit(1);
+      }
+    }
+
+    return null;
+  }
+
+  async _bundle(filePath) {
+    if (filePath) {
+      await fs.copy(filePath, path.join(this.bundlerDir, this.fileName));
+    }
+  }
+
+  async bundle() {
+    const filePath = await this._find();
+    await this._bundle(filePath);
+  }
+}
+
+module.exports = PreferenceBundler;

package/lib/commands/blocklet/bundle/bundlers/readme.js

@@ -0,0 +1,43 @@
+const fg = require('fast-glob');
+const MarkdownBundler = require('./markdown');
+
+class ReadmeMdBundler extends MarkdownBundler {
+  static README_MD_REGEX = /README\.[\S]*md$/i;
+
+  constructor({ blockletDir }) {
+    super({ blockletDir });
+    this.readmeMdFileName = 'README.md';
+  }
+
+  /**
+   *
+   *
+   * @return {string[]} blocklet.md absolute path
+   * @memberof BlockletMdBundler
+   */
+  _find() {
+    const found = fg.sync(this.readmeMdFileName, {
+      cwd: this.blockletDir,
+      deep: 1,
+      caseSensitiveMatch: false,
+      onlyFiles: true,
+      absolute: true,
+    });
+
+    if (found.length > 1) {
+      throw new Error(`Only one ${this.readmeMdFileName}(not case sensitive) can exist in ${this.blockletDir}`);
+    }
+
+    return found;
+  }
+
+  bundle() {
+    return super.bundle();
+  }
+
+  async writeFile(dest, data) {
+    await super.writeFile(dest.replace(ReadmeMdBundler.README_MD_REGEX, this.readmeMdFileName), data);
+  }
+}
+
+module.exports = ReadmeMdBundler;

package/lib/commands/blocklet/bundle/bundlers/screenshots.js

@@ -0,0 +1,94 @@
+const { pathExists, emptyDir, ensureDir, copy } = require('fs-extra');
+const { join, extname } = require('path');
+const chalk = require('chalk');
+const { createReadStream } = require('fs');
+const { onlyHash } = require('@arcblock/ipfs-only-hash');
+const { validateScreenshots } = require('@blocklet/images');
+const { print, printWarning, printInfo, printError } = require('../../../../util');
+
+class ScreenshotsBundler {
+  /**
+   * Creates an instance of ScreenshotsBundler.
+   * @param {{
+   *  blockletDir: string,
+   *  bundleDir: string,
+   *  meta: import('@abtnode/client').BlockletMeta
+   * }} { blockletDir, bundleDir, meta }
+   * @memberof ScreenshotsBundler
+   */
+  constructor({ blockletDir = '', bundleDir = '', meta }) {
+    this.blockletDir = blockletDir;
+    this.bundleDir = bundleDir;
+    this.meta = meta;
+
+    this.screenshotsBlockletFolder = join(this.blockletDir, 'screenshots');
+    this.screenshotsBundleFolder = join(this.bundleDir, 'screenshots');
+  }
+
+  /**
+   *
+   * @return {Promise<void>}
+   * @memberof ScreenshotsBundler
+   */
+  async _bundle() {
+    await this._clean();
+
+    if (!this.meta?.screenshots?.length) {
+      printWarning('No screenshots found!');
+    } else {
+      const errorMessages = validateScreenshots(this.meta.screenshots, {
+        extractedFilepath: this.blockletDir,
+        minWidth: 600,
+      });
+
+      if (errorMessages.length) {
+        print('');
+        printWarning('Blocklet bundle failed! Please check the following errors about screenshots:');
+        errorMessages.forEach((errorMessage) => printError(errorMessage));
+        printInfo(
+          `You can run the command "${chalk.cyan('blocklet dev studio')}" to help you generate more standard screenshots.`
+        );
+        print('');
+        process.exit(1);
+      }
+
+      const entryList = await Promise.all(
+        this.meta.screenshots.map(async (fileName) => {
+          const source = join(this.screenshotsBlockletFolder, fileName);
+          if (pathExists(source)) {
+            const hash = await onlyHash(createReadStream(source));
+            const filename = `${hash}${extname(fileName)}`;
+            const target = join(this.screenshotsBundleFolder, filename);
+            return { source, target, filename };
+          }
+          return null;
+        })
+      );
+      await Promise.all(entryList.map((item) => item && copy(item.source, item.target)));
+      const filenames = [];
+      entryList.forEach((item) => {
+        if (item?.filename) {
+          filenames.push(item.filename);
+        }
+      });
+      this.meta.screenshots = filenames;
+    }
+  }
+
+  /**
+   * @description 清理 screenshots bundle 的文件夹
+   * @memberof ScreenshotsBundler
+   */
+  async _clean() {
+    if (await pathExists(this.screenshotsBundleFolder)) {
+      await emptyDir(this.screenshotsBundleFolder);
+    }
+    await ensureDir(this.screenshotsBundleFolder);
+  }
+
+  bundle() {
+    return this._bundle();
+  }
+}
+
+module.exports = ScreenshotsBundler;

package/lib/commands/blocklet/bundle/bundlers/simple.js

@@ -0,0 +1,70 @@
+const ISO6391 = require('iso-639-1');
+
+const { BLOCKLET_BUNDLE_FOLDER } = require('@blocklet/constant');
+const { join } = require('path');
+const BlockletMdBundler = require('./blocklet');
+const ChangelogBundler = require('./changelog');
+const LogoBundler = require('./logo');
+const ReadmeMdBundler = require('./readme');
+const PreferenceBundler = require('./preference');
+const ScreenshotsBundler = require('./screenshots');
+
+class SimpleBundler {
+  /**
+   * Creates an instance of SimpleBundler.
+   * @param {{
+   *  blockletDir: string,
+   *  inMonoRepo: boolean,
+   *  withChangeLog: boolean,
+   *  meta: {
+   *    logo: string,
+   *  }
+   * }} { blockletDir, inMonoRepo, withChangeLog, meta }
+   * @memberof SimpleBundler
+   */
+  constructor({ blockletDir, inMonoRepo, withChangeLog, meta }) {
+    this.blockletDir = blockletDir;
+    this.inMonoRepo = inMonoRepo;
+    this.withChangeLog = withChangeLog;
+    this.meta = meta;
+  }
+
+  async bundle() {
+    const locales = ISO6391.getAllCodes();
+    const { blockletDir, inMonoRepo } = this;
+    const bundleDir = join(this.blockletDir, BLOCKLET_BUNDLE_FOLDER);
+
+    const blockletMdPromises = locales.map((locale) => {
+      if (locale === 'en') {
+        return new BlockletMdBundler({
+          blockletDir,
+          blockletMdFileName: 'blocklet.md',
+          backupMdFileNames: ['blocklet.en.md', 'README.md'],
+          duplicateMdFileName: 'blocklet.en.md',
+          required: true,
+        }).bundle();
+      }
+      return new BlockletMdBundler({
+        blockletDir,
+        blockletMdFileName: `blocklet.${locale}.md`,
+      }).bundle();
+    });
+
+    await new ScreenshotsBundler({ blockletDir, bundleDir, meta: this.meta }).bundle();
+    await new LogoBundler({ blockletDir, logoFileName: this.meta.logo }).bundle();
+
+    await Promise.all([
+      ...blockletMdPromises,
+      this.withChangeLog
+        ? new ChangelogBundler({
+            blockletDir,
+            inMonoRepo,
+          }).bundle()
+        : Promise.resolve(null),
+      new ReadmeMdBundler({ blockletDir }).bundle(),
+      new PreferenceBundler({ blockletDir }).bundle(),
+    ]);
+  }
+}
+
+module.exports = SimpleBundler;

package/lib/commands/blocklet/bundle/compact/bundle-compact-file.js

@@ -0,0 +1,48 @@
+const ncc = require('@vercel/ncc');
+const fs = require('fs-extra');
+const { dirname, resolve } = require('path');
+const convertToNoSourcesSourceMap = require('../../../../util/convert-to-nosources-sourcemap');
+
+async function bundleCompactFile({
+  srcPath,
+  distDir,
+  enterFile,
+  sourceMap,
+  nosourcesSourceMap,
+  externals,
+  minify = true,
+}) {
+  const { code, assets } = await ncc(resolve(srcPath, enterFile), {
+    sourceMap: sourceMap || nosourcesSourceMap,
+    externals,
+    minify,
+    quiet: true,
+    cache: false,
+    sourceMapBasePrefix: '',
+  });
+
+  const codeFile = resolve(distDir, enterFile);
+  const outDir = dirname(codeFile);
+  if (!fs.existsSync(dirname(codeFile))) {
+    fs.mkdirSync(dirname(codeFile), { recursive: true });
+  }
+
+  fs.writeFileSync(codeFile, code);
+
+  Object.keys(assets).forEach((name) => {
+    const distDirByFile = dirname(resolve(outDir, name));
+    if (!fs.existsSync(distDirByFile)) {
+      fs.mkdirSync(distDirByFile, { recursive: true });
+    }
+    fs.writeFileSync(resolve(outDir, name), assets[name].source);
+  });
+
+  if (!sourceMap && nosourcesSourceMap) {
+    const mapFilePath = resolve(distDir, 'index.js.map');
+    await convertToNoSourcesSourceMap(mapFilePath, mapFilePath);
+  }
+}
+
+module.exports = {
+  bundleCompactFile,
+};

package/lib/commands/blocklet/bundle/compact/bundle-merge-extra.js

@@ -0,0 +1,66 @@
+const fs = require('fs-extra');
+const path = require('path');
+const pAll = require('p-all');
+const { bundleCompactFile } = require('./bundle-compact-file');
+const { parseExternalDependencies } = require('../parse-external-dependencies');
+
+const compactFileName = 'blocklet-compact.js';
+
+async function bundleMergeExtra({
+  extraFiles,
+  srcPath,
+  distDir,
+  sourceMap,
+  nosourcesSourceMap,
+  enterFile,
+  externals,
+  externalManager,
+  minify,
+}) {
+  const entryFiles = [...extraFiles, enterFile];
+
+  const compactCode = `
+module.exports = {
+  ${entryFiles.map((file) => `'${file}':()=> require('${path.resolve(srcPath, file)}'),`).join('\n')}
+};
+  `;
+
+  if (!fs.existsSync(distDir)) {
+    fs.mkdirpSync(distDir);
+  }
+  fs.writeFileSync(path.resolve(distDir, compactFileName), compactCode);
+  await bundleCompactFile({
+    minify,
+    externals,
+    srcPath: distDir,
+    distDir,
+    enterFile: compactFileName,
+    sourceMap,
+    nosourcesSourceMap,
+  });
+
+  const promises = entryFiles.map((file) => {
+    return async () => {
+      const distDirByFile = path.dirname(path.resolve(distDir, file));
+      try {
+        await fs.access(distDirByFile);
+      } catch (err) {
+        await fs.mkdir(distDirByFile, { recursive: true });
+      }
+      await fs.writeFile(
+        path.resolve(distDir, file),
+        `module.exports = require('${path.relative(
+          distDirByFile,
+          path.resolve(distDir, compactFileName)
+        )}')['${file}']();`
+      );
+    };
+  });
+  await pAll(promises, { concurrency: 4 });
+
+  parseExternalDependencies({ externals, distDir, externalManager });
+}
+
+module.exports = {
+  bundleMergeExtra,
+};

package/lib/commands/blocklet/bundle/compact/default-external.js

@@ -0,0 +1,5 @@
+const defaultExternals = ['sqlite3', 'sharp', '@abtnode/models'];
+
+module.exports = {
+  defaultExternals,
+};