@blocklet/cli 1.16.33 → 1.16.34-beta-20241120-080738-bbbe036c

package/lib/commands/blocklet/bundle/zip/dependencies.js

@@ -0,0 +1,333 @@
+/* eslint-disable no-use-before-define */
+/* eslint-disable func-names */
+const { dirname, basename, normalize, extname } = require('path');
+
+const { glob } = require('glob');
+const { not: notJunk } = require('junk');
+const pkgDir = require('pkg-dir');
+const precinct = require('precinct');
+const requirePackageName = require('require-package-name');
+const formatBackSlash = require('@abtnode/util/lib/format-back-slash');
+
+const debug = require('../../../../debug')('bundle:zip');
+const { resolvePathPreserveSymlinks, resolvePackage } = require('./resolve');
+
+// Retrieve the paths to the Node.js files to zip.
+// We only include the files actually needed by the function because AWS Lambda
+// has a size limit for the zipped file. It also makes cold starts faster.
+const listNodeFiles = async function ({ srcPath, mainFile, srcDir, stat, extraPaths }) {
+  const [treeFiles, depFiles] = await Promise.all([getTreeFiles(srcPath, stat), getDependencies(mainFile, srcDir)]);
+  const extraFiles = [];
+  for (let i = 0; i < extraPaths.length; i++) {
+    const extraPath = extraPaths[i];
+    const extension = extname(extraPath);
+    if (extension === '.js') {
+      // eslint-disable-next-line no-await-in-loop
+      const [extraTreeFiles, extraDepFiles] = await Promise.all([
+        getTreeFiles(extraPath, stat),
+        getDependencies(extraPath, srcDir),
+      ]);
+      extraFiles.push(...extraTreeFiles);
+      extraFiles.push(...extraDepFiles);
+    } else {
+      extraFiles.push(extraPath);
+    }
+  }
+
+  const files = [...treeFiles, ...depFiles, ...extraFiles].map(normalize);
+
+  const uniqueFiles = [...new Set(files)];
+
+  // We sort so that the archive's checksum is deterministic.
+  const filteredFiles = uniqueFiles.filter(isNotJunk).sort();
+  debug('listNodeFiles', {
+    srcPath,
+    mainFile,
+    extraPaths,
+    uniqueFiles: uniqueFiles.length,
+    filteredFiles: filteredFiles.length,
+  });
+
+  return filteredFiles;
+};
+
+// When using a directory, we include all its descendants except `node_modules`
+const getTreeFiles = function (srcPath, stat) {
+  if (!stat.isDirectory()) {
+    return [srcPath];
+  }
+
+  const tmpPath = formatBackSlash(srcPath);
+  return glob(`${tmpPath}/**`, {
+    ignore: `${tmpPath}/**/node_modules/**`,
+    nodir: true,
+    absolute: true,
+  });
+};
+
+// Remove temporary files like *~, *.swp, etc.
+const isNotJunk = function (file) {
+  if (file.includes('.pnpm')) {
+    console.warn('Maybe a dependency is missing in your package.json', file);
+  }
+  return file.includes('.pnpm') === false && notJunk(basename(file));
+};
+
+// Retrieve all the files recursively required by a Node.js file
+const getDependencies = async function (mainFile, srcDir) {
+  const packageRoot = await pkgDir(srcDir);
+  const packageJson = getPackageJson(packageRoot);
+
+  const state = { localFiles: [], modulePaths: [] };
+
+  try {
+    return await getFileDependencies(mainFile, packageJson, state);
+  } catch (error) {
+    error.message = `In file "${mainFile}": ${error.message}`;
+    throw error;
+  }
+};
+
+const getPackageJson = function (packageRoot) {
+  if (packageRoot === undefined) {
+    return {};
+  }
+
+  const packageJsonPath = `${packageRoot}/package.json`;
+  try {
+    // eslint-disable-next-line
+    return require(packageJsonPath);
+  } catch (error) {
+    throw new Error(`${packageJsonPath} is invalid JSON: ${error.message}`);
+  }
+};
+
+const getFileDependencies = async function (path, packageJson, state) {
+  if (state.localFiles.includes(path)) {
+    return [];
+  }
+
+  state.localFiles.push(path);
+
+  const basedir = dirname(path);
+  // This parses JavaScript in `path` to retrieve all the `require()` statements
+  // TODO: `precinct.paperwork()` uses `fs.readFileSync()` under the hood,
+  // but should use `fs.readFile()` instead
+  const dependencies = precinct.paperwork(path, { includeCore: false });
+
+  const depsPaths = await Promise.all(
+    dependencies.map((dependency) => getImportDependencies(dependency, basedir, packageJson, state))
+  );
+  // debug('getFileDependencies', { path, depsPaths });
+  return [].concat(...depsPaths);
+};
+
+// `require()` statements can be either `require('moduleName')` or
+// `require(path)`
+const getImportDependencies = function (dependency, basedir, packageJson, state) {
+  if (LOCAL_IMPORT_REGEXP.test(dependency)) {
+    return getLocalImportDependencies(dependency, basedir, packageJson, state);
+  }
+
+  return getModuleDependencies(dependency, basedir, state, packageJson);
+};
+
+const LOCAL_IMPORT_REGEXP = /^(\.|\/)/;
+
+// When a file requires another one, we apply the top-level logic recursively
+const getLocalImportDependencies = async function (dependency, basedir, packageJson, state) {
+  const dependencyPath = await resolvePathPreserveSymlinks(dependency, basedir);
+  const depsPath = await getFileDependencies(dependencyPath, packageJson, state);
+  return [dependencyPath, ...depsPath];
+};
+
+// When a file requires a module, we find its path inside `node_modules` and
+// use all its published files. We also recurse on the module's dependencies.
+const getModuleDependencies = async function (dependency, basedir, state, packageJson) {
+  const moduleName = getModuleName(dependency);
+
+  // Happens when doing require("@scope") (not "@scope/name") or other oddities
+  // Ignore those.
+  if (moduleName === null) {
+    return [];
+  }
+
+  try {
+    return await getModuleNameDependencies(moduleName, basedir, state);
+  } catch (error) {
+    return handleModuleNotFound({ error, moduleName, packageJson });
+  }
+};
+
+// When doing require("moduleName/file/path"), only keep `moduleName`
+const getModuleName = function (dependency) {
+  const dependencyA = dependency.replace(BACKSLASH_REGEXP, '/');
+  const moduleName = requirePackageName(dependencyA);
+  return moduleName;
+};
+
+// Windows path normalization
+const BACKSLASH_REGEXP = /\\/g;
+
+const getModuleNameDependencies = async function (moduleName, basedir, state) {
+  if (isExcludedModule(moduleName)) {
+    return [];
+  }
+
+  // Find the Node.js module directory path
+  const packagePath = await resolvePackage(moduleName, basedir);
+
+  if (packagePath === undefined) {
+    return [];
+  }
+
+  const modulePath = dirname(packagePath);
+
+  if (state.modulePaths.includes(modulePath)) {
+    return [];
+  }
+
+  state.modulePaths.push(modulePath);
+
+  // eslint-disable-next-line
+  const pkg = require(packagePath);
+
+  const [publishedFiles, sideFiles, depsPaths] = await Promise.all([
+    getPublishedFiles(modulePath),
+    getSideFiles(modulePath, moduleName),
+    getNestedModules(modulePath, state, pkg),
+  ]);
+  // debug('getModuleNameDependencies', { moduleName, basedir, packagePath, modulePath, publishedFiles, depsPaths });
+  return [...publishedFiles, ...sideFiles, ...depsPaths];
+};
+
+const EXCLUDED_MODULES = [];
+const isExcludedModule = function (moduleName) {
+  return (
+    EXCLUDED_MODULES.includes(moduleName) || moduleName.startsWith('@types/') || moduleName.startsWith('@cypress/')
+  );
+};
+
+// Some modules generate source files on `postinstall` that are not located
+// inside the module's directory itself.
+const getSideFiles = function (modulePath, moduleName) {
+  const sideFiles = SIDE_FILES[moduleName];
+  if (sideFiles === undefined) {
+    return [];
+  }
+
+  return getPublishedFiles(`${modulePath}/${sideFiles}`);
+};
+
+const SIDE_FILES = {
+  '@prisma/client': '../../.prisma',
+};
+
+// We use all the files published by the Node.js except some that are not needed
+const getPublishedFiles = async function (modulePath) {
+  const ignore = getIgnoredFiles(modulePath);
+  const publishedFiles = await glob(formatBackSlash(`${modulePath}/**`), {
+    ignore,
+    nodir: true,
+    absolute: true,
+    dot: false,
+  });
+  return publishedFiles;
+};
+
+const getIgnoredFiles = function (modulePath) {
+  return IGNORED_FILES.map((ignoreFile) => `${modulePath}/${ignoreFile}`);
+};
+
+// To make the zip archive smaller, we remove those.
+const IGNORED_FILES = [
+  'node_modules/**',
+  '.blocklet/**',
+  '.abtnode/**',
+  'coverage/**',
+  'tests/**',
+  'test/**',
+  'doc/**',
+  'docs/**',
+  '.npmignore',
+  'package-lock.json',
+  'yarn.lock',
+  '**/*.js.map',
+  '**/*.css.map',
+  '**/*.d.ts',
+  '**/*.d.ts.map',
+  '*.log',
+  '*.lock',
+  '*~',
+  '*.ts',
+  '*.patch',
+];
+
+// Apply the Node.js module logic recursively on its own dependencies, using
+// the `package.json` `dependencies`, `peerDependencies` and
+// `optionalDependencies` keys
+const getNestedModules = async function (modulePath, state, pkg) {
+  const dependencies = getNestedDependencies(pkg);
+  const depsPaths = await Promise.all(
+    dependencies.map((dependency) => getModuleDependencies(dependency, modulePath, state, pkg))
+  );
+  return [].concat(...depsPaths);
+};
+
+const getNestedDependencies = function ({ dependencies = {}, peerDependencies = {}, optionalDependencies = {} }) {
+  return [
+    ...Object.keys(dependencies),
+    ...Object.keys(peerDependencies).filter(shouldIncludePeerDependency),
+    ...Object.keys(optionalDependencies),
+  ];
+};
+
+const shouldIncludePeerDependency = function (name) {
+  return !EXCLUDED_PEER_DEPENDENCIES.includes(name);
+};
+
+const EXCLUDED_PEER_DEPENDENCIES = ['@prisma/cli', 'prisma2'];
+
+// Modules can be required conditionally (inside an `if` or `try`/`catch` block).
+// When a `require()` statement is found but the module is not found, it is
+// possible that that block either always evaluates to:
+//  - `false`: in which case, we should not bundle the dependency
+//  - `true`: in which case, we should report the dependency as missing
+// Those conditional modules might be:
+//  - present in the `package.json` `dependencies`
+//  - present in the `package.json` `optionalDependencies`
+//  - present in the `package.json` `peerDependencies`
+//  - not present in the `package.json`, if the module author wants its users
+//    to explicitly install it as an optional dependency.
+// The current implementation:
+//  - when parsing `require()` statements inside function files, always consider
+//    conditional modules to be included, i.e. report them if not found.
+//    This is because our current parsing logic does not know whether a
+//    `require()` is conditional or not.
+//  - when parsing module dependencies, ignore `require()` statements if not
+//    present in the `package.json` `*dependencies`. I.e. user must manually
+//    install them if the module is used.
+// `optionalDependencies`:
+//  - are not reported when missing
+//  - are included in module dependencies
+const handleModuleNotFound = function ({ error, moduleName, packageJson }) {
+  if (error.code === 'MODULE_NOT_FOUND' && isOptionalModule(moduleName, packageJson)) {
+    return [];
+  }
+
+  throw error;
+};
+
+const isOptionalModule = function (
+  moduleName,
+  { optionalDependencies = {}, peerDependenciesMeta = {}, peerDependencies = {} }
+) {
+  return (
+    optionalDependencies[moduleName] !== undefined ||
+    (peerDependenciesMeta[moduleName] &&
+      peerDependenciesMeta[moduleName].optional &&
+      peerDependencies[moduleName] !== undefined)
+  );
+};
+
+module.exports = { listNodeFiles };

package/lib/commands/blocklet/bundle/zip/index.js

@@ -0,0 +1,165 @@
+/* eslint-disable no-console */
+const fs = require('fs-extra');
+const chalk = require('chalk');
+const { join, dirname, basename, extname, relative } = require('path');
+const { BLOCKLET_BUNDLE_FOLDER, BLOCKLET_BUNDLE_FILE, BLOCKLET_ENTRY_FILE } = require('@blocklet/constant');
+const parseBlockletMeta = require('@blocklet/meta/lib/parse');
+const detectWorkspace = require('@abtnode/util/lib/detect-workspace');
+
+const { print, printInfo, printSuccess, printError } = require('../../../../util');
+const { wrapSpinner } = require('../../../../ui');
+const main = require('./main');
+const pack = require('../pack');
+const { createBlockletBundle, createBlockletEntry, getExtraFiles } = require('../bundle');
+const { logTar, getContents } = require('../../../../util/blocklet/tar');
+
+const debug = require('../../../../debug')('bundle:zip');
+
+module.exports.run = async ({
+  meta,
+  blockletDir,
+  createRelease = false,
+  inMonoRepo = false,
+  withChangeLog = true,
+  externals = [],
+  externalManager = 'npm',
+}) => {
+  // eslint-disable-next-line no-param-reassign
+  meta = meta || parseBlockletMeta(blockletDir, { ensureFiles: true });
+
+  const destFolder = join(blockletDir, BLOCKLET_BUNDLE_FOLDER);
+
+  if (fs.existsSync(destFolder)) {
+    fs.removeSync(destFolder);
+  } else {
+    fs.mkdirSync(destFolder, { recursive: true });
+  }
+
+  const workspace = detectWorkspace(blockletDir);
+  const extraFiles = getExtraFiles(blockletDir, meta);
+
+  const zipConfig = {
+    options: {},
+    destFolder,
+    extraPaths: [],
+    skippable: true,
+  };
+
+  const mainPath = join(blockletDir, meta.main || '/');
+  const hasMainFile = fs.statSync(mainPath).isFile();
+
+  // 1. for dapp entry
+  if (hasMainFile) {
+    zipConfig.srcFolder = dirname(mainPath);
+    zipConfig.srcFile = basename(mainPath);
+    zipConfig.srcFilter = (x) => [zipConfig.srcFile].includes(x);
+    zipConfig.skippable = false;
+  }
+
+  // 2. for hook scripts
+  if (extraFiles.length) {
+    zipConfig.extraPaths = extraFiles.map((x) => join(blockletDir, x));
+    zipConfig.skippable = false;
+  }
+
+  // 3. static blocklets may also include hook scripts
+  if (!zipConfig.srcFile && extraFiles.length) {
+    const srcFile = zipConfig.extraPaths.shift();
+    zipConfig.srcFolder = dirname(srcFile);
+    zipConfig.srcFile = basename(srcFile);
+    zipConfig.srcFilter = (x) => [zipConfig.srcFile].includes(x);
+    zipConfig.skippable = false;
+  }
+  debug('config', zipConfig);
+  debug('workspace', workspace);
+
+  // 4. create zip bundles
+  let archive;
+  if (zipConfig.skippable === false) {
+    const archives = await wrapSpinner(`Bundle blocklet from entry: ${chalk.cyan(meta.main)}`, () =>
+      main.zipEntries(zipConfig)
+    );
+    debug('archives', archives);
+
+    [archive] = archives;
+    if (!archive) {
+      printError('Failed to create zipped bundle');
+      process.exit(1);
+    }
+  }
+
+  // 5. handle extraFiles
+  const entryParts = ['.'];
+  let relativePath = '';
+
+  if (inMonoRepo && workspace) {
+    relativePath = relative(workspace.dir, blockletDir);
+  } else if (archive) {
+    relativePath = relative(archive.prefix, blockletDir);
+  }
+  if (relativePath) {
+    entryParts.push(relativePath);
+    extraFiles.forEach((f) => {
+      if (extname(f) === '.js') {
+        const absoluteFilepath = join(blockletDir, BLOCKLET_BUNDLE_FOLDER, f);
+        fs.ensureDirSync(dirname(absoluteFilepath));
+
+        const requirePath = relative(
+          dirname(absoluteFilepath),
+          join(blockletDir, BLOCKLET_BUNDLE_FOLDER, relativePath, f)
+        );
+        fs.writeFileSync(absoluteFilepath, createBlockletEntry(blockletDir, requirePath));
+      }
+    });
+  }
+
+  // 6. produce a `blocklet.js` file
+  let outputFile;
+  if (hasMainFile) {
+    entryParts.push(meta.main);
+    outputFile = join(blockletDir, BLOCKLET_BUNDLE_FOLDER, BLOCKLET_ENTRY_FILE);
+    fs.writeFileSync(outputFile, createBlockletEntry(blockletDir, entryParts.filter(Boolean).join('/')));
+  }
+
+  // 7. rename bundle to blocklet.zip
+  let outputZip;
+  if (archive) {
+    outputZip = join(blockletDir, BLOCKLET_BUNDLE_FOLDER, BLOCKLET_BUNDLE_FILE);
+    fs.renameSync(archive.path, outputZip);
+  }
+
+  print('');
+  printInfo(`Source: ${mainPath}`);
+  if (outputFile) {
+    printInfo(`Entry: ${outputFile}`);
+  }
+  if (outputZip) {
+    printInfo(`Archive: ${outputZip}`);
+  }
+
+  // 8. create bundle
+  await wrapSpinner(`Creating blocklet bundle in ${chalk.cyan(BLOCKLET_BUNDLE_FOLDER)}...`, async () => {
+    await createBlockletBundle({
+      blockletDir,
+      meta,
+      updates: hasMainFile ? { main: basename(outputZip) } : {},
+      inMonoRepo,
+      withChangeLog,
+      externals,
+      externalManager,
+    });
+  });
+
+  printSuccess(`Blocklet ${chalk.cyan(`${meta.name}@${meta.version}`)} is successfully bundled!`);
+
+  // 9. create release
+  if (createRelease) {
+    print('');
+    // eslint-disable-next-line no-shadow
+    const { tarball, meta } = await pack(blockletDir);
+    const pkgContents = await getContents(meta, tarball);
+    logTar(pkgContents);
+  }
+
+  process.exit(0);
+};

package/lib/commands/blocklet/bundle/zip/main.js

@@ -0,0 +1,124 @@
+/* eslint-disable no-use-before-define */
+/* eslint-disable func-names */
+const { readdir, lstat } = require('fs');
+const { join, resolve, dirname, basename, extname } = require('path');
+
+const cpFile = require('cp-file');
+const locatePath = require('locate-path');
+const makeDir = require('make-dir');
+const pMap = require('p-map');
+const { promisify } = require('util');
+
+const { listNodeFiles } = require('./dependencies');
+const { zipNodeJs } = require('./node');
+
+const debug = require('../../../../debug')('bundle:zip');
+
+const pReaddir = promisify(readdir);
+const pLstat = promisify(lstat);
+
+// reduce `srcFolder/*` (Node.js files) to `destFolder/*.zip` so it can be used by Blocklet Server
+const zipEntries = async function ({ srcFolder, destFolder, srcFilter, extraPaths, options = {} }) {
+  const { parallelLimit = 5 } = options;
+  const srcPaths = await getSrcPaths(srcFolder, srcFilter);
+
+  const zipped = await pMap(srcPaths, (srcPath) => zipEntry({ srcPath, destFolder, extraPaths }), {
+    concurrency: parallelLimit,
+  });
+  return zipped.filter(Boolean);
+};
+
+const zipEntry = async function ({ srcPath, destFolder, extraPaths }) {
+  const { runtime, filename, extension, srcDir, stat, mainFile } = await getEntryInfo(srcPath);
+
+  if (runtime === undefined) {
+    return;
+  }
+
+  const srcFiles = await getSrcFiles({ runtime, filename, stat, mainFile, extension, srcPath, srcDir, extraPaths });
+
+  await makeDir(destFolder);
+
+  if (runtime === 'js') {
+    if (extension === '.zip') {
+      const destPath = join(destFolder, filename);
+      await cpFile(srcPath, destPath);
+      // eslint-disable-next-line consistent-return
+      return { path: destPath, runtime };
+    }
+
+    const destPath = join(destFolder, `${basename(filename, '.js')}.zip`);
+    const prefix = await zipNodeJs(srcFiles, destPath, filename, mainFile);
+    // eslint-disable-next-line consistent-return
+    return { path: destPath, prefix, runtime };
+  }
+};
+
+const getSrcPaths = async function (srcFolder, srcFilter) {
+  if (!srcFolder) {
+    return [];
+  }
+  const filenames = await listFilenames(srcFolder);
+  const srcPaths = filenames.filter(srcFilter).map((x) => resolve(srcFolder, x));
+  debug('getSrcPaths', { srcFolder, srcPaths });
+  return srcPaths;
+};
+
+const listFilenames = async function (srcFolder) {
+  try {
+    return await pReaddir(srcFolder);
+  } catch (error) {
+    throw new Error(`entry folder does not exist: ${srcFolder}`);
+  }
+};
+
+const getEntryInfo = async function (srcPath) {
+  const { filename, stat, mainFile, extension, srcDir } = await getSrcInfo(srcPath);
+
+  if (mainFile === undefined) {
+    return {};
+  }
+
+  if (extension === '.zip' || extension === '.js') {
+    return { runtime: 'js', filename, stat, mainFile, extension, srcPath, srcDir };
+  }
+
+  return {};
+};
+
+const getSrcInfo = async function (srcPath) {
+  const filename = basename(srcPath);
+  if (filename === 'node_modules') {
+    return {};
+  }
+
+  const stat = await pLstat(srcPath);
+  const mainFile = await getMainFile(srcPath, filename, stat);
+  if (mainFile === undefined) {
+    return {};
+  }
+
+  const extension = extname(mainFile);
+  const srcDir = stat.isDirectory() ? srcPath : dirname(srcPath);
+  return { filename, stat, mainFile, extension, srcDir };
+};
+
+// Each `srcPath` can also be a directory with an `index.js` file or a file
+// using the same filename as its directory
+const getMainFile = function (srcPath, filename, stat) {
+  if (!stat.isDirectory()) {
+    return srcPath;
+  }
+
+  return locatePath([join(srcPath, `${filename}.js`), join(srcPath, 'index.js')], { type: 'file' });
+};
+
+const getSrcFiles = function ({ runtime, filename, stat, mainFile, extension, srcPath, srcDir, extraPaths }) {
+  if (runtime === 'js' && extension === '.js') {
+    return listNodeFiles({ srcPath, filename, mainFile, srcDir, stat, extraPaths });
+  }
+
+  return [srcPath];
+};
+
+module.exports = { zipEntries, zipEntry };

package/lib/commands/blocklet/bundle/zip/node.js

@@ -0,0 +1,59 @@
+/* eslint-disable no-use-before-define */
+/* eslint-disable func-names */
+const { stat } = require('fs');
+const { dirname, normalize, sep } = require('path');
+
+const commonPathPrefix = require('common-path-prefix');
+const unixify = require('unixify');
+const { promisify } = require('util');
+
+const { startZip, addZipFile, endZip } = require('./archive');
+
+const pStat = promisify(stat);
+
+// zip a Node.js function file
+const zipNodeJs = async function (srcFiles, destPath) {
+  const { archive, output } = startZip(destPath);
+
+  const dirnames = srcFiles.map(dirname);
+  const commonPrefix = commonPathPrefix(dirnames);
+
+  const srcFilesInfos = await Promise.all(srcFiles.map(addStat));
+
+  // We ensure this is not async, so that the archive's checksum is
+  // deterministic. Otherwise it depends on the order the files were added.
+  // eslint-disable-next-line no-shadow
+  srcFilesInfos.forEach(({ srcFile, stat }) => {
+    zipJsFile({ srcFile, commonPrefix, archive, stat });
+  });
+
+  await endZip(archive, output);
+
+  return commonPrefix;
+};
+
+const addStat = async function (srcFile) {
+  // eslint-disable-next-line no-shadow
+  const stat = await pStat(srcFile);
+  return { srcFile, stat };
+};
+
+// eslint-disable-next-line no-shadow
+const zipJsFile = function ({ srcFile, commonPrefix, archive, stat }) {
+  const filename = normalizeFilePath(srcFile, commonPrefix);
+  addZipFile(archive, srcFile, filename, stat);
+};
+
+// `adm-zip` and `require()` expect Unix paths.
+// We remove the common path prefix.
+// With files on different Windows drives, we remove the drive letter.
+const normalizeFilePath = function (path, commonPrefix) {
+  const pathA = normalize(path);
+  const pathB = pathA.replace(commonPrefix, `${ZIP_ROOT_DIR}${sep}`);
+  const pathC = unixify(pathB);
+  return pathC;
+};
+
+const ZIP_ROOT_DIR = '.';
+
+module.exports = { zipNodeJs, ZIP_ROOT_DIR };