@rushstack/package-extractor 0.8.1 → 0.9.0

package/lib/PackageExtractor.js

@@ -30,7 +30,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PackageExtractor = exports.TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING = void 0;
 const path = __importStar(require("path"));
-const fs = __importStar(require("fs"));
 const minimatch_1 = require("minimatch");
 const semver_1 = __importDefault(require("semver"));
 const npm_packlist_1 = __importDefault(require("npm-packlist"));
@@ -38,8 +37,8 @@ const link_bins_1 = __importDefault(require("@pnpm/link-bins"));
 const ignore_1 = __importDefault(require("ignore"));
 const node_core_library_1 = require("@rushstack/node-core-library");
 const terminal_1 = require("@rushstack/terminal");
-const ArchiveManager_1 = require("./ArchiveManager");
 const SymlinkAnalyzer_1 = require("./SymlinkAnalyzer");
+const AssetHandler_1 = require("./AssetHandler");
 const Utils_1 = require("./Utils");
 const PathConstants_1 = require("./PathConstants");
 exports.TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING = '{TARGET_ROOT_SCRIPT_RELATIVE_PATH}';
@@ -71,54 +70,31 @@ class PackageExtractor {
      */
     async extractAsync(options) {
         options = PackageExtractor._normalizeOptions(options);
-        const { terminal, projectConfigurations, sourceRootFolder, targetRootFolder, mainProjectName, overwriteExisting, createArchiveFilePath, createArchiveOnly, dependencyConfigurations } = options;
-        if (createArchiveOnly) {
-            if (options.linkCreation !== 'script' && options.linkCreation !== 'none') {
-                throw new Error('createArchiveOnly is only supported when linkCreation is "script" or "none"');
-            }
-            if (!createArchiveFilePath) {
-                throw new Error('createArchiveOnly is only supported when createArchiveFilePath is specified');
-            }
-        }
-        let archiver;
-        let archiveFilePath;
-        if (createArchiveFilePath) {
-            if (path.extname(createArchiveFilePath) !== '.zip') {
-                throw new Error('Only archives with the .zip file extension are currently supported.');
-            }
-            archiveFilePath = path.resolve(targetRootFolder, createArchiveFilePath);
-            archiver = new ArchiveManager_1.ArchiveManager();
-        }
-        await node_core_library_1.FileSystem.ensureFolderAsync(targetRootFolder);
+        const { terminal, projectConfigurations, sourceRootFolder, targetRootFolder, mainProjectName, overwriteExisting, dependencyConfigurations } = options;
         terminal.writeLine(terminal_1.Colorize.cyan(`Extracting to target folder:  ${targetRootFolder}`));
         terminal.writeLine(terminal_1.Colorize.cyan(`Main project for extraction: ${mainProjectName}`));
-        try {
-            const existingExtraction = (await node_core_library_1.FileSystem.readFolderItemNamesAsync(targetRootFolder)).length > 0;
-            if (existingExtraction) {
-                if (!overwriteExisting) {
-                    throw new Error('The extraction target folder is not empty. Overwrite must be explicitly requested');
-                }
-                else {
-                    terminal.writeLine('Deleting target folder contents...');
-                    terminal.writeLine('');
-                    await node_core_library_1.FileSystem.ensureEmptyFolderAsync(targetRootFolder);
-                }
-            }
-        }
-        catch (error) {
-            if (!node_core_library_1.FileSystem.isFolderDoesNotExistError(error)) {
-                throw error;
+        await node_core_library_1.FileSystem.ensureFolderAsync(targetRootFolder);
+        const existingExtraction = (await node_core_library_1.FileSystem.readFolderItemNamesAsync(targetRootFolder)).length > 0;
+        if (existingExtraction) {
+            if (!overwriteExisting) {
+                throw new Error('The extraction target folder is not empty. Overwrite must be explicitly requested');
             }
+            terminal.writeLine('Deleting target folder contents...');
+            terminal.writeLine('');
+            await node_core_library_1.FileSystem.ensureEmptyFolderAsync(targetRootFolder);
         }
         // Create a new state for each run
+        const symlinkAnalyzer = new SymlinkAnalyzer_1.SymlinkAnalyzer({
+            requiredSourceParentPath: sourceRootFolder
+        });
         const state = {
+            symlinkAnalyzer,
+            assetHandler: new AssetHandler_1.AssetHandler(Object.assign(Object.assign({}, options), { symlinkAnalyzer })),
             foldersToCopy: new Set(),
             packageJsonByPath: new Map(),
             projectConfigurationsByName: new Map(projectConfigurations.map((p) => [p.projectName, p])),
             projectConfigurationsByPath: new Map(projectConfigurations.map((p) => [p.projectFolder, p])),
-            dependencyConfigurationsByName: new Map(),
-            symlinkAnalyzer: new SymlinkAnalyzer_1.SymlinkAnalyzer({ requiredSourceParentPath: sourceRootFolder }),
-            archiver
+            dependencyConfigurationsByName: new Map()
         };
         // set state dependencyConfigurationsByName
         for (const dependencyConfiguration of dependencyConfigurations || []) {
@@ -131,10 +107,7 @@ class PackageExtractor {
             existingDependencyConfigurations.push(dependencyConfiguration);
         }
         await this._performExtractionAsync(options, state);
-        if (archiver && archiveFilePath) {
-            terminal.writeLine(`Creating archive at "${archiveFilePath}"`);
-            await archiver.createArchiveAsync(archiveFilePath);
-        }
+        await state.assetHandler.finalizeAsync();
     }
     static _normalizeOptions(options) {
         if (options.subspaces) {
@@ -159,9 +132,8 @@ class PackageExtractor {
         return normalizedOptions;
     }
     async _performExtractionAsync(options, state) {
-        var _a;
-        const { terminal, mainProjectName, sourceRootFolder, targetRootFolder, folderToCopy: additionalFolderToCopy, linkCreation, linkCreationScriptPath, createArchiveOnly } = options;
-        const { projectConfigurationsByName, foldersToCopy, symlinkAnalyzer } = state;
+        const { terminal, mainProjectName, sourceRootFolder, targetRootFolder, folderToCopy: additionalFolderToCopy, linkCreation, createArchiveOnly } = options;
+        const { projectConfigurationsByName, foldersToCopy } = state;
         const mainProjectConfiguration = projectConfigurationsByName.get(mainProjectName);
         if (!mainProjectConfiguration) {
             throw new Error(`Main project "${mainProjectName}" was not found in the list of projects`);
@@ -198,38 +170,11 @@ class PackageExtractor {
             const additionalFolderExtractorOptions = Object.assign(Object.assign({}, options), { sourceRootFolder: additionalFolderPath, targetRootFolder });
             await this._extractFolderAsync(additionalFolderPath, additionalFolderExtractorOptions, state);
         }
-        switch (linkCreation) {
-            case 'script': {
-                terminal.writeLine(`Creating ${PathConstants_1.createLinksScriptFilename}`);
-                const createLinksSourceFilePath = `${PathConstants_1.scriptsFolderPath}/${PathConstants_1.createLinksScriptFilename}`;
-                const createLinksTargetFilePath = linkCreationScriptPath
-                    ? path.resolve(targetRootFolder, linkCreationScriptPath)
-                    : `${targetRootFolder}/${PathConstants_1.createLinksScriptFilename}`;
-                let createLinksScriptContent = await node_core_library_1.FileSystem.readFileAsync(createLinksSourceFilePath);
-                createLinksScriptContent = createLinksScriptContent.replace(exports.TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, node_core_library_1.Path.convertToSlashes(path.relative(path.dirname(createLinksTargetFilePath), targetRootFolder)));
-                if (!createArchiveOnly) {
-                    await node_core_library_1.FileSystem.writeFileAsync(createLinksTargetFilePath, createLinksScriptContent, {
-                        ensureFolderExists: true
-                    });
-                }
-                await ((_a = state.archiver) === null || _a === void 0 ? void 0 : _a.addToArchiveAsync({
-                    fileData: createLinksScriptContent,
-                    archivePath: path.relative(targetRootFolder, createLinksTargetFilePath)
-                }));
-                break;
-            }
-            case 'default': {
-                terminal.writeLine('Creating symlinks');
-                const linksToCopy = symlinkAnalyzer.reportSymlinks();
-                await node_core_library_1.Async.forEachAsync(linksToCopy, async (linkToCopy) => {
-                    await this._extractSymlinkAsync(linkToCopy, options, state);
-                });
-                await this._makeBinLinksAsync(options, state);
-                break;
-            }
-            default: {
-                break;
-            }
+        if (linkCreation === 'default') {
+            await this._makeBinLinksAsync(options, state);
+        }
+        else if (linkCreation === 'script') {
+            await this._writeCreateLinksScriptAsync(options, state);
         }
         terminal.writeLine('Creating extractor-metadata.json');
         await this._writeExtractorMetadataAsync(options, state);
@@ -381,47 +326,18 @@ class PackageExtractor {
         }
         return allDependencyNames;
     }
-    /**
-     * Maps a file path from IExtractorOptions.sourceRootFolder to IExtractorOptions.targetRootFolder
-     *
-     * Example input: "C:\\MyRepo\\libraries\\my-lib"
-     * Example output: "C:\\MyRepo\\common\\deploy\\libraries\\my-lib"
-     */
-    _remapPathForExtractorFolder(absolutePathInSourceFolder, options) {
-        const { sourceRootFolder, targetRootFolder } = options;
-        const relativePath = path.relative(sourceRootFolder, absolutePathInSourceFolder);
-        if (relativePath.startsWith('..')) {
-            throw new Error(`Source path "${absolutePathInSourceFolder}" is not under "${sourceRootFolder}"`);
-        }
-        const absolutePathInTargetFolder = path.join(targetRootFolder, relativePath);
-        return absolutePathInTargetFolder;
-    }
-    /**
-     * Maps a file path from IExtractorOptions.sourceRootFolder to relative path
-     *
-     * Example input: "C:\\MyRepo\\libraries\\my-lib"
-     * Example output: "libraries/my-lib"
-     */
-    _remapPathForExtractorMetadata(absolutePathInSourceFolder, options) {
-        const { sourceRootFolder } = options;
-        const relativePath = path.relative(sourceRootFolder, absolutePathInSourceFolder);
-        if (relativePath.startsWith('..')) {
-            throw new Error(`Source path "${absolutePathInSourceFolder}" is not under "${sourceRootFolder}"`);
-        }
-        return node_core_library_1.Path.convertToSlashes(relativePath);
-    }
     /**
      * Copy one package folder to the extractor target folder.
      */
     async _extractFolderAsync(sourceFolderPath, options, state) {
-        const { includeNpmIgnoreFiles, targetRootFolder } = options;
-        const { projectConfigurationsByPath, packageJsonByPath, dependencyConfigurationsByName, archiver } = state;
+        const { includeNpmIgnoreFiles } = options;
+        const { projectConfigurationsByPath, packageJsonByPath, dependencyConfigurationsByName, assetHandler } = state;
         let useNpmIgnoreFilter = false;
         const sourceFolderRealPath = await node_core_library_1.FileSystem.getRealPathAsync(sourceFolderPath);
         const sourceProjectConfiguration = projectConfigurationsByPath.get(sourceFolderRealPath);
         const packagesJson = packageJsonByPath.get(sourceFolderRealPath);
-        // As this function will be used to copy folder for both project inside monorepo and third party dependencies insides node_modules
-        // Third party dependencies won't have project configurations
+        // As this function will be used to copy folder for both project inside monorepo and third party
+        // dependencies insides node_modules. Third party dependencies won't have project configurations
         const isLocalProject = !!sourceProjectConfiguration;
         // Function to filter files inside local project or third party dependencies.
         const isFileExcluded = (filePath) => {
@@ -464,10 +380,9 @@ class PackageExtractor {
             // Only use the npmignore filter if the project configuration explicitly asks for it
             useNpmIgnoreFilter = true;
         }
-        const targetFolderPath = this._remapPathForExtractorFolder(sourceFolderPath, options);
+        const targetFolderPath = (0, Utils_1.remapSourcePathForTargetFolder)(Object.assign(Object.assign({}, options), { sourcePath: sourceFolderPath }));
         if (useNpmIgnoreFilter) {
             const npmPackFiles = await PackageExtractor.getPackageIncludedFilesAsync(sourceFolderPath);
-            const alreadyCopiedSourcePaths = new Set();
             await node_core_library_1.Async.forEachAsync(npmPackFiles, async (npmPackFile) => {
                 // In issue https://github.com/microsoft/rushstack/issues/2121 we found that npm-packlist sometimes returns
                 // duplicate file paths, for example:
@@ -479,31 +394,17 @@ class PackageExtractor {
                 if (isFileExcluded(npmPackFile)) {
                     return;
                 }
-                // We can detect the duplicates by comparing the path.resolve() result.
-                const copySourcePath = path.resolve(sourceFolderPath, npmPackFile);
-                if (alreadyCopiedSourcePaths.has(copySourcePath)) {
-                    return;
-                }
-                alreadyCopiedSourcePaths.add(copySourcePath);
-                const copyDestinationPath = path.join(targetFolderPath, npmPackFile);
-                const copySourcePathNode = await state.symlinkAnalyzer.analyzePathAsync({
-                    inputPath: copySourcePath
+                const sourceFilePath = path.resolve(sourceFolderPath, npmPackFile);
+                const { kind, linkStats: sourceFileStats } = await state.symlinkAnalyzer.analyzePathAsync({
+                    inputPath: sourceFilePath
                 });
-                if (copySourcePathNode.kind !== 'link') {
-                    if (!options.createArchiveOnly) {
-                        await node_core_library_1.FileSystem.ensureFolderAsync(path.dirname(copyDestinationPath));
-                        // Use the fs.copyFile API instead of FileSystem.copyFileAsync() since copyFileAsync performs
-                        // a needless stat() call to determine if it's a file or folder, and we already know it's a file.
-                        await fs.promises.copyFile(copySourcePath, copyDestinationPath, fs.constants.COPYFILE_EXCL);
-                    }
-                    if (archiver) {
-                        const archivePath = path.relative(targetRootFolder, copyDestinationPath);
-                        await archiver.addToArchiveAsync({
-                            filePath: copySourcePath,
-                            archivePath,
-                            stats: copySourcePathNode.linkStats
-                        });
-                    }
+                if (kind === 'file') {
+                    const targetFilePath = path.resolve(targetFolderPath, npmPackFile);
+                    await assetHandler.includeAssetAsync({
+                        sourceFilePath,
+                        sourceFileStats,
+                        targetFilePath
+                    });
                 }
             }, {
                 concurrency: 10
@@ -553,22 +454,12 @@ class PackageExtractor {
                         callback();
                         return;
                     }
-                    const targetPath = path.join(targetFolderPath, relativeSourcePath);
-                    if (!options.createArchiveOnly) {
-                        // Manually call fs.copyFile to avoid unnecessary stat calls.
-                        const targetParentPath = path.dirname(targetPath);
-                        await node_core_library_1.FileSystem.ensureFolderAsync(targetParentPath);
-                        await fs.promises.copyFile(sourcePath, targetPath, fs.constants.COPYFILE_EXCL);
-                    }
-                    // Add the file to the archive. Only need to add files since directories will be auto-created
-                    if (archiver) {
-                        const archivePath = path.relative(targetRootFolder, targetPath);
-                        await archiver.addToArchiveAsync({
-                            filePath: sourcePath,
-                            archivePath: archivePath,
-                            stats: sourcePathNode.linkStats
-                        });
-                    }
+                    const targetFilePath = path.resolve(targetFolderPath, relativeSourcePath);
+                    await assetHandler.includeAssetAsync({
+                        sourceFilePath: sourcePath,
+                        sourceFileStats: sourcePathNode.linkStats,
+                        targetFilePath
+                    });
                 }
                 else if (sourcePathNode.kind === 'folder') {
                     const children = await node_core_library_1.FileSystem.readFolderItemNamesAsync(sourcePath);
@@ -582,58 +473,11 @@ class PackageExtractor {
             });
         }
     }
-    /**
-     * Create a symlink as described by the ILinkInfo object.
-     */
-    async _extractSymlinkAsync(originalLinkInfo, options, state) {
-        var _a;
-        const linkInfo = {
-            kind: originalLinkInfo.kind,
-            linkPath: this._remapPathForExtractorFolder(originalLinkInfo.linkPath, options),
-            targetPath: this._remapPathForExtractorFolder(originalLinkInfo.targetPath, options)
-        };
-        const newLinkFolder = path.dirname(linkInfo.linkPath);
-        await node_core_library_1.FileSystem.ensureFolderAsync(newLinkFolder);
-        // Link to the relative path for symlinks
-        const relativeTargetPath = path.relative(newLinkFolder, linkInfo.targetPath);
-        // NOTE: This logic is based on NpmLinkManager._createSymlink()
-        if (linkInfo.kind === 'fileLink') {
-            // For files, we use a Windows "hard link", because creating a symbolic link requires
-            // administrator permission. However hard links seem to cause build failures on Mac,
-            // so for all other operating systems we use symbolic links for this case.
-            if (process.platform === 'win32') {
-                await node_core_library_1.FileSystem.createHardLinkAsync({
-                    linkTargetPath: relativeTargetPath,
-                    newLinkPath: linkInfo.linkPath
-                });
-            }
-            else {
-                await node_core_library_1.FileSystem.createSymbolicLinkFileAsync({
-                    linkTargetPath: relativeTargetPath,
-                    newLinkPath: linkInfo.linkPath
-                });
-            }
-        }
-        else {
-            // Junctions are only supported on Windows. This will create a symbolic link on other platforms.
-            await node_core_library_1.FileSystem.createSymbolicLinkJunctionAsync({
-                linkTargetPath: relativeTargetPath,
-                newLinkPath: linkInfo.linkPath
-            });
-        }
-        // Since the created symlinks have the required relative paths, they can be added directly to
-        // the archive.
-        await ((_a = state.archiver) === null || _a === void 0 ? void 0 : _a.addToArchiveAsync({
-            filePath: linkInfo.linkPath,
-            archivePath: path.relative(options.targetRootFolder, linkInfo.linkPath)
-        }));
-    }
     /**
      * Write the common/deploy/deploy-metadata.json file.
      */
     async _writeExtractorMetadataAsync(options, state) {
-        var _a;
-        const { mainProjectName, targetRootFolder, linkCreation, linkCreationScriptPath } = options;
+        const { mainProjectName, sourceRootFolder, targetRootFolder, linkCreation, linkCreationScriptPath } = options;
         const { projectConfigurationsByPath } = state;
         const extractorMetadataFileName = 'extractor-metadata.json';
         const extractorMetadataFolderPath = linkCreation === 'script' && linkCreationScriptPath
@@ -643,51 +487,50 @@ class PackageExtractor {
         const extractorMetadataJson = {
             mainProjectName,
             projects: [],
-            links: []
+            links: [],
+            files: []
         };
         for (const { projectFolder, projectName } of projectConfigurationsByPath.values()) {
             if (state.foldersToCopy.has(projectFolder)) {
                 extractorMetadataJson.projects.push({
                     projectName,
-                    path: this._remapPathForExtractorMetadata(projectFolder, options)
+                    path: (0, Utils_1.remapPathForExtractorMetadata)(sourceRootFolder, projectFolder)
                 });
             }
         }
         // Remap the links to be relative to target folder
-        for (const absoluteLinkInfo of state.symlinkAnalyzer.reportSymlinks()) {
-            const relativeInfo = {
-                kind: absoluteLinkInfo.kind,
-                linkPath: this._remapPathForExtractorMetadata(absoluteLinkInfo.linkPath, options),
-                targetPath: this._remapPathForExtractorMetadata(absoluteLinkInfo.targetPath, options)
-            };
-            extractorMetadataJson.links.push(relativeInfo);
-        }
-        const extractorMetadataFileContent = JSON.stringify(extractorMetadataJson, undefined, 0);
-        if (!options.createArchiveOnly) {
-            await node_core_library_1.FileSystem.writeFileAsync(extractorMetadataFilePath, extractorMetadataFileContent, {
-                ensureFolderExists: true
+        for (const { kind, linkPath, targetPath } of state.symlinkAnalyzer.reportSymlinks()) {
+            extractorMetadataJson.links.push({
+                kind,
+                linkPath: (0, Utils_1.remapPathForExtractorMetadata)(sourceRootFolder, linkPath),
+                targetPath: (0, Utils_1.remapPathForExtractorMetadata)(sourceRootFolder, targetPath)
             });
         }
-        const archivePath = path.relative(targetRootFolder, extractorMetadataFilePath);
-        await ((_a = state.archiver) === null || _a === void 0 ? void 0 : _a.addToArchiveAsync({ archivePath, fileData: extractorMetadataFileContent }));
+        for (const assetPath of state.assetHandler.assetPaths) {
+            extractorMetadataJson.files.push((0, Utils_1.remapPathForExtractorMetadata)(targetRootFolder, assetPath));
+        }
+        const extractorMetadataFileContent = JSON.stringify(extractorMetadataJson, undefined, 0);
+        await state.assetHandler.includeAssetAsync({
+            sourceFileContent: extractorMetadataFileContent,
+            targetFilePath: extractorMetadataFilePath
+        });
     }
     async _makeBinLinksAsync(options, state) {
         const { terminal } = options;
         const extractedProjectFolders = Array.from(state.projectConfigurationsByPath.keys()).filter((folderPath) => state.foldersToCopy.has(folderPath));
         await node_core_library_1.Async.forEachAsync(extractedProjectFolders, async (projectFolder) => {
-            const extractedProjectFolder = this._remapPathForExtractorFolder(projectFolder, options);
+            const extractedProjectFolder = (0, Utils_1.remapSourcePathForTargetFolder)(Object.assign(Object.assign({}, options), { sourcePath: projectFolder }));
             const extractedProjectNodeModulesFolder = path.join(extractedProjectFolder, 'node_modules');
             const extractedProjectBinFolder = path.join(extractedProjectNodeModulesFolder, '.bin');
             const linkedBinPackageNames = await (0, link_bins_1.default)(extractedProjectNodeModulesFolder, extractedProjectBinFolder, {
                 warn: (msg) => terminal.writeLine(terminal_1.Colorize.yellow(msg))
             });
-            if (linkedBinPackageNames.length && state.archiver) {
+            if (linkedBinPackageNames.length) {
                 const binFolderItems = await node_core_library_1.FileSystem.readFolderItemNamesAsync(extractedProjectBinFolder);
                 for (const binFolderItem of binFolderItems) {
-                    const binFilePath = path.join(extractedProjectBinFolder, binFolderItem);
-                    await state.archiver.addToArchiveAsync({
-                        filePath: binFilePath,
-                        archivePath: path.relative(options.targetRootFolder, binFilePath)
+                    const binFilePath = path.resolve(extractedProjectBinFolder, binFolderItem);
+                    await state.assetHandler.includeAssetAsync({
+                        targetFilePath: binFilePath
                     });
                 }
             }
@@ -695,6 +538,19 @@ class PackageExtractor {
             concurrency: 10
         });
     }
+    async _writeCreateLinksScriptAsync(options, state) {
+        const { terminal, targetRootFolder, linkCreationScriptPath } = options;
+        const { assetHandler } = state;
+        terminal.writeLine(`Creating ${PathConstants_1.createLinksScriptFilename}`);
+        const createLinksSourceFilePath = `${PathConstants_1.scriptsFolderPath}/${PathConstants_1.createLinksScriptFilename}`;
+        const createLinksTargetFilePath = path.resolve(targetRootFolder, linkCreationScriptPath || PathConstants_1.createLinksScriptFilename);
+        let createLinksScriptContent = await node_core_library_1.FileSystem.readFileAsync(createLinksSourceFilePath);
+        createLinksScriptContent = createLinksScriptContent.replace(exports.TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, node_core_library_1.Path.convertToSlashes(path.relative(path.dirname(createLinksTargetFilePath), targetRootFolder)));
+        await assetHandler.includeAssetAsync({
+            sourceFileContent: createLinksScriptContent,
+            targetFilePath: createLinksTargetFilePath
+        });
+    }
 }
 exports.PackageExtractor = PackageExtractor;
 //# sourceMappingURL=PackageExtractor.js.map