@theia/cli 1.18.0-next.d3501165 → 1.19.0

package/src/download-plugins.ts

@@ -26,7 +26,7 @@ declare global {
 import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
 import { green, red, yellow } from 'colors/safe';
 import * as decompress from 'decompress';
-import { createWriteStream, existsSync, promises as fs } from 'fs';
+import { createWriteStream, promises as fs } from 'fs';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import fetch, { RequestInit, Response } from 'node-fetch';
 import * as path from 'path';
@@ -34,13 +34,12 @@ import { getProxyForUrl } from 'proxy-from-env';
 import * as stream from 'stream';
 import * as temp from 'temp';
 import { promisify } from 'util';
+import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';
 
 const pipelineAsPromised = promisify(stream.pipeline);
 
 temp.track();
 
-export const extensionPackCacheName = '.packs';
-
 /**
  * Available options when downloading.
  */
@@ -73,7 +72,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
     const {
         packed = false,
         ignoreErrors = false,
-        apiVersion = '1.50.0',
+        apiVersion = DEFAULT_SUPPORTED_API_VERSION,
         apiUrl = 'https://open-vsx.org/api'
     } = options;
 
@@ -96,32 +95,22 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         return;
     }
     try {
-        // Retrieve the cached extension-packs in order to not re-download them.
-        const extensionPackCachePath = path.resolve(pluginsDir, extensionPackCacheName);
-        const cachedExtensionPacks = new Set<string>(
-            existsSync(extensionPackCachePath)
-                ? await fs.readdir(extensionPackCachePath)
-                : []
-        );
         console.warn('--- downloading plugins ---');
         // Download the raw plugins defined by the `theiaPlugins` property.
         // This will include both "normal" plugins as well as "extension packs".
         const downloads = [];
         for (const [plugin, pluginUrl] of Object.entries(pck.theiaPlugins)) {
-            // Skip extension packs that were moved to `.packs`:
-            if (cachedExtensionPacks.has(plugin) || typeof pluginUrl !== 'string') {
+            if (typeof pluginUrl !== 'string') {
                 continue;
             }
             downloads.push(downloadPluginAsync(failures, plugin, pluginUrl, pluginsDir, packed));
         }
         await Promise.all(downloads);
+
         console.warn('--- collecting extension-packs ---');
         const extensionPacks = await collectExtensionPacks(pluginsDir, excludedIds);
         if (extensionPacks.size > 0) {
-            console.warn(`--- found ${extensionPacks.size} extension-packs ---`);
-            // Move extension-packs to `.packs`
-            await cacheExtensionPacks(pluginsDir, extensionPacks);
-            console.warn('--- resolving extension-packs ---');
+            console.warn(`--- resolving ${extensionPacks.size} extension-packs ---`);
             const client = new OVSXClient({ apiVersion, apiUrl });
             // De-duplicate extension ids to only download each once:
             const ids = new Set<string>(Array.from(extensionPacks.values()).flat());
@@ -129,10 +118,27 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
                 const extension = await client.getLatestCompatibleExtensionVersion(id);
                 const downloadUrl = extension?.files.download;
                 if (downloadUrl) {
-                    await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed);
+                    await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed, extension?.version);
                 }
             }));
         }
+
+        console.warn('--- collecting extension dependencies ---');
+        const pluginDependencies = await collectPluginDependencies(pluginsDir, excludedIds);
+        if (pluginDependencies.length > 0) {
+            console.warn(`--- resolving ${pluginDependencies.length} extension dependencies ---`);
+            const client = new OVSXClient({ apiVersion, apiUrl });
+            // De-duplicate extension ids to only download each once:
+            const ids = new Set<string>(pluginDependencies);
+            await Promise.all(Array.from(ids, async id => {
+                const extension = await client.getLatestCompatibleExtensionVersion(id);
+                const downloadUrl = extension?.files.download;
+                if (downloadUrl) {
+                    await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed, extension?.version);
+                }
+            }));
+        }
+
     } finally {
         temp.cleanupSync();
     }
@@ -153,7 +159,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
  * @param packed whether to decompress or not.
  * @param cachedExtensionPacks the list of cached extension packs already downloaded.
  */
-async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean): Promise<void> {
+async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean, version?: string): Promise<void> {
     if (!plugin) {
         return;
     }
@@ -162,6 +168,8 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         fileExt = '.tar.gz';
     } else if (pluginUrl.endsWith('vsix')) {
         fileExt = '.vsix';
+    } else if (pluginUrl.endsWith('theia')) {
+        fileExt = '.theia'; // theia plugins.
     } else {
         failures.push(red(`error: '${plugin}' has an unsupported file type: '${pluginUrl}'`));
         return;
@@ -210,7 +218,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         return;
     }
 
-    if (fileExt === '.vsix' && packed === true) {
+    if ((fileExt === '.vsix' || fileExt === '.theia') && packed === true) {
         // Download .vsix without decompressing.
         const file = createWriteStream(targetPath);
         await pipelineAsPromised(response.body, file);
@@ -221,7 +229,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         await decompress(tempFile.path, targetPath);
     }
 
-    console.warn(green(`+ ${plugin}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
+    console.warn(green(`+ ${plugin}${version ? `@${version}` : ''}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`));
 }
 
 /**
@@ -257,8 +265,7 @@ async function collectPackageJsonPaths(pluginDir: string): Promise<string[]> {
     // Recursively fetch the list of extension `package.json` files.
     for (const file of files) {
         const filePath = path.join(pluginDir, file);
-        // Exclude the `.packs` folder used to store extension-packs after being resolved.
-        if (!filePath.startsWith(extensionPackCacheName) && (await fs.stat(filePath)).isDirectory()) {
+        if ((await fs.stat(filePath)).isDirectory()) {
             packageJsonPathList.push(...await collectPackageJsonPaths(filePath));
         } else if (path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules')) {
             packageJsonPathList.push(filePath);
@@ -280,7 +287,7 @@ async function collectExtensionPacks(pluginDir: string, excludedIds: Set<string>
     await Promise.all(packageJsonPaths.map(async packageJsonPath => {
         const json = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
         const extensionPack: unknown = json.extensionPack;
-        if (extensionPack && Array.isArray(extensionPack)) {
+        if (Array.isArray(extensionPack)) {
             extensionPackPaths.set(packageJsonPath, extensionPack.filter(id => {
                 if (excludedIds.has(id)) {
                     console.log(yellow(`'${id}' referenced by '${json.name}' (ext pack) is excluded because of 'theiaPluginsExcludeIds'`));
@@ -294,50 +301,27 @@ async function collectExtensionPacks(pluginDir: string, excludedIds: Set<string>
 }
 
 /**
- * Move extension-packs downloaded from `pluginsDir/x` to `pluginsDir/.packs/x`.
- *
- * The issue we are trying to solve is the following:
- * We may skip some extensions declared in a pack due to the `theiaPluginsExcludeIds` list. But once we start
- * a Theia application the plugin system will detect the pack and install the missing extensions.
- *
- * By moving the packs to a subdirectory it should make it invisible to the plugin system, only leaving
- * the plugins that were installed under `pluginsDir` directly.
- *
- * @param extensionPacksPaths the list of extension-pack paths.
+ * Get the mapping of  paths and their included plugin ids.
+ * - If an extension-pack references an explicitly excluded `id` the `id` will be omitted.
+ * @param pluginDir the plugin directory.
+ * @param excludedIds the list of plugin ids to exclude.
+ * @returns the mapping of extension-pack paths and their included plugin ids.
  */
-async function cacheExtensionPacks(pluginsDir: string, extensionPacks: Map<string, unknown>): Promise<void> {
-    const packsFolderPath = path.resolve(pluginsDir, extensionPackCacheName);
-    await fs.mkdir(packsFolderPath, { recursive: true });
-    await Promise.all(Array.from(extensionPacks.entries(), async ([extensionPackPath, value]) => {
-        extensionPackPath = path.resolve(extensionPackPath);
-        // Skip entries found in `.packs`
-        if (extensionPackPath.startsWith(packsFolderPath)) {
-            return; // skip
-        }
-        try {
-            const oldPath = getExtensionRoot(pluginsDir, extensionPackPath);
-            const newPath = path.resolve(packsFolderPath, path.basename(oldPath));
-            if (!existsSync(newPath)) {
-                await fs.rename(oldPath, newPath);
+async function collectPluginDependencies(pluginDir: string, excludedIds: Set<string>): Promise<string[]> {
+    const dependencyIds: string[] = [];
+    const packageJsonPaths = await collectPackageJsonPaths(pluginDir);
+    await Promise.all(packageJsonPaths.map(async packageJsonPath => {
+        const json = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+        const extensionDependencies: unknown = json.extensionDependencies;
+        if (Array.isArray(extensionDependencies)) {
+            for (const dependency of extensionDependencies) {
+                if (excludedIds.has(dependency)) {
+                    console.log(yellow(`'${dependency}' referenced by '${json.name}' is excluded because of 'theiaPluginsExcludeIds'`));
+                } else {
+                    dependencyIds.push(dependency);
+                }
             }
-        } catch (error) {
-            console.error(error);
         }
     }));
-}
-
-/**
- * Walk back to the root of an extension starting from its `package.json`. e.g.
- *
- * ```ts
- * getExtensionRoot('/a/b/c', '/a/b/c/EXT/d/e/f/package.json') === '/a/b/c/EXT'
- * ```
- */
-function getExtensionRoot(root: string, packageJsonPath: string): string {
-    root = path.resolve(root);
-    packageJsonPath = path.resolve(packageJsonPath);
-    if (!packageJsonPath.startsWith(root)) {
-        throw new Error(`unexpected paths:\n root: ${root}\n package.json: ${packageJsonPath}`);
-    }
-    return packageJsonPath.substr(0, packageJsonPath.indexOf(path.sep, root.length + 1));
+    return dependencyIds;
 }

package/src/theia.ts

@@ -18,10 +18,11 @@ import * as temp from 'temp';
 import * as yargs from 'yargs';
 import yargsFactory = require('yargs/yargs');
 import { ApplicationPackageManager, rebuild } from '@theia/application-manager';
-import { ApplicationProps } from '@theia/application-package';
+import { ApplicationProps, DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package';
 import checkHoisted from './check-hoisting';
 import downloadPlugins from './download-plugins';
 import runTest from './run-test';
+import { extract } from '@theia/localization-manager';
 
 process.on('unhandledRejection', (reason, promise) => {
     throw reason;
@@ -45,17 +46,46 @@ function toStringArray(argv?: (string | number)[]): string[] | undefined {
         : argv.map(arg => String(arg));
 }
 
-function rebuildCommand(command: string, target: ApplicationProps.Target): yargs.CommandModule<unknown, { modules: string[] }> {
+function rebuildCommand(command: string, target: ApplicationProps.Target): yargs.CommandModule<unknown, {
+    modules: string[]
+    cacheRoot?: string
+    force?: boolean
+}> {
     return {
         command,
-        describe: `Rebuild native node modules for the ${target}`,
+        describe: `Rebuild/revert native node modules for "${target}"`,
         builder: {
+            'cacheRoot': {
+                type: 'string',
+                describe: 'Root folder where to store the .browser_modules cache'
+            },
             'modules': {
-                array: true,
+                alias: 'm',
+                array: true, // === `--modules/-m` can be specified multiple times
+                describe: 'List of modules to rebuild/revert'
             },
+            'force': {
+                alias: 'f',
+                boolean: true,
+                describe: 'Rebuild modules for Electron anyway',
+            }
         },
-        handler: args => {
-            rebuild(target, args.modules);
+        handler: ({ cacheRoot, modules, force }) => {
+            // Note: `modules` is actually `string[] | undefined`.
+            if (modules) {
+                // It is ergonomic to pass arguments as --modules="a,b,c,..."
+                // but yargs doesn't parse it this way by default.
+                const flattened: string[] = [];
+                for (const value of modules) {
+                    if (value.includes(',')) {
+                        flattened.push(...value.split(',').map(mod => mod.trim()));
+                    } else {
+                        flattened.push(value);
+                    }
+                }
+                modules = flattened;
+            }
+            rebuild(target, { cacheRoot, modules, force });
         }
     };
 }
@@ -183,7 +213,7 @@ function theiaCli(): void {
                 'api-version': {
                     alias: 'v',
                     describe: 'Supported API version for plugins',
-                    default: '1.50.0'
+                    default: DEFAULT_SUPPORTED_API_VERSION
                 },
                 'api-url': {
                     alias: 'u',
@@ -194,7 +224,53 @@ function theiaCli(): void {
             handler: async ({ packed }) => {
                 await downloadPlugins({ packed });
             },
-        }).command<{
+        })
+        .command<{
+            root: string,
+            output: string,
+            merge: boolean,
+            exclude?: string,
+            logs?: string,
+            files?: string[]
+        }>({
+            command: 'nls-extract',
+            describe: 'Extract translation key/value pairs from source code',
+            builder: {
+                'output': {
+                    alias: 'o',
+                    describe: 'Output file for the extracted translations',
+                    demandOption: true
+                },
+                'root': {
+                    alias: 'r',
+                    describe: 'The directory which contains the source code',
+                    default: '.'
+                },
+                'merge': {
+                    alias: 'm',
+                    describe: 'Whether to merge new with existing translation values',
+                    boolean: true,
+                    default: false
+                },
+                'exclude': {
+                    alias: 'e',
+                    describe: 'Allows to exclude translation keys starting with this value'
+                },
+                'files': {
+                    alias: 'f',
+                    describe: 'Glob pattern matching the files to extract from (starting from --root).',
+                    array: true
+                },
+                'logs': {
+                    alias: 'l',
+                    describe: 'File path to a log file'
+                }
+            },
+            handler: async options => {
+                await extract(options);
+            }
+        })
+        .command<{
             testInspect: boolean,
             testExtension: string[],
             testFile: string[],