@steedos/service-package-registry 2.1.50 → 2.1.54

package/main/default/manager/loader.js

@@ -63,7 +63,7 @@ const loadPackages = async ()=>{
                     }
                 } catch (error) {
                     try {
-                        await registry.installModule(packageName, package.version, package.url)
+                        await registry.installModule(packageName, package.version, package.url, package.registry_url || '')
                         const packageInfo = await loadPackage(packageName);
                         appendToPackagesConfig(packageInfo.name, {version: packageInfo.version, description: packageInfo.description, local: false});
                     } catch (error) {

package/main/default/manager/login.js

@@ -0,0 +1,96 @@
+var ncl = require('./npm_login.js');
+const path = require('path');
+var fs = require('fs');
+const _ = require('lodash');
+const configPath = path.join(process.cwd(), '.steedos', '.npmrc');
+const yarnrcConfigPath = path.join(process.cwd(), '.steedos', '.yarnrc');
+const timeout = 10 * 1000;
+const NPM_TOKENS = {};
+const npmLogin = async function (user, pass, email, registry, scope, quotes, configPath) {
+    return await new Promise((resolve, reject) => {
+        var finalArgs = ncl.processArguments(user, pass, email, registry, scope, quotes, configPath);
+        var response;
+        var contents;
+        var newContents;
+
+        ncl.login(finalArgs, function (err, data) {
+            if (err) {
+                reject(err);
+            }
+            else {
+                response = data;
+                NPM_TOKENS[registry] = response.token;
+                ncl.readFile(finalArgs, function (err, data) {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        contents = data;
+                        newContents = ncl.generateFileContents(finalArgs, contents, response);
+                        ncl.writeFile(finalArgs, newContents, function (err) {
+                            if (err) {
+                                // let users know that it didn't work (could be prettier)
+                                reject(err);
+                            }
+                            resolve();
+                        });
+                    }
+                });
+            }
+        });
+
+        setTimeout(() => {
+            reject({ message: `connect ECONNREFUSED ${registry}` })
+        }, timeout)
+    })
+};
+
+async function login(username, password, email, registry, scope) {
+    return await npmLogin(username, password, email, registry, scope, true, configPath);
+}
+
+function setYarnrcScopes(scopes, registry) {
+    const lines = [];
+    _.each(scopes, (scope) => {
+        lines.push(`"@${scope}:registry" "${registry}"`);
+    })
+    fs.writeFileSync(yarnrcConfigPath, lines.join('\n') + '\n');
+}
+
+function getYarnrcScopes() {
+    const data = fs.readFileSync(yarnrcConfigPath, 'utf-8');
+    const scopes = {};
+    if (data) {
+        // console.log(`data`, data);
+        try {
+            const lines = data.split('\n');
+            _.each(lines, (line) => {
+                try {
+                    let s = line.split(' ');
+                    if (s.length == 2) {
+                        scopes[s[0].replace(/\"/g, "")] = s[1].replace(/\"/g, "");
+                    }
+                } catch (error) {
+                    console.log(error)
+                }
+            })
+        } catch (error) {
+            console.log(error)
+        }
+    }
+    return scopes
+}
+
+function getNpmAuthInfo(registry) {
+    const token = NPM_TOKENS[registry];
+    if (token) {
+        return { token: token, type: 'Bearer' }
+    }
+}
+
+module.exports = {
+    loginToRegistry: login,
+    setYarnrcScopes,
+    getYarnrcScopes,
+    getNpmAuthInfo
+}

package/main/default/manager/npm_login.js

@@ -0,0 +1,145 @@
+var RegClient = require('npm-registry-client');
+function noop() { }
+var client = new RegClient({
+    log: {
+        error: noop,
+        warn: noop,
+        info: noop,
+        verbose: noop,
+        silly: noop,
+        http: noop,
+        pause: noop,
+        resume: noop
+    }
+});
+
+var fs = require('fs');
+var path = require('path');
+
+// jshint freeze:false
+if (!Array.prototype.findIndex) {
+    Array.prototype.findIndex = function (predicate) {
+        if (this === null) {
+            throw new TypeError('Array.prototype.findIndex called on null or undefined');
+        }
+        if (typeof predicate !== 'function') {
+            throw new TypeError('predicate must be a function');
+        }
+        var list = Object(this);
+        var length = list.length >>> 0;
+        var thisArg = arguments[1];
+        var value;
+
+        for (var i = 0; i < length; i++) {
+            value = list[i];
+            if (predicate.call(thisArg, value, i, list)) {
+                return i;
+            }
+        }
+        return -1;
+    };
+}
+// jshint freeze:true
+
+function removeSuffix(pattern, suffix) {
+    if (pattern.endsWith(suffix)) {
+        return pattern.slice(0, -suffix.length);
+    }
+    return pattern;
+}
+
+module.exports = {
+    processArguments: function (npmUser, npmPass, npmEmail, npmRegistry, npmScope, quotes, configPath) {
+        var registry = npmRegistry || 'https://registry.npmjs.org';
+        registry = removeSuffix(registry, '/')
+        var homePath = process.env.HOME ? process.env.HOME : process.env.USERPROFILE;
+        var finalPath = configPath ? configPath : path.join(homePath, '.npmrc');
+        var hasQuotes = quotes ? quotes : false;
+        var args = {
+            user: npmUser,
+            pass: npmPass,
+            email: npmEmail,
+            registry: registry,
+            scope: npmScope,
+            quotes: hasQuotes,
+            configPath: finalPath
+        };
+
+        return args;
+    },
+
+    login: function (args, callback) {
+        client.adduser(args.registry, {
+            auth: {
+                username: args.user,
+                password: args.pass,
+                email: args.email
+            }
+        }, function (err, data) {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, data);
+        });
+    },
+
+    readFile: function (args, callback) {
+        fs.readFile(args.configPath, 'utf-8', function (err, contents) {
+            if (err) {
+                contents = '';
+            }
+            return callback(null, contents);
+        });
+    },
+
+    generateFileContents: function (args, contents, response) {
+        // `contents` holds the initial content of the NPMRC file
+        // Convert the file contents into an array
+        var lines = contents ? contents.split('\n') : [];
+        // Regex pattern to detect end of registry
+        const registryEndRegexPattern = /\:\//;
+        if (args.scope !== undefined) {
+            var scopeWrite = lines.findIndex(function (element) {
+                if (element.indexOf(args.scope + ':registry=' + args.registry) !== -1) {
+                    // If an entry for the scope is found, replace it
+                    element = args.scope + ':registry=' + args.registry;
+                    return true;
+                }
+            });
+
+            // If no entry for the scope is found, add one
+            if (scopeWrite === -1) {
+                lines.push(args.scope + ':registry=' + args.registry);
+            }
+        }
+
+        var authWrite = lines.findIndex(function (element, index, array) {
+            if (element.indexOf(args.registry.slice(args.registry.search(registryEndRegexPattern, '') + 1) +
+                '/:_authToken=') !== -1) {
+                // If an entry for the auth token is found, replace it
+                array[index] = element.replace(/authToken\=.*/, 'authToken=' + (args.quotes ? '"' : '') +
+                    response.token + (args.quotes ? '"' : ''));
+                return true;
+            }
+        });
+
+        // If no entry for the auth token is found, add one
+        if (authWrite === -1) {
+            lines.push(args.registry.slice(args.registry.search(registryEndRegexPattern, '') +
+                1) + '/:_authToken=' + (args.quotes ? '"' : '') + response.token + (args.quotes ? '"' : ''));
+        }
+
+        var toWrite = lines.filter(function (element) {
+            if (element === '') {
+                return false;
+            }
+            return true;
+        });
+
+        return toWrite;
+    },
+
+    writeFile: function (args, lines, callback) {
+        fs.writeFile(args.configPath, lines.join('\n') + '\n', callback);
+    }
+};

package/main/default/manager/package-json.js

@@ -0,0 +1,111 @@
+'use strict';
+const { Agent: HttpAgent } = require('http');
+const { Agent: HttpsAgent } = require('https');
+const got = require('got');
+const registryUrl = require('registry-url');
+const registryAuthToken = require('registry-auth-token');
+const semver = require('semver');
+
+// These agent options are chosen to match the npm client defaults and help with performance
+// See: `npm config get maxsockets` and #50
+const agentOptions = {
+    keepAlive: true,
+    maxSockets: 50
+};
+const httpAgent = new HttpAgent(agentOptions);
+const httpsAgent = new HttpsAgent(agentOptions);
+
+class PackageNotFoundError extends Error {
+    constructor(packageName) {
+        super(`Package \`${packageName}\` could not be found`);
+        this.name = 'PackageNotFoundError';
+    }
+}
+
+class VersionNotFoundError extends Error {
+    constructor(packageName, version) {
+        super(`Version \`${version}\` for package \`${packageName}\` could not be found`);
+        this.name = 'VersionNotFoundError';
+    }
+}
+
+const packageJson = async (packageName, options) => {
+    options = {
+        version: 'latest',
+        ...options
+    };
+
+    const scope = packageName.split('/')[0];
+    const registryUrl_ = options.registryUrl || registryUrl(scope);
+    const packageUrl = new URL(encodeURIComponent(packageName).replace(/^%40/, '@'), registryUrl_);
+    const authInfo = options.authInfo || registryAuthToken(registryUrl_.toString(), { recursive: true });
+
+    const headers = {
+        accept: 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*'
+    };
+
+    if (options.fullMetadata) {
+        delete headers.accept;
+    }
+
+    if (authInfo) {
+        headers.authorization = `${authInfo.type} ${authInfo.token}`;
+    }
+
+    const gotOptions = {
+        headers,
+        agent: {
+            http: httpAgent,
+            https: httpsAgent
+        }
+    };
+
+    if (options.agent) {
+        gotOptions.agent = options.agent;
+    }
+
+    let data;
+    try {
+        data = await got(packageUrl, gotOptions).json();
+    } catch (error) {
+        if (error.response.statusCode === 404) {
+            throw new PackageNotFoundError(packageName);
+        }
+
+        throw error;
+    }
+
+    if (options.allVersions) {
+        return data;
+    }
+
+    let { version } = options;
+    const versionError = new VersionNotFoundError(packageName, version);
+
+    if (data['dist-tags'][version]) {
+        data = data.versions[data['dist-tags'][version]];
+    } else if (version) {
+        if (!data.versions[version]) {
+            const versions = Object.keys(data.versions);
+            version = semver.maxSatisfying(versions, version);
+
+            if (!version) {
+                throw versionError;
+            }
+        }
+
+        data = data.versions[version];
+
+        if (!data) {
+            throw versionError;
+        }
+    }
+
+    return data;
+};
+
+module.exports = packageJson;
+// TODO: remove this in the next major version
+module.exports.default = packageJson;
+module.exports.PackageNotFoundError = PackageNotFoundError;
+module.exports.VersionNotFoundError = VersionNotFoundError;

package/main/default/manager/packages.js

@@ -3,8 +3,21 @@ const loader = require('./loader');
 const registry = require('./registry');
 const path = require("path");
 const objectql = require('@steedos/objectql');
-const packageJson = require('package-json');
+const packageJson = require('./package-json');
 const _ = require('lodash');
+const login = require('./login');
+
+function registryUrl(scope) {
+    try {
+        const result = login.getYarnrcScopes();
+        const url = result[`${scope}:registry`];
+        if (url) {
+            return url.slice(-1) === '/' ? url : `${url}/`;
+        }
+    } catch (Exception) {
+
+    }
+}
 
 const getAllPackages = async ()=>{
     // const installPackages = loader.loadPackagesConfig();
@@ -106,8 +119,14 @@ const maintainSystemFiles = ()=>{
 }
 
 const getPackageVersions = async (packageName, options)=>{
+    //TODO 处理 registry_url
+
+    let registry_url = registryUrl(packageName.toLowerCase().split('/')[0]);
+
     const result = await packageJson(packageName.toLowerCase(), {
-        allVersions: true
+        allVersions: true,
+        registryUrl: registry_url,
+        authInfo: login.getNpmAuthInfo(registry_url)
     });
 
     const versions = [];

package/main/default/manager/registry.js

@@ -112,7 +112,7 @@ function checkModuleIsInstall(module){
     return false
 }
 
-async function installModule(module,version,url) {
+async function installModule(module, version, url, registry_url) {
     if (Buffer.isBuffer(module)) {
         // return installTarball(module)
         return 
@@ -193,7 +193,11 @@ async function installModule(module,version,url) {
 
         var installDir = settings.userDir || ".";
         var args = ['install','--no-audit','--no-update-notifier','--no-fund','--save','--save-prefix=~','--production',installName];
-        var yarnArgs = ['add', '-E', installName]; //yarnCommand  , '--json' --registry
+        var yarnArgs = ['add', '-E', installName, '--json']; //yarnCommand  , '--json' --registry
+        if (false && registry_url) {
+            yarnArgs.push('--registry')
+            yarnArgs.push(registry_url)
+        }
         return exec.run(yarnCommand,yarnArgs,{
             cwd: installDir
         }, true).then(result => {
@@ -233,8 +237,6 @@ async function installModule(module,version,url) {
                 // return require("./registry").setModulePendingUpdated(module,version);
             }
         }).catch(result => {
-            console.log(`result error`, result)
-            log.error(`result`, result);
             activePromise = Promise.resolve();
             throw result;
             // var output = result.stderr;

package/main/default/routes/registry.router.js

@@ -0,0 +1,14 @@
+const express = require("express");
+const router = express.Router();
+const core = require("@steedos/core");
+
+let STEEDOS_REGISTRY_URL = process.env.STEEDOS_REGISTRY_URL ? process.env.STEEDOS_REGISTRY_URL : 'https://registry.steedos.cn/';
+
+router.get("/api/nodes/registry", core.requireAuthentication, async function (req, res) {
+    res.status(200).send([
+        { label: "https://registry.npmjs.org/", value: "https://registry.npmjs.org/" },
+        { label: "https://registry.npm.taobao.org/", value: "https://registry.npm.taobao.org/" },
+        { label: STEEDOS_REGISTRY_URL, value: STEEDOS_REGISTRY_URL },
+    ]);
+});
+exports.default = router;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steedos/service-package-registry",
-  "version": "2.1.50",
+  "version": "2.1.54",
   "description": "",
   "main": "package.service.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@steedos/service-package-loader": "2.1.50",
+    "@steedos/service-package-loader": "2.1.54",
     "fs-extra": "8.1.0",
     "i18next": "20.3.2",
     "json-stringify-safe": "5.0.1",
@@ -18,6 +18,7 @@
     "lodash.clonedeep": "^4.5.0",
     "moment-timezone": "0.5.33",
     "npm-package-arg": "~8.1.5",
+    "npm-registry-client": "8.6.0",
     "package-json": "^7.0.0",
     "radwag-mass": "~1.0.5"
   },
@@ -25,5 +26,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "3df06ee3928dd49857dc6b98ae66fa4330459f81"
+  "gitHead": "6b261ebf1924b3ec2dc9d1951ad3e0449de9fdef"
 }

package/package.service.js

@@ -6,6 +6,7 @@ const loader = require('./main/default/manager/loader');
 const packages = require('./main/default/manager/packages');
 const registry = require('./main/default/manager/registry');
 const metadata = require('@steedos/metadata-core')
+const axios = require('axios');
 const _ = require(`lodash`);
 const path = require(`path`);
 const objectql = require('@steedos/objectql');
@@ -14,6 +15,8 @@ const util = require('./main/default/manager/util');
 const fetch = require('node-fetch');
 const { MoleculerError } = require("moleculer").Errors;
 
+const login = require('./main/default/manager/login');
+
 const HEADER_AUTH = 'Authorization';
 const AUTH_TYPE = 'Bearer';
 
@@ -68,7 +71,8 @@ module.exports = {
 		},
 		STEEDOS_CLOUD_URL: process.env.STEEDOS_CLOUD_URL ? process.env.STEEDOS_CLOUD_URL : 'https://console.steedos.cn',
 		STEEDOS_CLOUD_SPACE_ID: process.env.STEEDOS_CLOUD_SPACE_ID,
-		STEEDOS_CLOUD_API_KEY: process.env.STEEDOS_CLOUD_API_KEY
+		STEEDOS_CLOUD_API_KEY: process.env.STEEDOS_CLOUD_API_KEY,
+		STEEDOS_REGISTRY_URL: process.env.STEEDOS_REGISTRY_URL ? process.env.STEEDOS_REGISTRY_URL : 'https://registry.steedos.cn/',
 	},
 
 	/**
@@ -221,6 +225,7 @@ module.exports = {
 						try {
 							const { name, version, label, description } = _package
 							let enable = true; //安装已购买的软件包时先默认启用
+							//TODO 处理 registry_url
 							const packageInfo = await this.installPackage(name, version, label, description, enable, ctx.broker);
 							installPackages.push(packageInfo)
 						} catch (error) {
@@ -263,11 +268,26 @@ module.exports = {
 					if(!user.is_space_admin){
 						throw new Error('not permission!');
 					}
-					let { module, version, url, auth } = ctx.params
+					let { module, version, url, auth, registry_url } = ctx.params
 					const enable = true;
-					return await this.installPackageFromUrl(module, version, url, auth, enable, ctx.broker)
+					return await this.installPackageFromUrl(module, version, url, auth, enable, registry_url, ctx.broker)
 				} catch (error) {
-					throw new MoleculerError(error.message, 500, "ERR_SOMETHING");
+					let errorInfo = error.message || '';
+					if (error.stderr) {
+						const errors = error.stderr.split('\n');
+						errors.forEach(element => {
+							if (element) {
+								try {
+									const item = JSON.parse(element);
+									if (item.type === 'error') {
+										errorInfo = errorInfo + item.data + '\n'
+									}
+								} catch (error) {
+								}
+							}
+						});
+					}
+					throw new MoleculerError(errorInfo, 500, "ERR_SOMETHING");
 				}
 			}
 		}
@@ -362,6 +382,7 @@ module.exports = {
 		},
 		upgradePackage: {
 			async handler(module, version) {
+				//TODO 处理 registry_url
                 const packagePath = await registry.installModule(module, version);
 				const enable = true;
 				if(enable){
@@ -389,7 +410,7 @@ module.exports = {
             }
 		},
 		installPackageFromUrl: {
-			async handler(module, version, url, auth, enable, broker) {
+			async handler(module, version, url, auth, enable, registry_url, broker) {
 				if(!module || !_.isString(module) || !module.trim()){
 					throw new Error(`无效的软件包名称`);
 				} else {
@@ -432,8 +453,7 @@ module.exports = {
 					url = `${url}/${result.token}`
 					console.log(`url`, url);
 				}
-
-				const packagePath = await registry.installModule(module, version, url);
+				const packagePath = await registry.installModule(module, version, url, registry_url);
 				const packageInfo = loader.getPackageInfo(null, packagePath);
 				const packageName = packageInfo.name;
 				if(enable){
@@ -448,6 +468,7 @@ module.exports = {
 					local: false, 
 					enable: enable,
 					url: url,
+					// registry_url: registry_url,
 					path: util.getPackageRelativePath(process.cwd(), packagePath)
 				}
 				loader.appendToPackagesConfig(packageName, packageConfig);
@@ -462,7 +483,61 @@ module.exports = {
 					})
 				})
 				return packageConfig;
-            }
+			}
+		},
+		getSafeScopes: {
+			async handler(spaceId, apiKey, consoleUrl) {
+				let result = await axios({
+					url: `${consoleUrl}/api/npm-scopes/safe`,
+					method: 'post',
+					data: {
+						cloudSpaceId: spaceId
+					},
+					timeout: 5 * 1000,
+					headers: { "Content-Type": "application/json", "Authorization": `Bearer apikey,${apiKey}` }
+				})
+				if (!result.data.success) {
+					throw new Error(result.data.error);
+				}
+				return result.data;
+			}
+		},
+		loginSteedosRegistry: {
+			async handler() {
+				const settings = this.settings;
+				// 配置主控地址
+				const consoleUrl = settings.STEEDOS_CLOUD_URL;
+				if (!consoleUrl) {
+					console.log(chalk.red('请配置主控地址'));
+					return;
+				}
+
+				// 初始化工作区数据
+				// 获取环境变量中工作区信息
+				const spaceId = settings.STEEDOS_CLOUD_SPACE_ID;
+				const apiKey = settings.STEEDOS_CLOUD_API_KEY;
+
+				if (!spaceId || !apiKey) {
+					console.log(chalk.red('请配置环境变量STEEDOS_CLOUD_SPACE_ID和STEEDOS_CLOUD_API_KEY。'));
+					return;
+				}
+
+				const registryUrl = settings.STEEDOS_REGISTRY_URL
+
+				// 调用接口获取初始化信息
+				const { info, scopes } = await this.getSafeScopes(spaceId, apiKey, consoleUrl);
+				const { adminPhone } = info;
+				if (!adminPhone) {
+					console.log(chalk.red('缺少工作区信息 工作区名称、管理员姓名、管理员手机号，请检查'));
+					return;
+				}
+				// let scope = '';
+				// if (scopes && scopes.length > 0) {
+				// 	scope = scopes[0];
+				// }
+				await login.loginToRegistry(adminPhone, apiKey, `${adminPhone}@steedos.com`, registryUrl, undefined);
+				login.setYarnrcScopes(scopes, registryUrl);
+			}
 		}
 	},
 
@@ -477,6 +552,12 @@ module.exports = {
 	 * Service started lifecycle event handler
 	 */
 	async started() {
+		try {
+			await this.loginSteedosRegistry();
+			console.info(`login steedos registry success`);
+		} catch (error) {
+			console.error(`login steedos registry fail: `, error.message);
+		}
 		const PACKAGE_INSTALL_NODE = process.env.PACKAGE_INSTALL_NODE
 		if(PACKAGE_INSTALL_NODE){
 			await this.broker.call('metadata.add', {key: `#package_install_node.${this.broker.nodeID}`, data: {nodeID: PACKAGE_INSTALL_NODE}}, {meta: {}})