@nodesecure/scanner 2.2.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -94
- package/index.js +4 -4
- package/package.json +17 -34
- package/src/{dependency.class.js → class/dependency.class.js} +22 -21
- package/src/{logger.class.js → class/logger.class.js} +0 -0
- package/src/depWalker.js +298 -370
- package/src/manifest.js +57 -0
- package/src/npmRegistry.js +68 -0
- package/src/tarball.js +85 -189
- package/src/utils/addMissingVersionFlags.js +24 -0
- package/src/utils/analyzeDependencies.js +40 -0
- package/src/utils/booleanToFlags.js +12 -0
- package/src/utils/filterDependencyKind.js +44 -0
- package/src/utils/getPackageName.js +18 -2
- package/src/utils/getTarballComposition.js +8 -10
- package/src/utils/index.js +16 -14
- package/src/utils/isGitDependency.js +20 -0
- package/src/utils/isSensitiveFile.js +6 -0
- package/src/utils/mergeDependencies.js +26 -23
- package/src/utils/semver.js +3 -3
- package/types/logger.d.ts +2 -0
- package/types/scanner.d.ts +128 -32
package/src/manifest.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Import Node.js Dependencies
|
|
2
|
+
import fs from "fs/promises";
|
|
3
|
+
import path from "path";
|
|
4
|
+
|
|
5
|
+
// Import Third-party Dependencies
|
|
6
|
+
import { parseManifestAuthor } from "@nodesecure/utils";
|
|
7
|
+
|
|
8
|
+
// CONSTANTS
|
|
9
|
+
// PR welcome to contribute to this list!
|
|
10
|
+
const kNativeNpmPackages = new Set([
|
|
11
|
+
"node-gyp", "node-pre-gyp", "node-gyp-build", "node-addon-api"
|
|
12
|
+
]);
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @see https://www.nerdycode.com/prevent-npm-executing-scripts-security/
|
|
16
|
+
*/
|
|
17
|
+
const kUnsafeNpmScripts = new Set([
|
|
18
|
+
"install",
|
|
19
|
+
"preinstall", "postinstall",
|
|
20
|
+
"preuninstall", "postuninstall"
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @param {!string} location
|
|
25
|
+
* @returns {import("@npm/types").PackageJson}
|
|
26
|
+
*/
|
|
27
|
+
export async function read(location) {
|
|
28
|
+
const packageStr = await fs.readFile(
|
|
29
|
+
path.join(location, "package.json"),
|
|
30
|
+
"utf-8"
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
return JSON.parse(packageStr);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// TODO: PR @npm/types to fix dependencies typo
|
|
37
|
+
export async function readAnalyze(location) {
|
|
38
|
+
const {
|
|
39
|
+
description = "", author = {}, scripts = {},
|
|
40
|
+
dependencies = {}, devDependencies = {}, gypfile = false
|
|
41
|
+
} = await read(location);
|
|
42
|
+
|
|
43
|
+
const packageDeps = Object.keys(dependencies);
|
|
44
|
+
const packageDevDeps = Object.keys(devDependencies);
|
|
45
|
+
const hasNativePackage = [...packageDevDeps, ...packageDeps]
|
|
46
|
+
.some((pkg) => kNativeNpmPackages.has(pkg));
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
author: typeof author === "string" ? parseManifestAuthor(author) : author,
|
|
50
|
+
description,
|
|
51
|
+
hasScript: Object.keys(scripts)
|
|
52
|
+
.some((value) => kUnsafeNpmScripts.has(value.toLowerCase())),
|
|
53
|
+
packageDeps,
|
|
54
|
+
packageDevDeps,
|
|
55
|
+
hasNativeElements: hasNativePackage || gypfile
|
|
56
|
+
};
|
|
57
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Import Third-party Dependencies
|
|
2
|
+
import semver from "semver";
|
|
3
|
+
import { parseManifestAuthor } from "@nodesecure/utils";
|
|
4
|
+
import { packument } from "@nodesecure/npm-registry-sdk";
|
|
5
|
+
|
|
6
|
+
export function parseAuthor(author) {
|
|
7
|
+
return typeof author === "string" ? parseManifestAuthor(author) : author;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export async function packageMetadata(name, version, options) {
|
|
11
|
+
const { ref, logger } = options;
|
|
12
|
+
|
|
13
|
+
try {
|
|
14
|
+
const pkg = await packument(name);
|
|
15
|
+
|
|
16
|
+
const oneYearFromToday = new Date();
|
|
17
|
+
oneYearFromToday.setFullYear(oneYearFromToday.getFullYear() - 1);
|
|
18
|
+
|
|
19
|
+
const lastVersion = pkg["dist-tags"].latest;
|
|
20
|
+
const lastUpdateAt = new Date(pkg.time[lastVersion]);
|
|
21
|
+
const metadata = {
|
|
22
|
+
author: parseAuthor(pkg.author) ?? {},
|
|
23
|
+
homepage: pkg.homepage || null,
|
|
24
|
+
publishedCount: Object.values(pkg.versions).length,
|
|
25
|
+
lastVersion,
|
|
26
|
+
lastUpdateAt,
|
|
27
|
+
hasReceivedUpdateInOneYear: !(oneYearFromToday > lastUpdateAt),
|
|
28
|
+
maintainers: pkg.maintainers,
|
|
29
|
+
publishers: []
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const isOutdated = semver.neq(version, lastVersion);
|
|
33
|
+
if (isOutdated) {
|
|
34
|
+
ref.versions[version].flags.push("isOutdated");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const publishers = new Set();
|
|
38
|
+
for (const ver of Object.values(pkg.versions).reverse()) {
|
|
39
|
+
const { _npmUser: npmUser, version } = ver;
|
|
40
|
+
const isNullOrUndefined = typeof npmUser === "undefined" || npmUser === null;
|
|
41
|
+
if (isNullOrUndefined || !("name" in npmUser) || typeof npmUser.name !== "string") {
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const authorName = metadata.author?.name ?? null;
|
|
46
|
+
if (authorName === null) {
|
|
47
|
+
metadata.author = npmUser;
|
|
48
|
+
}
|
|
49
|
+
else if (npmUser.name !== metadata.author.name) {
|
|
50
|
+
metadata.hasManyPublishers = true;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// TODO: add npmUser.email
|
|
54
|
+
if (!publishers.has(npmUser.name)) {
|
|
55
|
+
publishers.add(npmUser.name);
|
|
56
|
+
metadata.publishers.push({ ...npmUser, version, at: new Date(pkg.time[version]) });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
Object.assign(ref.metadata, metadata);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// ignore
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
logger.tick("registry");
|
|
67
|
+
}
|
|
68
|
+
}
|
package/src/tarball.js
CHANGED
|
@@ -1,100 +1,63 @@
|
|
|
1
1
|
// Import Node.js Dependencies
|
|
2
|
-
import
|
|
3
|
-
import fs from "fs/promises";
|
|
2
|
+
import path from "path";
|
|
4
3
|
import os from "os";
|
|
5
4
|
import timers from "timers/promises";
|
|
6
5
|
|
|
7
6
|
// Import Third-party Dependencies
|
|
8
|
-
import {
|
|
9
|
-
import { parseManifestAuthor } from "@nodesecure/utils";
|
|
10
|
-
import difference from "lodash.difference";
|
|
11
|
-
import isMinified from "is-minified-code";
|
|
7
|
+
import { runASTAnalysisOnFile } from "@nodesecure/js-x-ray";
|
|
12
8
|
import pacote from "pacote";
|
|
13
9
|
import ntlp from "@nodesecure/ntlp";
|
|
14
|
-
import builtins from "builtins";
|
|
15
10
|
|
|
16
11
|
// Import Internal Dependencies
|
|
17
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
getTarballComposition, isSensitiveFile, filterDependencyKind, analyzeDependencies, booleanToFlags,
|
|
14
|
+
NPM_TOKEN
|
|
15
|
+
} from "./utils/index.js";
|
|
16
|
+
import * as manifest from "./manifest.js";
|
|
18
17
|
import { getLocalRegistryURL } from "@nodesecure/npm-registry-sdk";
|
|
19
18
|
|
|
20
19
|
// CONSTANTS
|
|
21
|
-
const
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
20
|
+
const kNativeCodeExtensions = new Set([".gyp", ".c", ".cpp", ".node", ".so", ".h"]);
|
|
21
|
+
const kJsExtname = new Set([".js", ".mjs", ".cjs"]);
|
|
22
|
+
|
|
23
|
+
export async function scanJavascriptFile(dest, file, packageName) {
|
|
24
|
+
const result = await runASTAnalysisOnFile(path.join(dest, file), { packageName });
|
|
25
|
+
|
|
26
|
+
const warnings = result.warnings.map((curr) => Object.assign({}, curr, { file }));
|
|
27
|
+
if (!result.ok) {
|
|
28
|
+
return {
|
|
29
|
+
file,
|
|
30
|
+
warnings,
|
|
31
|
+
isMinified: false,
|
|
32
|
+
tryDependencies: [],
|
|
33
|
+
dependencies: [],
|
|
34
|
+
filesDependencies: []
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const { packages, files } = filterDependencyKind(result.dependencies, path.dirname(file));
|
|
39
38
|
|
|
40
39
|
return {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
40
|
+
file,
|
|
41
|
+
warnings,
|
|
42
|
+
isMinified: result.isMinified,
|
|
43
|
+
tryDependencies: [...result.dependencies.getDependenciesInTryStatement()],
|
|
44
|
+
dependencies: packages,
|
|
45
|
+
filesDependencies: files
|
|
44
46
|
};
|
|
45
47
|
}
|
|
46
48
|
|
|
47
|
-
export async function scanFile(dest, file, options) {
|
|
48
|
-
const { name, ref } = options;
|
|
49
|
-
|
|
50
|
-
try {
|
|
51
|
-
const str = await fs.readFile(join(dest, file), "utf-8");
|
|
52
|
-
const isMin = file.includes(".min") || isMinified(str);
|
|
53
|
-
|
|
54
|
-
const ASTAnalysis = runASTAnalysis(str, { isMinified: isMin });
|
|
55
|
-
ASTAnalysis.dependencies.removeByName(name);
|
|
56
|
-
|
|
57
|
-
const dependencies = [];
|
|
58
|
-
const filesDependencies = [];
|
|
59
|
-
for (const depName of ASTAnalysis.dependencies) {
|
|
60
|
-
if (depName.startsWith(".")) {
|
|
61
|
-
const indexName = DIRECT_PATH.has(depName) ? join(depName, "index.js") : join(dirname(file), depName);
|
|
62
|
-
filesDependencies.push(indexName);
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
dependencies.push(depName);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
const inTryDeps = [...ASTAnalysis.dependencies.getDependenciesInTryStatement()];
|
|
69
|
-
|
|
70
|
-
if (!ASTAnalysis.isOneLineRequire && isMin) {
|
|
71
|
-
ref.composition.minified.push(file);
|
|
72
|
-
}
|
|
73
|
-
ref.warnings.push(...ASTAnalysis.warnings.map((curr) => Object.assign({}, curr, { file })));
|
|
74
|
-
|
|
75
|
-
return { inTryDeps, dependencies, filesDependencies };
|
|
76
|
-
}
|
|
77
|
-
catch (error) {
|
|
78
|
-
if (!("code" in error)) {
|
|
79
|
-
ref.warnings.push({ file, kind: "parsing-error", value: error.message, location: [[0, 0], [0, 0]] });
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
return null;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
49
|
export async function scanDirOrArchive(name, version, options) {
|
|
87
50
|
const { ref, tmpLocation, locker } = options;
|
|
88
51
|
|
|
89
52
|
const isNpmTarball = !(tmpLocation === null);
|
|
90
|
-
const dest = isNpmTarball ? join(tmpLocation, `${name}@${version}`) : process.cwd();
|
|
53
|
+
const dest = isNpmTarball ? path.join(tmpLocation, `${name}@${version}`) : process.cwd();
|
|
91
54
|
const free = await locker.acquireOne();
|
|
92
55
|
|
|
93
56
|
try {
|
|
94
57
|
// If this is an NPM tarball then we extract it on the disk with pacote.
|
|
95
58
|
if (isNpmTarball) {
|
|
96
59
|
await pacote.extract(ref.flags.includes("isGit") ? ref.gitUrl : `${name}@${version}`, dest, {
|
|
97
|
-
...
|
|
60
|
+
...NPM_TOKEN,
|
|
98
61
|
registry: getLocalRegistryURL(),
|
|
99
62
|
cache: `${os.homedir()}/.npm`
|
|
100
63
|
});
|
|
@@ -102,103 +65,67 @@ export async function scanDirOrArchive(name, version, options) {
|
|
|
102
65
|
}
|
|
103
66
|
|
|
104
67
|
// Read the package.json at the root of the directory or archive.
|
|
105
|
-
const { packageDeps, packageDevDeps,
|
|
68
|
+
const { packageDeps, packageDevDeps, author, description, hasScript, hasNativeElements } = await manifest.readAnalyze(dest);
|
|
69
|
+
ref.author = author;
|
|
70
|
+
ref.description = description;
|
|
106
71
|
|
|
107
72
|
// Get the composition of the (extracted) directory
|
|
108
73
|
const { ext, files, size } = await getTarballComposition(dest);
|
|
109
74
|
ref.size = size;
|
|
110
75
|
ref.composition.extensions.push(...ext);
|
|
111
76
|
ref.composition.files.push(...files);
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
77
|
+
const hasBannedFile = files.some((path) => isSensitiveFile(path));
|
|
78
|
+
const hasNativeCode = hasNativeElements || files.some((file) => kNativeCodeExtensions.has(path.extname(file)));
|
|
115
79
|
|
|
116
80
|
// Search for minified and runtime dependencies
|
|
117
81
|
// Run a JS-X-Ray analysis on each JavaScript files of the project!
|
|
118
|
-
const
|
|
119
|
-
const fileAnalysisResults = await Promise.all(
|
|
82
|
+
const fileAnalysisRaw = await Promise.allSettled(
|
|
120
83
|
files
|
|
121
|
-
.filter((name) =>
|
|
122
|
-
.map((file) =>
|
|
84
|
+
.filter((name) => kJsExtname.has(path.extname(name)))
|
|
85
|
+
.map((file) => scanJavascriptFile(dest, file, name))
|
|
123
86
|
);
|
|
124
87
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
result.filesDependencies.forEach((dep) => filesDependencies.add(dep));
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
// Search for native code
|
|
132
|
-
{
|
|
133
|
-
const hasNativeFile = files.some((file) => NATIVE_CODE_EXTENSIONS.has(extname(file)));
|
|
134
|
-
const hasNativePackage = hasNativeFile ? null : [
|
|
135
|
-
...new Set([...packageDevDeps, ...(packageDeps || [])])
|
|
136
|
-
].some((pkg) => NATIVE_NPM_PACKAGES.has(pkg));
|
|
88
|
+
const fileAnalysisResults = fileAnalysisRaw
|
|
89
|
+
.filter((promiseSettledResult) => promiseSettledResult.status === "fulfilled")
|
|
90
|
+
.map((promiseSettledResult) => promiseSettledResult.value);
|
|
137
91
|
|
|
138
|
-
|
|
139
|
-
ref.flags.push("hasNativeCode");
|
|
140
|
-
}
|
|
141
|
-
}
|
|
92
|
+
ref.warnings.push(...fileAnalysisResults.flatMap((row) => row.warnings));
|
|
142
93
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const
|
|
94
|
+
const dependencies = [...new Set(fileAnalysisResults.flatMap((row) => row.dependencies))];
|
|
95
|
+
const filesDependencies = [...new Set(fileAnalysisResults.flatMap((row) => row.filesDependencies))];
|
|
96
|
+
const tryDependencies = new Set(fileAnalysisResults.flatMap((row) => row.tryDependencies));
|
|
97
|
+
const minifiedFiles = fileAnalysisResults.filter((row) => row.isMinified).flatMap((row) => row.file);
|
|
147
98
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
.filter((name) => !name.startsWith("."))
|
|
152
|
-
.filter((name) => !NODE_CORE_LIBS.has(name))
|
|
153
|
-
.filter((name) => !packageDevDeps.includes(name))
|
|
154
|
-
.filter((name) => !inTryDeps.has(name));
|
|
155
|
-
ref.composition.required_thirdparty = thirdPartyDependencies;
|
|
99
|
+
const {
|
|
100
|
+
nodeDependencies, thirdPartyDependencies, missingDependencies, unusedDependencies, flags
|
|
101
|
+
} = analyzeDependencies(dependencies, { packageDeps, packageDevDeps, tryDependencies });
|
|
156
102
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
}
|
|
164
|
-
ref.composition.unused.push(...unusedDeps);
|
|
165
|
-
ref.composition.missing.push(...missingDeps);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
ref.composition.required_files = [...filesDependencies]
|
|
169
|
-
.filter((depName) => depName.trim() !== "")
|
|
170
|
-
// .map((depName) => {
|
|
171
|
-
// return files.includes(depName) ? depName : join(depName, "index.js");
|
|
172
|
-
// })
|
|
173
|
-
.map((depName) => (extname(depName) === "" ? `${depName}.js` : depName));
|
|
174
|
-
ref.composition.required_nodejs = required.filter((name) => NODE_CORE_LIBS.has(name));
|
|
175
|
-
|
|
176
|
-
if (ref.composition.minified.length > 0) {
|
|
177
|
-
ref.flags.push("hasMinifiedCode");
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
const hasExternalCapacity = ref.composition.required_nodejs
|
|
181
|
-
.some((depName) => constants.EXT_DEPS.has(depName));
|
|
182
|
-
if (hasExternalCapacity) {
|
|
183
|
-
ref.flags.push("hasExternalCapacity");
|
|
184
|
-
}
|
|
103
|
+
ref.composition.required_thirdparty = thirdPartyDependencies;
|
|
104
|
+
ref.composition.unused.push(...unusedDependencies);
|
|
105
|
+
ref.composition.missing.push(...missingDependencies);
|
|
106
|
+
ref.composition.required_files = filesDependencies;
|
|
107
|
+
ref.composition.required_nodejs = nodeDependencies;
|
|
108
|
+
ref.composition.minified = minifiedFiles;
|
|
185
109
|
|
|
186
110
|
// License
|
|
187
111
|
await timers.setImmediate();
|
|
188
112
|
const licenses = await ntlp(dest);
|
|
189
|
-
|
|
190
113
|
const uniqueLicenseIds = Array.isArray(licenses.uniqueLicenseIds) ? licenses.uniqueLicenseIds : [];
|
|
191
|
-
if (uniqueLicenseIds.length === 0) {
|
|
192
|
-
ref.flags.push("hasNoLicense");
|
|
193
|
-
}
|
|
194
|
-
if (licenses.hasMultipleLicenses) {
|
|
195
|
-
ref.flags.push("hasMultipleLicenses");
|
|
196
|
-
}
|
|
197
|
-
|
|
198
114
|
ref.license = licenses;
|
|
199
115
|
ref.license.uniqueLicenseIds = uniqueLicenseIds;
|
|
116
|
+
|
|
117
|
+
ref.flags.push(...booleanToFlags({
|
|
118
|
+
...flags,
|
|
119
|
+
hasNoLicense: uniqueLicenseIds.length === 0,
|
|
120
|
+
hasMultipleLicenses: licenses.hasMultipleLicenses,
|
|
121
|
+
hasMinifiedCode: minifiedFiles.length > 0,
|
|
122
|
+
hasWarnings: ref.warnings.length > 0 && !ref.flags.includes("hasWarnings"),
|
|
123
|
+
hasBannedFile,
|
|
124
|
+
hasNativeCode,
|
|
125
|
+
hasScript
|
|
126
|
+
}));
|
|
200
127
|
}
|
|
201
|
-
catch
|
|
128
|
+
catch {
|
|
202
129
|
// Ignore
|
|
203
130
|
}
|
|
204
131
|
finally {
|
|
@@ -206,66 +133,35 @@ export async function scanDirOrArchive(name, version, options) {
|
|
|
206
133
|
}
|
|
207
134
|
}
|
|
208
135
|
|
|
209
|
-
async function readJSFile(dest, file) {
|
|
210
|
-
const str = await fs.readFile(join(dest, file), "utf-8");
|
|
211
|
-
|
|
212
|
-
return [file, str];
|
|
213
|
-
}
|
|
214
|
-
|
|
215
136
|
export async function scanPackage(dest, packageName) {
|
|
216
|
-
|
|
217
|
-
let isProjectUsingESM = false;
|
|
218
|
-
let localPackageName = packageName;
|
|
219
|
-
{
|
|
220
|
-
const packageStr = await fs.readFile(join(dest, "package.json"), "utf-8");
|
|
221
|
-
const { type = "script", name } = JSON.parse(packageStr);
|
|
222
|
-
isProjectUsingESM = type === "module";
|
|
223
|
-
if (localPackageName === null) {
|
|
224
|
-
localPackageName = name;
|
|
225
|
-
}
|
|
226
|
-
}
|
|
137
|
+
const { type = "script", name } = await manifest.read(dest);
|
|
227
138
|
|
|
228
|
-
// Get the tarball composition
|
|
229
139
|
await timers.setImmediate();
|
|
230
140
|
const { ext, files, size } = await getTarballComposition(dest);
|
|
141
|
+
ext.delete("");
|
|
231
142
|
|
|
232
143
|
// Search for runtime dependencies
|
|
233
144
|
const dependencies = Object.create(null);
|
|
234
|
-
const minified = [];
|
|
235
|
-
const warnings = [];
|
|
145
|
+
const [minified, warnings] = [[], []];
|
|
236
146
|
|
|
237
|
-
const JSFiles = files.filter((name) =>
|
|
238
|
-
const
|
|
239
|
-
|
|
147
|
+
const JSFiles = files.filter((name) => kJsExtname.has(path.extname(name)));
|
|
148
|
+
for (const file of JSFiles) {
|
|
149
|
+
const result = await runASTAnalysisOnFile(path.join(dest, file), {
|
|
150
|
+
packageName: packageName ?? name,
|
|
151
|
+
module: type === "module"
|
|
152
|
+
});
|
|
240
153
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
const ASTAnalysis = runASTAnalysis(str, {
|
|
245
|
-
module: extname(file) === ".mjs" ? true : isProjectUsingESM
|
|
246
|
-
});
|
|
247
|
-
ASTAnalysis.dependencies.removeByName(localPackageName);
|
|
248
|
-
dependencies[file] = ASTAnalysis.dependencies.dependencies;
|
|
249
|
-
warnings.push(...ASTAnalysis.warnings.map((warn) => {
|
|
250
|
-
warn.file = file;
|
|
251
|
-
|
|
252
|
-
return warn;
|
|
253
|
-
}));
|
|
254
|
-
|
|
255
|
-
if (!ASTAnalysis.isOneLineRequire && !file.includes(".min") && isMinified(str)) {
|
|
256
|
-
minified.push(file);
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
catch (err) {
|
|
260
|
-
if (!Reflect.has(err, "code")) {
|
|
261
|
-
warnings.push({ file, kind: "parsing-error", value: err.message, location: [[0, 0], [0, 0]] });
|
|
262
|
-
}
|
|
154
|
+
warnings.push(...result.warnings.map((curr) => Object.assign({}, curr, { file })));
|
|
155
|
+
if (!result.ok) {
|
|
156
|
+
continue;
|
|
263
157
|
}
|
|
158
|
+
|
|
159
|
+
dependencies[file] = result.dependencies.dependencies;
|
|
160
|
+
result.isMinified && minified.push(file);
|
|
264
161
|
}
|
|
265
162
|
|
|
266
163
|
await timers.setImmediate();
|
|
267
164
|
const { uniqueLicenseIds, licenses } = await ntlp(dest);
|
|
268
|
-
ext.delete("");
|
|
269
165
|
|
|
270
166
|
return {
|
|
271
167
|
files: { list: files, extensions: [...ext], minified },
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @param {Set<string>} flags
|
|
3
|
+
* @param {import("../../types/scanner").Dependency} descriptor
|
|
4
|
+
*/
|
|
5
|
+
export function* addMissingVersionFlags(flags, descriptor) {
|
|
6
|
+
const { metadata, vulnerabilities = [], versions } = descriptor;
|
|
7
|
+
const semverVersions = Object.keys(versions);
|
|
8
|
+
|
|
9
|
+
if (!metadata.hasReceivedUpdateInOneYear && flags.has("hasOutdatedDependency") && !flags.has("isDead")) {
|
|
10
|
+
yield "isDead";
|
|
11
|
+
}
|
|
12
|
+
if (metadata.hasManyPublishers && !flags.has("hasManyPublishers")) {
|
|
13
|
+
yield "hasManyPublishers";
|
|
14
|
+
}
|
|
15
|
+
if (metadata.hasChangedAuthor && !flags.has("hasChangedAuthor")) {
|
|
16
|
+
yield "hasChangedAuthor";
|
|
17
|
+
}
|
|
18
|
+
if (vulnerabilities.length > 0 && !flags.has("hasVulnerabilities")) {
|
|
19
|
+
yield "hasVulnerabilities";
|
|
20
|
+
}
|
|
21
|
+
if (semverVersions.length > 1 && !flags.has("hasDuplicate")) {
|
|
22
|
+
yield "hasDuplicate";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Import Third-party Dependencies
|
|
2
|
+
import difference from "lodash.difference";
|
|
3
|
+
import builtins from "builtins";
|
|
4
|
+
|
|
5
|
+
// Import Internal Dependencies
|
|
6
|
+
import { getPackageName } from "./getPackageName.js";
|
|
7
|
+
|
|
8
|
+
// CONSTANTS
|
|
9
|
+
const kNodeModules = new Set(builtins({ experimental: true }));
|
|
10
|
+
const kExternalModules = new Set(["http", "https", "net", "http2", "dgram", "child_process"]);
|
|
11
|
+
|
|
12
|
+
export function analyzeDependencies(dependencies, deps = {}) {
|
|
13
|
+
const { packageDeps, packageDevDeps, tryDependencies } = deps;
|
|
14
|
+
|
|
15
|
+
const thirdPartyDependencies = dependencies
|
|
16
|
+
.map((name) => (packageDeps.includes(name) ? name : getPackageName(name)))
|
|
17
|
+
.filter((name) => !name.startsWith("."))
|
|
18
|
+
.filter((name) => !kNodeModules.has(name))
|
|
19
|
+
.filter((name) => !packageDevDeps.includes(name))
|
|
20
|
+
.filter((name) => !tryDependencies.has(name));
|
|
21
|
+
|
|
22
|
+
const unusedDependencies = difference(
|
|
23
|
+
packageDeps.filter((name) => !name.startsWith("@types")),
|
|
24
|
+
thirdPartyDependencies
|
|
25
|
+
);
|
|
26
|
+
const missingDependencies = [...new Set(difference(thirdPartyDependencies, packageDeps))];
|
|
27
|
+
const nodeDependencies = dependencies.filter((name) => kNodeModules.has(name));
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
nodeDependencies,
|
|
31
|
+
thirdPartyDependencies,
|
|
32
|
+
unusedDependencies,
|
|
33
|
+
missingDependencies,
|
|
34
|
+
|
|
35
|
+
flags: {
|
|
36
|
+
hasExternalCapacity: nodeDependencies.some((depName) => kExternalModules.has(depName)),
|
|
37
|
+
hasMissingOrUnusedDependency: unusedDependencies.length > 0 || missingDependencies.length > 0
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @param {Record<string, boolean>} flagsRecord
|
|
3
|
+
* @example
|
|
4
|
+
* console.log(...booleanToFlags({ hasScript: true })); // "hasScript"
|
|
5
|
+
*/
|
|
6
|
+
export function* booleanToFlags(flagsRecord) {
|
|
7
|
+
for (const [flagName, boolValue] of Object.entries(flagsRecord)) {
|
|
8
|
+
if (boolValue) {
|
|
9
|
+
yield flagName;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Import Node.js Dependencies
|
|
2
|
+
import path from "path";
|
|
3
|
+
|
|
4
|
+
// CONSTANTS
|
|
5
|
+
const kRelativeImportPath = new Set([".", "..", "./", "../"]);
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @see https://nodejs.org/docs/latest/api/modules.html#file-modules
|
|
9
|
+
*
|
|
10
|
+
* @param {IterableIterator<string>} dependencies
|
|
11
|
+
* @param {!string} relativeFileLocation
|
|
12
|
+
*/
|
|
13
|
+
export function filterDependencyKind(dependencies, relativeFileLocation) {
|
|
14
|
+
const packages = [];
|
|
15
|
+
const files = [];
|
|
16
|
+
|
|
17
|
+
for (const moduleNameOrPath of dependencies) {
|
|
18
|
+
const firstChar = moduleNameOrPath.charAt(0);
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @example
|
|
22
|
+
* require("..");
|
|
23
|
+
* require("/home/marco/foo.js");
|
|
24
|
+
*/
|
|
25
|
+
if (firstChar === "." || firstChar === "/") {
|
|
26
|
+
// Note: condition only possible for CJS
|
|
27
|
+
if (kRelativeImportPath.has(moduleNameOrPath)) {
|
|
28
|
+
files.push(path.join(moduleNameOrPath, "index.js"));
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
// Note: we are speculating that the extension is .js (but it could be .json or .node)
|
|
32
|
+
const fixedFileName = path.extname(moduleNameOrPath) === "" ?
|
|
33
|
+
`${moduleNameOrPath}.js` : moduleNameOrPath;
|
|
34
|
+
|
|
35
|
+
files.push(path.join(relativeFileLocation, fixedFileName));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
packages.push(moduleNameOrPath);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return { packages, files };
|
|
44
|
+
}
|
|
@@ -1,5 +1,21 @@
|
|
|
1
|
+
// CONSTANTS
|
|
2
|
+
const kPackageSeparator = "/";
|
|
3
|
+
const kPackageOrgSymbol = "@";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @see https://github.com/npm/validate-npm-package-name#naming-rules
|
|
7
|
+
*
|
|
8
|
+
* @param {!string} name full package name
|
|
9
|
+
* @returns {string}
|
|
10
|
+
*
|
|
11
|
+
* @example
|
|
12
|
+
* getPackageName("foo"); // foo
|
|
13
|
+
* getPackageName("foo/bar"); // foo
|
|
14
|
+
* getPackageName("@org/bar"); // @org/bar
|
|
15
|
+
*/
|
|
1
16
|
export function getPackageName(name) {
|
|
2
|
-
const parts = name.split(
|
|
17
|
+
const parts = name.split(kPackageSeparator);
|
|
3
18
|
|
|
4
|
-
|
|
19
|
+
// Note: only scoped package are allowed to start with @
|
|
20
|
+
return name.startsWith(kPackageOrgSymbol) ? `${parts[0]}/${parts[1]}` : parts[0];
|
|
5
21
|
}
|
|
@@ -21,16 +21,14 @@ export async function getTarballComposition(tarballDir) {
|
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
// ignore
|
|
33
|
-
}
|
|
24
|
+
const sizeUnfilteredResult = await Promise.allSettled([
|
|
25
|
+
...files.map((file) => fs.stat(file)),
|
|
26
|
+
...dirs.map((file) => fs.stat(file))
|
|
27
|
+
]);
|
|
28
|
+
const sizeAll = sizeUnfilteredResult
|
|
29
|
+
.filter((promiseSettledResult) => promiseSettledResult.status === "fulfilled")
|
|
30
|
+
.map((promiseSettledResult) => promiseSettledResult.value);
|
|
31
|
+
size += sizeAll.reduce((prev, curr) => prev + curr.size, 0);
|
|
34
32
|
|
|
35
33
|
return {
|
|
36
34
|
ext,
|