linguist-js 2.9.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +1 -1
- package/dist/analyser/classifiers/byAttributes.d.ts +3 -0
- package/dist/analyser/classifiers/byAttributes.js +13 -0
- package/dist/analyser/classifiers/byExtension.d.ts +3 -0
- package/dist/analyser/classifiers/byExtension.js +31 -0
- package/dist/analyser/classifiers/byFilename.d.ts +3 -0
- package/dist/analyser/classifiers/byFilename.js +10 -0
- package/dist/analyser/classifiers/byHeuristics.d.ts +3 -0
- package/dist/analyser/classifiers/byHeuristics.js +67 -0
- package/dist/analyser/classifiers/byModeline.d.ts +3 -0
- package/dist/analyser/classifiers/byModeline.js +22 -0
- package/dist/analyser/classifiers/byShebang.d.ts +3 -0
- package/dist/analyser/classifiers/byShebang.js +27 -0
- package/dist/analyser/index.d.ts +3 -0
- package/dist/analyser/index.js +12 -0
- package/dist/analyser/pipeline/aggregate.d.ts +4 -0
- package/dist/analyser/pipeline/aggregate.js +98 -0
- package/dist/analyser/pipeline/classify.d.ts +3 -0
- package/dist/analyser/pipeline/classify.js +32 -0
- package/dist/analyser/pipeline/filter.d.ts +2 -0
- package/dist/analyser/pipeline/filter.js +14 -0
- package/dist/analyser/pipeline/heuristics.d.ts +3 -0
- package/dist/analyser/pipeline/heuristics.js +12 -0
- package/dist/analyser/pipeline/normalise.d.ts +2 -0
- package/dist/analyser/pipeline/normalise.js +9 -0
- package/dist/cli/output/default.d.ts +3 -0
- package/dist/cli/output/default.js +106 -0
- package/dist/cli/output/tree.d.ts +3 -0
- package/dist/cli/output/tree.js +11 -0
- package/dist/cli/runCliAnalysis.d.ts +2 -0
- package/dist/cli/runCliAnalysis.js +26 -0
- package/dist/cli/utils.d.ts +2 -0
- package/dist/cli/utils.js +9 -0
- package/dist/cli.js +14 -143
- package/dist/entry/analyseFs.d.ts +2 -0
- package/dist/entry/analyseFs.js +11 -0
- package/dist/entry/analyseRaw.d.ts +4 -0
- package/dist/entry/analyseRaw.js +11 -0
- package/dist/index.d.ts +19 -4
- package/dist/index.js +18 -483
- package/dist/input/fromFilesystem.d.ts +4 -0
- package/dist/input/fromFilesystem.js +115 -0
- package/dist/input/fromRawContent.d.ts +2 -0
- package/dist/input/fromRawContent.js +19 -0
- package/dist/input/normaliseOpts.d.ts +2 -0
- package/dist/input/normaliseOpts.js +13 -0
- package/dist/program/classes/attributes.d.ts +11 -0
- package/dist/program/classes/attributes.js +37 -0
- package/dist/program/data/loadDataFiles.d.ts +4 -0
- package/dist/program/data/loadDataFiles.js +32 -0
- package/dist/program/data/retrieveData.d.ts +12 -0
- package/dist/program/data/retrieveData.js +27 -0
- package/dist/program/fs/normalisedPath.d.ts +3 -0
- package/dist/program/fs/normalisedPath.js +17 -0
- package/dist/program/fs/readFile.js +17 -0
- package/dist/program/fs/walkTree.d.ts +20 -0
- package/dist/program/fs/walkTree.js +82 -0
- package/dist/program/parsing/parseGitattributes.d.ts +17 -0
- package/dist/program/parsing/parseGitattributes.js +33 -0
- package/dist/program/parsing/parseGitignore.js +9 -0
- package/dist/program/processFiles.d.ts +9 -0
- package/dist/program/processFiles.js +111 -0
- package/dist/program/utils/pcre.js +35 -0
- package/dist/src/cli/output/default.d.ts +3 -0
- package/dist/src/cli/output/default.js +106 -0
- package/dist/src/cli/output/tree.d.ts +3 -0
- package/dist/src/cli/output/tree.js +11 -0
- package/dist/src/cli/runCliAnalysis.d.ts +2 -0
- package/dist/src/cli/runCliAnalysis.js +25 -0
- package/dist/src/cli/utils.d.ts +2 -0
- package/dist/src/cli/utils.js +9 -0
- package/dist/src/cli.d.ts +1 -0
- package/dist/src/cli.js +56 -0
- package/dist/src/helpers/convert-pcre.d.ts +2 -0
- package/dist/src/helpers/parse-gitignore.d.ts +1 -0
- package/dist/src/helpers/read-file.d.ts +5 -0
- package/dist/{helpers → src/helpers}/read-file.js +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.js +470 -0
- package/dist/src/program/data/loadData.d.ts +4 -0
- package/dist/src/program/data/loadData.js +32 -0
- package/dist/src/program/fs/normalisedPath.d.ts +2 -0
- package/dist/src/program/fs/normalisedPath.js +7 -0
- package/dist/src/program/fs/readFile.d.ts +5 -0
- package/dist/src/program/fs/readFile.js +17 -0
- package/dist/src/program/fs/walkTree.d.ts +20 -0
- package/dist/src/program/fs/walkTree.js +82 -0
- package/dist/src/program/parsing/parseGitattributes.d.ts +17 -0
- package/dist/src/program/parsing/parseGitattributes.js +33 -0
- package/dist/src/program/parsing/parseGitignore.d.ts +1 -0
- package/dist/src/program/parsing/parseGitignore.js +9 -0
- package/dist/src/program/utils/pcre.d.ts +2 -0
- package/dist/src/program/utils/pcre.js +35 -0
- package/dist/src/types/schema.d.ts +37 -0
- package/dist/src/types/schema.js +1 -0
- package/dist/src/types/types.d.ts +65 -0
- package/dist/src/types/types.js +1 -0
- package/dist/types/schema.d.ts +37 -0
- package/dist/types/schema.js +1 -0
- package/dist/types/types.d.ts +84 -0
- package/dist/types/types.js +1 -0
- package/ext/generated.rb +6 -0
- package/ext/heuristics.yml +102 -11
- package/ext/languages.yml +526 -88
- package/ext/vendor.yml +1 -0
- package/package.json +14 -13
- package/readme.md +99 -111
- /package/dist/{helpers/read-file.d.ts → program/fs/readFile.d.ts} +0 -0
- /package/dist/{helpers/parse-gitignore.d.ts → program/parsing/parseGitignore.d.ts} +0 -0
- /package/dist/{helpers/convert-pcre.d.ts → program/utils/pcre.d.ts} +0 -0
- /package/dist/{helpers → src/helpers}/convert-pcre.js +0 -0
- /package/dist/{helpers → src/helpers}/load-data.d.ts +0 -0
- /package/dist/{helpers → src/helpers}/load-data.js +0 -0
- /package/dist/{helpers → src/helpers}/norm-path.d.ts +0 -0
- /package/dist/{helpers → src/helpers}/norm-path.js +0 -0
- /package/dist/{helpers → src/helpers}/parse-gitattributes.d.ts +0 -0
- /package/dist/{helpers → src/helpers}/parse-gitattributes.js +0 -0
- /package/dist/{helpers → src/helpers}/parse-gitignore.js +0 -0
- /package/dist/{helpers → src/helpers}/walk-tree.d.ts +0 -0
- /package/dist/{helpers → src/helpers}/walk-tree.js +0 -0
- /package/dist/{schema.d.ts → src/schema.d.ts} +0 -0
- /package/dist/{schema.js → src/schema.js} +0 -0
- /package/dist/{types.d.ts → src/types.d.ts} +0 -0
- /package/dist/{types.js → src/types.js} +0 -0
package/bin/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
2
|
+
import '../dist/cli.js';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export default function byAttributes(file, langData) {
|
|
2
|
+
// Check manual override from gitattributes or explicit language metadata
|
|
3
|
+
const requestedLang = file.attributes?.language;
|
|
4
|
+
if (!requestedLang)
|
|
5
|
+
return [];
|
|
6
|
+
// If specified language is an alias, associate it with its full name
|
|
7
|
+
const normalizedLang = Object.keys(langData).find((lang) => {
|
|
8
|
+
const aliases = langData[lang]?.aliases ?? [];
|
|
9
|
+
return (lang.toLowerCase() === requestedLang.toLowerCase() ||
|
|
10
|
+
aliases.some((alias) => alias.toLowerCase() === requestedLang.toLowerCase()));
|
|
11
|
+
});
|
|
12
|
+
return [normalizedLang ?? requestedLang];
|
|
13
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { getFileExtension } from '../../program/fs/normalisedPath.js';
|
|
2
|
+
function isComplexExt(ext) {
|
|
3
|
+
return /\..+\./.test(ext);
|
|
4
|
+
}
|
|
5
|
+
export default function byExtension(file, langData) {
|
|
6
|
+
// Check if extension is a match
|
|
7
|
+
const extension = file.extension ?? getFileExtension(file.path);
|
|
8
|
+
if (!extension)
|
|
9
|
+
return [];
|
|
10
|
+
const possible = [];
|
|
11
|
+
for (const [lang, data] of Object.entries(langData)) {
|
|
12
|
+
const extMatches = data.extensions?.filter((ext) => file.path.toLowerCase().endsWith(ext.toLowerCase()));
|
|
13
|
+
if (extMatches?.length) {
|
|
14
|
+
for (const ext of extMatches) {
|
|
15
|
+
possible.push({ ext, lang });
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
// Apply more specific extension if available
|
|
20
|
+
const hasComplexExt = possible.some((entry) => isComplexExt(entry.ext));
|
|
21
|
+
return possible
|
|
22
|
+
.filter((entry) => {
|
|
23
|
+
const complex = isComplexExt(entry.ext);
|
|
24
|
+
if (hasComplexExt && !complex)
|
|
25
|
+
return false;
|
|
26
|
+
if (!hasComplexExt && complex)
|
|
27
|
+
return false;
|
|
28
|
+
return true;
|
|
29
|
+
})
|
|
30
|
+
.map((entry) => entry.lang);
|
|
31
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import Path from 'node:path';
|
|
2
|
+
export default function byFilename(file, langData) {
|
|
3
|
+
const filename = Path.basename(file.path).toLowerCase();
|
|
4
|
+
// Check if filename is a match
|
|
5
|
+
return Object.entries(langData)
|
|
6
|
+
.flatMap(([lang, data]) => {
|
|
7
|
+
const matches = data.filenames?.some((name) => name.toLowerCase() === filename);
|
|
8
|
+
return matches ? [lang] : [];
|
|
9
|
+
});
|
|
10
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import pcre from '../../program/utils/pcre.js';
|
|
2
|
+
import byFilename from './byFilename.js';
|
|
3
|
+
import byModeline from './byModeline.js';
|
|
4
|
+
import byShebang from './byShebang.js';
|
|
5
|
+
function collectPatterns(heuristic, heuristicsData) {
|
|
6
|
+
const patterns = [];
|
|
7
|
+
const add = (value) => {
|
|
8
|
+
if (!value)
|
|
9
|
+
return;
|
|
10
|
+
if (Array.isArray(value)) {
|
|
11
|
+
patterns.push(...value);
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
patterns.push(value);
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
add(heuristic.pattern);
|
|
18
|
+
add(heuristicsData.named_patterns?.[heuristic.named_pattern]);
|
|
19
|
+
if (heuristic.and) {
|
|
20
|
+
for (const entry of heuristic.and) {
|
|
21
|
+
add(entry.pattern);
|
|
22
|
+
add(heuristicsData.named_patterns?.[entry.named_pattern]);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return patterns;
|
|
26
|
+
}
|
|
27
|
+
export default function byHeuristics(file, candidateLanguages, heuristicsData, langData, opts) {
|
|
28
|
+
// Parse heuristics if applicable
|
|
29
|
+
if (!opts.checkHeuristics || file.content === undefined)
|
|
30
|
+
return [];
|
|
31
|
+
// Skip if file has explicit association
|
|
32
|
+
if (file.attributes?.language)
|
|
33
|
+
return [];
|
|
34
|
+
if (byFilename(file, langData).length)
|
|
35
|
+
return [];
|
|
36
|
+
if (byShebang(file, langData, opts).length)
|
|
37
|
+
return [];
|
|
38
|
+
if (byModeline(file, langData, opts).length)
|
|
39
|
+
return [];
|
|
40
|
+
if (candidateLanguages.length <= 1)
|
|
41
|
+
return [];
|
|
42
|
+
const extension = file.extension ?? '';
|
|
43
|
+
for (const heuristics of heuristicsData.disambiguations ?? []) {
|
|
44
|
+
// Make sure the extension matches the current file
|
|
45
|
+
if (!heuristics.extensions.includes(extension))
|
|
46
|
+
continue;
|
|
47
|
+
// Load heuristic rules
|
|
48
|
+
for (const heuristic of heuristics.rules) {
|
|
49
|
+
// Make sure the language is not an array
|
|
50
|
+
const language = Array.isArray(heuristic.language) ? heuristic.language[0] : heuristic.language;
|
|
51
|
+
const languageGroup = langData[language]?.group;
|
|
52
|
+
const matchesLang = candidateLanguages.includes(language);
|
|
53
|
+
const matchesParent = languageGroup ? candidateLanguages.includes(languageGroup) : false;
|
|
54
|
+
// Make sure the results includes this language
|
|
55
|
+
if (!matchesLang && !matchesParent)
|
|
56
|
+
continue;
|
|
57
|
+
// Normalise heuristic data
|
|
58
|
+
const patterns = collectPatterns(heuristic, heuristicsData);
|
|
59
|
+
// Check file contents and apply heuristic patterns
|
|
60
|
+
if (!patterns.length || patterns.some((pattern) => pcre(pattern).test(file.content))) {
|
|
61
|
+
// Apply heuristics
|
|
62
|
+
return [language];
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return [];
|
|
67
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
const modelineRegex = /-\*-|(?:syntax|filetype|ft)\s*=/;
|
|
2
|
+
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
3
|
+
export default function byModeline(file, langData, opts) {
|
|
4
|
+
if (!opts.checkModeline || opts.quick)
|
|
5
|
+
return [];
|
|
6
|
+
const firstLine = file.firstLine;
|
|
7
|
+
// Check modeline declaration
|
|
8
|
+
if (!firstLine || !modelineRegex.test(firstLine))
|
|
9
|
+
return [];
|
|
10
|
+
const modelineText = firstLine.toLowerCase().split(modelineRegex)[1] ?? '';
|
|
11
|
+
// Add identified language(s)
|
|
12
|
+
return Object.entries(langData).flatMap(([lang, data]) => {
|
|
13
|
+
const safeLang = escapeRegExp(lang.toLowerCase());
|
|
14
|
+
const matchesLang = new RegExp(`\\b${safeLang}(?![\\w#+*]|-\\*-)`).test(modelineText);
|
|
15
|
+
const aliases = data.aliases ?? [];
|
|
16
|
+
const matchesAlias = aliases.some((alias) => {
|
|
17
|
+
const safeAlias = escapeRegExp(alias.toLowerCase());
|
|
18
|
+
return new RegExp(`\\b${safeAlias}(?![\\w#+*]|-\\*-)`).test(modelineText);
|
|
19
|
+
});
|
|
20
|
+
return matchesLang || matchesAlias ? [lang] : [];
|
|
21
|
+
});
|
|
22
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
2
|
+
const categoryPriority = {
|
|
3
|
+
programming: 0,
|
|
4
|
+
markup: 1,
|
|
5
|
+
prose: 2,
|
|
6
|
+
data: 3,
|
|
7
|
+
};
|
|
8
|
+
export default function byShebang(file, langData, opts) {
|
|
9
|
+
if (!opts.checkShebang || opts.quick)
|
|
10
|
+
return [];
|
|
11
|
+
const firstLine = file.firstLine;
|
|
12
|
+
if (!firstLine || !firstLine.startsWith('#!'))
|
|
13
|
+
return [];
|
|
14
|
+
const matches = Object.entries(langData).flatMap(([lang, data]) => {
|
|
15
|
+
const interpreters = data.interpreters ?? [];
|
|
16
|
+
// Check for interpreter match
|
|
17
|
+
const matchesInterpretor = interpreters.some((interpreter) => new RegExp(`\\b${escapeRegExp(interpreter)}\\b`).test(firstLine));
|
|
18
|
+
return matchesInterpretor ? [lang] : [];
|
|
19
|
+
});
|
|
20
|
+
return [...new Set(matches)].sort((a, b) => {
|
|
21
|
+
const aType = langData[a]?.type;
|
|
22
|
+
const bType = langData[b]?.type;
|
|
23
|
+
const aPriority = categoryPriority[aType ?? ''] ?? 4;
|
|
24
|
+
const bPriority = categoryPriority[bType ?? ''] ?? 4;
|
|
25
|
+
return aPriority - bPriority;
|
|
26
|
+
});
|
|
27
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { aggregateResults } from './pipeline/aggregate.js';
|
|
2
|
+
import { classifyFiles } from './pipeline/classify.js';
|
|
3
|
+
import { filterFiles } from './pipeline/filter.js';
|
|
4
|
+
import { resolveHeuristics } from './pipeline/heuristics.js';
|
|
5
|
+
import { normaliseFiles } from './pipeline/normalise.js';
|
|
6
|
+
export default function analyseVirtualFiles(files, langData, heuristicsData, opts) {
|
|
7
|
+
const normalizedFiles = normaliseFiles(files);
|
|
8
|
+
const filteredFiles = filterFiles(normalizedFiles, opts);
|
|
9
|
+
const classifications = classifyFiles(filteredFiles, langData, opts);
|
|
10
|
+
const heuristicResolutions = resolveHeuristics(filteredFiles, classifications, heuristicsData, langData, opts);
|
|
11
|
+
return aggregateResults(filteredFiles, classifications, heuristicResolutions, langData, opts);
|
|
12
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import * as T from '../../types/types.js';
|
|
2
|
+
export declare function aggregateResults(files: T.VirtualFile[], classifications: Record<string, string[]>, heuristicResolutions: Record<string, string | undefined>, langData: {
|
|
3
|
+
[language: string]: T.LanguageMetadata;
|
|
4
|
+
}, opts: T.Options): T.Results;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import Path from 'node:path';
|
|
2
|
+
const categoryKeys = ['data', 'markup', 'programming', 'prose'];
|
|
3
|
+
function pickBestLanguage(classifications) {
|
|
4
|
+
// Assign first language as a default option
|
|
5
|
+
return classifications[0] ?? null;
|
|
6
|
+
}
|
|
7
|
+
function makeRelPath(path) {
|
|
8
|
+
let relPath = normPath(Path.relative(process.cwd(), path));
|
|
9
|
+
if (!relPath.startsWith('../') && !relPath.startsWith('./')) {
|
|
10
|
+
relPath = `./${relPath}`;
|
|
11
|
+
}
|
|
12
|
+
return relPath;
|
|
13
|
+
}
|
|
14
|
+
function normPath(filePath) {
|
|
15
|
+
return filePath.replace(/\\\\/g, '/');
|
|
16
|
+
}
|
|
17
|
+
export function aggregateResults(files, classifications, heuristicResolutions, langData, opts) {
|
|
18
|
+
const results = {
|
|
19
|
+
files: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, results: {} },
|
|
20
|
+
languages: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, results: {} },
|
|
21
|
+
unknown: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, extensions: {}, filenames: {} },
|
|
22
|
+
repository: {},
|
|
23
|
+
};
|
|
24
|
+
// Skip specified categories
|
|
25
|
+
const allowedCategories = opts.categories ?? categoryKeys;
|
|
26
|
+
const hiddenCategories = categoryKeys.filter((category) => !allowedCategories.includes(category));
|
|
27
|
+
for (const file of files) {
|
|
28
|
+
// Narrow down file associations to the best fit
|
|
29
|
+
const candidates = classifications[file.path] ?? [];
|
|
30
|
+
// If no heuristics, assign a language
|
|
31
|
+
const bestLanguage = heuristicResolutions[file.path] ?? pickBestLanguage(candidates);
|
|
32
|
+
const selectedLanguage = bestLanguage
|
|
33
|
+
? opts.childLanguages
|
|
34
|
+
? bestLanguage // use the child language
|
|
35
|
+
: (langData[bestLanguage]?.group ?? bestLanguage) // use the parent language, if it exists
|
|
36
|
+
: null;
|
|
37
|
+
// Load language bytes size
|
|
38
|
+
const size = file.size ?? file.content?.length ?? 0;
|
|
39
|
+
// Calculate lines of code
|
|
40
|
+
const lineContent = file.content ?? '';
|
|
41
|
+
const allLines = opts.calculateLines === false ? [] : lineContent.split(/\r?\n/gm);
|
|
42
|
+
const loc = {
|
|
43
|
+
total: opts.calculateLines === false ? NaN : allLines.length,
|
|
44
|
+
content: opts.calculateLines === false ? NaN : allLines.filter((line) => line.trim().length > 0).length,
|
|
45
|
+
};
|
|
46
|
+
const outputPath = opts.relativePaths && Path.isAbsolute(file.path) ? makeRelPath(file.path) : file.path;
|
|
47
|
+
if (!selectedLanguage) {
|
|
48
|
+
const extension = file.extension || Path.extname(file.path);
|
|
49
|
+
const unknownType = extension ? 'extensions' : 'filenames';
|
|
50
|
+
const name = extension || Path.basename(file.path);
|
|
51
|
+
results.files.results[outputPath] = null;
|
|
52
|
+
results.files.bytes += size;
|
|
53
|
+
results.files.lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
|
|
54
|
+
results.files.lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
|
|
55
|
+
results.unknown[unknownType][name] ??= 0;
|
|
56
|
+
results.unknown[unknownType][name] += size;
|
|
57
|
+
results.unknown.bytes += size;
|
|
58
|
+
results.unknown.lines.total += loc.total || 0;
|
|
59
|
+
results.unknown.lines.content += loc.content || 0;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
const languageMeta = langData[selectedLanguage];
|
|
63
|
+
const category = languageMeta?.type;
|
|
64
|
+
const allowed = !hiddenCategories.includes(category ?? 'programming') || file.attributes?.detectable === true;
|
|
65
|
+
if (!allowed) {
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (!results.repository[selectedLanguage]) {
|
|
69
|
+
results.repository[selectedLanguage] = {
|
|
70
|
+
type: languageMeta?.type ?? 'programming',
|
|
71
|
+
color: languageMeta?.color,
|
|
72
|
+
};
|
|
73
|
+
if (opts.childLanguages) {
|
|
74
|
+
results.repository[selectedLanguage].parent = languageMeta?.group;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
results.files.results[outputPath] = selectedLanguage;
|
|
78
|
+
// Apply to files totals
|
|
79
|
+
results.files.bytes += size;
|
|
80
|
+
results.files.lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
|
|
81
|
+
results.files.lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
|
|
82
|
+
results.languages.results[selectedLanguage] ??= { count: 0, bytes: 0, lines: { total: 0, content: 0 } };
|
|
83
|
+
results.languages.results[selectedLanguage].count += 1;
|
|
84
|
+
results.languages.results[selectedLanguage].bytes += size;
|
|
85
|
+
results.languages.results[selectedLanguage].lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
|
|
86
|
+
results.languages.results[selectedLanguage].lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
|
|
87
|
+
results.languages.bytes += size;
|
|
88
|
+
results.languages.lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
|
|
89
|
+
results.languages.lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
|
|
90
|
+
}
|
|
91
|
+
if (opts.calculateLines === false) {
|
|
92
|
+
results.files.lines = { total: NaN, content: NaN };
|
|
93
|
+
}
|
|
94
|
+
results.files.count = Object.keys(results.files.results).length;
|
|
95
|
+
results.languages.count = Object.keys(results.languages.results).length;
|
|
96
|
+
results.unknown.count = Object.keys({ ...results.unknown.extensions, ...results.unknown.filenames }).length;
|
|
97
|
+
return results;
|
|
98
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import byAttributes from '../classifiers/byAttributes.js';
|
|
2
|
+
import byExtension from '../classifiers/byExtension.js';
|
|
3
|
+
import byFilename from '../classifiers/byFilename.js';
|
|
4
|
+
import byModeline from '../classifiers/byModeline.js';
|
|
5
|
+
import byShebang from '../classifiers/byShebang.js';
|
|
6
|
+
function dedupeClassifications(classifications) {
|
|
7
|
+
const seen = new Set();
|
|
8
|
+
const ordered = [];
|
|
9
|
+
for (const classification of classifications) {
|
|
10
|
+
if (!seen.has(classification)) {
|
|
11
|
+
seen.add(classification);
|
|
12
|
+
ordered.push(classification);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return ordered;
|
|
16
|
+
}
|
|
17
|
+
export function classifyFiles(files, langData, opts) {
|
|
18
|
+
const classifications = {};
|
|
19
|
+
for (const file of files) {
|
|
20
|
+
// Search each language
|
|
21
|
+
// Note: order here is important; the first match will be the one chosen
|
|
22
|
+
const candidates = [
|
|
23
|
+
...byAttributes(file, langData),
|
|
24
|
+
...byModeline(file, langData, opts),
|
|
25
|
+
...byFilename(file, langData),
|
|
26
|
+
...byShebang(file, langData, opts),
|
|
27
|
+
...byExtension(file, langData),
|
|
28
|
+
];
|
|
29
|
+
classifications[file.path] = dedupeClassifications(candidates);
|
|
30
|
+
}
|
|
31
|
+
return classifications;
|
|
32
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function filterFiles(files, opts) {
|
|
2
|
+
return files.filter((file) => {
|
|
3
|
+
// Skip binary files
|
|
4
|
+
if (!opts.keepBinary && (file.isBinary || file.attributes?.binary === true)) {
|
|
5
|
+
return false;
|
|
6
|
+
}
|
|
7
|
+
if (!opts.keepVendored &&
|
|
8
|
+
(file.metadata?.vendored === true || file.metadata?.generated === true || file.metadata?.documentation === true)) {
|
|
9
|
+
// Skip vendored, generated, or documentation files
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
return true;
|
|
13
|
+
});
|
|
14
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { LanguagesScema } from '../../types/schema.js';
|
|
2
|
+
import * as T from '../../types/types.js';
|
|
3
|
+
export declare function resolveHeuristics(files: T.VirtualFile[], classifications: Record<string, string[]>, heuristicsData: unknown, langData: LanguagesScema, opts: T.Options): Record<string, string | undefined>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import byHeuristics from '../classifiers/byHeuristics.js';
|
|
2
|
+
export function resolveHeuristics(files, classifications, heuristicsData, langData, opts) {
|
|
3
|
+
const resolved = {};
|
|
4
|
+
for (const file of files) {
|
|
5
|
+
const candidateLanguages = classifications[file.path] ?? [];
|
|
6
|
+
const heuristicLanguage = byHeuristics(file, candidateLanguages, heuristicsData, langData, opts)[0];
|
|
7
|
+
if (heuristicLanguage) {
|
|
8
|
+
resolved[file.path] = heuristicLanguage;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
return resolved;
|
|
12
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { getFileExtension } from '../../program/fs/normalisedPath.js';
|
|
2
|
+
export function normaliseFiles(files) {
|
|
3
|
+
return files.map((file) => ({
|
|
4
|
+
...file,
|
|
5
|
+
extension: file.extension ?? getFileExtension(file.path),
|
|
6
|
+
firstLine: file.firstLine ?? file.content?.split(/\r?\n/)[0],
|
|
7
|
+
size: file.size ?? file.content?.length,
|
|
8
|
+
}));
|
|
9
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import FS from 'node:fs';
|
|
2
|
+
import Path from 'node:path';
|
|
3
|
+
import { normPath } from '../../program/fs/normalisedPath.js';
|
|
4
|
+
import { colouredMsg, hexToRgb } from '../utils.js';
|
|
5
|
+
export default async function defaultOutput(args, data) {
|
|
6
|
+
const { files, languages, unknown, repository } = data;
|
|
7
|
+
if (args.minSize) {
|
|
8
|
+
// Ignore languages with a bytes/% size less than the declared min size
|
|
9
|
+
const totalSize = languages.bytes;
|
|
10
|
+
const minSizeAmt = parseFloat(args.minSize.replace(/[a-z]+$/i, '')); // '2KB' -> 2
|
|
11
|
+
const minSizeUnit = args.minSize.replace(/^\d+/, '').toLowerCase(); // '2KB' -> 'kb'
|
|
12
|
+
const checkBytes = minSizeUnit !== 'loc'; // whether to check bytes or loc
|
|
13
|
+
const conversionFactors = {
|
|
14
|
+
['b']: (n) => n,
|
|
15
|
+
['kb']: (n) => n * 1e3,
|
|
16
|
+
['mb']: (n) => n * 1e6,
|
|
17
|
+
['%']: (n) => (n * totalSize) / 100,
|
|
18
|
+
['loc']: (n) => n,
|
|
19
|
+
};
|
|
20
|
+
const minBytesSize = conversionFactors[minSizeUnit](+minSizeAmt);
|
|
21
|
+
const other = { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 } };
|
|
22
|
+
// Apply specified minimums: delete language results that do not reach the threshold
|
|
23
|
+
for (const [lang, data] of Object.entries(languages.results)) {
|
|
24
|
+
const checkUnit = checkBytes ? data.bytes : data.lines.content;
|
|
25
|
+
if (checkUnit < minBytesSize) {
|
|
26
|
+
// Add to 'other' count
|
|
27
|
+
other.count++;
|
|
28
|
+
other.bytes += data.bytes;
|
|
29
|
+
other.lines.total += data.lines.total;
|
|
30
|
+
other.lines.content += data.lines.content;
|
|
31
|
+
// Remove language result
|
|
32
|
+
delete languages.results[lang];
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (other.bytes) {
|
|
36
|
+
languages.results['Other'] = other;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const sortedEntries = Object.entries(languages.results).sort((a, b) => (a[1].bytes < b[1].bytes ? +1 : -1));
|
|
40
|
+
const totalBytes = languages.bytes;
|
|
41
|
+
console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
|
|
42
|
+
console.log(`\n Language analysis results: \n`);
|
|
43
|
+
let count = 0;
|
|
44
|
+
if (sortedEntries.length === 0)
|
|
45
|
+
console.log(` None`);
|
|
46
|
+
// Collate files per language
|
|
47
|
+
const filesPerLanguage = {};
|
|
48
|
+
if (args.listFiles) {
|
|
49
|
+
for (const language of Object.keys(languages.results)) {
|
|
50
|
+
filesPerLanguage[language] = [];
|
|
51
|
+
}
|
|
52
|
+
for (const [file, lang] of Object.entries(files.results)) {
|
|
53
|
+
if (lang) {
|
|
54
|
+
filesPerLanguage[lang].push(file);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// List parsed results
|
|
59
|
+
for (const [lang, { bytes, lines }] of sortedEntries) {
|
|
60
|
+
const colour = hexToRgb(repository[lang].color ?? '#ededed');
|
|
61
|
+
const percent = (bytes) => (bytes / (totalBytes || 1)) * 100;
|
|
62
|
+
const fmtd = {
|
|
63
|
+
index: (++count).toString().padStart(2, ' '),
|
|
64
|
+
lang: lang.padEnd(24, ' '),
|
|
65
|
+
percent: percent(bytes).toFixed(2).padStart(5, ' '),
|
|
66
|
+
bytes: bytes.toLocaleString().padStart(10, ' '),
|
|
67
|
+
loc: lines.content.toLocaleString().padStart(10, ' '),
|
|
68
|
+
icon: colouredMsg(colour, '\u2588'),
|
|
69
|
+
};
|
|
70
|
+
console.log(` ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B ${fmtd.loc} LOC`);
|
|
71
|
+
// If using `listFiles` option, list all files tagged as this language
|
|
72
|
+
if (args.listFiles) {
|
|
73
|
+
console.log(); // padding
|
|
74
|
+
for (const file of filesPerLanguage[lang]) {
|
|
75
|
+
let relFile = normPath(Path.relative(Path.resolve('.'), file));
|
|
76
|
+
if (!relFile.startsWith('../')) {
|
|
77
|
+
relFile = './' + relFile;
|
|
78
|
+
}
|
|
79
|
+
const fileStat = await FS.promises.stat(file);
|
|
80
|
+
const bytes = fileStat.size;
|
|
81
|
+
const fmtd2 = {
|
|
82
|
+
file: relFile.padEnd(42, ' '),
|
|
83
|
+
percent: percent(bytes).toFixed(2).padStart(5, ' '),
|
|
84
|
+
bytes: bytes.toLocaleString().padStart(10, ' '),
|
|
85
|
+
};
|
|
86
|
+
console.log(` ${fmtd.icon} ${fmtd2.file} ${fmtd2.percent}% ${fmtd2.bytes} B`);
|
|
87
|
+
}
|
|
88
|
+
console.log(); // padding
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (!args.listFiles) {
|
|
92
|
+
console.log(); // padding
|
|
93
|
+
}
|
|
94
|
+
console.log(` Total: ${totalBytes.toLocaleString()} B`);
|
|
95
|
+
// List unknown files/extensions
|
|
96
|
+
if (unknown.bytes > 0) {
|
|
97
|
+
console.log(`\n Unknown files and extensions:`);
|
|
98
|
+
for (const [name, bytes] of Object.entries(unknown.filenames)) {
|
|
99
|
+
console.log(` '${name}': ${bytes.toLocaleString()} B`);
|
|
100
|
+
}
|
|
101
|
+
for (const [ext, bytes] of Object.entries(unknown.extensions)) {
|
|
102
|
+
console.log(` '*${ext}': ${bytes.toLocaleString()} B`);
|
|
103
|
+
}
|
|
104
|
+
console.log(` Total: ${unknown.bytes.toLocaleString()} B`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export default function treeOutput(args, data) {
|
|
2
|
+
const treeParts = args.tree.split('.');
|
|
3
|
+
let nestedData = data;
|
|
4
|
+
for (const part of treeParts) {
|
|
5
|
+
if (!nestedData[part]) {
|
|
6
|
+
throw Error(`TraversalError: Key '${part}' cannot be found on output object.`);
|
|
7
|
+
}
|
|
8
|
+
nestedData = nestedData[part];
|
|
9
|
+
}
|
|
10
|
+
console.log(nestedData);
|
|
11
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import analyseFs from '../entry/analyseFs.js';
|
|
2
|
+
import normaliseOpts from '../input/normaliseOpts.js';
|
|
3
|
+
import defaultOutput from './output/default.js';
|
|
4
|
+
import treeOutput from './output/tree.js';
|
|
5
|
+
const validCategories = ['data', 'programming', 'prose', 'markup'];
|
|
6
|
+
export default async function runCliAnalysis(args) {
|
|
7
|
+
// Check arguments
|
|
8
|
+
if (args.categories?.some((category) => !validCategories.includes(category))) {
|
|
9
|
+
console.log(`Error: '${args.categories.join(', ')}' contains an invalid category.`);
|
|
10
|
+
console.log(`\tValid options: ${validCategories.join(', ')}.`);
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
// Analyse language data
|
|
14
|
+
const folders = args.analyse === true ? ['.'] : args.analyse;
|
|
15
|
+
const data = await analyseFs(folders, normaliseOpts(args));
|
|
16
|
+
// Print output
|
|
17
|
+
if (!args.json) {
|
|
18
|
+
defaultOutput(args, data);
|
|
19
|
+
}
|
|
20
|
+
else if (args.tree) {
|
|
21
|
+
treeOutput(args, data);
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
console.dir(data, { depth: null });
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export function colouredMsg([r, g, b], msg) {
|
|
2
|
+
return `\u001B[${38};2;${r};${g};${b}m${msg}\u001b[0m`;
|
|
3
|
+
}
|
|
4
|
+
export function hexToRgb(hex) {
|
|
5
|
+
const r = parseInt(hex.slice(1, 3), 16);
|
|
6
|
+
const g = parseInt(hex.slice(3, 5), 16);
|
|
7
|
+
const b = parseInt(hex.slice(5, 7), 16);
|
|
8
|
+
return [r, g, b];
|
|
9
|
+
}
|