linguist-js 2.9.1 → 3.0.0-pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/bin/index.js +1 -1
  2. package/dist/analyser/classifiers/byAttributes.d.ts +3 -0
  3. package/dist/analyser/classifiers/byAttributes.js +13 -0
  4. package/dist/analyser/classifiers/byExtension.d.ts +3 -0
  5. package/dist/analyser/classifiers/byExtension.js +31 -0
  6. package/dist/analyser/classifiers/byFilename.d.ts +3 -0
  7. package/dist/analyser/classifiers/byFilename.js +10 -0
  8. package/dist/analyser/classifiers/byHeuristics.d.ts +3 -0
  9. package/dist/analyser/classifiers/byHeuristics.js +67 -0
  10. package/dist/analyser/classifiers/byModeline.d.ts +3 -0
  11. package/dist/analyser/classifiers/byModeline.js +22 -0
  12. package/dist/analyser/classifiers/byShebang.d.ts +3 -0
  13. package/dist/analyser/classifiers/byShebang.js +27 -0
  14. package/dist/analyser/index.d.ts +3 -0
  15. package/dist/analyser/index.js +12 -0
  16. package/dist/analyser/pipeline/aggregate.d.ts +2 -0
  17. package/dist/analyser/pipeline/aggregate.js +98 -0
  18. package/dist/analyser/pipeline/classify.d.ts +3 -0
  19. package/dist/analyser/pipeline/classify.js +31 -0
  20. package/dist/analyser/pipeline/filter.d.ts +2 -0
  21. package/dist/analyser/pipeline/filter.js +14 -0
  22. package/dist/analyser/pipeline/heuristics.d.ts +3 -0
  23. package/dist/analyser/pipeline/heuristics.js +12 -0
  24. package/dist/analyser/pipeline/normalise.d.ts +2 -0
  25. package/dist/analyser/pipeline/normalise.js +9 -0
  26. package/dist/cli/output/default.d.ts +3 -0
  27. package/dist/cli/output/default.js +106 -0
  28. package/dist/cli/output/tree.d.ts +3 -0
  29. package/dist/cli/output/tree.js +11 -0
  30. package/dist/cli/runCliAnalysis.d.ts +2 -0
  31. package/dist/cli/runCliAnalysis.js +26 -0
  32. package/dist/cli/utils.d.ts +2 -0
  33. package/dist/cli/utils.js +9 -0
  34. package/dist/cli.js +14 -143
  35. package/dist/entry/analyseFs.d.ts +2 -0
  36. package/dist/entry/analyseFs.js +11 -0
  37. package/dist/entry/analyseRaw.d.ts +4 -0
  38. package/dist/entry/analyseRaw.js +11 -0
  39. package/dist/index.d.ts +19 -4
  40. package/dist/index.js +18 -483
  41. package/dist/input/fromFilesystem.d.ts +4 -0
  42. package/dist/input/fromFilesystem.js +115 -0
  43. package/dist/input/fromRawContent.d.ts +2 -0
  44. package/dist/input/fromRawContent.js +19 -0
  45. package/dist/input/normaliseOpts.d.ts +2 -0
  46. package/dist/input/normaliseOpts.js +13 -0
  47. package/dist/program/classes/attributes.d.ts +11 -0
  48. package/dist/program/classes/attributes.js +37 -0
  49. package/dist/{helpers/load-data.d.ts → program/data/loadDataFiles.d.ts} +1 -1
  50. package/dist/program/data/loadDataFiles.js +32 -0
  51. package/dist/program/data/retrieveData.d.ts +12 -0
  52. package/dist/program/data/retrieveData.js +27 -0
  53. package/dist/{helpers/norm-path.d.ts → program/fs/normalisedPath.d.ts} +1 -0
  54. package/dist/program/fs/normalisedPath.js +17 -0
  55. package/dist/program/fs/readFile.js +17 -0
  56. package/dist/{helpers/walk-tree.d.ts → program/fs/walkTree.d.ts} +1 -1
  57. package/dist/{helpers/walk-tree.js → program/fs/walkTree.js} +22 -30
  58. package/dist/{helpers/parse-gitattributes.d.ts → program/parsing/parseGitattributes.d.ts} +2 -2
  59. package/dist/{helpers/parse-gitattributes.js → program/parsing/parseGitattributes.js} +4 -8
  60. package/dist/{helpers/parse-gitignore.js → program/parsing/parseGitignore.js} +1 -4
  61. package/dist/program/processFiles.d.ts +9 -0
  62. package/dist/program/processFiles.js +111 -0
  63. package/dist/{helpers/convert-pcre.js → program/utils/pcre.js} +1 -4
  64. package/dist/{schema.d.ts → types/schema.d.ts} +1 -1
  65. package/dist/types/schema.js +1 -0
  66. package/dist/{types.d.ts → types/types.d.ts} +35 -24
  67. package/dist/types/types.js +1 -0
  68. package/ext/generated.rb +6 -0
  69. package/ext/heuristics.yml +103 -12
  70. package/ext/languages.yml +573 -97
  71. package/ext/vendor.yml +1 -0
  72. package/package.json +14 -13
  73. package/readme.md +82 -87
  74. package/dist/helpers/load-data.js +0 -37
  75. package/dist/helpers/norm-path.js +0 -15
  76. package/dist/helpers/read-file.js +0 -23
  77. package/dist/schema.js +0 -2
  78. package/dist/types.js +0 -2
  79. /package/dist/{helpers/read-file.d.ts → program/fs/readFile.d.ts} +0 -0
  80. /package/dist/{helpers/parse-gitignore.d.ts → program/parsing/parseGitignore.d.ts} +0 -0
  81. /package/dist/{helpers/convert-pcre.d.ts → program/utils/pcre.d.ts} +0 -0
package/bin/index.js CHANGED
@@ -1,2 +1,2 @@
1
1
  #!/usr/bin/env node
2
- require('../dist/cli.js');
2
+ import '../dist/cli.js';
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export default function byAttributes(file: T.VirtualFile, langData: LanguagesScema): string[];
@@ -0,0 +1,13 @@
1
+ export default function byAttributes(file, langData) {
2
+ // Check manual override from gitattributes or explicit language metadata
3
+ const requestedLang = file.attributes?.language;
4
+ if (!requestedLang)
5
+ return [];
6
+ // If specified language is an alias, associate it with its full name
7
+ const normalizedLang = Object.keys(langData).find((lang) => {
8
+ const aliases = langData[lang]?.aliases ?? [];
9
+ return (lang.toLowerCase() === requestedLang.toLowerCase() ||
10
+ aliases.some((alias) => alias.toLowerCase() === requestedLang.toLowerCase()));
11
+ });
12
+ return [normalizedLang ?? requestedLang];
13
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export default function byExtension(file: T.VirtualFile, langData: LanguagesScema): string[];
@@ -0,0 +1,31 @@
1
+ import { getFileExtension } from '../../program/fs/normalisedPath.js';
2
+ function isComplexExt(ext) {
3
+ return /\..+\./.test(ext);
4
+ }
5
+ export default function byExtension(file, langData) {
6
+ // Check if extension is a match
7
+ const extension = file.extension ?? getFileExtension(file.path);
8
+ if (!extension)
9
+ return [];
10
+ const possible = [];
11
+ for (const [lang, data] of Object.entries(langData)) {
12
+ const extMatches = data.extensions?.filter((ext) => file.path.toLowerCase().endsWith(ext.toLowerCase()));
13
+ if (extMatches?.length) {
14
+ for (const ext of extMatches) {
15
+ possible.push({ ext, lang });
16
+ }
17
+ }
18
+ }
19
+ // Apply more specific extension if available
20
+ const hasComplexExt = possible.some((entry) => isComplexExt(entry.ext));
21
+ return possible
22
+ .filter((entry) => {
23
+ const complex = isComplexExt(entry.ext);
24
+ if (hasComplexExt && !complex)
25
+ return false;
26
+ if (!hasComplexExt && complex)
27
+ return false;
28
+ return true;
29
+ })
30
+ .map((entry) => entry.lang);
31
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export default function byFilename(file: T.VirtualFile, langData: LanguagesScema): string[];
@@ -0,0 +1,10 @@
1
+ import Path from 'node:path';
2
+ export default function byFilename(file, langData) {
3
+ const filename = Path.basename(file.path).toLowerCase();
4
+ // Check if filename is a match
5
+ return Object.entries(langData)
6
+ .flatMap(([lang, data]) => {
7
+ const matches = data.filenames?.some((name) => name.toLowerCase() === filename);
8
+ return matches ? [lang] : [];
9
+ });
10
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export default function byHeuristics(file: T.VirtualFile, candidateLanguages: string[], heuristicsData: any, langData: LanguagesScema, opts: T.Options): string[];
@@ -0,0 +1,67 @@
1
+ import pcre from '../../program/utils/pcre.js';
2
+ import byFilename from './byFilename.js';
3
+ import byModeline from './byModeline.js';
4
+ import byShebang from './byShebang.js';
5
+ function collectPatterns(heuristic, heuristicsData) {
6
+ const patterns = [];
7
+ const add = (value) => {
8
+ if (!value)
9
+ return;
10
+ if (Array.isArray(value)) {
11
+ patterns.push(...value);
12
+ }
13
+ else {
14
+ patterns.push(value);
15
+ }
16
+ };
17
+ add(heuristic.pattern);
18
+ add(heuristicsData.named_patterns?.[heuristic.named_pattern]);
19
+ if (heuristic.and) {
20
+ for (const entry of heuristic.and) {
21
+ add(entry.pattern);
22
+ add(heuristicsData.named_patterns?.[entry.named_pattern]);
23
+ }
24
+ }
25
+ return patterns;
26
+ }
27
+ export default function byHeuristics(file, candidateLanguages, heuristicsData, langData, opts) {
28
+ // Parse heuristics if applicable
29
+ if (!opts.checkHeuristics || file.content === undefined)
30
+ return [];
31
+ // Skip if file has explicit association
32
+ if (file.attributes?.language)
33
+ return [];
34
+ if (byFilename(file, langData).length)
35
+ return [];
36
+ if (byShebang(file, langData, opts).length)
37
+ return [];
38
+ if (byModeline(file, langData, opts).length)
39
+ return [];
40
+ if (candidateLanguages.length <= 1)
41
+ return [];
42
+ const extension = file.extension ?? '';
43
+ for (const heuristics of heuristicsData.disambiguations ?? []) {
44
+ // Make sure the extension matches the current file
45
+ if (!heuristics.extensions.includes(extension))
46
+ continue;
47
+ // Load heuristic rules
48
+ for (const heuristic of heuristics.rules) {
49
+ // Make sure the language is not an array
50
+ const language = Array.isArray(heuristic.language) ? heuristic.language[0] : heuristic.language;
51
+ const languageGroup = langData[language]?.group;
52
+ const matchesLang = candidateLanguages.includes(language);
53
+ const matchesParent = languageGroup ? candidateLanguages.includes(languageGroup) : false;
54
+ // Make sure the results includes this language
55
+ if (!matchesLang && !matchesParent)
56
+ continue;
57
+ // Normalise heuristic data
58
+ const patterns = collectPatterns(heuristic, heuristicsData);
59
+ // Check file contents and apply heuristic patterns
60
+ if (!patterns.length || patterns.some((pattern) => pcre(pattern).test(file.content))) {
61
+ // Apply heuristics
62
+ return [language];
63
+ }
64
+ }
65
+ }
66
+ return [];
67
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export default function byModeline(file: T.VirtualFile, langData: LanguagesScema, opts: T.Options): string[];
@@ -0,0 +1,22 @@
1
+ const modelineRegex = /-\*-|(?:syntax|filetype|ft)\s*=/;
2
+ const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
3
+ export default function byModeline(file, langData, opts) {
4
+ if (!opts.checkModeline || opts.quick)
5
+ return [];
6
+ const firstLine = file.firstLine;
7
+ // Check modeline declaration
8
+ if (!firstLine || !modelineRegex.test(firstLine))
9
+ return [];
10
+ const modelineText = firstLine.toLowerCase().split(modelineRegex)[1] ?? '';
11
+ // Add identified language(s)
12
+ return Object.entries(langData).flatMap(([lang, data]) => {
13
+ const safeLang = escapeRegExp(lang.toLowerCase());
14
+ const matchesLang = new RegExp(`\\b${safeLang}(?![\\w#+*]|-\\*-)`).test(modelineText);
15
+ const aliases = data.aliases ?? [];
16
+ const matchesAlias = aliases.some((alias) => {
17
+ const safeAlias = escapeRegExp(alias.toLowerCase());
18
+ return new RegExp(`\\b${safeAlias}(?![\\w#+*]|-\\*-)`).test(modelineText);
19
+ });
20
+ return matchesLang || matchesAlias ? [lang] : [];
21
+ });
22
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export default function byShebang(file: T.VirtualFile, langData: LanguagesScema, opts: T.Options): string[];
@@ -0,0 +1,27 @@
1
+ const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
2
+ const categoryPriority = {
3
+ programming: 0,
4
+ markup: 1,
5
+ prose: 2,
6
+ data: 3,
7
+ };
8
+ export default function byShebang(file, langData, opts) {
9
+ if (!opts.checkShebang || opts.quick)
10
+ return [];
11
+ const firstLine = file.firstLine;
12
+ if (!firstLine || !firstLine.startsWith('#!'))
13
+ return [];
14
+ const matches = Object.entries(langData).flatMap(([lang, data]) => {
15
+ const interpreters = data.interpreters ?? [];
16
+ // Check for interpreter match
17
+ const matchesInterpretor = interpreters.some((interpreter) => new RegExp(`\\b${escapeRegExp(interpreter)}\\b`).test(firstLine));
18
+ return matchesInterpretor ? [lang] : [];
19
+ });
20
+ return [...new Set(matches)].sort((a, b) => {
21
+ const aType = langData[a]?.type;
22
+ const bType = langData[b]?.type;
23
+ const aPriority = categoryPriority[aType ?? ''] ?? 4;
24
+ const bPriority = categoryPriority[bType ?? ''] ?? 4;
25
+ return aPriority - bPriority;
26
+ });
27
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../types/schema.js';
2
+ import * as T from '../types/types.js';
3
+ export default function analyseVirtualFiles(files: T.VirtualFile[], langData: LanguagesScema, heuristicsData: unknown, opts: T.Options): T.Results;
@@ -0,0 +1,12 @@
1
+ import { aggregateResults } from './pipeline/aggregate.js';
2
+ import { classifyFiles } from './pipeline/classify.js';
3
+ import { filterFiles } from './pipeline/filter.js';
4
+ import { resolveHeuristics } from './pipeline/heuristics.js';
5
+ import { normaliseFiles } from './pipeline/normalise.js';
6
+ export default function analyseVirtualFiles(files, langData, heuristicsData, opts) {
7
+ const normalizedFiles = normaliseFiles(files);
8
+ const filteredFiles = filterFiles(normalizedFiles, opts);
9
+ const classifications = classifyFiles(filteredFiles, langData, opts);
10
+ const heuristicResolutions = resolveHeuristics(filteredFiles, classifications, heuristicsData, langData, opts);
11
+ return aggregateResults(filteredFiles, classifications, heuristicResolutions, langData, opts);
12
+ }
@@ -0,0 +1,2 @@
1
+ import * as T from '../../types/types.js';
2
+ export declare function aggregateResults(files: T.VirtualFile[], classifications: Record<string, string[]>, heuristicResolutions: Record<string, string | undefined>, langData: Record<string, unknown>, opts: T.Options): T.Results;
@@ -0,0 +1,98 @@
1
+ import Path from 'node:path';
2
+ const categoryKeys = ['data', 'markup', 'programming', 'prose'];
3
+ function pickBestLanguage(classifications) {
4
+ return classifications[0] ?? null;
5
+ }
6
+ function makeRelPath(path) {
7
+ let relPath = normPath(Path.relative(process.cwd(), path));
8
+ if (!relPath.startsWith('../') && !relPath.startsWith('./')) {
9
+ relPath = `./${relPath}`;
10
+ }
11
+ return relPath;
12
+ }
13
+ function normPath(filePath) {
14
+ return filePath.replace(/\\\\/g, '/');
15
+ }
16
+ export function aggregateResults(files, classifications, heuristicResolutions, langData, opts) {
17
+ const results = {
18
+ files: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, results: {}, alternatives: {} },
19
+ languages: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, results: {} },
20
+ unknown: { count: 0, bytes: 0, lines: { total: 0, content: 0 }, extensions: {}, filenames: {} },
21
+ repository: {},
22
+ };
23
+ // Skip specified categories
24
+ const allowedCategories = opts.categories ?? categoryKeys;
25
+ const hiddenCategories = categoryKeys.filter((category) => !allowedCategories.includes(category));
26
+ for (const file of files) {
27
+ // Narrow down file associations to the best fit
28
+ const candidates = classifications[file.path] ?? [];
29
+ // If no heuristics, assign a language
30
+ const selectedLanguage = heuristicResolutions[file.path] ?? pickBestLanguage(candidates);
31
+ const alternativeLanguages = [...new Set(candidates.filter((lang) => lang !== selectedLanguage))];
32
+ // Assign first language as a default option
33
+ // List alternative languages if there are any
34
+ // Load language bytes size
35
+ const size = file.size ?? file.content?.length ?? 0;
36
+ // Calculate lines of code
37
+ const lineContent = file.content ?? '';
38
+ const allLines = opts.calculateLines === false ? [] : lineContent.split(/\r?\n/gm);
39
+ const loc = {
40
+ total: opts.calculateLines === false ? NaN : allLines.length,
41
+ content: opts.calculateLines === false ? NaN : allLines.filter((line) => line.trim().length > 0).length,
42
+ };
43
+ const outputPath = opts.relativePaths && Path.isAbsolute(file.path) ? makeRelPath(file.path) : file.path;
44
+ if (!selectedLanguage) {
45
+ const extension = file.extension || Path.extname(file.path);
46
+ const unknownType = extension ? 'extensions' : 'filenames';
47
+ const name = extension || Path.basename(file.path);
48
+ results.files.results[outputPath] = null;
49
+ results.files.bytes += size;
50
+ results.files.lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
51
+ results.files.lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
52
+ results.unknown[unknownType][name] ??= 0;
53
+ results.unknown[unknownType][name] += size;
54
+ results.unknown.bytes += size;
55
+ results.unknown.lines.total += loc.total || 0;
56
+ results.unknown.lines.content += loc.content || 0;
57
+ continue;
58
+ }
59
+ const languageMeta = langData[selectedLanguage];
60
+ const category = languageMeta?.type;
61
+ const allowed = !hiddenCategories.includes(category ?? 'programming') || file.attributes?.detectable === true;
62
+ if (!allowed) {
63
+ continue;
64
+ }
65
+ if (!results.repository[selectedLanguage]) {
66
+ results.repository[selectedLanguage] = {
67
+ type: languageMeta?.type ?? 'programming',
68
+ color: languageMeta?.color,
69
+ };
70
+ if (opts.childLanguages) {
71
+ results.repository[selectedLanguage].parent = languageMeta?.group;
72
+ }
73
+ }
74
+ results.files.results[outputPath] = selectedLanguage;
75
+ if (alternativeLanguages.length) {
76
+ results.files.alternatives[outputPath] = alternativeLanguages;
77
+ }
78
+ // Apply to files totals
79
+ results.files.bytes += size;
80
+ results.files.lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
81
+ results.files.lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
82
+ results.languages.results[selectedLanguage] ??= { count: 0, bytes: 0, lines: { total: 0, content: 0 } };
83
+ results.languages.results[selectedLanguage].count += 1;
84
+ results.languages.results[selectedLanguage].bytes += size;
85
+ results.languages.results[selectedLanguage].lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
86
+ results.languages.results[selectedLanguage].lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
87
+ results.languages.bytes += size;
88
+ results.languages.lines.total += Number.isNaN(loc.total) ? 0 : loc.total;
89
+ results.languages.lines.content += Number.isNaN(loc.content) ? 0 : loc.content;
90
+ }
91
+ if (opts.calculateLines === false) {
92
+ results.files.lines = { total: NaN, content: NaN };
93
+ }
94
+ results.files.count = Object.keys(results.files.results).length;
95
+ results.languages.count = Object.keys(results.languages.results).length;
96
+ results.unknown.count = Object.keys({ ...results.unknown.extensions, ...results.unknown.filenames }).length;
97
+ return results;
98
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export declare function classifyFiles(files: T.VirtualFile[], langData: LanguagesScema, opts: T.Options): Record<string, string[]>;
@@ -0,0 +1,31 @@
1
+ import byAttributes from '../classifiers/byAttributes.js';
2
+ import byExtension from '../classifiers/byExtension.js';
3
+ import byFilename from '../classifiers/byFilename.js';
4
+ import byModeline from '../classifiers/byModeline.js';
5
+ import byShebang from '../classifiers/byShebang.js';
6
+ function dedupeClassifications(classifications) {
7
+ const seen = new Set();
8
+ const ordered = [];
9
+ for (const classification of classifications) {
10
+ if (!seen.has(classification)) {
11
+ seen.add(classification);
12
+ ordered.push(classification);
13
+ }
14
+ }
15
+ return ordered;
16
+ }
17
+ export function classifyFiles(files, langData, opts) {
18
+ const classifications = {};
19
+ for (const file of files) {
20
+ // Search each language
21
+ const candidates = [
22
+ ...byAttributes(file, langData),
23
+ ...byFilename(file, langData),
24
+ ...byShebang(file, langData, opts),
25
+ ...byModeline(file, langData, opts),
26
+ ...byExtension(file, langData),
27
+ ];
28
+ classifications[file.path] = dedupeClassifications(candidates);
29
+ }
30
+ return classifications;
31
+ }
@@ -0,0 +1,2 @@
1
+ import * as T from '../../types/types.js';
2
+ export declare function filterFiles(files: T.VirtualFile[], opts: T.Options): T.VirtualFile[];
@@ -0,0 +1,14 @@
1
+ export function filterFiles(files, opts) {
2
+ return files.filter((file) => {
3
+ // Skip binary files
4
+ if (!opts.keepBinary && (file.isBinary || file.attributes?.binary === true)) {
5
+ return false;
6
+ }
7
+ if (!opts.keepVendored &&
8
+ (file.metadata?.vendored === true || file.metadata?.generated === true || file.metadata?.documentation === true)) {
9
+ // Skip vendored, generated, or documentation files
10
+ return false;
11
+ }
12
+ return true;
13
+ });
14
+ }
@@ -0,0 +1,3 @@
1
+ import { LanguagesScema } from '../../types/schema.js';
2
+ import * as T from '../../types/types.js';
3
+ export declare function resolveHeuristics(files: T.VirtualFile[], classifications: Record<string, string[]>, heuristicsData: unknown, langData: LanguagesScema, opts: T.Options): Record<string, string | undefined>;
@@ -0,0 +1,12 @@
1
+ import byHeuristics from '../classifiers/byHeuristics.js';
2
+ export function resolveHeuristics(files, classifications, heuristicsData, langData, opts) {
3
+ const resolved = {};
4
+ for (const file of files) {
5
+ const candidateLanguages = classifications[file.path] ?? [];
6
+ const heuristicLanguage = byHeuristics(file, candidateLanguages, heuristicsData, langData, opts)[0];
7
+ if (heuristicLanguage) {
8
+ resolved[file.path] = heuristicLanguage;
9
+ }
10
+ }
11
+ return resolved;
12
+ }
@@ -0,0 +1,2 @@
1
+ import * as T from '../../types/types.js';
2
+ export declare function normaliseFiles(files: T.VirtualFile[]): T.VirtualFile[];
@@ -0,0 +1,9 @@
1
+ import { getFileExtension } from '../../program/fs/normalisedPath.js';
2
+ export function normaliseFiles(files) {
3
+ return files.map((file) => ({
4
+ ...file,
5
+ extension: file.extension ?? getFileExtension(file.path),
6
+ firstLine: file.firstLine ?? file.content?.split(/\r?\n/)[0],
7
+ size: file.size ?? file.content?.length,
8
+ }));
9
+ }
@@ -0,0 +1,3 @@
1
+ import { OptionValues } from 'commander';
2
+ import { Results } from '../../types/types.js';
3
+ export default function defaultOutput(args: OptionValues, data: Results): Promise<void>;
@@ -0,0 +1,106 @@
1
+ import FS from 'node:fs';
2
+ import Path from 'node:path';
3
+ import { normPath } from '../../program/fs/normalisedPath.js';
4
+ import { colouredMsg, hexToRgb } from '../utils.js';
5
+ export default async function defaultOutput(args, data) {
6
+ const { files, languages, unknown, repository } = data;
7
+ if (args.minSize) {
8
+ // Ignore languages with a bytes/% size less than the declared min size
9
+ const totalSize = languages.bytes;
10
+ const minSizeAmt = parseFloat(args.minSize.replace(/[a-z]+$/i, '')); // '2KB' -> 2
11
+ const minSizeUnit = args.minSize.replace(/^\d+/, '').toLowerCase(); // '2KB' -> 'kb'
12
+ const checkBytes = minSizeUnit !== 'loc'; // whether to check bytes or loc
13
+ const conversionFactors = {
14
+ ['b']: (n) => n,
15
+ ['kb']: (n) => n * 1e3,
16
+ ['mb']: (n) => n * 1e6,
17
+ ['%']: (n) => (n * totalSize) / 100,
18
+ ['loc']: (n) => n,
19
+ };
20
+ const minBytesSize = conversionFactors[minSizeUnit](+minSizeAmt);
21
+ const other = { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 } };
22
+ // Apply specified minimums: delete language results that do not reach the threshold
23
+ for (const [lang, data] of Object.entries(languages.results)) {
24
+ const checkUnit = checkBytes ? data.bytes : data.lines.content;
25
+ if (checkUnit < minBytesSize) {
26
+ // Add to 'other' count
27
+ other.count++;
28
+ other.bytes += data.bytes;
29
+ other.lines.total += data.lines.total;
30
+ other.lines.content += data.lines.content;
31
+ // Remove language result
32
+ delete languages.results[lang];
33
+ }
34
+ }
35
+ if (other.bytes) {
36
+ languages.results['Other'] = other;
37
+ }
38
+ }
39
+ const sortedEntries = Object.entries(languages.results).sort((a, b) => (a[1].bytes < b[1].bytes ? +1 : -1));
40
+ const totalBytes = languages.bytes;
41
+ console.log(`\n Analysed ${files.bytes.toLocaleString()} B from ${files.count} files with linguist-js`);
42
+ console.log(`\n Language analysis results: \n`);
43
+ let count = 0;
44
+ if (sortedEntries.length === 0)
45
+ console.log(` None`);
46
+ // Collate files per language
47
+ const filesPerLanguage = {};
48
+ if (args.listFiles) {
49
+ for (const language of Object.keys(languages.results)) {
50
+ filesPerLanguage[language] = [];
51
+ }
52
+ for (const [file, lang] of Object.entries(files.results)) {
53
+ if (lang) {
54
+ filesPerLanguage[lang].push(file);
55
+ }
56
+ }
57
+ }
58
+ // List parsed results
59
+ for (const [lang, { bytes, lines }] of sortedEntries) {
60
+ const colour = hexToRgb(repository[lang].color ?? '#ededed');
61
+ const percent = (bytes) => (bytes / (totalBytes || 1)) * 100;
62
+ const fmtd = {
63
+ index: (++count).toString().padStart(2, ' '),
64
+ lang: lang.padEnd(24, ' '),
65
+ percent: percent(bytes).toFixed(2).padStart(5, ' '),
66
+ bytes: bytes.toLocaleString().padStart(10, ' '),
67
+ loc: lines.content.toLocaleString().padStart(10, ' '),
68
+ icon: colouredMsg(colour, '\u2588'),
69
+ };
70
+ console.log(` ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B ${fmtd.loc} LOC`);
71
+ // If using `listFiles` option, list all files tagged as this language
72
+ if (args.listFiles) {
73
+ console.log(); // padding
74
+ for (const file of filesPerLanguage[lang]) {
75
+ let relFile = normPath(Path.relative(Path.resolve('.'), file));
76
+ if (!relFile.startsWith('../')) {
77
+ relFile = './' + relFile;
78
+ }
79
+ const fileStat = await FS.promises.stat(file);
80
+ const bytes = fileStat.size;
81
+ const fmtd2 = {
82
+ file: relFile.padEnd(42, ' '),
83
+ percent: percent(bytes).toFixed(2).padStart(5, ' '),
84
+ bytes: bytes.toLocaleString().padStart(10, ' '),
85
+ };
86
+ console.log(` ${fmtd.icon} ${fmtd2.file} ${fmtd2.percent}% ${fmtd2.bytes} B`);
87
+ }
88
+ console.log(); // padding
89
+ }
90
+ }
91
+ if (!args.listFiles) {
92
+ console.log(); // padding
93
+ }
94
+ console.log(` Total: ${totalBytes.toLocaleString()} B`);
95
+ // List unknown files/extensions
96
+ if (unknown.bytes > 0) {
97
+ console.log(`\n Unknown files and extensions:`);
98
+ for (const [name, bytes] of Object.entries(unknown.filenames)) {
99
+ console.log(` '${name}': ${bytes.toLocaleString()} B`);
100
+ }
101
+ for (const [ext, bytes] of Object.entries(unknown.extensions)) {
102
+ console.log(` '*${ext}': ${bytes.toLocaleString()} B`);
103
+ }
104
+ console.log(` Total: ${unknown.bytes.toLocaleString()} B`);
105
+ }
106
+ }
@@ -0,0 +1,3 @@
1
+ import { OptionValues } from 'commander';
2
+ import { Results } from '../../types/types.js';
3
+ export default function treeOutput(args: OptionValues, data: Results): void;
@@ -0,0 +1,11 @@
1
+ export default function treeOutput(args, data) {
2
+ const treeParts = args.tree.split('.');
3
+ let nestedData = data;
4
+ for (const part of treeParts) {
5
+ if (!nestedData[part]) {
6
+ throw Error(`TraversalError: Key '${part}' cannot be found on output object.`);
7
+ }
8
+ nestedData = nestedData[part];
9
+ }
10
+ console.log(nestedData);
11
+ }
@@ -0,0 +1,2 @@
1
+ import { OptionValues } from 'commander';
2
+ export default function runCliAnalysis(args: OptionValues): Promise<void>;
@@ -0,0 +1,26 @@
1
+ import analyseFs from '../entry/analyseFs.js';
2
+ import normaliseOpts from '../input/normaliseOpts.js';
3
+ import defaultOutput from './output/default.js';
4
+ import treeOutput from './output/tree.js';
5
+ const validCategories = ['data', 'programming', 'prose', 'markup'];
6
+ export default async function runCliAnalysis(args) {
7
+ // Check arguments
8
+ if (args.categories?.some((category) => !validCategories.includes(category))) {
9
+ console.log(`Error: '${args.categories.join(', ')}' contains an invalid category.`);
10
+ console.log(`\tValid options: ${validCategories.join(', ')}.`);
11
+ return;
12
+ }
13
+ // Analyse language data
14
+ const folders = args.analyse === true ? ['.'] : args.analyse;
15
+ const data = await analyseFs(folders, normaliseOpts(args));
16
+ // Print output
17
+ if (!args.json) {
18
+ defaultOutput(args, data);
19
+ }
20
+ else if (args.tree) {
21
+ treeOutput(args, data);
22
+ }
23
+ else {
24
+ console.dir(data, { depth: null });
25
+ }
26
+ }
@@ -0,0 +1,2 @@
1
+ export declare function colouredMsg([r, g, b]: [number, number, number], msg: string): string;
2
+ export declare function hexToRgb(hex: string): [number, number, number];
@@ -0,0 +1,9 @@
1
+ export function colouredMsg([r, g, b], msg) {
2
+ return `\u001B[${38};2;${r};${g};${b}m${msg}\u001b[0m`;
3
+ }
4
+ export function hexToRgb(hex) {
5
+ const r = parseInt(hex.slice(1, 3), 16);
6
+ const g = parseInt(hex.slice(3, 5), 16);
7
+ const b = parseInt(hex.slice(5, 7), 16);
8
+ return [r, g, b];
9
+ }