linguist-js 2.9.2 → 3.0.0-pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +1 -1
- package/dist/analyser/classifiers/byAttributes.d.ts +3 -0
- package/dist/analyser/classifiers/byAttributes.js +13 -0
- package/dist/analyser/classifiers/byExtension.d.ts +3 -0
- package/dist/analyser/classifiers/byExtension.js +31 -0
- package/dist/analyser/classifiers/byFilename.d.ts +3 -0
- package/dist/analyser/classifiers/byFilename.js +10 -0
- package/dist/analyser/classifiers/byHeuristics.d.ts +3 -0
- package/dist/analyser/classifiers/byHeuristics.js +67 -0
- package/dist/analyser/classifiers/byModeline.d.ts +3 -0
- package/dist/analyser/classifiers/byModeline.js +22 -0
- package/dist/analyser/classifiers/byShebang.d.ts +3 -0
- package/dist/analyser/classifiers/byShebang.js +27 -0
- package/dist/analyser/index.d.ts +3 -0
- package/dist/analyser/index.js +12 -0
- package/dist/analyser/pipeline/aggregate.d.ts +2 -0
- package/dist/analyser/pipeline/aggregate.js +98 -0
- package/dist/analyser/pipeline/classify.d.ts +3 -0
- package/dist/analyser/pipeline/classify.js +31 -0
- package/dist/analyser/pipeline/filter.d.ts +2 -0
- package/dist/analyser/pipeline/filter.js +14 -0
- package/dist/analyser/pipeline/heuristics.d.ts +3 -0
- package/dist/analyser/pipeline/heuristics.js +12 -0
- package/dist/analyser/pipeline/normalise.d.ts +2 -0
- package/dist/analyser/pipeline/normalise.js +9 -0
- package/dist/cli/output/default.d.ts +3 -0
- package/dist/cli/output/default.js +106 -0
- package/dist/cli/output/tree.d.ts +3 -0
- package/dist/cli/output/tree.js +11 -0
- package/dist/cli/runCliAnalysis.d.ts +2 -0
- package/dist/cli/runCliAnalysis.js +26 -0
- package/dist/cli/utils.d.ts +2 -0
- package/dist/cli/utils.js +9 -0
- package/dist/cli.js +14 -143
- package/dist/entry/analyseFs.d.ts +2 -0
- package/dist/entry/analyseFs.js +11 -0
- package/dist/entry/analyseRaw.d.ts +4 -0
- package/dist/entry/analyseRaw.js +11 -0
- package/dist/index.d.ts +19 -4
- package/dist/index.js +18 -483
- package/dist/input/fromFilesystem.d.ts +4 -0
- package/dist/input/fromFilesystem.js +115 -0
- package/dist/input/fromRawContent.d.ts +2 -0
- package/dist/input/fromRawContent.js +19 -0
- package/dist/input/normaliseOpts.d.ts +2 -0
- package/dist/input/normaliseOpts.js +13 -0
- package/dist/program/classes/attributes.d.ts +11 -0
- package/dist/program/classes/attributes.js +37 -0
- package/dist/{helpers/load-data.d.ts → program/data/loadDataFiles.d.ts} +1 -1
- package/dist/program/data/loadDataFiles.js +32 -0
- package/dist/program/data/retrieveData.d.ts +12 -0
- package/dist/program/data/retrieveData.js +27 -0
- package/dist/{helpers/norm-path.d.ts → program/fs/normalisedPath.d.ts} +1 -0
- package/dist/program/fs/normalisedPath.js +17 -0
- package/dist/program/fs/readFile.js +17 -0
- package/dist/{helpers/walk-tree.d.ts → program/fs/walkTree.d.ts} +1 -1
- package/dist/{helpers/walk-tree.js → program/fs/walkTree.js} +22 -30
- package/dist/{helpers/parse-gitattributes.d.ts → program/parsing/parseGitattributes.d.ts} +2 -2
- package/dist/{helpers/parse-gitattributes.js → program/parsing/parseGitattributes.js} +4 -8
- package/dist/{helpers/parse-gitignore.js → program/parsing/parseGitignore.js} +1 -4
- package/dist/program/processFiles.d.ts +9 -0
- package/dist/program/processFiles.js +111 -0
- package/dist/{helpers/convert-pcre.js → program/utils/pcre.js} +1 -4
- package/dist/{schema.d.ts → types/schema.d.ts} +1 -1
- package/dist/types/schema.js +1 -0
- package/dist/{types.d.ts → types/types.d.ts} +35 -24
- package/dist/types/types.js +1 -0
- package/ext/generated.rb +6 -0
- package/ext/heuristics.yml +102 -11
- package/ext/languages.yml +518 -88
- package/ext/vendor.yml +1 -0
- package/package.json +14 -13
- package/readme.md +82 -87
- package/dist/helpers/load-data.js +0 -37
- package/dist/helpers/norm-path.js +0 -15
- package/dist/helpers/read-file.js +0 -23
- package/dist/schema.js +0 -2
- package/dist/types.js +0 -2
- /package/dist/{helpers/read-file.d.ts → program/fs/readFile.d.ts} +0 -0
- /package/dist/{helpers/parse-gitignore.d.ts → program/parsing/parseGitignore.d.ts} +0 -0
- /package/dist/{helpers/convert-pcre.d.ts → program/utils/pcre.d.ts} +0 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { FileGlob, RelFile } from '../../types/types.js';
|
|
2
|
+
import { FlagAttributes } from '../parsing/parseGitattributes.js';
|
|
3
|
+
/** Stores parsed attribute information per file glob */
|
|
4
|
+
export default class Attributes {
|
|
5
|
+
#private;
|
|
6
|
+
constructor();
|
|
7
|
+
get attributes(): Record<FileGlob, FlagAttributes>;
|
|
8
|
+
add(glob: FileGlob, attributes: FlagAttributes): void;
|
|
9
|
+
getFlaggedGlobs(attr: keyof FlagAttributes, val: boolean): string[];
|
|
10
|
+
findAttrsForPath(relFilePath: RelFile): FlagAttributes | null;
|
|
11
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import ignore from 'ignore';
|
|
2
|
+
/** Stores parsed attribute information per file glob */
|
|
3
|
+
export default class Attributes {
|
|
4
|
+
#attributes;
|
|
5
|
+
constructor() {
|
|
6
|
+
this.#attributes = {};
|
|
7
|
+
}
|
|
8
|
+
get attributes() {
|
|
9
|
+
return this.#attributes;
|
|
10
|
+
}
|
|
11
|
+
add(glob, attributes) {
|
|
12
|
+
this.#attributes[glob] = attributes;
|
|
13
|
+
}
|
|
14
|
+
getFlaggedGlobs(attr, val) {
|
|
15
|
+
return Object.entries(this.#attributes)
|
|
16
|
+
.filter(([, attrs]) => attrs[attr] === val)
|
|
17
|
+
.map(([glob]) => glob);
|
|
18
|
+
}
|
|
19
|
+
findAttrsForPath(relFilePath) {
|
|
20
|
+
const resultAttrs = {};
|
|
21
|
+
for (const glob in this.#attributes) {
|
|
22
|
+
const matchingAttrs = this.#attributes[glob];
|
|
23
|
+
// Check if glob matches rel path
|
|
24
|
+
if (ignore().add(glob).ignores(relFilePath)) {
|
|
25
|
+
for (const [attr, val] of Object.entries(matchingAttrs)) {
|
|
26
|
+
if (val != null) {
|
|
27
|
+
resultAttrs[attr] = val;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (!JSON.stringify(resultAttrs).length) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
return resultAttrs;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/** Nukes unused `generated.rb` file content. */
|
|
2
2
|
export declare function parseGeneratedDataFile(fileContent: string): string[];
|
|
3
3
|
/** Load a data file from github-linguist. */
|
|
4
|
-
export
|
|
4
|
+
export declare function loadFile(file: string, offline?: boolean): Promise<string>;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import Cache from 'node-cache';
|
|
2
|
+
import FS from 'node:fs';
|
|
3
|
+
import Path from 'node:path';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
const cache = new Cache({});
|
|
6
|
+
const dirname = Path.dirname(fileURLToPath(import.meta.url));
|
|
7
|
+
async function loadWebFile(file) {
|
|
8
|
+
// Return cache if it exists
|
|
9
|
+
const cachedContent = cache.get(file);
|
|
10
|
+
if (cachedContent)
|
|
11
|
+
return cachedContent;
|
|
12
|
+
// Otherwise cache the request
|
|
13
|
+
const dataUrl = (file) => `https://raw.githubusercontent.com/github/linguist/HEAD/lib/linguist/${file}`;
|
|
14
|
+
// Load file content, falling back to the local file if the request fails
|
|
15
|
+
const fileContent = await fetch(dataUrl(file))
|
|
16
|
+
.then((data) => data.text())
|
|
17
|
+
.catch(async () => await loadLocalFile(file));
|
|
18
|
+
cache.set(file, fileContent);
|
|
19
|
+
return fileContent;
|
|
20
|
+
}
|
|
21
|
+
async function loadLocalFile(file) {
|
|
22
|
+
const filePath = Path.resolve(dirname, '../../../ext', file);
|
|
23
|
+
return FS.promises.readFile(filePath).then((buffer) => buffer.toString());
|
|
24
|
+
}
|
|
25
|
+
/** Nukes unused `generated.rb` file content. */
|
|
26
|
+
export function parseGeneratedDataFile(fileContent) {
|
|
27
|
+
return [...(fileContent.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm) ?? [])];
|
|
28
|
+
}
|
|
29
|
+
/** Load a data file from github-linguist. */
|
|
30
|
+
export function loadFile(file, offline = false) {
|
|
31
|
+
return offline ? loadLocalFile(file) : loadWebFile(file);
|
|
32
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { HeuristicsSchema, LanguagesScema, VendorSchema } from '../../types/schema.js';
|
|
2
|
+
type LoadedData = {
|
|
3
|
+
langData: LanguagesScema;
|
|
4
|
+
vendorData: VendorSchema;
|
|
5
|
+
docData: VendorSchema;
|
|
6
|
+
heuristicsData: HeuristicsSchema;
|
|
7
|
+
generatedData: string[];
|
|
8
|
+
vendorPaths: string[];
|
|
9
|
+
};
|
|
10
|
+
/** Load data from github-linguist web repo or cached local file. */
|
|
11
|
+
export default function retrieveData(offline: boolean): Promise<LoadedData>;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import YAML from 'js-yaml';
|
|
2
|
+
import { loadFile, parseGeneratedDataFile } from './loadDataFiles.js';
|
|
3
|
+
let data = null;
|
|
4
|
+
async function initRetrieveData(offline) {
|
|
5
|
+
// Only load the data on mont
|
|
6
|
+
if (data)
|
|
7
|
+
return;
|
|
8
|
+
const langData = (await loadFile('languages.yml', offline).then(YAML.load));
|
|
9
|
+
const vendorData = (await loadFile('vendor.yml', offline).then(YAML.load));
|
|
10
|
+
const docData = (await loadFile('documentation.yml', offline).then(YAML.load));
|
|
11
|
+
const heuristicsData = (await loadFile('heuristics.yml', offline).then(YAML.load));
|
|
12
|
+
const generatedData = (await loadFile('generated.rb', offline).then(parseGeneratedDataFile));
|
|
13
|
+
const vendorPaths = [...vendorData, ...docData, ...generatedData];
|
|
14
|
+
data = {
|
|
15
|
+
langData,
|
|
16
|
+
vendorData,
|
|
17
|
+
docData,
|
|
18
|
+
heuristicsData,
|
|
19
|
+
generatedData,
|
|
20
|
+
vendorPaths,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/** Load data from github-linguist web repo or cached local file. */
|
|
24
|
+
export default async function retrieveData(offline) {
|
|
25
|
+
await initRetrieveData(offline);
|
|
26
|
+
return data;
|
|
27
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import Path from 'node:path';
|
|
2
|
+
export const normPath = function normalisedPath(...inputPaths) {
|
|
3
|
+
return Path.join(...inputPaths).replace(/\\/g, '/');
|
|
4
|
+
};
|
|
5
|
+
export const normAbsPath = function normalisedAbsolutePath(...inputPaths) {
|
|
6
|
+
return Path.resolve(...inputPaths).replace(/\\/g, '/');
|
|
7
|
+
};
|
|
8
|
+
export const getFileExtension = function getFileExtension(filePath) {
|
|
9
|
+
const extension = Path.extname(filePath).toLowerCase();
|
|
10
|
+
if (extension)
|
|
11
|
+
return extension;
|
|
12
|
+
const basename = Path.basename(filePath);
|
|
13
|
+
if (basename.startsWith('.') && basename.length > 1 && basename.indexOf('.', 1) === -1) {
|
|
14
|
+
return basename.toLowerCase();
|
|
15
|
+
}
|
|
16
|
+
return '';
|
|
17
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import FS from 'node:fs';
|
|
2
|
+
/**
|
|
3
|
+
* Read part of a file on disc.
|
|
4
|
+
* @throws 'EPERM' if the file is not readable.
|
|
5
|
+
*/
|
|
6
|
+
export default async function readFileChunk(filename, onlyFirstLine = false) {
|
|
7
|
+
const chunkSize = 100;
|
|
8
|
+
const stream = FS.createReadStream(filename, { highWaterMark: chunkSize });
|
|
9
|
+
let content = '';
|
|
10
|
+
for await (const data of stream) { // may throw
|
|
11
|
+
content += data.toString();
|
|
12
|
+
if (onlyFirstLine) {
|
|
13
|
+
return content.split(/\r?\n/)[0];
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return content;
|
|
17
|
+
}
|
|
@@ -1,19 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.default = walk;
|
|
7
|
-
const fs_1 = __importDefault(require("fs"));
|
|
8
|
-
const path_1 = __importDefault(require("path"));
|
|
9
|
-
const parse_gitignore_1 = __importDefault(require("./parse-gitignore"));
|
|
10
|
-
const norm_path_1 = require("./norm-path");
|
|
1
|
+
import FS from 'node:fs';
|
|
2
|
+
import Path from 'node:path';
|
|
3
|
+
import parseGitignore from '../parsing/parseGitignore.js';
|
|
4
|
+
import { normAbsPath, normPath } from './normalisedPath.js';
|
|
11
5
|
let allFiles;
|
|
12
6
|
let allFolders;
|
|
13
|
-
;
|
|
14
|
-
;
|
|
15
7
|
/** Generate list of files in a directory. */
|
|
16
|
-
function
|
|
8
|
+
export default function walkTree(data) {
|
|
17
9
|
const { init, commonRoot, folderRoots, folders, ignored } = data;
|
|
18
10
|
// Initialise files and folders lists
|
|
19
11
|
if (init) {
|
|
@@ -25,37 +17,37 @@ function walk(data) {
|
|
|
25
17
|
const folder = folders[0];
|
|
26
18
|
const localRoot = folderRoots[0].replace(commonRoot, '').replace(/^\//, '');
|
|
27
19
|
// Get list of files and folders inside this folder
|
|
28
|
-
const files =
|
|
20
|
+
const files = FS.readdirSync(folder).map((file) => {
|
|
29
21
|
// Create path relative to root
|
|
30
|
-
const base =
|
|
22
|
+
const base = normAbsPath(folder, file).replace(commonRoot, '.');
|
|
31
23
|
// Add trailing slash to mark directories
|
|
32
|
-
const isDir =
|
|
24
|
+
const isDir = FS.lstatSync(Path.resolve(commonRoot, base)).isDirectory();
|
|
33
25
|
return isDir ? `${base}/` : base;
|
|
34
26
|
});
|
|
35
27
|
// Read and apply gitignores
|
|
36
|
-
const gitignoreFilename =
|
|
37
|
-
if (
|
|
38
|
-
const gitignoreContents =
|
|
39
|
-
const ignoredPaths = (
|
|
40
|
-
const rootRelIgnoredPaths = ignoredPaths.map(ignorePath =>
|
|
28
|
+
const gitignoreFilename = normPath(folder, '.gitignore');
|
|
29
|
+
if (FS.existsSync(gitignoreFilename)) {
|
|
30
|
+
const gitignoreContents = FS.readFileSync(gitignoreFilename, 'utf-8');
|
|
31
|
+
const ignoredPaths = parseGitignore(gitignoreContents);
|
|
32
|
+
const rootRelIgnoredPaths = ignoredPaths.map((ignorePath) =>
|
|
41
33
|
// get absolute path of the ignore glob
|
|
42
|
-
|
|
34
|
+
normPath(folder, ignorePath)
|
|
43
35
|
// convert abs ignore glob to be relative to the root folder
|
|
44
36
|
.replace(commonRoot + '/', ''));
|
|
45
37
|
ignored.add(rootRelIgnoredPaths);
|
|
46
38
|
}
|
|
47
39
|
// Add gitattributes if present
|
|
48
|
-
const gitattributesPath =
|
|
49
|
-
if (
|
|
40
|
+
const gitattributesPath = normPath(folder, '.gitattributes');
|
|
41
|
+
if (FS.existsSync(gitattributesPath)) {
|
|
50
42
|
allFiles.add(gitattributesPath);
|
|
51
43
|
}
|
|
52
44
|
// Loop through files and folders
|
|
53
45
|
for (const file of files) {
|
|
54
46
|
// Create absolute path for disc operations
|
|
55
|
-
const path =
|
|
47
|
+
const path = normAbsPath(commonRoot, file);
|
|
56
48
|
const localPath = localRoot ? file.replace(`./${localRoot}/`, '') : file.replace('./', '');
|
|
57
49
|
// Skip if nonexistant
|
|
58
|
-
const nonExistant = !
|
|
50
|
+
const nonExistant = !FS.existsSync(path);
|
|
59
51
|
if (nonExistant)
|
|
60
52
|
continue;
|
|
61
53
|
// Skip if marked in gitignore
|
|
@@ -63,12 +55,12 @@ function walk(data) {
|
|
|
63
55
|
if (isIgnored)
|
|
64
56
|
continue;
|
|
65
57
|
// Add absolute folder path to list
|
|
66
|
-
allFolders.add(
|
|
58
|
+
allFolders.add(normAbsPath(folder));
|
|
67
59
|
// Check if this is a folder or file
|
|
68
60
|
if (file.endsWith('/')) {
|
|
69
61
|
// Recurse into subfolders
|
|
70
62
|
allFolders.add(path);
|
|
71
|
-
|
|
63
|
+
walkTree({ init: false, commonRoot, folderRoots, folders: [path], ignored });
|
|
72
64
|
}
|
|
73
65
|
else {
|
|
74
66
|
// Add file path to list
|
|
@@ -79,12 +71,12 @@ function walk(data) {
|
|
|
79
71
|
// Recurse into all folders
|
|
80
72
|
else {
|
|
81
73
|
for (const i in folders) {
|
|
82
|
-
|
|
74
|
+
walkTree({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored });
|
|
83
75
|
}
|
|
84
76
|
}
|
|
85
77
|
// Return absolute files and folders lists
|
|
86
78
|
return {
|
|
87
|
-
files: [...allFiles].map(file => file.replace(/^\./, commonRoot)),
|
|
79
|
+
files: [...allFiles].map((file) => file.replace(/^\./, commonRoot)),
|
|
88
80
|
folders: [...allFolders],
|
|
89
81
|
};
|
|
90
82
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import * as T from '
|
|
1
|
+
import * as T from '../../types/types.js';
|
|
2
2
|
export type FlagAttributes = {
|
|
3
3
|
'vendored': boolean | null;
|
|
4
4
|
'generated': boolean | null;
|
|
@@ -14,4 +14,4 @@ export type ParsedGitattributes = Array<{
|
|
|
14
14
|
/**
|
|
15
15
|
* Parses a gitattributes file.
|
|
16
16
|
*/
|
|
17
|
-
export default function
|
|
17
|
+
export default function parseGitattributes(content: string, folderRoot?: string): ParsedGitattributes;
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.default = parseAttributes;
|
|
4
|
-
const norm_path_1 = require("./norm-path");
|
|
1
|
+
import { normPath } from '../fs/normalisedPath.js';
|
|
5
2
|
/**
|
|
6
3
|
* Parses a gitattributes file.
|
|
7
4
|
*/
|
|
8
|
-
function
|
|
9
|
-
var _a, _b;
|
|
5
|
+
export default function parseGitattributes(content, folderRoot = '.') {
|
|
10
6
|
const output = [];
|
|
11
7
|
for (const rawLine of content.split('\n')) {
|
|
12
8
|
const line = rawLine.replace(/#.*/, '').trim();
|
|
@@ -14,7 +10,7 @@ function parseAttributes(content, folderRoot = '.') {
|
|
|
14
10
|
continue;
|
|
15
11
|
const parts = line.split(/\s+/g);
|
|
16
12
|
const fileGlob = parts[0];
|
|
17
|
-
const relFileGlob =
|
|
13
|
+
const relFileGlob = normPath(folderRoot, fileGlob);
|
|
18
14
|
const attrParts = parts.slice(1);
|
|
19
15
|
const isTrue = (str) => !str.startsWith('-') && !str.endsWith('=false');
|
|
20
16
|
const isFalse = (str) => str.startsWith('-') || str.endsWith('=false');
|
|
@@ -29,7 +25,7 @@ function parseAttributes(content, folderRoot = '.') {
|
|
|
29
25
|
'documentation': boolOrNullVal('linguist-documentation'),
|
|
30
26
|
'detectable': boolOrNullVal('linguist-detectable'),
|
|
31
27
|
'binary': hasTrueParts('binary') || hasFalseParts('text') ? true : hasFalseParts('binary') || hasTrueParts('text') ? false : null,
|
|
32
|
-
'language':
|
|
28
|
+
'language': trueParts('linguist-language').at(-1)?.split('=')[1] ?? null,
|
|
33
29
|
};
|
|
34
30
|
output.push({ glob: relFileGlob, attrs });
|
|
35
31
|
}
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.default = parseGitignore;
|
|
4
|
-
function parseGitignore(content) {
|
|
1
|
+
export default function parseGitignore(content) {
|
|
5
2
|
const readableData = content
|
|
6
3
|
// Remove comments unless escaped
|
|
7
4
|
.replace(/(?<!\\)#.+/g, '')
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import * as T from '../types/types.js';
|
|
2
|
+
import Attributes from './classes/attributes.js';
|
|
3
|
+
type ProcessFilesResult = {
|
|
4
|
+
files: T.AbsFile[];
|
|
5
|
+
manualAttributes: Attributes;
|
|
6
|
+
relPath(file: T.AbsFile): T.RelFile;
|
|
7
|
+
};
|
|
8
|
+
export default function processFiles(input: string[], opts: T.Options, useRawContent: boolean, vendorPaths: string[]): Promise<ProcessFilesResult>;
|
|
9
|
+
export {};
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import commonPrefix from 'common-path-prefix';
|
|
2
|
+
import ignore from 'ignore';
|
|
3
|
+
import FS from 'node:fs';
|
|
4
|
+
import Path from 'node:path';
|
|
5
|
+
import Attributes from './classes/attributes.js';
|
|
6
|
+
import { normPath } from './fs/normalisedPath.js';
|
|
7
|
+
import readFileChunk from './fs/readFile.js';
|
|
8
|
+
import walkTree from './fs/walkTree.js';
|
|
9
|
+
import parseGitattributes from './parsing/parseGitattributes.js';
|
|
10
|
+
const binaryData = JSON.parse(FS.readFileSync(new URL('../../node_modules/binary-extensions/binary-extensions.json', import.meta.url), 'utf-8'));
|
|
11
|
+
export default async function processFiles(input, opts, useRawContent, vendorPaths) {
|
|
12
|
+
// Set a common root path so that vendor paths do not incorrectly match parent folders
|
|
13
|
+
const resolvedInput = input.map((path) => normPath(Path.resolve(path)));
|
|
14
|
+
const commonRoot = (input.length > 1 ? commonPrefix(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
|
|
15
|
+
const relPath = (file) => (useRawContent ? file : normPath(Path.relative(commonRoot, file)));
|
|
16
|
+
const unRelPath = (file) => (useRawContent ? file : normPath(Path.resolve(commonRoot, file)));
|
|
17
|
+
// Other helper functions
|
|
18
|
+
const filterOutIgnored = (files, ignored) => ignored.filter(files.map((file) => relPath(file))).map((file) => unRelPath(file));
|
|
19
|
+
//*PREPARE FILES AND DATA*//
|
|
20
|
+
// Prepare list of ignored files
|
|
21
|
+
const ignored = ignore();
|
|
22
|
+
ignored.add('.git/');
|
|
23
|
+
ignored.add(opts.ignoredFiles ?? []);
|
|
24
|
+
const regexIgnores = opts.keepVendored ? [] : vendorPaths.map((path) => RegExp(path, 'i'));
|
|
25
|
+
// Load file paths and folders
|
|
26
|
+
let files;
|
|
27
|
+
if (useRawContent) {
|
|
28
|
+
// Uses raw file content
|
|
29
|
+
files = input;
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
// Uses directory on disc
|
|
33
|
+
const data = walkTree({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored });
|
|
34
|
+
files = data.files;
|
|
35
|
+
}
|
|
36
|
+
// Fetch and normalise gitattributes data of all subfolders and save to metadata
|
|
37
|
+
const manualAttributes = new Attributes();
|
|
38
|
+
if (!useRawContent && opts.checkAttributes) {
|
|
39
|
+
const nestedAttrFiles = files.filter((file) => file.endsWith('.gitattributes'));
|
|
40
|
+
for (const attrFile of nestedAttrFiles) {
|
|
41
|
+
const relAttrFile = relPath(attrFile);
|
|
42
|
+
const relAttrFolder = Path.dirname(relAttrFile);
|
|
43
|
+
const contents = await readFileChunk(attrFile);
|
|
44
|
+
const parsed = parseGitattributes(contents, relAttrFolder);
|
|
45
|
+
for (const { glob, attrs } of parsed) {
|
|
46
|
+
manualAttributes.add(glob, attrs);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
// Remove files that are linguist-ignored via regex by default unless explicitly unignored in gitattributes
|
|
51
|
+
const filesToIgnore = [];
|
|
52
|
+
for (const file of files) {
|
|
53
|
+
const relFile = relPath(file);
|
|
54
|
+
const isRegexIgnored = regexIgnores.some((pattern) => pattern.test(relFile));
|
|
55
|
+
if (!isRegexIgnored) {
|
|
56
|
+
// Checking overrides is moot if file is not even marked as ignored by default
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const fileAttrs = manualAttributes.findAttrsForPath(relPath(file));
|
|
60
|
+
if (fileAttrs?.generated === false || fileAttrs?.vendored === false) {
|
|
61
|
+
// File is explicitly marked as *not* to be ignored
|
|
62
|
+
// do nothing
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
filesToIgnore.push(file);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
files = files.filter((file) => !filesToIgnore.includes(file));
|
|
69
|
+
// Apply vendor file path matches and filter out vendored files
|
|
70
|
+
if (!opts.keepVendored) {
|
|
71
|
+
// Get data of files that have been manually marked with metadata
|
|
72
|
+
const vendorTrueGlobs = [
|
|
73
|
+
...manualAttributes.getFlaggedGlobs('vendored', true),
|
|
74
|
+
...manualAttributes.getFlaggedGlobs('generated', true),
|
|
75
|
+
...manualAttributes.getFlaggedGlobs('documentation', true),
|
|
76
|
+
];
|
|
77
|
+
const vendorFalseGlobs = [
|
|
78
|
+
...manualAttributes.getFlaggedGlobs('vendored', false),
|
|
79
|
+
...manualAttributes.getFlaggedGlobs('generated', false),
|
|
80
|
+
...manualAttributes.getFlaggedGlobs('documentation', false),
|
|
81
|
+
];
|
|
82
|
+
// Set up glob ignore object to use for expanding globs to match files
|
|
83
|
+
const vendorTrueIgnore = ignore().add(vendorTrueGlobs);
|
|
84
|
+
const vendorFalseIgnore = ignore().add(vendorFalseGlobs);
|
|
85
|
+
// Remove all files marked as vendored by default
|
|
86
|
+
const excludedFiles = files.filter((file) => vendorPaths.some((pathPtn) => RegExp(pathPtn, 'i').test(relPath(file))));
|
|
87
|
+
files = files.filter((file) => !excludedFiles.includes(file));
|
|
88
|
+
// Re-add removed files that are overridden manually in gitattributes
|
|
89
|
+
const overriddenExcludedFiles = excludedFiles.filter((file) => vendorFalseIgnore.ignores(relPath(file)));
|
|
90
|
+
files.push(...overriddenExcludedFiles);
|
|
91
|
+
// Remove files explicitly marked as vendored in gitattributes
|
|
92
|
+
files = files.filter((file) => !vendorTrueIgnore.ignores(relPath(file)));
|
|
93
|
+
}
|
|
94
|
+
// Filter out binary files
|
|
95
|
+
if (!opts.keepBinary) {
|
|
96
|
+
// Filter out files that are binary by default
|
|
97
|
+
files = files.filter((file) => !binaryData.some((ext) => file.endsWith('.' + ext)));
|
|
98
|
+
// Filter out manually specified binary files
|
|
99
|
+
const binaryIgnored = ignore().add(manualAttributes.getFlaggedGlobs('binary', true));
|
|
100
|
+
files = filterOutIgnored(files, binaryIgnored);
|
|
101
|
+
// Re-add files manually marked not as binary
|
|
102
|
+
const binaryUnignored = ignore().add(manualAttributes.getFlaggedGlobs('binary', false));
|
|
103
|
+
const unignoredList = filterOutIgnored(files, binaryUnignored);
|
|
104
|
+
files.push(...unignoredList);
|
|
105
|
+
}
|
|
106
|
+
return {
|
|
107
|
+
files,
|
|
108
|
+
manualAttributes,
|
|
109
|
+
relPath,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.default = pcre;
|
|
4
1
|
/** Convert a PCRE regex into JS. */
|
|
5
|
-
function pcre(regex) {
|
|
2
|
+
export default function pcre(regex) {
|
|
6
3
|
let finalRegex = regex;
|
|
7
4
|
const replace = (search, replace) => finalRegex = finalRegex.replace(search, replace);
|
|
8
5
|
const finalFlags = new Set();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
+
import type { LanguagesScema } from './schema.js';
|
|
1
2
|
export type LanguageResult = string | null;
|
|
2
3
|
export type Language = string;
|
|
3
4
|
export type Category = 'data' | 'markup' | 'programming' | 'prose';
|
|
4
5
|
export type FilePath = string;
|
|
5
6
|
export type Bytes = Integer;
|
|
6
7
|
export type Integer = number;
|
|
8
|
+
export type LanguageMetadata = LanguagesScema[string];
|
|
7
9
|
export type RelFile = string & {};
|
|
8
10
|
export type AbsFile = string & {};
|
|
9
11
|
export type AbsFolder = string & {};
|
|
10
12
|
export type FileGlob = string & {};
|
|
11
13
|
export interface Options {
|
|
12
|
-
fileContent?: string | string[];
|
|
13
14
|
ignoredFiles?: string[];
|
|
14
15
|
ignoredLanguages?: Language[];
|
|
15
16
|
categories?: Category[];
|
|
@@ -27,15 +28,33 @@ export interface Options {
|
|
|
27
28
|
checkShebang?: boolean;
|
|
28
29
|
checkModeline?: boolean;
|
|
29
30
|
}
|
|
31
|
+
export interface VirtualFile {
|
|
32
|
+
path: string;
|
|
33
|
+
content?: string;
|
|
34
|
+
firstLine?: string;
|
|
35
|
+
size?: number;
|
|
36
|
+
extension?: string;
|
|
37
|
+
isBinary?: boolean;
|
|
38
|
+
metadata?: {
|
|
39
|
+
vendored?: boolean;
|
|
40
|
+
generated?: boolean;
|
|
41
|
+
documentation?: boolean;
|
|
42
|
+
};
|
|
43
|
+
attributes?: {
|
|
44
|
+
language?: LanguageResult;
|
|
45
|
+
binary?: boolean;
|
|
46
|
+
detectable?: boolean;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
type LinesOfCode = {
|
|
50
|
+
total: Integer;
|
|
51
|
+
content: Integer;
|
|
52
|
+
};
|
|
30
53
|
export interface Results {
|
|
31
54
|
files: {
|
|
32
55
|
count: Integer;
|
|
33
56
|
bytes: Bytes;
|
|
34
|
-
lines:
|
|
35
|
-
total: Integer;
|
|
36
|
-
content: Integer;
|
|
37
|
-
code: Integer;
|
|
38
|
-
};
|
|
57
|
+
lines: LinesOfCode;
|
|
39
58
|
/** Note: Results use slashes as delimiters even on Windows. */
|
|
40
59
|
results: Record<FilePath, LanguageResult>;
|
|
41
60
|
alternatives: Record<FilePath, LanguageResult[]>;
|
|
@@ -43,32 +62,24 @@ export interface Results {
|
|
|
43
62
|
languages: {
|
|
44
63
|
count: Integer;
|
|
45
64
|
bytes: Bytes;
|
|
46
|
-
lines:
|
|
47
|
-
total: Integer;
|
|
48
|
-
content: Integer;
|
|
49
|
-
code: Integer;
|
|
50
|
-
};
|
|
65
|
+
lines: LinesOfCode;
|
|
51
66
|
results: Record<Language, {
|
|
67
|
+
count: Integer;
|
|
52
68
|
bytes: Bytes;
|
|
53
|
-
lines:
|
|
54
|
-
total: Integer;
|
|
55
|
-
content: Integer;
|
|
56
|
-
code: Integer;
|
|
57
|
-
};
|
|
58
|
-
type: Category;
|
|
59
|
-
parent?: Language;
|
|
60
|
-
color?: `#${string}`;
|
|
69
|
+
lines: LinesOfCode;
|
|
61
70
|
}>;
|
|
62
71
|
};
|
|
63
72
|
unknown: {
|
|
64
73
|
count: Integer;
|
|
65
74
|
bytes: Bytes;
|
|
66
|
-
lines:
|
|
67
|
-
total: Integer;
|
|
68
|
-
content: Integer;
|
|
69
|
-
code: Integer;
|
|
70
|
-
};
|
|
75
|
+
lines: LinesOfCode;
|
|
71
76
|
extensions: Record<string, Bytes>;
|
|
72
77
|
filenames: Record<string, Bytes>;
|
|
73
78
|
};
|
|
79
|
+
repository: Record<Language, {
|
|
80
|
+
type: Category;
|
|
81
|
+
parent?: Language;
|
|
82
|
+
color?: `#${string}`;
|
|
83
|
+
}>;
|
|
74
84
|
}
|
|
85
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/ext/generated.rb
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
- node_modules\/
|
|
7
7
|
- vendor\/((?!-)[-0-9A-Za-z]+(?<!-)\.)+(com|edu|gov|in|me|net|org|fm|io)
|
|
8
8
|
- (Gopkg|glide)\.lock
|
|
9
|
+
- Package\.resolved
|
|
9
10
|
- poetry\.lock
|
|
10
11
|
- pdm\.lock
|
|
11
12
|
- uv\.lock
|
|
@@ -15,6 +16,7 @@
|
|
|
15
16
|
- npm-shrinkwrap\.json
|
|
16
17
|
- package-lock\.json
|
|
17
18
|
- pnpm-lock\.yaml
|
|
19
|
+
- (?:^|\/)bun\.lockb?$
|
|
18
20
|
- (^|\/)\.pnp\..*$
|
|
19
21
|
- Godeps\/
|
|
20
22
|
- composer\.lock
|
|
@@ -23,6 +25,10 @@
|
|
|
23
25
|
- Cargo\.toml\.orig
|
|
24
26
|
- (^|\/)flake\.lock$
|
|
25
27
|
- (^|\/)MODULE\.bazel\.lock$
|
|
28
|
+
- (?:^|\/)gradlew(?:\.bat)?$
|
|
29
|
+
- (?:^|\/)mvnw(?:\.cmd)?$
|
|
30
|
+
- (?:^|\/)mise(?:\.[^\/]+)?\.lock$
|
|
31
|
+
- (?:^|\/)(Julia)?Manifest(-v\d+\.\d+)?\.toml$
|
|
26
32
|
- Pipfile\.lock
|
|
27
33
|
- (?:^|\/)\.terraform\.lock\.hcl$
|
|
28
34
|
- ppport\.h$
|