linguist-js 2.9.2 → 3.0.0-pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/bin/index.js +1 -1
  2. package/dist/analyser/classifiers/byAttributes.d.ts +3 -0
  3. package/dist/analyser/classifiers/byAttributes.js +13 -0
  4. package/dist/analyser/classifiers/byExtension.d.ts +3 -0
  5. package/dist/analyser/classifiers/byExtension.js +31 -0
  6. package/dist/analyser/classifiers/byFilename.d.ts +3 -0
  7. package/dist/analyser/classifiers/byFilename.js +10 -0
  8. package/dist/analyser/classifiers/byHeuristics.d.ts +3 -0
  9. package/dist/analyser/classifiers/byHeuristics.js +67 -0
  10. package/dist/analyser/classifiers/byModeline.d.ts +3 -0
  11. package/dist/analyser/classifiers/byModeline.js +22 -0
  12. package/dist/analyser/classifiers/byShebang.d.ts +3 -0
  13. package/dist/analyser/classifiers/byShebang.js +27 -0
  14. package/dist/analyser/index.d.ts +3 -0
  15. package/dist/analyser/index.js +12 -0
  16. package/dist/analyser/pipeline/aggregate.d.ts +2 -0
  17. package/dist/analyser/pipeline/aggregate.js +98 -0
  18. package/dist/analyser/pipeline/classify.d.ts +3 -0
  19. package/dist/analyser/pipeline/classify.js +31 -0
  20. package/dist/analyser/pipeline/filter.d.ts +2 -0
  21. package/dist/analyser/pipeline/filter.js +14 -0
  22. package/dist/analyser/pipeline/heuristics.d.ts +3 -0
  23. package/dist/analyser/pipeline/heuristics.js +12 -0
  24. package/dist/analyser/pipeline/normalise.d.ts +2 -0
  25. package/dist/analyser/pipeline/normalise.js +9 -0
  26. package/dist/cli/output/default.d.ts +3 -0
  27. package/dist/cli/output/default.js +106 -0
  28. package/dist/cli/output/tree.d.ts +3 -0
  29. package/dist/cli/output/tree.js +11 -0
  30. package/dist/cli/runCliAnalysis.d.ts +2 -0
  31. package/dist/cli/runCliAnalysis.js +26 -0
  32. package/dist/cli/utils.d.ts +2 -0
  33. package/dist/cli/utils.js +9 -0
  34. package/dist/cli.js +14 -143
  35. package/dist/entry/analyseFs.d.ts +2 -0
  36. package/dist/entry/analyseFs.js +11 -0
  37. package/dist/entry/analyseRaw.d.ts +4 -0
  38. package/dist/entry/analyseRaw.js +11 -0
  39. package/dist/index.d.ts +19 -4
  40. package/dist/index.js +18 -483
  41. package/dist/input/fromFilesystem.d.ts +4 -0
  42. package/dist/input/fromFilesystem.js +115 -0
  43. package/dist/input/fromRawContent.d.ts +2 -0
  44. package/dist/input/fromRawContent.js +19 -0
  45. package/dist/input/normaliseOpts.d.ts +2 -0
  46. package/dist/input/normaliseOpts.js +13 -0
  47. package/dist/program/classes/attributes.d.ts +11 -0
  48. package/dist/program/classes/attributes.js +37 -0
  49. package/dist/{helpers/load-data.d.ts → program/data/loadDataFiles.d.ts} +1 -1
  50. package/dist/program/data/loadDataFiles.js +32 -0
  51. package/dist/program/data/retrieveData.d.ts +12 -0
  52. package/dist/program/data/retrieveData.js +27 -0
  53. package/dist/{helpers/norm-path.d.ts → program/fs/normalisedPath.d.ts} +1 -0
  54. package/dist/program/fs/normalisedPath.js +17 -0
  55. package/dist/program/fs/readFile.js +17 -0
  56. package/dist/{helpers/walk-tree.d.ts → program/fs/walkTree.d.ts} +1 -1
  57. package/dist/{helpers/walk-tree.js → program/fs/walkTree.js} +22 -30
  58. package/dist/{helpers/parse-gitattributes.d.ts → program/parsing/parseGitattributes.d.ts} +2 -2
  59. package/dist/{helpers/parse-gitattributes.js → program/parsing/parseGitattributes.js} +4 -8
  60. package/dist/{helpers/parse-gitignore.js → program/parsing/parseGitignore.js} +1 -4
  61. package/dist/program/processFiles.d.ts +9 -0
  62. package/dist/program/processFiles.js +111 -0
  63. package/dist/{helpers/convert-pcre.js → program/utils/pcre.js} +1 -4
  64. package/dist/{schema.d.ts → types/schema.d.ts} +1 -1
  65. package/dist/types/schema.js +1 -0
  66. package/dist/{types.d.ts → types/types.d.ts} +35 -24
  67. package/dist/types/types.js +1 -0
  68. package/ext/generated.rb +6 -0
  69. package/ext/heuristics.yml +102 -11
  70. package/ext/languages.yml +518 -88
  71. package/ext/vendor.yml +1 -0
  72. package/package.json +14 -13
  73. package/readme.md +82 -87
  74. package/dist/helpers/load-data.js +0 -37
  75. package/dist/helpers/norm-path.js +0 -15
  76. package/dist/helpers/read-file.js +0 -23
  77. package/dist/schema.js +0 -2
  78. package/dist/types.js +0 -2
  79. /package/dist/{helpers/read-file.d.ts → program/fs/readFile.d.ts} +0 -0
  80. /package/dist/{helpers/parse-gitignore.d.ts → program/parsing/parseGitignore.d.ts} +0 -0
  81. /package/dist/{helpers/convert-pcre.d.ts → program/utils/pcre.d.ts} +0 -0
@@ -0,0 +1,11 @@
1
+ import { FileGlob, RelFile } from '../../types/types.js';
2
+ import { FlagAttributes } from '../parsing/parseGitattributes.js';
3
+ /** Stores parsed attribute information per file glob */
4
+ export default class Attributes {
5
+ #private;
6
+ constructor();
7
+ get attributes(): Record<FileGlob, FlagAttributes>;
8
+ add(glob: FileGlob, attributes: FlagAttributes): void;
9
+ getFlaggedGlobs(attr: keyof FlagAttributes, val: boolean): string[];
10
+ findAttrsForPath(relFilePath: RelFile): FlagAttributes | null;
11
+ }
@@ -0,0 +1,37 @@
1
+ import ignore from 'ignore';
2
+ /** Stores parsed attribute information per file glob */
3
+ export default class Attributes {
4
+ #attributes;
5
+ constructor() {
6
+ this.#attributes = {};
7
+ }
8
+ get attributes() {
9
+ return this.#attributes;
10
+ }
11
+ add(glob, attributes) {
12
+ this.#attributes[glob] = attributes;
13
+ }
14
+ getFlaggedGlobs(attr, val) {
15
+ return Object.entries(this.#attributes)
16
+ .filter(([, attrs]) => attrs[attr] === val)
17
+ .map(([glob]) => glob);
18
+ }
19
+ findAttrsForPath(relFilePath) {
20
+ const resultAttrs = {};
21
+ for (const glob in this.#attributes) {
22
+ const matchingAttrs = this.#attributes[glob];
23
+ // Check if glob matches rel path
24
+ if (ignore().add(glob).ignores(relFilePath)) {
25
+ for (const [attr, val] of Object.entries(matchingAttrs)) {
26
+ if (val != null) {
27
+ resultAttrs[attr] = val;
28
+ }
29
+ }
30
+ }
31
+ }
32
+ if (!JSON.stringify(resultAttrs).length) {
33
+ return null;
34
+ }
35
+ return resultAttrs;
36
+ }
37
+ }
@@ -1,4 +1,4 @@
1
1
  /** Nukes unused `generated.rb` file content. */
2
2
  export declare function parseGeneratedDataFile(fileContent: string): string[];
3
3
  /** Load a data file from github-linguist. */
4
- export default function loadFile(file: string, offline?: boolean): Promise<string>;
4
+ export declare function loadFile(file: string, offline?: boolean): Promise<string>;
@@ -0,0 +1,32 @@
1
+ import Cache from 'node-cache';
2
+ import FS from 'node:fs';
3
+ import Path from 'node:path';
4
+ import { fileURLToPath } from 'node:url';
5
+ const cache = new Cache({});
6
+ const dirname = Path.dirname(fileURLToPath(import.meta.url));
7
+ async function loadWebFile(file) {
8
+ // Return cache if it exists
9
+ const cachedContent = cache.get(file);
10
+ if (cachedContent)
11
+ return cachedContent;
12
+ // Otherwise cache the request
13
+ const dataUrl = (file) => `https://raw.githubusercontent.com/github/linguist/HEAD/lib/linguist/${file}`;
14
+ // Load file content, falling back to the local file if the request fails
15
+ const fileContent = await fetch(dataUrl(file))
16
+ .then((data) => data.text())
17
+ .catch(async () => await loadLocalFile(file));
18
+ cache.set(file, fileContent);
19
+ return fileContent;
20
+ }
21
+ async function loadLocalFile(file) {
22
+ const filePath = Path.resolve(dirname, '../../../ext', file);
23
+ return FS.promises.readFile(filePath).then((buffer) => buffer.toString());
24
+ }
25
+ /** Nukes unused `generated.rb` file content. */
26
+ export function parseGeneratedDataFile(fileContent) {
27
+ return [...(fileContent.match(/(?<=name\.match\(\/).+?(?=(?<!\\)\/)/gm) ?? [])];
28
+ }
29
+ /** Load a data file from github-linguist. */
30
+ export function loadFile(file, offline = false) {
31
+ return offline ? loadLocalFile(file) : loadWebFile(file);
32
+ }
@@ -0,0 +1,12 @@
1
+ import { HeuristicsSchema, LanguagesScema, VendorSchema } from '../../types/schema.js';
2
+ type LoadedData = {
3
+ langData: LanguagesScema;
4
+ vendorData: VendorSchema;
5
+ docData: VendorSchema;
6
+ heuristicsData: HeuristicsSchema;
7
+ generatedData: string[];
8
+ vendorPaths: string[];
9
+ };
10
+ /** Load data from github-linguist web repo or cached local file. */
11
+ export default function retrieveData(offline: boolean): Promise<LoadedData>;
12
+ export {};
@@ -0,0 +1,27 @@
1
+ import YAML from 'js-yaml';
2
+ import { loadFile, parseGeneratedDataFile } from './loadDataFiles.js';
3
+ let data = null;
4
+ async function initRetrieveData(offline) {
5
+ // Only load the data on mont
6
+ if (data)
7
+ return;
8
+ const langData = (await loadFile('languages.yml', offline).then(YAML.load));
9
+ const vendorData = (await loadFile('vendor.yml', offline).then(YAML.load));
10
+ const docData = (await loadFile('documentation.yml', offline).then(YAML.load));
11
+ const heuristicsData = (await loadFile('heuristics.yml', offline).then(YAML.load));
12
+ const generatedData = (await loadFile('generated.rb', offline).then(parseGeneratedDataFile));
13
+ const vendorPaths = [...vendorData, ...docData, ...generatedData];
14
+ data = {
15
+ langData,
16
+ vendorData,
17
+ docData,
18
+ heuristicsData,
19
+ generatedData,
20
+ vendorPaths,
21
+ };
22
+ }
23
+ /** Load data from github-linguist web repo or cached local file. */
24
+ export default async function retrieveData(offline) {
25
+ await initRetrieveData(offline);
26
+ return data;
27
+ }
@@ -1,2 +1,3 @@
1
1
  export declare const normPath: (...inputPaths: string[]) => string;
2
2
  export declare const normAbsPath: (...inputPaths: string[]) => string;
3
+ export declare const getFileExtension: (filePath: string) => string;
@@ -0,0 +1,17 @@
1
+ import Path from 'node:path';
2
+ export const normPath = function normalisedPath(...inputPaths) {
3
+ return Path.join(...inputPaths).replace(/\\/g, '/');
4
+ };
5
+ export const normAbsPath = function normalisedAbsolutePath(...inputPaths) {
6
+ return Path.resolve(...inputPaths).replace(/\\/g, '/');
7
+ };
8
+ export const getFileExtension = function getFileExtension(filePath) {
9
+ const extension = Path.extname(filePath).toLowerCase();
10
+ if (extension)
11
+ return extension;
12
+ const basename = Path.basename(filePath);
13
+ if (basename.startsWith('.') && basename.length > 1 && basename.indexOf('.', 1) === -1) {
14
+ return basename.toLowerCase();
15
+ }
16
+ return '';
17
+ };
@@ -0,0 +1,17 @@
1
+ import FS from 'node:fs';
2
+ /**
3
+ * Read part of a file on disc.
4
+ * @throws 'EPERM' if the file is not readable.
5
+ */
6
+ export default async function readFileChunk(filename, onlyFirstLine = false) {
7
+ const chunkSize = 100;
8
+ const stream = FS.createReadStream(filename, { highWaterMark: chunkSize });
9
+ let content = '';
10
+ for await (const data of stream) { // may throw
11
+ content += data.toString();
12
+ if (onlyFirstLine) {
13
+ return content.split(/\r?\n/)[0];
14
+ }
15
+ }
16
+ return content;
17
+ }
@@ -16,5 +16,5 @@ interface WalkOutput {
16
16
  folders: string[];
17
17
  }
18
18
  /** Generate list of files in a directory. */
19
- export default function walk(data: WalkInput): WalkOutput;
19
+ export default function walkTree(data: WalkInput): WalkOutput;
20
20
  export {};
@@ -1,19 +1,11 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.default = walk;
7
- const fs_1 = __importDefault(require("fs"));
8
- const path_1 = __importDefault(require("path"));
9
- const parse_gitignore_1 = __importDefault(require("./parse-gitignore"));
10
- const norm_path_1 = require("./norm-path");
1
+ import FS from 'node:fs';
2
+ import Path from 'node:path';
3
+ import parseGitignore from '../parsing/parseGitignore.js';
4
+ import { normAbsPath, normPath } from './normalisedPath.js';
11
5
  let allFiles;
12
6
  let allFolders;
13
- ;
14
- ;
15
7
  /** Generate list of files in a directory. */
16
- function walk(data) {
8
+ export default function walkTree(data) {
17
9
  const { init, commonRoot, folderRoots, folders, ignored } = data;
18
10
  // Initialise files and folders lists
19
11
  if (init) {
@@ -25,37 +17,37 @@ function walk(data) {
25
17
  const folder = folders[0];
26
18
  const localRoot = folderRoots[0].replace(commonRoot, '').replace(/^\//, '');
27
19
  // Get list of files and folders inside this folder
28
- const files = fs_1.default.readdirSync(folder).map(file => {
20
+ const files = FS.readdirSync(folder).map((file) => {
29
21
  // Create path relative to root
30
- const base = (0, norm_path_1.normAbsPath)(folder, file).replace(commonRoot, '.');
22
+ const base = normAbsPath(folder, file).replace(commonRoot, '.');
31
23
  // Add trailing slash to mark directories
32
- const isDir = fs_1.default.lstatSync(path_1.default.resolve(commonRoot, base)).isDirectory();
24
+ const isDir = FS.lstatSync(Path.resolve(commonRoot, base)).isDirectory();
33
25
  return isDir ? `${base}/` : base;
34
26
  });
35
27
  // Read and apply gitignores
36
- const gitignoreFilename = (0, norm_path_1.normPath)(folder, '.gitignore');
37
- if (fs_1.default.existsSync(gitignoreFilename)) {
38
- const gitignoreContents = fs_1.default.readFileSync(gitignoreFilename, 'utf-8');
39
- const ignoredPaths = (0, parse_gitignore_1.default)(gitignoreContents);
40
- const rootRelIgnoredPaths = ignoredPaths.map(ignorePath =>
28
+ const gitignoreFilename = normPath(folder, '.gitignore');
29
+ if (FS.existsSync(gitignoreFilename)) {
30
+ const gitignoreContents = FS.readFileSync(gitignoreFilename, 'utf-8');
31
+ const ignoredPaths = parseGitignore(gitignoreContents);
32
+ const rootRelIgnoredPaths = ignoredPaths.map((ignorePath) =>
41
33
  // get absolute path of the ignore glob
42
- (0, norm_path_1.normPath)(folder, ignorePath)
34
+ normPath(folder, ignorePath)
43
35
  // convert abs ignore glob to be relative to the root folder
44
36
  .replace(commonRoot + '/', ''));
45
37
  ignored.add(rootRelIgnoredPaths);
46
38
  }
47
39
  // Add gitattributes if present
48
- const gitattributesPath = (0, norm_path_1.normPath)(folder, '.gitattributes');
49
- if (fs_1.default.existsSync(gitattributesPath)) {
40
+ const gitattributesPath = normPath(folder, '.gitattributes');
41
+ if (FS.existsSync(gitattributesPath)) {
50
42
  allFiles.add(gitattributesPath);
51
43
  }
52
44
  // Loop through files and folders
53
45
  for (const file of files) {
54
46
  // Create absolute path for disc operations
55
- const path = (0, norm_path_1.normAbsPath)(commonRoot, file);
47
+ const path = normAbsPath(commonRoot, file);
56
48
  const localPath = localRoot ? file.replace(`./${localRoot}/`, '') : file.replace('./', '');
57
49
  // Skip if nonexistant
58
- const nonExistant = !fs_1.default.existsSync(path);
50
+ const nonExistant = !FS.existsSync(path);
59
51
  if (nonExistant)
60
52
  continue;
61
53
  // Skip if marked in gitignore
@@ -63,12 +55,12 @@ function walk(data) {
63
55
  if (isIgnored)
64
56
  continue;
65
57
  // Add absolute folder path to list
66
- allFolders.add((0, norm_path_1.normAbsPath)(folder));
58
+ allFolders.add(normAbsPath(folder));
67
59
  // Check if this is a folder or file
68
60
  if (file.endsWith('/')) {
69
61
  // Recurse into subfolders
70
62
  allFolders.add(path);
71
- walk({ init: false, commonRoot, folderRoots, folders: [path], ignored });
63
+ walkTree({ init: false, commonRoot, folderRoots, folders: [path], ignored });
72
64
  }
73
65
  else {
74
66
  // Add file path to list
@@ -79,12 +71,12 @@ function walk(data) {
79
71
  // Recurse into all folders
80
72
  else {
81
73
  for (const i in folders) {
82
- walk({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored });
74
+ walkTree({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored });
83
75
  }
84
76
  }
85
77
  // Return absolute files and folders lists
86
78
  return {
87
- files: [...allFiles].map(file => file.replace(/^\./, commonRoot)),
79
+ files: [...allFiles].map((file) => file.replace(/^\./, commonRoot)),
88
80
  folders: [...allFolders],
89
81
  };
90
82
  }
@@ -1,4 +1,4 @@
1
- import * as T from '../types';
1
+ import * as T from '../../types/types.js';
2
2
  export type FlagAttributes = {
3
3
  'vendored': boolean | null;
4
4
  'generated': boolean | null;
@@ -14,4 +14,4 @@ export type ParsedGitattributes = Array<{
14
14
  /**
15
15
  * Parses a gitattributes file.
16
16
  */
17
- export default function parseAttributes(content: string, folderRoot?: string): ParsedGitattributes;
17
+ export default function parseGitattributes(content: string, folderRoot?: string): ParsedGitattributes;
@@ -1,12 +1,8 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.default = parseAttributes;
4
- const norm_path_1 = require("./norm-path");
1
+ import { normPath } from '../fs/normalisedPath.js';
5
2
  /**
6
3
  * Parses a gitattributes file.
7
4
  */
8
- function parseAttributes(content, folderRoot = '.') {
9
- var _a, _b;
5
+ export default function parseGitattributes(content, folderRoot = '.') {
10
6
  const output = [];
11
7
  for (const rawLine of content.split('\n')) {
12
8
  const line = rawLine.replace(/#.*/, '').trim();
@@ -14,7 +10,7 @@ function parseAttributes(content, folderRoot = '.') {
14
10
  continue;
15
11
  const parts = line.split(/\s+/g);
16
12
  const fileGlob = parts[0];
17
- const relFileGlob = (0, norm_path_1.normPath)(folderRoot, fileGlob);
13
+ const relFileGlob = normPath(folderRoot, fileGlob);
18
14
  const attrParts = parts.slice(1);
19
15
  const isTrue = (str) => !str.startsWith('-') && !str.endsWith('=false');
20
16
  const isFalse = (str) => str.startsWith('-') || str.endsWith('=false');
@@ -29,7 +25,7 @@ function parseAttributes(content, folderRoot = '.') {
29
25
  'documentation': boolOrNullVal('linguist-documentation'),
30
26
  'detectable': boolOrNullVal('linguist-detectable'),
31
27
  'binary': hasTrueParts('binary') || hasFalseParts('text') ? true : hasFalseParts('binary') || hasTrueParts('text') ? false : null,
32
- 'language': (_b = (_a = trueParts('linguist-language').at(-1)) === null || _a === void 0 ? void 0 : _a.split('=')[1]) !== null && _b !== void 0 ? _b : null,
28
+ 'language': trueParts('linguist-language').at(-1)?.split('=')[1] ?? null,
33
29
  };
34
30
  output.push({ glob: relFileGlob, attrs });
35
31
  }
@@ -1,7 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.default = parseGitignore;
4
- function parseGitignore(content) {
1
+ export default function parseGitignore(content) {
5
2
  const readableData = content
6
3
  // Remove comments unless escaped
7
4
  .replace(/(?<!\\)#.+/g, '')
@@ -0,0 +1,9 @@
1
+ import * as T from '../types/types.js';
2
+ import Attributes from './classes/attributes.js';
3
+ type ProcessFilesResult = {
4
+ files: T.AbsFile[];
5
+ manualAttributes: Attributes;
6
+ relPath(file: T.AbsFile): T.RelFile;
7
+ };
8
+ export default function processFiles(input: string[], opts: T.Options, useRawContent: boolean, vendorPaths: string[]): Promise<ProcessFilesResult>;
9
+ export {};
@@ -0,0 +1,111 @@
1
+ import commonPrefix from 'common-path-prefix';
2
+ import ignore from 'ignore';
3
+ import FS from 'node:fs';
4
+ import Path from 'node:path';
5
+ import Attributes from './classes/attributes.js';
6
+ import { normPath } from './fs/normalisedPath.js';
7
+ import readFileChunk from './fs/readFile.js';
8
+ import walkTree from './fs/walkTree.js';
9
+ import parseGitattributes from './parsing/parseGitattributes.js';
10
+ const binaryData = JSON.parse(FS.readFileSync(new URL('../../node_modules/binary-extensions/binary-extensions.json', import.meta.url), 'utf-8'));
11
+ export default async function processFiles(input, opts, useRawContent, vendorPaths) {
12
+ // Set a common root path so that vendor paths do not incorrectly match parent folders
13
+ const resolvedInput = input.map((path) => normPath(Path.resolve(path)));
14
+ const commonRoot = (input.length > 1 ? commonPrefix(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
15
+ const relPath = (file) => (useRawContent ? file : normPath(Path.relative(commonRoot, file)));
16
+ const unRelPath = (file) => (useRawContent ? file : normPath(Path.resolve(commonRoot, file)));
17
+ // Other helper functions
18
+ const filterOutIgnored = (files, ignored) => ignored.filter(files.map((file) => relPath(file))).map((file) => unRelPath(file));
19
+ //*PREPARE FILES AND DATA*//
20
+ // Prepare list of ignored files
21
+ const ignored = ignore();
22
+ ignored.add('.git/');
23
+ ignored.add(opts.ignoredFiles ?? []);
24
+ const regexIgnores = opts.keepVendored ? [] : vendorPaths.map((path) => RegExp(path, 'i'));
25
+ // Load file paths and folders
26
+ let files;
27
+ if (useRawContent) {
28
+ // Uses raw file content
29
+ files = input;
30
+ }
31
+ else {
32
+ // Uses directory on disc
33
+ const data = walkTree({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored });
34
+ files = data.files;
35
+ }
36
+ // Fetch and normalise gitattributes data of all subfolders and save to metadata
37
+ const manualAttributes = new Attributes();
38
+ if (!useRawContent && opts.checkAttributes) {
39
+ const nestedAttrFiles = files.filter((file) => file.endsWith('.gitattributes'));
40
+ for (const attrFile of nestedAttrFiles) {
41
+ const relAttrFile = relPath(attrFile);
42
+ const relAttrFolder = Path.dirname(relAttrFile);
43
+ const contents = await readFileChunk(attrFile);
44
+ const parsed = parseGitattributes(contents, relAttrFolder);
45
+ for (const { glob, attrs } of parsed) {
46
+ manualAttributes.add(glob, attrs);
47
+ }
48
+ }
49
+ }
50
+ // Remove files that are linguist-ignored via regex by default unless explicitly unignored in gitattributes
51
+ const filesToIgnore = [];
52
+ for (const file of files) {
53
+ const relFile = relPath(file);
54
+ const isRegexIgnored = regexIgnores.some((pattern) => pattern.test(relFile));
55
+ if (!isRegexIgnored) {
56
+ // Checking overrides is moot if file is not even marked as ignored by default
57
+ continue;
58
+ }
59
+ const fileAttrs = manualAttributes.findAttrsForPath(relPath(file));
60
+ if (fileAttrs?.generated === false || fileAttrs?.vendored === false) {
61
+ // File is explicitly marked as *not* to be ignored
62
+ // do nothing
63
+ }
64
+ else {
65
+ filesToIgnore.push(file);
66
+ }
67
+ }
68
+ files = files.filter((file) => !filesToIgnore.includes(file));
69
+ // Apply vendor file path matches and filter out vendored files
70
+ if (!opts.keepVendored) {
71
+ // Get data of files that have been manually marked with metadata
72
+ const vendorTrueGlobs = [
73
+ ...manualAttributes.getFlaggedGlobs('vendored', true),
74
+ ...manualAttributes.getFlaggedGlobs('generated', true),
75
+ ...manualAttributes.getFlaggedGlobs('documentation', true),
76
+ ];
77
+ const vendorFalseGlobs = [
78
+ ...manualAttributes.getFlaggedGlobs('vendored', false),
79
+ ...manualAttributes.getFlaggedGlobs('generated', false),
80
+ ...manualAttributes.getFlaggedGlobs('documentation', false),
81
+ ];
82
+ // Set up glob ignore object to use for expanding globs to match files
83
+ const vendorTrueIgnore = ignore().add(vendorTrueGlobs);
84
+ const vendorFalseIgnore = ignore().add(vendorFalseGlobs);
85
+ // Remove all files marked as vendored by default
86
+ const excludedFiles = files.filter((file) => vendorPaths.some((pathPtn) => RegExp(pathPtn, 'i').test(relPath(file))));
87
+ files = files.filter((file) => !excludedFiles.includes(file));
88
+ // Re-add removed files that are overridden manually in gitattributes
89
+ const overriddenExcludedFiles = excludedFiles.filter((file) => vendorFalseIgnore.ignores(relPath(file)));
90
+ files.push(...overriddenExcludedFiles);
91
+ // Remove files explicitly marked as vendored in gitattributes
92
+ files = files.filter((file) => !vendorTrueIgnore.ignores(relPath(file)));
93
+ }
94
+ // Filter out binary files
95
+ if (!opts.keepBinary) {
96
+ // Filter out files that are binary by default
97
+ files = files.filter((file) => !binaryData.some((ext) => file.endsWith('.' + ext)));
98
+ // Filter out manually specified binary files
99
+ const binaryIgnored = ignore().add(manualAttributes.getFlaggedGlobs('binary', true));
100
+ files = filterOutIgnored(files, binaryIgnored);
101
+ // Re-add files manually marked not as binary
102
+ const binaryUnignored = ignore().add(manualAttributes.getFlaggedGlobs('binary', false));
103
+ const unignoredList = filterOutIgnored(files, binaryUnignored);
104
+ files.push(...unignoredList);
105
+ }
106
+ return {
107
+ files,
108
+ manualAttributes,
109
+ relPath,
110
+ };
111
+ }
@@ -1,8 +1,5 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.default = pcre;
4
1
  /** Convert a PCRE regex into JS. */
5
- function pcre(regex) {
2
+ export default function pcre(regex) {
6
3
  let finalRegex = regex;
7
4
  const replace = (search, replace) => finalRegex = finalRegex.replace(search, replace);
8
5
  const finalFlags = new Set();
@@ -1,4 +1,4 @@
1
- import { Category, Language } from './types';
1
+ import { Category, Language } from './types.js';
2
2
  export interface LanguagesScema {
3
3
  [name: string]: {
4
4
  language_id: number;
@@ -0,0 +1 @@
1
+ export {};
@@ -1,15 +1,16 @@
1
+ import type { LanguagesScema } from './schema.js';
1
2
  export type LanguageResult = string | null;
2
3
  export type Language = string;
3
4
  export type Category = 'data' | 'markup' | 'programming' | 'prose';
4
5
  export type FilePath = string;
5
6
  export type Bytes = Integer;
6
7
  export type Integer = number;
8
+ export type LanguageMetadata = LanguagesScema[string];
7
9
  export type RelFile = string & {};
8
10
  export type AbsFile = string & {};
9
11
  export type AbsFolder = string & {};
10
12
  export type FileGlob = string & {};
11
13
  export interface Options {
12
- fileContent?: string | string[];
13
14
  ignoredFiles?: string[];
14
15
  ignoredLanguages?: Language[];
15
16
  categories?: Category[];
@@ -27,15 +28,33 @@ export interface Options {
27
28
  checkShebang?: boolean;
28
29
  checkModeline?: boolean;
29
30
  }
31
+ export interface VirtualFile {
32
+ path: string;
33
+ content?: string;
34
+ firstLine?: string;
35
+ size?: number;
36
+ extension?: string;
37
+ isBinary?: boolean;
38
+ metadata?: {
39
+ vendored?: boolean;
40
+ generated?: boolean;
41
+ documentation?: boolean;
42
+ };
43
+ attributes?: {
44
+ language?: LanguageResult;
45
+ binary?: boolean;
46
+ detectable?: boolean;
47
+ };
48
+ }
49
+ type LinesOfCode = {
50
+ total: Integer;
51
+ content: Integer;
52
+ };
30
53
  export interface Results {
31
54
  files: {
32
55
  count: Integer;
33
56
  bytes: Bytes;
34
- lines: {
35
- total: Integer;
36
- content: Integer;
37
- code: Integer;
38
- };
57
+ lines: LinesOfCode;
39
58
  /** Note: Results use slashes as delimiters even on Windows. */
40
59
  results: Record<FilePath, LanguageResult>;
41
60
  alternatives: Record<FilePath, LanguageResult[]>;
@@ -43,32 +62,24 @@ export interface Results {
43
62
  languages: {
44
63
  count: Integer;
45
64
  bytes: Bytes;
46
- lines: {
47
- total: Integer;
48
- content: Integer;
49
- code: Integer;
50
- };
65
+ lines: LinesOfCode;
51
66
  results: Record<Language, {
67
+ count: Integer;
52
68
  bytes: Bytes;
53
- lines: {
54
- total: Integer;
55
- content: Integer;
56
- code: Integer;
57
- };
58
- type: Category;
59
- parent?: Language;
60
- color?: `#${string}`;
69
+ lines: LinesOfCode;
61
70
  }>;
62
71
  };
63
72
  unknown: {
64
73
  count: Integer;
65
74
  bytes: Bytes;
66
- lines: {
67
- total: Integer;
68
- content: Integer;
69
- code: Integer;
70
- };
75
+ lines: LinesOfCode;
71
76
  extensions: Record<string, Bytes>;
72
77
  filenames: Record<string, Bytes>;
73
78
  };
79
+ repository: Record<Language, {
80
+ type: Category;
81
+ parent?: Language;
82
+ color?: `#${string}`;
83
+ }>;
74
84
  }
85
+ export {};
@@ -0,0 +1 @@
1
+ export {};
package/ext/generated.rb CHANGED
@@ -6,6 +6,7 @@
6
6
  - node_modules\/
7
7
  - vendor\/((?!-)[-0-9A-Za-z]+(?<!-)\.)+(com|edu|gov|in|me|net|org|fm|io)
8
8
  - (Gopkg|glide)\.lock
9
+ - Package\.resolved
9
10
  - poetry\.lock
10
11
  - pdm\.lock
11
12
  - uv\.lock
@@ -15,6 +16,7 @@
15
16
  - npm-shrinkwrap\.json
16
17
  - package-lock\.json
17
18
  - pnpm-lock\.yaml
19
+ - (?:^|\/)bun\.lockb?$
18
20
  - (^|\/)\.pnp\..*$
19
21
  - Godeps\/
20
22
  - composer\.lock
@@ -23,6 +25,10 @@
23
25
  - Cargo\.toml\.orig
24
26
  - (^|\/)flake\.lock$
25
27
  - (^|\/)MODULE\.bazel\.lock$
28
+ - (?:^|\/)gradlew(?:\.bat)?$
29
+ - (?:^|\/)mvnw(?:\.cmd)?$
30
+ - (?:^|\/)mise(?:\.[^\/]+)?\.lock$
31
+ - (?:^|\/)(Julia)?Manifest(-v\d+\.\d+)?\.toml$
26
32
  - Pipfile\.lock
27
33
  - (?:^|\/)\.terraform\.lock\.hcl$
28
34
  - ppport\.h$