linguist-js 2.9.2 → 3.0.0

package/dist/src/program/fs/walkTree.d.ts

@@ -0,0 +1,20 @@
+import { Ignore } from 'ignore';
+interface WalkInput {
+    /** Whether this is walking the tree from the root */
+    init: boolean;
+    /** The common root absolute path of all folders being checked */
+    commonRoot: string;
+    /** The absolute path that each folder is relative to */
+    folderRoots: string[];
+    /** The absolute path of folders being checked */
+    folders: string[];
+    /** An instantiated Ignore object listing ignored files */
+    ignored: Ignore;
+}
+interface WalkOutput {
+    files: string[];
+    folders: string[];
+}
+/** Generate list of files in a directory. */
+export default function walkTree(data: WalkInput): WalkOutput;
+export {};

package/dist/src/program/fs/walkTree.js

@@ -0,0 +1,82 @@
+import FS from 'node:fs';
+import Path from 'node:path';
+import parseGitignore from '../parsing/parseGitignore.js';
+import { normAbsPath, normPath } from './normalisedPath.js';
+let allFiles;
+let allFolders;
+/** Generate list of files in a directory. */
+export default function walkTree(data) {
+    const { init, commonRoot, folderRoots, folders, ignored } = data;
+    // Initialise files and folders lists
+    if (init) {
+        allFiles = new Set();
+        allFolders = new Set();
+    }
+    // Walk tree of a folder
+    if (folders.length === 1) {
+        const folder = folders[0];
+        const localRoot = folderRoots[0].replace(commonRoot, '').replace(/^\//, '');
+        // Get list of files and folders inside this folder
+        const files = FS.readdirSync(folder).map((file) => {
+            // Create path relative to root
+            const base = normAbsPath(folder, file).replace(commonRoot, '.');
+            // Add trailing slash to mark directories
+            const isDir = FS.lstatSync(Path.resolve(commonRoot, base)).isDirectory();
+            return isDir ? `${base}/` : base;
+        });
+        // Read and apply gitignores
+        const gitignoreFilename = normPath(folder, '.gitignore');
+        if (FS.existsSync(gitignoreFilename)) {
+            const gitignoreContents = FS.readFileSync(gitignoreFilename, 'utf-8');
+            const ignoredPaths = parseGitignore(gitignoreContents);
+            const rootRelIgnoredPaths = ignoredPaths.map((ignorePath) => 
+            // get absolute path of the ignore glob
+            normPath(folder, ignorePath)
+                // convert abs ignore glob to be relative to the root folder
+                .replace(commonRoot + '/', ''));
+            ignored.add(rootRelIgnoredPaths);
+        }
+        // Add gitattributes if present
+        const gitattributesPath = normPath(folder, '.gitattributes');
+        if (FS.existsSync(gitattributesPath)) {
+            allFiles.add(gitattributesPath);
+        }
+        // Loop through files and folders
+        for (const file of files) {
+            // Create absolute path for disc operations
+            const path = normAbsPath(commonRoot, file);
+            const localPath = localRoot ? file.replace(`./${localRoot}/`, '') : file.replace('./', '');
+            // Skip if nonexistant
+            const nonExistant = !FS.existsSync(path);
+            if (nonExistant)
+                continue;
+            // Skip if marked in gitignore
+            const isIgnored = ignored.test(localPath).ignored;
+            if (isIgnored)
+                continue;
+            // Add absolute folder path to list
+            allFolders.add(normAbsPath(folder));
+            // Check if this is a folder or file
+            if (file.endsWith('/')) {
+                // Recurse into subfolders
+                allFolders.add(path);
+                walkTree({ init: false, commonRoot, folderRoots, folders: [path], ignored });
+            }
+            else {
+                // Add file path to list
+                allFiles.add(path);
+            }
+        }
+    }
+    // Recurse into all folders
+    else {
+        for (const i in folders) {
+            walkTree({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored });
+        }
+    }
+    // Return absolute files and folders lists
+    return {
+        files: [...allFiles].map((file) => file.replace(/^\./, commonRoot)),
+        folders: [...allFolders],
+    };
+}

package/dist/src/program/parsing/parseGitattributes.d.ts

@@ -0,0 +1,17 @@
+import * as T from '../../types/types.js';
+export type FlagAttributes = {
+    'vendored': boolean | null;
+    'generated': boolean | null;
+    'documentation': boolean | null;
+    'detectable': boolean | null;
+    'binary': boolean | null;
+    'language': T.LanguageResult;
+};
+export type ParsedGitattributes = Array<{
+    glob: T.FileGlob;
+    attrs: FlagAttributes;
+}>;
+/**
+ * Parses a gitattributes file.
+ */
+export default function parseGitattributes(content: string, folderRoot?: string): ParsedGitattributes;

package/dist/src/program/parsing/parseGitattributes.js

@@ -0,0 +1,33 @@
+import { normPath } from '../fs/normalisedPath.js';
+/**
+ * Parses a gitattributes file.
+ */
+export default function parseGitattributes(content, folderRoot = '.') {
+    const output = [];
+    for (const rawLine of content.split('\n')) {
+        const line = rawLine.replace(/#.*/, '').trim();
+        if (!line)
+            continue;
+        const parts = line.split(/\s+/g);
+        const fileGlob = parts[0];
+        const relFileGlob = normPath(folderRoot, fileGlob);
+        const attrParts = parts.slice(1);
+        const isTrue = (str) => !str.startsWith('-') && !str.endsWith('=false');
+        const isFalse = (str) => str.startsWith('-') || str.endsWith('=false');
+        const trueParts = (str) => attrParts.filter(part => part.includes(str) && isTrue(part));
+        const falseParts = (str) => attrParts.filter(part => part.includes(str) && isFalse(part));
+        const hasTrueParts = (str) => trueParts(str).length > 0;
+        const hasFalseParts = (str) => falseParts(str).length > 0;
+        const boolOrNullVal = (str) => hasTrueParts(str) ? true : hasFalseParts(str) ? false : null;
+        const attrs = {
+            'generated': boolOrNullVal('linguist-generated'),
+            'vendored': boolOrNullVal('linguist-vendored'),
+            'documentation': boolOrNullVal('linguist-documentation'),
+            'detectable': boolOrNullVal('linguist-detectable'),
+            'binary': hasTrueParts('binary') || hasFalseParts('text') ? true : hasFalseParts('binary') || hasTrueParts('text') ? false : null,
+            'language': trueParts('linguist-language').at(-1)?.split('=')[1] ?? null,
+        };
+        output.push({ glob: relFileGlob, attrs });
+    }
+    return output;
+}

package/dist/src/program/parsing/parseGitignore.d.ts

@@ -0,0 +1 @@
+export default function parseGitignore(content: string): string[];

package/dist/src/program/parsing/parseGitignore.js

@@ -0,0 +1,9 @@
+export default function parseGitignore(content) {
+    const readableData = content
+        // Remove comments unless escaped
+        .replace(/(?<!\\)#.+/g, '')
+        // Remove whitespace unless escaped
+        .replace(/(?:(?<!\\)\s)+$/g, '');
+    const arrayData = readableData.split(/\r?\n/).filter(data => data);
+    return arrayData;
+}

package/dist/src/program/utils/pcre.d.ts

@@ -0,0 +1,2 @@
+/** Convert a PCRE regex into JS. */
+export default function pcre(regex: string): RegExp;

package/dist/src/program/utils/pcre.js

@@ -0,0 +1,35 @@
+/** Convert a PCRE regex into JS. */
+export default function pcre(regex) {
+    let finalRegex = regex;
+    const replace = (search, replace) => finalRegex = finalRegex.replace(search, replace);
+    const finalFlags = new Set();
+    // Convert inline flag declarations
+    const inlineMatches = regex.matchAll(/\?(-)?([a-z]):/g);
+    const startMatches = regex.matchAll(/\(\?(-)?([a-z]+)\)/g);
+    for (const [match, isNegative, flags] of [...inlineMatches, ...startMatches]) {
+        replace(match, '');
+        const func = (flag) => isNegative ? finalFlags.delete(flag) : finalFlags.add(flag);
+        [...flags].forEach(func);
+    }
+    // Remove PCRE-only syntax
+    replace(/([*+]){2}/g, '$1');
+    replace(/\(\?>/g, '(?:');
+    // Remove start/end-of-file markers
+    if (/\\[AZ]/.test(finalRegex)) {
+        replace(/\\A/g, '^');
+        replace(/\\Z/g, '$');
+        finalFlags.delete('m');
+    }
+    else {
+        finalFlags.add('m');
+    }
+    // Reformat free-spacing mode
+    if (finalFlags.has('x')) {
+        finalFlags.delete('x');
+        replace(/#.+/g, '');
+        replace(/^\s+|\s+$|\n/gm, '');
+        replace(/\s+/g, ' ');
+    }
+    // Return final regex
+    return RegExp(finalRegex, [...finalFlags].join(''));
+}

package/dist/src/types/schema.d.ts

@@ -0,0 +1,37 @@
+import { Category, Language } from './types.js';
+export interface LanguagesScema {
+    [name: string]: {
+        language_id: number;
+        fs_name?: Language;
+        type: Category;
+        group?: Language;
+        color?: `#${string}`;
+        aliases?: string[];
+        extensions?: `.${string}`[];
+        filenames?: string[];
+        interpreters?: string[];
+        tm_scope?: `${'source' | 'text'}.${string}`;
+        ace_mode?: string;
+        codemirror_mode?: string;
+        codemirror_mime_type?: `${string}/${string}`;
+        wrap?: boolean;
+    };
+}
+interface HeuristicsRules {
+    pattern?: string;
+    named_pattern?: string;
+    named_patterns?: string[];
+    negative_pattern?: string;
+}
+export interface HeuristicsSchema {
+    disambiguations: Array<{
+        extensions: string[];
+        rules: Array<HeuristicsRules & {
+            language: string | string[];
+            and?: HeuristicsRules[];
+        }>;
+    }>;
+    named_patterns: Record<string, string | string[]>;
+}
+export type VendorSchema = string[];
+export {};

package/dist/src/types/schema.js

@@ -0,0 +1 @@
+export {};

package/dist/src/types/types.d.ts

@@ -0,0 +1,65 @@
+export type LanguageResult = string | null;
+export type Language = string;
+export type Category = 'data' | 'markup' | 'programming' | 'prose';
+export type FilePath = string;
+export type Bytes = Integer;
+export type Integer = number;
+export type RelFile = string & {};
+export type AbsFile = string & {};
+export type AbsFolder = string & {};
+export type FileGlob = string & {};
+export interface Options {
+    ignoredFiles?: string[];
+    ignoredLanguages?: Language[];
+    categories?: Category[];
+    keepVendored?: boolean;
+    keepBinary?: boolean;
+    relativePaths?: boolean;
+    childLanguages?: boolean;
+    quick?: boolean;
+    offline?: boolean;
+    calculateLines?: boolean;
+    checkIgnored?: boolean;
+    checkDetected?: boolean;
+    checkAttributes?: boolean;
+    checkHeuristics?: boolean;
+    checkShebang?: boolean;
+    checkModeline?: boolean;
+}
+type LinesOfCode = {
+    total: Integer;
+    content: Integer;
+};
+export interface Results {
+    files: {
+        count: Integer;
+        bytes: Bytes;
+        lines: LinesOfCode;
+        /** Note: Results use slashes as delimiters even on Windows. */
+        results: Record<FilePath, LanguageResult>;
+        alternatives: Record<FilePath, LanguageResult[]>;
+    };
+    languages: {
+        count: Integer;
+        bytes: Bytes;
+        lines: LinesOfCode;
+        results: Record<Language, {
+            count: Integer;
+            bytes: Bytes;
+            lines: LinesOfCode;
+        }>;
+    };
+    unknown: {
+        count: Integer;
+        bytes: Bytes;
+        lines: LinesOfCode;
+        extensions: Record<string, Bytes>;
+        filenames: Record<string, Bytes>;
+    };
+    repository: Record<Language, {
+        type: Category;
+        parent?: Language;
+        color?: `#${string}`;
+    }>;
+}
+export {};

package/dist/src/types/types.js

@@ -0,0 +1 @@
+export {};

package/dist/types/schema.d.ts

@@ -0,0 +1,37 @@
+import { Category, Language } from './types.js';
+export interface LanguagesScema {
+    [name: string]: {
+        language_id: number;
+        fs_name?: Language;
+        type: Category;
+        group?: Language;
+        color?: `#${string}`;
+        aliases?: string[];
+        extensions?: `.${string}`[];
+        filenames?: string[];
+        interpreters?: string[];
+        tm_scope?: `${'source' | 'text'}.${string}`;
+        ace_mode?: string;
+        codemirror_mode?: string;
+        codemirror_mime_type?: `${string}/${string}`;
+        wrap?: boolean;
+    };
+}
+interface HeuristicsRules {
+    pattern?: string;
+    named_pattern?: string;
+    named_patterns?: string[];
+    negative_pattern?: string;
+}
+export interface HeuristicsSchema {
+    disambiguations: Array<{
+        extensions: string[];
+        rules: Array<HeuristicsRules & {
+            language: string | string[];
+            and?: HeuristicsRules[];
+        }>;
+    }>;
+    named_patterns: Record<string, string | string[]>;
+}
+export type VendorSchema = string[];
+export {};

package/dist/types/schema.js

@@ -0,0 +1 @@
+export {};

package/dist/types/types.d.ts

@@ -0,0 +1,84 @@
+import type { LanguagesScema } from './schema.js';
+export type LanguageResult = string | null;
+export type Language = string;
+export type Category = 'data' | 'markup' | 'programming' | 'prose';
+export type FilePath = string;
+export type Bytes = Integer;
+export type Integer = number;
+export type LanguageMetadata = LanguagesScema[string];
+export type RelFile = string & {};
+export type AbsFile = string & {};
+export type AbsFolder = string & {};
+export type FileGlob = string & {};
+export interface Options {
+    ignoredFiles?: string[];
+    ignoredLanguages?: Language[];
+    categories?: Category[];
+    keepVendored?: boolean;
+    keepBinary?: boolean;
+    relativePaths?: boolean;
+    childLanguages?: boolean;
+    quick?: boolean;
+    offline?: boolean;
+    calculateLines?: boolean;
+    checkIgnored?: boolean;
+    checkDetected?: boolean;
+    checkAttributes?: boolean;
+    checkHeuristics?: boolean;
+    checkShebang?: boolean;
+    checkModeline?: boolean;
+}
+export interface VirtualFile {
+    path: string;
+    content?: string;
+    firstLine?: string;
+    size?: number;
+    extension?: string;
+    isBinary?: boolean;
+    metadata?: {
+        vendored?: boolean;
+        generated?: boolean;
+        documentation?: boolean;
+    };
+    attributes?: {
+        language?: LanguageResult;
+        binary?: boolean;
+        detectable?: boolean;
+    };
+}
+type LinesOfCode = {
+    total: Integer;
+    content: Integer;
+};
+export interface Results {
+    files: {
+        count: Integer;
+        bytes: Bytes;
+        lines: LinesOfCode;
+        /** Note: Results use slashes as delimiters even on Windows. */
+        results: Record<FilePath, LanguageResult>;
+    };
+    languages: {
+        count: Integer;
+        bytes: Bytes;
+        lines: LinesOfCode;
+        results: Record<Language, {
+            count: Integer;
+            bytes: Bytes;
+            lines: LinesOfCode;
+        }>;
+    };
+    unknown: {
+        count: Integer;
+        bytes: Bytes;
+        lines: LinesOfCode;
+        extensions: Record<string, Bytes>;
+        filenames: Record<string, Bytes>;
+    };
+    repository: Record<Language, {
+        type: Category;
+        parent?: Language;
+        color?: `#${string}`;
+    }>;
+}
+export {};

package/dist/types/types.js

@@ -0,0 +1 @@
+export {};

package/ext/generated.rb

@@ -6,6 +6,7 @@
 - node_modules\/
 - vendor\/((?!-)[-0-9A-Za-z]+(?<!-)\.)+(com|edu|gov|in|me|net|org|fm|io)
 - (Gopkg|glide)\.lock
+- Package\.resolved
 - poetry\.lock
 - pdm\.lock
 - uv\.lock
@@ -15,6 +16,7 @@
 - npm-shrinkwrap\.json
 - package-lock\.json
 - pnpm-lock\.yaml
+- (?:^|\/)bun\.lockb?$
 - (^|\/)\.pnp\..*$
 - Godeps\/
 - composer\.lock
@@ -23,6 +25,10 @@
 - Cargo\.toml\.orig
 - (^|\/)flake\.lock$
 - (^|\/)MODULE\.bazel\.lock$
+- (?:^|\/)gradlew(?:\.bat)?$
+- (?:^|\/)mvnw(?:\.cmd)?$
+- (?:^|\/)mise(?:\.[^\/]+)?\.lock$
+- (?:^|\/)(Julia)?Manifest(-v\d+\.\d+)?\.toml$
 - Pipfile\.lock
 - (?:^|\/)\.terraform\.lock\.hcl$
 - ppport\.h$