cspell-grammar 0.1.5 → 6.1.3

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Jason Dent
+Copyright (c) 2017 Jason Dent
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,15 +1,100 @@
-# Grammar Parser
-tmLanguage Grammar parser for cSpell
+# `cspell-grammar`
 
-**WIP** -- This is a *Work In Progress*
+CSpell Grammar is used to generate a parser. The Parser is used to add context / scope to parts of a document, making it easier to define the parts to spell spell checked.
 
-This language parser support tmLanguage style grammars.
+This is to address the issues and limitations related to `ignoreRegExpList` and `includeRegExpList`.
 
-## Notes
+The parser is use to add `scope` to sections of a document. The `scope` can then be used to apply spell checking rules.
 
-- Works ONLY with Node 8 and above or Javascript engines that support RegExp look back.
-- Most tmLanguage grammars use RegExp based upon Oniguruma engine. This grammar engine attempts to
-simulate the features of Oniguruma, but it may fail. At the grammar engine's core, XRegExp is used.
-So any limitations in XRegExp are also limitations in the grammar.
-  - `\A`, `\G`, `\Z`, `\z` flags are simulated in a best attempt, but it is not possible to get them
-exactly right.
+Example: Only check comments and strings
+
+```yaml
+rules:
+  '*': false
+  comment: true
+  string: true
+```
+
+It can be even more powerful like controlling the language settings based upon scope.
+
+```yaml
+rules:
+  comment:
+    language: en
+  string:
+    language: en,fr
+    dictionaries: ['marketing-terms'],
+    caseSensitive: true
+  string.javascript:
+    caseSensitive: false
+```
+
+Rules are applied in the order they match the scope of the text.
+
+When checking JavaScript files with the above example rules:
+
+- strings will:
+  - use the locale `en,fr`
+  - the `marketing-terms` dictionary will be enabled
+  - `caseSensitive` will be `true`
+- everything else:
+  - locale: `en`
+  - `caseSensitive` will be `false`
+
+At its core, `cspell-grammar` uses a simplified form of the TextMate grammar.
+
+## Reasoning
+
+Why use a grammar parser? Couldn't a colorizer / highlighter or a language AST be used?
+At one level, needs of the spell checker are simpler and different from colorizers or language AST parsers.
+The goal of a spell checker is to spell check **_relevant_** text. The spell check does not need to care about
+the syntactical correctness of a document or presentation.
+
+The goal of a grammar parser for the spell checker is to allow the user to decide:
+
+1. What text should be checked.
+1. Which dictionaries (or languages) should be used.
+1. Are accents and case important
+
+Note: CSpell is a pure JavaScript application, so including the Oniguruma is not an option.
+
+### Considerations
+
+- Parsing a document should be fast - meaning the grammar should be as simple as possible to meet
+  the needs of the spell checker and not focus on scope detail. This is where a colorizer grammar is
+  not a good fit to be used.
+- AST's are a bit of an overkill for a spell checker. They provide too much detail while not bringing much benefit
+  from the detail.
+
+## Transformation
+
+Consider the following bit of LaTeX:
+
+```latex
+k\"{o}nnen
+können
+```
+
+<!--- cspell:ignore können nnen   -->
+
+For the spell checker to work correctly, the `\"{o}` should be transformed into `ö` before it is checked against the German dictionary.
+
+This creates a few challenges.
+
+Possible options:
+
+1. Simple whole document substitution
+   - Challenges
+     - It is not context aware and might replace the wrong text.
+     - It changes the location of the words and messes up issue reporting (some sort of Map would be needed to get the correct line / character offset).
+
+- Advantages
+  - Easy to implement except for the context and mapping.
+
+1. Scope level substitution
+   Transformations occur at the scope level.
+   - Challenges
+     - offset mapping is still and issue (maybe)
+     - need a way to merge text with adjacent scopes after transformation
+   - Advantages
+     - it is context aware

package/bin.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+'use strict';
+
+const app = require('./dist/app.js');
+
+app.run(process.argv);

package/dist/app.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Run simple parser
+ * @param args -- command line arguments
+ * @returns Promise
+ */
+export declare function run(args: string[]): Promise<void>;
+//# sourceMappingURL=app.d.ts.map

package/dist/app.js

@@ -0,0 +1,67 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = void 0;
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const typescript_1 = require("./parsers/typescript");
+const parsers = {
+    '.ts': typescript_1.parser,
+};
+/**
+ * Run simple parser
+ * @param args -- command line arguments
+ * @returns Promise
+ */
+async function run(args) {
+    // early out if there are not enough arguments
+    if (args.length < 3) {
+        console.log('usage...');
+        return;
+    }
+    const filename = args.slice(2).filter((p) => !p.startsWith('-'))[0];
+    if (!filename) {
+        console.log('filename missing');
+        return;
+    }
+    const ext = path.extname(filename);
+    const parser = parsers[ext];
+    if (!parser) {
+        console.log(`No parser for ${path.basename(filename)}`);
+        return;
+    }
+    console.log(`File: ${path.basename(filename)} Parser: ${parser.name}`);
+    const content = await fs_1.promises.readFile(filename, 'utf-8');
+    const result = parser.parse(content, filename);
+    for (const pt of result.parsedTexts) {
+        emit(pt);
+    }
+}
+exports.run = run;
+function emit(pt) {
+    const t = pt.text.replace(/\t/g, '↦').replace(/\r?\n/g, '↩︎').replace(/\r/g, '⇠');
+    console.log(`${pt.range[0]}-${pt.range[1]}\t${t}\t${pt.scope?.toString() || ''}`);
+}
+//# sourceMappingURL=app.js.map

package/dist/grammars/index.d.ts

@@ -0,0 +1,3 @@
+export * as TypeScript from './typescript';
+export * as Markdown from './markdown';
+//# sourceMappingURL=index.d.ts.map

package/dist/grammars/index.js

@@ -0,0 +1,29 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Markdown = exports.TypeScript = void 0;
+exports.TypeScript = __importStar(require("./typescript"));
+exports.Markdown = __importStar(require("./markdown"));
+//# sourceMappingURL=index.js.map

package/dist/grammars/markdown.d.ts

@@ -0,0 +1,3 @@
+import { GrammarDef } from '..';
+export declare const grammar: GrammarDef;
+//# sourceMappingURL=markdown.d.ts.map

package/dist/grammars/markdown.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.grammar = void 0;
+const repository = {
+    frontMatter: { patterns: [] },
+    block: { patterns: ['#raw'] },
+    fenced_code_block: {
+        patterns: ['#fenced_code_block_unknown'],
+    },
+    fenced_code_block_unknown: {
+        name: 'markup.fenced_code.block.markdown',
+        begin: /(^)(\s*)(`{3,}|~{3,})\s*(?=([^`~]*)?$)/,
+        beginCaptures: {
+            '3': 'punctuation.definition.markdown',
+            '4': 'fenced_code.block.language',
+        },
+        // eslint-disable-next-line no-useless-backreference
+        end: /(^|\G)(\2|\s{0,3})(\3)\s*$/,
+        endCaptures: {
+            '3': 'punctuation.definition.markdown',
+        },
+    },
+    raw: {
+        name: 'markup.inline.raw.string.markdown',
+        captures: 'punctuation.definition.raw.markdown',
+        begin: /`+/,
+        end: '\\1',
+    },
+};
+exports.grammar = {
+    name: 'Markdown',
+    scopeName: 'text.html.markdown',
+    patterns: ['#frontMatter', '#block'],
+    repository,
+};
+//# sourceMappingURL=markdown.js.map

package/dist/grammars/simple.d.ts

@@ -0,0 +1,3 @@
+import { GrammarDef } from '..';
+export declare const grammar: GrammarDef;
+//# sourceMappingURL=simple.d.ts.map

package/dist/grammars/simple.js

@@ -0,0 +1,112 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.grammar = void 0;
+const repository = {
+    statements: {
+        patterns: ['#braces', '#comment', '#string'],
+    },
+    braces: {
+        name: 'braces.s',
+        patterns: ['#braces_paren', '#braces_curly'],
+    },
+    braces_paren: {
+        captures: 'paren.braces.s',
+        contentName: 'content.braces.s',
+        begin: '(',
+        end: ')',
+        patterns: ['$self'],
+    },
+    braces_curly: {
+        captures: 'curly.braces.s',
+        contentName: 'content.braces.s',
+        begin: '{',
+        end: '}',
+        patterns: ['$self'],
+    },
+    comment: {
+        patterns: ['#comment_line', '#comment_doc_block', '#comment_block'],
+    },
+    comment_line: {
+        patterns: ['#comment_line_double_slash', '#comment_line_hash'],
+    },
+    comment_line_hash: {
+        name: 'comment.line.s',
+        match: /(#).*\s?/,
+        captures: {
+            '1': 'punctuation.s',
+        },
+    },
+    comment_line_double_slash: {
+        name: 'comment.line.s',
+        match: /(\/\/).*\s?/,
+        captures: {
+            '1': 'punctuation.s',
+        },
+    },
+    comment_doc_block: {
+        name: 'comment.block.s',
+        begin: '/**',
+        end: '*/',
+        captures: {
+            '0': 'punctuation.s',
+        },
+    },
+    comment_block: {
+        name: 'comment.block.s',
+        begin: '/*',
+        end: '*/',
+        captures: {
+            '0': 'punctuation.s',
+        },
+    },
+    string: {
+        patterns: [{ include: '#string_q_single' }, { include: '#string_q_double' }, { include: '#string_template' }],
+    },
+    string_q_single: {
+        name: 'string.quoted.single.s',
+        begin: "'",
+        end: /'|((?:[^\\\n])$)/,
+        patterns: [{ include: '#string_character_escape' }],
+        captures: {
+            '0': 'punctuation.s',
+        },
+    },
+    string_q_double: {
+        name: 'string.quoted.double.s',
+        begin: '"',
+        end: /"|((?:[^\\\n])$)/,
+        patterns: [{ include: '#string_character_escape' }],
+        captures: {
+            '0': 'punctuation.s',
+        },
+    },
+    string_template: {
+        name: 'string.template.s',
+        begin: '`',
+        end: '`',
+        patterns: [{ include: '#string_character_escape' }, '#template_embedded'],
+        captures: {
+            '0': 'punctuation.s',
+        },
+    },
+    string_character_escape: {
+        name: 'constant.character.escape.s',
+        match: /\\(x[0-9A-Fa-f]{2}|[0-3][0-7]{0,2}|[4-7][0-7]?|.|$)/,
+    },
+    template_embedded: {
+        name: 'embedded.template.s',
+        begin: '${',
+        end: '}',
+        patterns: ['#statements'],
+        captures: {
+            '0': 'punctuation.s',
+        },
+    },
+};
+exports.grammar = {
+    name: 'Simple',
+    scopeName: 'source.s',
+    patterns: ['#statements'],
+    repository,
+};
+//# sourceMappingURL=simple.js.map

package/dist/grammars/typescript.d.ts

@@ -0,0 +1,3 @@
+import { GrammarDef } from '..';
+export declare const grammar: GrammarDef;
+//# sourceMappingURL=typescript.d.ts.map

package/dist/grammars/typescript.js

@@ -0,0 +1,165 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.grammar = void 0;
+const repository = {
+    statements: {
+        name: 'code.ts',
+        patterns: [
+            '#keyword',
+            '#regexp',
+            '#string',
+            '#comment',
+            '#braces',
+            '#punctuation',
+            '#space',
+            { name: 'identifier', match: /[^\s;,!|&:^%{}[\]()*/+=<>]+/ },
+        ],
+    },
+    keyword: {
+        patterns: ['#keywordBase', '#standardTypes', '#standardLib'],
+    },
+    keywordBase: {
+        name: 'keyword.typescript.ts',
+        match: /\b(?:any|as|async|await|bigint|boolean|break|case|catch|const|continue|do|else|enum|export|extends|false|finally|for|from|function|get|if|implements|in|instanceof|interface|import|let|map|module|new|new|null|number|of|package|private|public|require|return|set|static|string|super|switch|this|throw|true|try|type|typeof|unknown|undefined|var|void|while|yield)\b/,
+    },
+    standardTypes: {
+        name: 'keyword.type.ts',
+        match: /\b(?:Promise|Record|Omit|Extract|Exclude|BigInt|Array)\b/,
+    },
+    standardLib: {
+        name: 'keyword.lib.ts',
+        match: /\b(?:console|process|window)\b/,
+    },
+    string: {
+        patterns: ['#string_q_single', '#string_q_double', '#string_template'],
+    },
+    string_q_single: {
+        name: 'string.quoted.single.ts',
+        begin: "'",
+        end: /'|((?:[^\\\n])$)/,
+        captures: 'punctuation.string.ts',
+        patterns: [{ include: '#string_character_escape' }],
+    },
+    string_q_double: {
+        name: 'string.quoted.double.ts',
+        begin: '"',
+        end: /"|((?:[^\\\n])$)/,
+        captures: 'punctuation.string.ts',
+        patterns: [{ include: '#string_character_escape' }],
+    },
+    string_template: {
+        name: 'string.template.ts',
+        begin: '`',
+        end: '`',
+        captures: 'punctuation.string.ts',
+        patterns: [
+            {
+                name: 'meta.template.expression.ts',
+                contentName: 'meta.embedded.line.ts',
+                begin: '${',
+                end: '}',
+                patterns: ['#statements'],
+                captures: 'punctuation.definition.template.expression.ts',
+            },
+            { include: '#string_character_escape' },
+        ],
+    },
+    string_character_escape: {
+        name: 'constant.character.escape.ts',
+        match: /\\(x[0-9A-Fa-f]{2}|[0-3][0-7]{0,2}|[4-7][0-7]?|u[0-9A-Fa-f]{4}|.|\r?\n?$)/,
+    },
+    braces: {
+        patterns: [
+            {
+                begin: '(',
+                end: ')',
+                captures: 'punctuation.meta.brace.ts',
+                patterns: ['#statements'],
+                name: 'meta.brace.ts',
+                contentName: 'code.ts',
+            },
+            {
+                begin: '{',
+                end: '}',
+                captures: 'punctuation.meta.brace.ts',
+                patterns: ['#statements'],
+                name: 'meta.brace.ts',
+                contentName: 'code.ts',
+            },
+            {
+                begin: '[',
+                end: ']',
+                captures: 'punctuation.meta.brace.ts',
+                patterns: ['#statements'],
+                name: 'meta.brace.ts',
+                contentName: 'code.ts',
+            },
+        ],
+    },
+    punctuation: {
+        name: 'punctuation.ts',
+        match: /[-;:,!|&^%*/+=<>\n\r]/,
+    },
+    space: {
+        name: 'punctuation.space.ts',
+        match: /\s+/,
+    },
+    comment: {
+        patterns: [
+            {
+                name: 'comment.line.ts',
+                comment: 'line comment',
+                begin: '//',
+                end: /(?=$)/,
+                captures: 'punctuation.definition.comment.ts',
+            },
+            {
+                name: 'comment.block.documentation.ts',
+                comment: 'DocBlock',
+                begin: /\/\*\*(?!\/)/,
+                captures: 'punctuation.definition.comment.ts',
+                end: '*/',
+            },
+            {
+                name: 'comment.block.ts',
+                begin: '/*',
+                end: '*/',
+                captures: 'punctuation.definition.comment.ts',
+            },
+        ],
+    },
+    regexp: {
+        name: 'regexp.ts',
+        begin: /\/(?![/*])/,
+        end: /\/([a-z]*)/i,
+        beginCaptures: 'punctuation.begin.regexp.ts',
+        endCaptures: 'punctuation.end.regexp.ts',
+        patterns: ['#regexp_escape', '#regexp_brace'],
+    },
+    regexp_escape: {
+        name: 'escape.regexp.ts',
+        match: /\\./,
+    },
+    regexp_brace: {
+        name: 'brace.regexp.ts',
+        begin: '[',
+        end: ']',
+        contentName: 'character-class.regexp.ts',
+        patterns: ['#regexp_escape'],
+    },
+};
+exports.grammar = {
+    name: 'TypeScript',
+    scopeName: 'source.ts',
+    patterns: [
+        {
+            name: 'comment.line.shebang.ts',
+            match: /^#!.*(?=$)/,
+        },
+        {
+            include: '#statements',
+        },
+    ],
+    repository,
+};
+//# sourceMappingURL=typescript.js.map

package/dist/index.d.ts

@@ -1,2 +1,3 @@
-export * from './grammar';
-export * from './visualize';
+export { compileGrammar, tokenizeLine, tokenizeText } from './parser';
+export type { Grammar, GrammarDef, Pattern, Repository } from './parser';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,8 +1,8 @@
 "use strict";
-function __export(m) {
-    for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
-}
 Object.defineProperty(exports, "__esModule", { value: true });
-__export(require("./grammar"));
-__export(require("./visualize"));
+exports.tokenizeText = exports.tokenizeLine = exports.compileGrammar = void 0;
+var parser_1 = require("./parser");
+Object.defineProperty(exports, "compileGrammar", { enumerable: true, get: function () { return parser_1.compileGrammar; } });
+Object.defineProperty(exports, "tokenizeLine", { enumerable: true, get: function () { return parser_1.tokenizeLine; } });
+Object.defineProperty(exports, "tokenizeText", { enumerable: true, get: function () { return parser_1.tokenizeText; } });
 //# sourceMappingURL=index.js.map

package/dist/mappers/appendMappedText.d.ts

@@ -0,0 +1,3 @@
+import { MappedText } from './types';
+export declare function appendMappedText(a: MappedText, b: MappedText): MappedText;
+//# sourceMappingURL=appendMappedText.d.ts.map

package/dist/mappers/appendMappedText.js

@@ -0,0 +1,45 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendMappedText = void 0;
+const assert_1 = __importDefault(require("assert"));
+function appendMappedText(a, b) {
+    if (!a.map && !b.map) {
+        return { text: a.text + b.text };
+    }
+    const aLen = a.text.length;
+    const bLen = b.text.length;
+    const aMap = [0, 0].concat(a.map || [0, 0, aLen, aLen]);
+    const bMap = [0, 0].concat(b.map || [0, 0, bLen, bLen]);
+    (0, assert_1.default)(aMap[aMap.length - 1] === aLen);
+    (0, assert_1.default)(bMap[bMap.length - 1] === bLen);
+    (0, assert_1.default)((aMap.length & 1) === 0);
+    (0, assert_1.default)((bMap.length & 1) === 0);
+    return {
+        text: a.text + b.text,
+        map: joinMaps(aMap, bMap),
+    };
+}
+exports.appendMappedText = appendMappedText;
+function joinMaps(aMap, bMap) {
+    const n = aMap.length - 1;
+    const offsets = [aMap[n - 1], aMap[n]];
+    const ab = aMap.concat(bMap.map((v, i) => v + offsets[i & 1]));
+    // Normalize the map by removing duplicate entries
+    const r = [0, 0];
+    let last0 = 0, last1 = 0;
+    for (let i = 0; i < ab.length; i += 2) {
+        const v0 = ab[i];
+        const v1 = ab[i + 1];
+        if (v0 === last0 && v1 === last1) {
+            continue;
+        }
+        r.push(v0, v1);
+        last0 = v0;
+        last1 = v1;
+    }
+    return r;
+}
+//# sourceMappingURL=appendMappedText.js.map

package/dist/mappers/types.d.ts

@@ -0,0 +1,17 @@
+export interface MappedText {
+    text: string;
+    /**
+     * `(i, j)` number pairs where
+     * - `i` is the offset in the source
+     * - `j` is the offset in the destination
+     *
+     * Example:
+     * - source text = `"caf\xe9"`
+     * - mapped text = `"café"`
+     * - map = `[3, 3, 7, 4]`, which is equivalent to `[0, 0, 3, 3, 7, 4]`
+     *   where the `[0, 0]` is unnecessary.
+     *
+     */
+    map?: number[] | undefined;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/mappers/typescript.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Mappers for TypeScript and JavaScript
+ */
+import { MappedText } from './types';
+export declare function mapRawString(text: string): MappedText;
+//# sourceMappingURL=typescript.d.ts.map