@santi020k/lumen-core 0.1.0 → 0.2.0

package/dist/code.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"code.d.ts","sourceRoot":"","sources":["../src/code.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAA;AAEhG,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,EAAE,kBAAkB,CAAA;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,wBAAwB;;;;;;;CAOkB,CAAA;AA0EvD,eAAO,MAAM,kBAAkB,GAAI,MAAM,MAAM,WAc9C,CAAA;AAoBD,eAAO,MAAM,iBAAiB,GAAI,MAAM,MAAM,EAAE,iBAAa,KAAG,cAAc,EA0B7E,CAAA;AAgBD,eAAO,MAAM,mBAAmB,GAAI,MAAM,MAAM,EAAE,iBAAa,WACW,CAAA"}
+{"version":3,"file":"code.d.ts","sourceRoot":"","sources":["../src/code.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAA;AAEhG,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,EAAE,kBAAkB,CAAA;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,wBAAwB;;;;;;;CAOkB,CAAA;AA+NvD,eAAO,MAAM,kBAAkB,GAAI,MAAM,MAAM,WAc9C,CAAA;AAuKD,eAAO,MAAM,iBAAiB,GAAI,MAAM,MAAM,EAAE,iBAAa,KAAG,cAAc,EA0B7E,CAAA;AAgBD,eAAO,MAAM,mBAAmB,GAAI,MAAM,MAAM,EAAE,iBAAa,WACW,CAAA"}

package/dist/code.js

@@ -6,7 +6,20 @@ export const lumenCodeTokenClassNames = {
     symbol: 'ui-code__token--symbol',
     type: 'ui-code__token--type'
 };
-const jsLikeLanguages = new Set([
+const bashLikeLanguages = new Set([
+    'bash',
+    'console',
+    'shell',
+    'sh',
+    'terminal',
+    'zsh'
+]);
+const jsonLikeLanguages = new Set([
+    'json',
+    'json5',
+    'jsonc'
+]);
+const javascriptLikeLanguages = new Set([
     'js',
     'jsx',
     'javascript',
@@ -15,7 +28,30 @@ const jsLikeLanguages = new Set([
     'tsx',
     'typescript'
 ]);
-const jsKeywords = new Set([
+const luaLikeLanguages = new Set([
+    'lua'
+]);
+const markdownLikeLanguages = new Set([
+    'md',
+    'markdown',
+    'mdx'
+]);
+const markupLikeLanguages = new Set([
+    'astro',
+    'html',
+    'markup',
+    'svg',
+    'vue',
+    'xml'
+]);
+const sqlLikeLanguages = new Set([
+    'sql'
+]);
+const yamlLikeLanguages = new Set([
+    'yaml',
+    'yml'
+]);
+const javascriptKeywords = new Set([
     'as',
     'async',
     'await',
@@ -51,7 +87,7 @@ const jsKeywords = new Set([
     'var',
     'while'
 ]);
-const jsTypes = new Set([
+const javascriptTypes = new Set([
     'Array',
     'Boolean',
     'Date',
@@ -67,12 +103,108 @@ const jsTypes = new Set([
     'unknown',
     'void'
 ]);
-const jsTokenPattern = /\/\/[^\n]*|\/\*[\s\S]*?\*\/|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|\b\d[\d._]*\b|\b[A-Za-z_$][\w$]*\b|[=+\-*%<>!?:.,;()[\]{}|&/]+/g;
+const bashKeywords = new Set([
+    'case',
+    'do',
+    'done',
+    'elif',
+    'else',
+    'esac', // cspell:ignore esac
+    'export',
+    'fi',
+    'for',
+    'function',
+    'if',
+    'in',
+    'local',
+    'readonly',
+    'set',
+    'then',
+    'unset',
+    'while'
+]);
+const jsonKeywords = new Set([
+    'false',
+    'null',
+    'true'
+]);
+const luaKeywords = new Set([
+    'and',
+    'break',
+    'do',
+    'else',
+    'elseif',
+    'end',
+    'false',
+    'for',
+    'function',
+    'goto',
+    'if',
+    'in',
+    'local',
+    'nil',
+    'not',
+    'or',
+    'repeat',
+    'return',
+    'then',
+    'true',
+    'until',
+    'while'
+]);
+const sqlKeywords = new Set([
+    'alter',
+    'and',
+    'as',
+    'asc',
+    'by',
+    'create',
+    'delete',
+    'desc',
+    'distinct',
+    'drop',
+    'from',
+    'group',
+    'having',
+    'insert',
+    'into',
+    'join',
+    'limit',
+    'not',
+    'null',
+    'on',
+    'or',
+    'order',
+    'select',
+    'set',
+    'table',
+    'union',
+    'update',
+    'values',
+    'where'
+]);
+const yamlKeywords = new Set([
+    'false',
+    'no',
+    'null',
+    'off',
+    'on',
+    'true',
+    'yes'
+]);
+const bashTokenPattern = /#[^\n]*|"(?:\\.|[^"\\])*"|'[^']*'|\$\{?[A-Za-z_][\w]*\}?|--?[A-Za-z][\w-]*|\b[A-Za-z_][\w]*\b|\b\d+(?:\.\d+)?\b|[|&;<>()[\]{}=]+/g;
+const javascriptTokenPattern = /\/\/[^\n]*|\/\*[\s\S]*?\*\/|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|\b\d[\d._]*\b|\b[A-Za-z_$][\w$]*\b|[=+\-*%<>!?:.,;()[\]{}|&/]+/g;
+const jsonTokenPattern = /\/\/[^\n]*|\/\*[\s\S]*?\*\/|"(?:\\.|[^"\\])*"|\b(?:false|null|true)\b|-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b|[{},:]|\[|\]/gi;
+const luaTokenPattern = /--\[\[[\s\S]*?\]\]|--[^\n]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\b[A-Za-z_][\w]*\b|\b\d+(?:\.\d+)?\b|[=+\-*%<>~#.,:;()[\]{}]+/g;
+const markdownTokenPattern = /```[\s\S]*?```|`[^`\n]+`|<!--[\s\S]*?-->|^#{1,6}(?=\s)|^>\s?|^\s*(?:[-+*]|\d+\.)\s+|\*\*|__|\*|_|\[[^\]]+\]\([^)]+\)/gm;
+const markupTokenPattern = /<!--[\s\S]*?-->|---|<\/?|\/?>|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\b[A-Za-z_][\w:.-]*\b|[{}=]/g;
+const sqlTokenPattern = /--[^\n]*|\/\*[\s\S]*?\*\/|"(?:\\.|[^"\\])*"|'(?:''|[^'])*'|\b[A-Za-z_][\w$]*\b|\b\d+(?:\.\d+)?\b|[=+\-*%<>!.,;()[\]]+/g;
+const yamlTokenPattern = /#[^\n]*|"(?:\\.|[^"\\])*"|'(?:''|[^'])*'|\b[A-Za-z_][\w.-]*(?=\s*:)|\b(?:false|no|null|off|on|true|yes)\b|\b\d+(?:\.\d+)?\b|[{},:&*!?|>@`~-]+|\[|\]/gi;
 const leadingWhitespacePattern = /^[\t ]*/;
 const normalizeLanguage = (language = '') => language.trim().toLowerCase();
 const startsWithIdentifierPattern = /^[A-Za-z_$]/;
 const startsWithNumberPattern = /^\d/;
-const startsWithCommentPattern = /^\/[/*]/;
+const startsWithCommentPattern = /^(?:#|--|\/[/*]|<!--)/;
 const startsWithStringPattern = /^["'`]/;
 export const normalizeLumenCode = (code) => {
     const lines = code.replaceAll(/\r\n?/g, '\n').split('\n');
@@ -86,7 +218,7 @@ export const normalizeLumenCode = (code) => {
     const trimBy = indentation.length > 0 ? Math.min(...indentation) : 0;
     return lines.map(line => line.slice(trimBy)).join('\n');
 };
-const getTokenKind = (value) => {
+const getJavascriptTokenKind = (value) => {
     if (startsWithCommentPattern.exec(value))
         return 'comment';
     if (startsWithStringPattern.exec(value))
@@ -94,30 +226,155 @@ const getTokenKind = (value) => {
     if (startsWithNumberPattern.exec(value))
         return 'accent';
     if (startsWithIdentifierPattern.exec(value)) {
-        if (jsKeywords.has(value))
+        if (javascriptKeywords.has(value))
             return 'keyword';
-        if (jsTypes.has(value))
+        if (javascriptTypes.has(value))
             return 'type';
         return undefined;
     }
     return 'symbol';
 };
+const getBashTokenKind = (value) => {
+    if (value.startsWith('#'))
+        return 'comment';
+    if (startsWithStringPattern.exec(value))
+        return 'string';
+    if (value.startsWith('$'))
+        return 'accent';
+    if (value.startsWith('-'))
+        return 'type';
+    if (startsWithNumberPattern.exec(value))
+        return 'accent';
+    if (bashKeywords.has(value))
+        return 'keyword';
+    if (startsWithIdentifierPattern.exec(value))
+        return undefined;
+    return 'symbol';
+};
+const getJsonTokenKind = (value, source, start) => {
+    if (startsWithCommentPattern.exec(value))
+        return 'comment';
+    if (value.startsWith('"')) {
+        const trailingSource = source.slice(start + value.length);
+        return /^\s*:/.exec(trailingSource) ? 'keyword' : 'string';
+    }
+    if (jsonKeywords.has(value.toLowerCase()))
+        return 'type';
+    if (/^-?\d/.exec(value))
+        return 'accent';
+    return 'symbol';
+};
+const getLuaTokenKind = (value) => {
+    if (value.startsWith('--'))
+        return 'comment';
+    if (startsWithStringPattern.exec(value))
+        return 'string';
+    if (startsWithNumberPattern.exec(value))
+        return 'accent';
+    if (luaKeywords.has(value))
+        return 'keyword';
+    if (startsWithIdentifierPattern.exec(value))
+        return undefined;
+    return 'symbol';
+};
+const getMarkdownTokenKind = (value) => {
+    if (value.startsWith('<!--'))
+        return 'comment';
+    if (value.startsWith('`'))
+        return 'string';
+    if (value.startsWith('['))
+        return 'accent';
+    if (value.trimStart().startsWith('#'))
+        return 'keyword';
+    return 'symbol';
+};
+const getMarkupTokenKind = (value, source, start) => {
+    if (value.startsWith('<!--'))
+        return 'comment';
+    if (startsWithStringPattern.exec(value))
+        return 'string';
+    if (value === '---')
+        return 'comment';
+    if (/^(?:<\/?|\/?>|[{}=])$/.exec(value))
+        return 'symbol';
+    const openingTagIndex = source.lastIndexOf('<', start);
+    const closingTagIndex = source.lastIndexOf('>', start);
+    if (openingTagIndex < closingTagIndex)
+        return undefined;
+    const beforeValue = source.slice(openingTagIndex + 1, start).replace('/', '').trim();
+    return beforeValue === '' ? 'keyword' : 'type';
+};
+const getSqlTokenKind = (value) => {
+    if (startsWithCommentPattern.exec(value))
+        return 'comment';
+    if (startsWithStringPattern.exec(value))
+        return 'string';
+    if (/^\d/.exec(value))
+        return 'accent';
+    if (sqlKeywords.has(value.toLowerCase()))
+        return 'keyword';
+    if (startsWithIdentifierPattern.exec(value))
+        return undefined;
+    return 'symbol';
+};
+const getYamlTokenKind = (value, source, start) => {
+    if (value.startsWith('#'))
+        return 'comment';
+    if (startsWithStringPattern.exec(value))
+        return 'string';
+    if (startsWithNumberPattern.exec(value))
+        return 'accent';
+    if (yamlKeywords.has(value.toLowerCase()))
+        return 'type';
+    if (startsWithIdentifierPattern.exec(value)) {
+        const trailingSource = source.slice(start + value.length);
+        return /^\s*:/.exec(trailingSource) ? 'keyword' : undefined;
+    }
+    return 'symbol';
+};
+const getLumenCodeTokenizer = (language) => {
+    if (javascriptLikeLanguages.has(language)) {
+        return { getKind: getJavascriptTokenKind, pattern: javascriptTokenPattern };
+    }
+    if (bashLikeLanguages.has(language)) {
+        return { getKind: getBashTokenKind, pattern: bashTokenPattern };
+    }
+    if (jsonLikeLanguages.has(language)) {
+        return { getKind: getJsonTokenKind, pattern: jsonTokenPattern };
+    }
+    if (luaLikeLanguages.has(language)) {
+        return { getKind: getLuaTokenKind, pattern: luaTokenPattern };
+    }
+    if (markdownLikeLanguages.has(language)) {
+        return { getKind: getMarkdownTokenKind, pattern: markdownTokenPattern };
+    }
+    if (markupLikeLanguages.has(language)) {
+        return { getKind: getMarkupTokenKind, pattern: markupTokenPattern };
+    }
+    if (sqlLikeLanguages.has(language)) {
+        return { getKind: getSqlTokenKind, pattern: sqlTokenPattern };
+    }
+    if (yamlLikeLanguages.has(language)) {
+        return { getKind: getYamlTokenKind, pattern: yamlTokenPattern };
+    }
+    return undefined;
+};
 export const tokenizeLumenCode = (code, language = '') => {
     const source = normalizeLumenCode(code);
     const tokens = [];
-    const shouldTokenizeJs = jsLikeLanguages.has(normalizeLanguage(language));
+    const tokenizer = getLumenCodeTokenizer(normalizeLanguage(language));
     let offset = 0;
     const push = (value, start, kind) => {
         if (value)
             tokens.push(kind ? { kind, start, value } : { start, value });
     };
-    if (!shouldTokenizeJs)
+    if (!tokenizer)
         return source ? [{ start: 0, value: source }] : [];
-    for (const match of source.matchAll(jsTokenPattern)) {
+    for (const match of source.matchAll(tokenizer.pattern)) {
         const index = match.index;
         const value = match[0];
         push(source.slice(offset, index), offset);
-        push(value, index, getTokenKind(value));
+        push(value, index, tokenizer.getKind(value, source, index));
         offset = index + value.length;
     }
     push(source.slice(offset), offset);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@santi020k/lumen-core",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "publishConfig": {
     "access": "public"
   },
@@ -68,7 +68,7 @@
     "vitest": "^4.1.10"
   },
   "engines": {
-    "node": ">=22.18.0"
+    "node": ">=22.12.0"
   },
   "repository": {
     "type": "git",