juxscript 1.1.56 → 1.1.58

package/lib/utils/codeparser.d.ts

@@ -8,7 +8,7 @@ export interface ParsedLine {
  */
 declare function escapeHtml(text: string): string;
 /**
- * Parse code into lines with simple syntax highlighting
+ * Parse code into lines - CHARACTER-BY-CHARACTER TOKENIZATION
  */
 export declare function parseCode(code: string, language?: string): ParsedLine[];
 /**

package/lib/utils/codeparser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"codeparser.d.ts","sourceRoot":"","sources":["codeparser.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,UAAU;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,iBAAS,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOxC;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAqB,GAAG,UAAU,EAAE,CAQrF;AAuCD;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAEnE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,MAAM,CA8D9C;;;;;;;AAED,wBAKE"}
+{"version":3,"file":"codeparser.d.ts","sourceRoot":"","sources":["codeparser.ts"],"names":[],"mappings":"AA2DA,MAAM,WAAW,UAAU;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,iBAAS,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOxC;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAqB,GAAG,UAAU,EAAE,CAQrF;AAiLD;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAEnE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,MAAM,CAkG9C;;;;;;;AAED,wBAKE"}

package/lib/utils/codeparser.js

@@ -12,6 +12,43 @@ const KEYWORDS = new Set([
     '!=', '!==', '+', '-', '*', '/', '%', '++', '--', '&&', '||', '!', '<', '>',
     '<=', '>=', '=>', '...', '.', ',', '(', ')', '{', '}', '[', ']', ':', '?'
 ]);
+/**
+ * Categorized token sets for semantic highlighting
+ */
+// Control flow keywords (loops, conditionals, etc.)
+const CONTROL_FLOW = new Set([
+    'if', 'else', 'switch', 'case', 'default',
+    'for', 'while', 'do', 'break', 'continue',
+    'return', 'throw', 'try', 'catch', 'finally'
+]);
+// Declaration keywords
+const DECLARATIONS = new Set([
+    'const', 'let', 'var', 'function', 'class',
+    'import', 'export', 'from', 'as',
+    'interface', 'type', 'enum', 'namespace',
+    'static', 'get', 'set', 'extends'
+]);
+// Operator keywords
+const OPERATORS = new Set([
+    'new', 'delete', 'typeof', 'instanceof',
+    'void', 'await', 'yield', 'in', 'of'
+]);
+// Special keywords
+const SPECIAL = new Set([
+    'this', 'super', 'async', 'debugger', 'with'
+]);
+// JUX-specific keywords
+const JUX_KEYWORDS = new Set([
+    'jux', 'state', 'registry', 'render', 'bind', 'sync'
+]);
+// Combine all keywords for quick lookup
+const ALL_KEYWORDS = new Set([
+    ...CONTROL_FLOW,
+    ...DECLARATIONS,
+    ...OPERATORS,
+    ...SPECIAL,
+    ...JUX_KEYWORDS
+]);
 /**
  * Escape HTML entities
  */
@@ -24,47 +61,176 @@ function escapeHtml(text) {
         .replace(/'/g, '&#39;');
 }
 /**
- * Parse code into lines with simple syntax highlighting
+ * Parse code into lines - CHARACTER-BY-CHARACTER TOKENIZATION
  */
 export function parseCode(code, language = 'javascript') {
     const lines = code.split('\n');
     return lines.map((line, index) => ({
         lineNumber: index + 1,
-        html: highlightLine(line),
+        html: tokenizeLine(line),
         raw: line
     }));
 }
 /**
- * Highlight a single line - token-by-token approach to avoid nested spans
+ * Get the appropriate CSS class for a keyword
+ */
+function getKeywordClass(word) {
+    if (CONTROL_FLOW.has(word))
+        return 'token-control';
+    if (DECLARATIONS.has(word))
+        return 'token-declaration';
+    if (OPERATORS.has(word))
+        return 'token-operator-keyword';
+    if (SPECIAL.has(word))
+        return 'token-special';
+    if (JUX_KEYWORDS.has(word))
+        return 'token-jux';
+    return 'token-keyword'; // fallback
+}
+/**
+ * Check if character is a structural punctuation (braces, brackets, parens)
  */
-function highlightLine(line) {
+function isStructural(char) {
+    return '(){}[]'.includes(char);
+}
+/**
+ * Check if character is a delimiter (semicolon, comma, colon)
+ */
+function isDelimiter(char) {
+    return ';,:'.includes(char);
+}
+/**
+ * Check if character is an operator
+ */
+function isOperator(char) {
+    return '+-*/%=<>!&|^~?'.includes(char);
+}
+/**
+ * Tokenize a single line character-by-character with semantic categories
+ */
+function tokenizeLine(line) {
     if (!line.trim())
         return ' ';
-    // ✅ Split line into tokens (words, operators, whitespace)
-    const tokens = line.match(/\w+|[^\w\s]|\s+/g) || [];
     let result = '';
-    tokens.forEach(token => {
-        // Skip empty tokens
-        if (!token)
-            return;
-        // Whitespace - preserve as-is
-        if (/^\s+$/.test(token)) {
-            result += token;
-            return;
+    let i = 0;
+    const len = line.length;
+    // Check for comment at start
+    if (line.trim().startsWith('//')) {
+        return `<span class="token-comment">${escapeHtml(line)}</span>`;
+    }
+    while (i < len) {
+        const char = line[i];
+        // Whitespace - preserve
+        if (/\s/.test(char)) {
+            result += char;
+            i++;
+            continue;
         }
-        // Escape the token
-        const escaped = escapeHtml(token);
-        // Keywords
-        if (KEYWORDS.has(token)) {
-            result += `<span class="token-keyword">${escaped}</span>`;
+        // Comment (// or /* */)
+        if (char === '/' && i + 1 < len) {
+            if (line[i + 1] === '/') {
+                // Line comment - consume rest of line
+                const comment = line.slice(i);
+                result += `<span class="token-comment">${escapeHtml(comment)}</span>`;
+                break;
+            }
+            else if (line[i + 1] === '*') {
+                // Block comment start
+                const commentEnd = line.indexOf('*/', i + 2);
+                if (commentEnd !== -1) {
+                    const comment = line.slice(i, commentEnd + 2);
+                    result += `<span class="token-comment">${escapeHtml(comment)}</span>`;
+                    i = commentEnd + 2;
+                    continue;
+                }
+            }
         }
-        // Everything else - plain text
-        else {
-            result += escaped;
+        // String literals (single, double, backtick)
+        if (char === '"' || char === "'" || char === '`') {
+            const stringEnd = line.indexOf(char, i + 1);
+            if (stringEnd !== -1) {
+                const str = line.slice(i, stringEnd + 1);
+                result += `<span class="token-string">${escapeHtml(str)}</span>`;
+                i = stringEnd + 1;
+                continue;
+            }
+        }
+        // Word (identifier or keyword)
+        if (/[a-zA-Z_$]/.test(char)) {
+            const word = consumeWord(line, i);
+            const escaped = escapeHtml(word);
+            if (ALL_KEYWORDS.has(word)) {
+                const tokenClass = getKeywordClass(word);
+                result += `<span class="${tokenClass}">${escaped}</span>`;
+            }
+            else {
+                result += escaped;
+            }
+            i += word.length;
+            continue;
+        }
+        // Number
+        if (/\d/.test(char)) {
+            const num = consumeNumber(line, i);
+            result += `<span class="token-number">${escapeHtml(num)}</span>`;
+            i += num.length;
+            continue;
+        }
+        // Structural punctuation (braces, brackets, parens)
+        if (isStructural(char)) {
+            result += `<span class="token-structural">${escapeHtml(char)}</span>`;
+            i++;
+            continue;
+        }
+        // Delimiters (semicolon, comma, colon)
+        if (isDelimiter(char)) {
+            result += `<span class="token-delimiter">${escapeHtml(char)}</span>`;
+            i++;
+            continue;
         }
-    });
+        // Operators
+        if (isOperator(char)) {
+            result += `<span class="token-operator">${escapeHtml(char)}</span>`;
+            i++;
+            continue;
+        }
+        // Everything else - just escape and append
+        result += escapeHtml(char);
+        i++;
+    }
     return result || ' ';
 }
+/**
+ * Consume a complete word (identifier)
+ */
+function consumeWord(line, start) {
+    let i = start;
+    while (i < line.length && /[a-zA-Z0-9_$]/.test(line[i])) {
+        i++;
+    }
+    return line.substring(start, i);
+}
+/**
+ * Consume a complete number
+ */
+function consumeNumber(line, start) {
+    let i = start;
+    let hasDot = false;
+    while (i < line.length) {
+        const char = line[i];
+        if (/\d/.test(char)) {
+            i++;
+        }
+        else if (char === '.' && !hasDot) {
+            hasDot = true;
+            i++;
+        }
+        else {
+            break;
+        }
+    }
+    return line.substring(start, i);
+}
 /**
  * Render a parsed line
  */
@@ -76,7 +242,7 @@ export function renderLineWithTokens(parsedLine) {
  */
 export function getSyntaxHighlightCSS() {
     return `
-/* Simple Syntax Highlighting */
+/* Semantic Code Highlighting */
 .jux-code {
     font-family: 'Fira Code', 'Consolas', 'Monaco', monospace;
     font-size: 14px;
@@ -130,10 +296,46 @@ export function getSyntaxHighlightCSS() {
     display: none;
 }
 
-/* Token colors - JUST KEYWORDS */
-.token-keyword { 
-    color: #c678dd; 
+/* Token colors - Semantic categories */
+.token-control { 
+    color: #c678dd;  /* Purple - control flow (if, for, return) */
+    font-weight: 600; 
+}
+.token-declaration { 
+    color: #61afef;  /* Blue - declarations (const, let, function) */
+    font-weight: 600; 
+}
+.token-operator-keyword { 
+    color: #e06c75;  /* Red - operator keywords (typeof, instanceof) */
     font-weight: 600; 
+}
+.token-special { 
+    color: #e5c07b;  /* Yellow - special keywords (this, super, async) */
+    font-weight: 600; 
+}
+.token-jux { 
+    color: #56b6c2;  /* Cyan - JUX-specific (jux, state, render) */
+    font-weight: 700; 
+}
+.token-number {
+    color: #d19a66;  /* Orange - numbers */
+}
+.token-string {
+    color: #98c379;  /* Green - strings */
+}
+.token-comment {
+    color: #5c6370;  /* Gray - comments */
+    font-style: italic;
+}
+.token-structural {
+    color: #abb2bf;  /* Light gray - braces, brackets, parens */
+    font-weight: 600;
+}
+.token-delimiter {
+    color: #777;     /* Dark gray - semicolons, commas, colons */
+}
+.token-operator {
+    color: #56b6c2;  /* Cyan - operators (+, -, *, etc.) */
 }
   `;
 }

package/lib/utils/codeparser.ts

@@ -13,6 +13,50 @@ const KEYWORDS = new Set([
     '<=', '>=', '=>', '...', '.', ',', '(', ')', '{', '}', '[', ']', ':', '?'
 ]);
 
+/**
+ * Categorized token sets for semantic highlighting
+ */
+
+// Control flow keywords (loops, conditionals, etc.)
+const CONTROL_FLOW = new Set([
+    'if', 'else', 'switch', 'case', 'default',
+    'for', 'while', 'do', 'break', 'continue',
+    'return', 'throw', 'try', 'catch', 'finally'
+]);
+
+// Declaration keywords
+const DECLARATIONS = new Set([
+    'const', 'let', 'var', 'function', 'class',
+    'import', 'export', 'from', 'as',
+    'interface', 'type', 'enum', 'namespace',
+    'static', 'get', 'set', 'extends'
+]);
+
+// Operator keywords
+const OPERATORS = new Set([
+    'new', 'delete', 'typeof', 'instanceof',
+    'void', 'await', 'yield', 'in', 'of'
+]);
+
+// Special keywords
+const SPECIAL = new Set([
+    'this', 'super', 'async', 'debugger', 'with'
+]);
+
+// JUX-specific keywords
+const JUX_KEYWORDS = new Set([
+    'jux', 'state', 'registry', 'render', 'bind', 'sync'
+]);
+
+// Combine all keywords for quick lookup
+const ALL_KEYWORDS = new Set([
+    ...CONTROL_FLOW,
+    ...DECLARATIONS,
+    ...OPERATORS,
+    ...SPECIAL,
+    ...JUX_KEYWORDS
+]);
+
 export interface ParsedLine {
     lineNumber: number;
     html: string;
@@ -32,55 +76,193 @@ function escapeHtml(text: string): string {
 }
 
 /**
- * Parse code into lines with simple syntax highlighting
+ * Parse code into lines - CHARACTER-BY-CHARACTER TOKENIZATION
  */
 export function parseCode(code: string, language: string = 'javascript'): ParsedLine[] {
     const lines = code.split('\n');
 
     return lines.map((line, index) => ({
         lineNumber: index + 1,
-        html: highlightLine(line),
+        html: tokenizeLine(line),
         raw: line
     }));
 }
 
 /**
- * Highlight a single line - token-by-token approach to avoid nested spans
+ * Get the appropriate CSS class for a keyword
  */
-function highlightLine(line: string): string {
-    if (!line.trim()) return ' ';
+function getKeywordClass(word: string): string {
+    if (CONTROL_FLOW.has(word)) return 'token-control';
+    if (DECLARATIONS.has(word)) return 'token-declaration';
+    if (OPERATORS.has(word)) return 'token-operator-keyword';
+    if (SPECIAL.has(word)) return 'token-special';
+    if (JUX_KEYWORDS.has(word)) return 'token-jux';
+    return 'token-keyword'; // fallback
+}
+
+/**
+ * Check if character is a structural punctuation (braces, brackets, parens)
+ */
+function isStructural(char: string): boolean {
+    return '(){}[]'.includes(char);
+}
+
+/**
+ * Check if character is a delimiter (semicolon, comma, colon)
+ */
+function isDelimiter(char: string): boolean {
+    return ';,:'.includes(char);
+}
+
+/**
+ * Check if character is an operator
+ */
+function isOperator(char: string): boolean {
+    return '+-*/%=<>!&|^~?'.includes(char);
+}
 
-    // ✅ Split line into tokens (words, operators, whitespace)
-    const tokens = line.match(/\w+|[^\w\s]|\s+/g) || [];
+/**
+ * Tokenize a single line character-by-character with semantic categories
+ */
+function tokenizeLine(line: string): string {
+    if (!line.trim()) return ' ';
 
     let result = '';
+    let i = 0;
+    const len = line.length;
 
-    tokens.forEach(token => {
-        // Skip empty tokens
-        if (!token) return;
+    // Check for comment at start
+    if (line.trim().startsWith('//')) {
+        return `<span class="token-comment">${escapeHtml(line)}</span>`;
+    }
 
-        // Whitespace - preserve as-is
-        if (/^\s+$/.test(token)) {
-            result += token;
-            return;
+    while (i < len) {
+        const char = line[i];
+
+        // Whitespace - preserve
+        if (/\s/.test(char)) {
+            result += char;
+            i++;
+            continue;
         }
 
-        // Escape the token
-        const escaped = escapeHtml(token);
+        // Comment (// or /* */)
+        if (char === '/' && i + 1 < len) {
+            if (line[i + 1] === '/') {
+                // Line comment - consume rest of line
+                const comment = line.slice(i);
+                result += `<span class="token-comment">${escapeHtml(comment)}</span>`;
+                break;
+            } else if (line[i + 1] === '*') {
+                // Block comment start
+                const commentEnd = line.indexOf('*/', i + 2);
+                if (commentEnd !== -1) {
+                    const comment = line.slice(i, commentEnd + 2);
+                    result += `<span class="token-comment">${escapeHtml(comment)}</span>`;
+                    i = commentEnd + 2;
+                    continue;
+                }
+            }
+        }
 
-        // Keywords
-        if (KEYWORDS.has(token)) {
-            result += `<span class="token-keyword">${escaped}</span>`;
+        // String literals (single, double, backtick)
+        if (char === '"' || char === "'" || char === '`') {
+            const stringEnd = line.indexOf(char, i + 1);
+            if (stringEnd !== -1) {
+                const str = line.slice(i, stringEnd + 1);
+                result += `<span class="token-string">${escapeHtml(str)}</span>`;
+                i = stringEnd + 1;
+                continue;
+            }
         }
-        // Everything else - plain text
-        else {
-            result += escaped;
+
+        // Word (identifier or keyword)
+        if (/[a-zA-Z_$]/.test(char)) {
+            const word = consumeWord(line, i);
+            const escaped = escapeHtml(word);
+
+            if (ALL_KEYWORDS.has(word)) {
+                const tokenClass = getKeywordClass(word);
+                result += `<span class="${tokenClass}">${escaped}</span>`;
+            } else {
+                result += escaped;
+            }
+
+            i += word.length;
+            continue;
+        }
+
+        // Number
+        if (/\d/.test(char)) {
+            const num = consumeNumber(line, i);
+            result += `<span class="token-number">${escapeHtml(num)}</span>`;
+            i += num.length;
+            continue;
+        }
+
+        // Structural punctuation (braces, brackets, parens)
+        if (isStructural(char)) {
+            result += `<span class="token-structural">${escapeHtml(char)}</span>`;
+            i++;
+            continue;
+        }
+
+        // Delimiters (semicolon, comma, colon)
+        if (isDelimiter(char)) {
+            result += `<span class="token-delimiter">${escapeHtml(char)}</span>`;
+            i++;
+            continue;
+        }
+
+        // Operators
+        if (isOperator(char)) {
+            result += `<span class="token-operator">${escapeHtml(char)}</span>`;
+            i++;
+            continue;
         }
-    });
+
+        // Everything else - just escape and append
+        result += escapeHtml(char);
+        i++;
+    }
 
     return result || ' ';
 }
 
+/**
+ * Consume a complete word (identifier)
+ */
+function consumeWord(line: string, start: number): string {
+    let i = start;
+    while (i < line.length && /[a-zA-Z0-9_$]/.test(line[i])) {
+        i++;
+    }
+    return line.substring(start, i);
+}
+
+/**
+ * Consume a complete number
+ */
+function consumeNumber(line: string, start: number): string {
+    let i = start;
+    let hasDot = false;
+
+    while (i < line.length) {
+        const char = line[i];
+
+        if (/\d/.test(char)) {
+            i++;
+        } else if (char === '.' && !hasDot) {
+            hasDot = true;
+            i++;
+        } else {
+            break;
+        }
+    }
+
+    return line.substring(start, i);
+}
+
 /**
  * Render a parsed line
  */
@@ -93,7 +275,7 @@ export function renderLineWithTokens(parsedLine: ParsedLine): string {
  */
 export function getSyntaxHighlightCSS(): string {
     return `
-/* Simple Syntax Highlighting */
+/* Semantic Code Highlighting */
 .jux-code {
     font-family: 'Fira Code', 'Consolas', 'Monaco', monospace;
     font-size: 14px;
@@ -147,10 +329,46 @@ export function getSyntaxHighlightCSS(): string {
     display: none;
 }
 
-/* Token colors - JUST KEYWORDS */
-.token-keyword { 
-    color: #c678dd; 
+/* Token colors - Semantic categories */
+.token-control { 
+    color: #c678dd;  /* Purple - control flow (if, for, return) */
     font-weight: 600; 
+}
+.token-declaration { 
+    color: #61afef;  /* Blue - declarations (const, let, function) */
+    font-weight: 600; 
+}
+.token-operator-keyword { 
+    color: #e06c75;  /* Red - operator keywords (typeof, instanceof) */
+    font-weight: 600; 
+}
+.token-special { 
+    color: #e5c07b;  /* Yellow - special keywords (this, super, async) */
+    font-weight: 600; 
+}
+.token-jux { 
+    color: #56b6c2;  /* Cyan - JUX-specific (jux, state, render) */
+    font-weight: 700; 
+}
+.token-number {
+    color: #d19a66;  /* Orange - numbers */
+}
+.token-string {
+    color: #98c379;  /* Green - strings */
+}
+.token-comment {
+    color: #5c6370;  /* Gray - comments */
+    font-style: italic;
+}
+.token-structural {
+    color: #abb2bf;  /* Light gray - braces, brackets, parens */
+    font-weight: 600;
+}
+.token-delimiter {
+    color: #777;     /* Dark gray - semicolons, commas, colons */
+}
+.token-operator {
+    color: #56b6c2;  /* Cyan - operators (+, -, *, etc.) */
 }
   `;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "juxscript",
-  "version": "1.1.56",
+  "version": "1.1.58",
   "type": "module",
   "description": "A JavaScript UX authorship platform",
   "main": "index.js",