@principal-ai/principal-view-react 0.16.45 → 0.16.46

package/src/subsystem/tokenizeFormatted.ts

@@ -0,0 +1,92 @@
+/**
+ * Tokenize a formatted TypeScript declaration string into SubsystemDeclToken[]
+ * using highlight.js.
+ *
+ * highlight.js produces HTML with `<span class="hljs-...">` wrappers. This
+ * module parses that output into our token format so the renderer stays
+ * language-agnostic.
+ */
+
+import hljs from 'highlight.js/lib/core';
+import typescript from 'highlight.js/lib/languages/typescript';
+import type { SubsystemDeclToken, SubsystemDeclTokenKind } from './model';
+
+// Register once — safe to call multiple times.
+try {
+  hljs.registerLanguage('typescript', typescript);
+} catch {
+  // Already registered.
+}
+
+// hljs class → our token kind
+const CLASS_MAP: Record<string, SubsystemDeclTokenKind> = {
+  'hljs-keyword': 'keyword',
+  'hljs-title': 'name',
+  'hljs-title.function_': 'name',
+  'hljs-params': 'member',
+  'hljs-type': 'type',
+  'hljs-built_in': 'type',
+  'hljs-string': 'string',
+  'hljs-number': 'string',
+  'hljs-comment': 'punctuation',
+};
+
+/**
+ * Tokenize a formatted TypeScript string into SubsystemDeclToken[].
+ * The input should already be formatted by Prettier (or be valid TS).
+ */
+export function tokenizeFormatted(code: string): SubsystemDeclToken[] {
+  const result = hljs.highlight(code, { language: 'typescript' });
+  return parseHighlightedHtml(result.value);
+}
+
+// ---------------------------------------------------------------------------
+// HTML parser — walks the hljs output and extracts tokens
+// ---------------------------------------------------------------------------
+
+function unescapeHtml(text: string): string {
+  return text
+    .replace(/&quot;/g, '"')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&amp;/g, '&')
+    .replace(/&#39;/g, "'");
+}
+
+function parseHighlightedHtml(html: string): SubsystemDeclToken[] {
+  const tokens: SubsystemDeclToken[] = [];
+  // Regex matches either `<span class="hljs-...">`, `</span>`, or text content.
+  const re = /<span class="([^"]+)">|<\/span>|[^<]+/g;
+  const classStack: string[] = [];
+  let match: RegExpExecArray | null;
+
+  while ((match = re.exec(html)) !== null) {
+    const chunk = match[0];
+
+    if (chunk.startsWith('<span')) {
+      // Opening tag — push class onto stack.
+      classStack.push(match[1]);
+    } else if (chunk === '</span>') {
+      // Closing tag — pop class.
+      classStack.pop();
+    } else {
+      // Text content — emit tokens for each line segment.
+      const currentClass = classStack[classStack.length - 1] ?? '';
+      const kind = CLASS_MAP[currentClass] ?? 'punctuation';
+      const text = chunk;
+
+      // Split on newlines so each line becomes its own token set.
+      const lines = text.split('\n');
+      for (let i = 0; i < lines.length; i++) {
+        if (i > 0) {
+          tokens.push({ text: '', kind: 'newline' });
+        }
+        if (lines[i]) {
+          tokens.push({ text: unescapeHtml(lines[i]), kind });
+        }
+      }
+    }
+  }
+
+  return tokens;
+}