buninu 0.5.0 → 0.7.0

package/apps/jsmdcui/tests/heading-ids.test.js

@@ -1,4 +1,5 @@
 import { expect, test } from "bun:test";
+import { escapesSlashTags } from "./markdown-behavior.mjs";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -347,23 +348,27 @@ test("raw heading markers do not change quoted attribute parsing", () => {
   expect(rendered.html).toContain('<h1 id="中文-1">中文</h1>');
 });
 
-test("raw heading markers preserve slash-separated HTML attributes", () => {
-  const markdown = `<h1/id=中文>Raw</h1>
-
-# 中文
-`;
-  expect(collectRawHtmlHeadingDeclarations(markdown)).toEqual([
-    {
-      id: "中文",
-      kind: "raw HTML heading",
-      line: 1,
-      source: "<h1/id=中文>Raw</h1>",
-    },
+test("raw heading markers follow Bun's slash-separated HTML behavior", () => {
+  const markdown = "<h1/id=中文>Raw</h1>\n\n# 中文\n";
+  expect(collectRawHtmlHeadingDeclarations(markdown)).toEqual(escapesSlashTags ? [] : [
+    { id: "中文", kind: "raw HTML heading", line: 1, source: "<h1/id=中文>Raw</h1>" },
   ]);
   const rendered = renderMarkdownWithHeadingIds(markdown);
-  expect(rendered.headings.map(heading => heading.id)).toEqual(["中文-1"]);
-  expect(rendered.html).toContain("<h1/id=中文>Raw</h1>");
-  expect(rendered.html).toContain('<h1 id="中文-1">中文</h1>');
+  const id = escapesSlashTags ? "中文" : "中文-1";
+  expect(rendered.normalized).toBe(true);
+  expect(rendered.headings.map(heading => heading.id)).toEqual([id]);
+  expect(rendered.html).toContain(escapesSlashTags
+    ? "<p>&lt;h1/id=中文&gt;Raw</h1></p>"
+    : "<h1/id=中文>Raw</h1>");
+  expect(rendered.html).toContain(`<h1 id="${id}">中文</h1>`);
+});
+
+test("slash-tag ambiguity follows Bun's actual Markdown output", () => {
+  const rendered = renderMarkdownWithHeadingIds('<div/a< x=">TAIL\n\n# 中文\n');
+  expect(rendered.normalized).toBe(escapesSlashTags);
+  expect(rendered.headings.map(heading => heading.id)).toEqual(escapesSlashTags ? ["中文"] : []);
+  expect(rendered.html).toContain(escapesSlashTags
+    ? '<h1 id="中文">中文</h1>' : '<h1 id="">中文</h1>');
 });
 
 test("self-closing inline headings cannot suppress other raw collisions", () => {
@@ -511,7 +516,7 @@ test("quoted tag delimiters preserve raw IDs and Markdown heading metadata", ()
 });
 
 test("browser alignment failures never fall back to lexical heading rewrites", () => {
-  const markdown = '<div/a< x=">TAIL\n\n# 中文\n';
+  const markdown = '<div x=">TAIL\n\n# 中文\n';
   const rendered = renderMarkdownWithHeadingIds(markdown);
   expect(rendered.normalized).toBe(false);
   expect(rendered.headings).toEqual([]);

package/apps/jsmdcui/tests/highlight-regions.test.js

@@ -0,0 +1,73 @@
+import {test,expect} from 'bun:test';
+import {SyntaxDefinition,parseHeaderYaml} from '../src/highlight/parser.js';
+import {Highlighter} from '../src/highlight/highlighter.js';
+async function highlighter(name){
+ const source=Bun.YAML.parse(await Bun.file(new URL('../runtime/syntax/'+name+'.yaml',import.meta.url)).text());
+ return new Highlighter(new SyntaxDefinition(parseHeaderYaml(source),source));
+}
+function groupAt(result,index){let group='default';for(const [pos,value]of result.changes){if(pos>index)break;group=value;}return group;}
+for(const [syntax,first]of [['html','<div class="hello">Hello</div>'],['xml','<Project Sdk="Microsoft.NET.Sdk">']]){
+ test(syntax+' nested attribute returns to outer tag and does not leak to next line',async()=>{
+  const h=await highlighter(syntax),r=h.highlightLine(first);
+  expect(r.state).toBeNull();
+  expect(groupAt(r,first.indexOf('>'))).toBe('symbol.tag');
+  const next=h.highlightLine('<p>Next line</p>',r.state);
+  expect(groupAt(next,3)).toBe('default');
+  expect(next.state).toBeNull();
+  expect([...r.changes.keys()].every(p=>p<=first.length)).toBe(true);
+ });
+ test(syntax+' retains an unfinished attribute across lines then exits it',async()=>{
+  const h=await highlighter(syntax);
+  const a=h.highlightLine('<div title="first');expect(a.state?.group).toBe('constant.string');
+  const b=h.highlightLine('second">Text</div>',a.state);
+  expect(groupAt(b,0)).toBe('constant.string');
+  expect(groupAt(b,8)).toBe('default');expect(b.state).toBeNull();
+ });
+}
+test('HTML multiple attributes, adjacent tags and nested escaped quotes',async()=>{
+ const h=await highlighter('html');
+ const line='<a href="/path" title="say \\"hi\\"">Link</a><b>next</b>';
+ const r=h.highlightLine(line);
+ expect(r.state).toBeNull();expect(groupAt(r,line.indexOf('Link'))).toBe('default');
+ expect(groupAt(r,line.indexOf('next'))).toBe('default');
+});
+test('zero-width child/parent loop steps forward and keeps highlighting the line',()=>{
+ const h=new Highlighter({rules:[{type:'region',group:'outer',start:/</u,end:/>/u,rules:[{type:'region',group:'inner',start:/(?=x)/u,end:/(?=x)/u,rules:[]}]},{type:'pattern',group:'keyword',regex:/END/u}]});
+ let calls=0;const line='<x>END';const r=h.highlightLine(line,null,()=>{if(++calls>100)throw Error('loop');});
+ expect(r.state).toBeNull();expect(calls).toBeLessThan(100);
+ expect(groupAt(r,1)).toBe('outer');expect(groupAt(r,line.indexOf('END'))).toBe('keyword');
+});
+test('empty pattern matches advance past full Unicode characters, with or without progress',()=>{
+ const h=new Highlighter({rules:[{type:'pattern',group:'empty',regex:/(?=.)/u},{type:'pattern',group:'number',regex:/42/u}]});
+ for(const progress of [null,()=>{}]){
+  const r=h.highlightLine('😀\0\ud80042',null,progress);
+  expect(groupAt(r,4)).toBe('number');expect(r.state).toBeNull();
+ }
+});
+test('NUL and isolated surrogate units inside HTML do not leak state',async()=>{
+ const h=await highlighter('html');
+ const r=h.highlightLine('<a title="\0😀\ud800">hello\0\udc00</a>');
+ expect(r.state).toBeNull();
+ expect(h.highlightLine('\0'.repeat(10000)).state).toBeNull();
+});
+test('long lines finish at their final token without recursion or truncation',()=>{
+ const h=new Highlighter({rules:[{type:'region',group:'string',start:/"/u,end:/"/u,rules:[]},{type:'pattern',group:'keyword',regex:/END/u}]});
+ const line='"x";'.repeat(50000)+'END';
+ const r=h.highlightLine(line);
+ expect(r.state).toBeNull();expect(groupAt(r,line.length-1)).toBe('keyword');
+ expect([...r.changes.keys()].at(-1)).toBe(line.length);
+},30000);
+test('cyclic incoming state and cyclic syntax trees cannot recurse forever',()=>{
+ const region={type:'region',group:'cycle',start:/</u,end:/>/u,rules:[]};region.rules.push(region);
+ const h=new Highlighter({rules:[region]});
+ expect(h.highlightLine('plain').state).toBeNull();
+ const state={group:'cycle',rules:[]};state.parent=state;
+ expect(h.highlightLine('plain',state).state).toBeNull();
+});
+test('externally supplied zero-width recursive state has a no-progress bound',()=>{
+ const child={type:'region',group:'inner',start:/(?=x)/u,end:/$/u,rules:[]};child.rules.push(child);
+ const state={group:'outer',end:/$/u,rules:[child]};
+ const h=new Highlighter({rules:[]});let calls=0;
+ const r=h.highlightLine('x',state,()=>{if(++calls>5000)throw Error('loop');});
+ expect(r.state).toBeNull();expect(calls).toBeLessThan(5000);
+});

package/apps/jsmdcui/tests/id-collision.test.js

@@ -1,4 +1,5 @@
 import { expect, test } from "bun:test";
+import { escapesSlashTags } from "./markdown-behavior.mjs";
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -39,7 +40,7 @@ test("--check exits 0 and reports unique IDs", async () => {
 });
 
 test("--check fails closed when browser heading parsing is ambiguous", async () => {
-  const result = await runCheck('<div/a< x=">TAIL\n\n# 中文\n');
+  const result = await runCheck('<div x=">TAIL\n\n# 中文\n');
   expect(result.exitCode).toBe(2);
   expect(result.stderr.toString()).toContain(
     "Markdown headings could not be classified safely",
@@ -359,15 +360,37 @@ test("--check ignores heading-looking text inside raw attributes", async () => {
   expect(output).toContain("FAILED");
 });
 
-test("--check preserves slash-separated raw heading attributes", async () => {
+test("--check follows Bun's slash-separated raw heading behavior", async () => {
   const result = await runCheck("<h1/id=中文>Raw</h1>\n\n# 中文\n");
   const output = Bun.stripANSI(result.stdout.toString());
-  expect(result.exitCode).toBe(1);
-  expect(output).toContain("ID #中文");
-  expect(output).toContain("Type: raw HTML heading");
-  expect(output).toContain("Type: heading");
-  expect(output).toContain("FAILED");
-});
+  expect(result.exitCode).toBe(escapesSlashTags ? 0 : 1);
+  if (escapesSlashTags) {
+    expect(output).toContain("Selectable IDs: 1");
+    expect(output).toContain("PASSED");
+    expect(output).not.toContain("Type: raw HTML heading");
+  } else {
+    expect(output).toContain("ID #中文");
+    expect(output).toContain("Type: raw HTML heading");
+    expect(output).toContain("Type: heading");
+    expect(output).toContain("FAILED");
+  }
+});
+
+for (const [command, run] of [["--check", runCheck], ["--outline", runOutline]]) {
+  test(`${command} follows Bun's slash-tag ambiguity behavior`, async () => {
+    const result = await run('<div/a< x=">TAIL\n\n# 中文\n');
+    expect(result.exitCode).toBe(escapesSlashTags ? 0 : 2);
+    if (escapesSlashTags) {
+      expect(result.stderr.toString()).toBe("");
+      if (command === "--check") expect(result.stdout.toString()).toContain("PASSED");
+      else expect(result.stdout.toString()).toContain("中文");
+    } else {
+      expect(result.stderr.toString()).toContain("Markdown headings could not be classified safely");
+      expect(result.stdout.toString()).not.toContain("PASSED");
+      if (command === "--outline") expect(result.stdout.toString()).toBe("");
+    }
+  });
+}
 
 test("--check ignores heading-like custom tag names", async () => {
   const result = await runCheck(
@@ -494,7 +517,7 @@ value
 });
 
 test("--outline fails closed when browser heading parsing is ambiguous", async () => {
-  const result = await runOutline('<div/a< x=">TAIL\n\n# 中文\n');
+  const result = await runOutline('<div x=">TAIL\n\n# 中文\n');
   expect(result.exitCode).toBe(2);
   expect(result.stderr.toString()).toContain(
     "Markdown headings could not be classified safely",

package/apps/jsmdcui/tests/markdown-behavior.mjs

@@ -0,0 +1,9 @@
+// Probe the parser, not Bun.version: canaries and backports can differ.
+// CommonMark allows `/` after a block tag name only as part of `/>`.
+const probe = Bun.markdown.html("<h1/id=x>Raw</h1>\n");
+const raw = "<h1/id=x>Raw</h1>\n";
+const escaped = "<p>&lt;h1/id=x&gt;Raw</h1></p>\n";
+if (probe !== raw && probe !== escaped) {
+  throw new Error(`Unexpected Bun Markdown slash-tag behavior: ${JSON.stringify(probe)}`);
+}
+export const escapesSlashTags = probe === escaped;

package/apps/jsmdcui/tests/syntax-parity/NORMAL-HTML.md

@@ -0,0 +1,46 @@
+# Normal HTML follow-up
+
+Tested with the same unmodified micro highlighter oracle and upstream YAML as
+REPORT.md. See normal-html-report.json for counts and first differing positions.
+The full comparison.html was also passed to both engines as input, unchanged.
+
+The minimal reproduction is ordinary valid HTML:
+
+```html
+<div class="hello">Hello</div>
+<p>Next line</p>
+```
+
+With html.yaml (the definition selected by the .html filename), jmi ends the
+first line with a constant.string state and colors the entire second line as a
+string. This is a real nested-region state bug, not just a malformed generated
+sample. normal-html.html contains a complete document for manual reproduction.
+
+For the generated comparison.html, html.yaml produced 731354 differing character
+groups, including 438494 characters marked as strings only by jmi. This large
+one-line file amplifies the state issue; it is not a typical per-file error rate.
+The same full file matched with explicitly selected html4.yaml and html5.yaml,
+which have different rules and are not the default .html filename match.
+
+This check did not modify the highlighter or the syntax files.
+
+## After the nested-region fix
+
+The historical reproduction above is now covered by highlight-regions.test.js.
+The child scanner already resumes its parent and processes the rest of the line;
+returning that result directly avoids rescanning closing quotes as new openings.
+Both HTML and XML now return to the correct outer state, including when an
+attribute legitimately spans multiple lines. Original syntax YAML is unchanged.
+
+The current normal-html-report.json records the follow-up: simple HTML,
+two-attribute HTML and a complete normal document have no extra string-colored
+characters. Their remaining differences are local token groups, such as micro
+using symbol.tag where jmi recognizes an attribute as identifier. The current
+comparison.html still has local group differences and is not fully identical.
+It is regenerated from the latest results, so its total size differs from the
+historical input above; raw differing-character totals are not a fixed-corpus
+before/after benchmark.
+
+The 158-syntax corpus now has 130 identical samples and 28 with differences.
+XML and MSBuild samples match completely. Mixed POSIX character classes are
+also translated; default/email, mail and privoxy-action samples now match.

package/apps/jsmdcui/tests/syntax-parity/README.md

@@ -0,0 +1,85 @@
+# micro syntax parity fixtures
+
+This directory contains one lexical sample for each of **158 syntax definitions**.
+The current report uses upstream micro YAML for both engines, including
+JavaScript, Python and Markdown. For the first run those three local files were
+backed up, temporarily replaced and restored byte-for-byte after comparison.
+`--upstream-syntax` reproduces that comparison without modifying any syntax files.
+Without this flag, the comparator tests only byte-identical local definitions
+and excludes JavaScript, Python and Markdown; included definitions then come
+from each engine's own tree.
+
+- [REPORT.md](REPORT.md): counts and first differences, with sample links.
+- `comparison.html` (generated locally, not included): side-by-side colors using
+  the same Monokai palette; hover over a span for its exact group. No external
+  assets required.
+- [report.json](report.json): machine-readable per-language results.
+- `samples/*.txt`: generated rule witnesses, with hand-written Dockerfile and
+  MSBuild examples. They are lexical test fragments, **not generally valid
+  programs**. Region boundaries, escaping, and overlapping patterns are included.
+
+The oracle invokes the actual, unmodified upstream `pkg/highlight` Go package,
+including its YAML parser and include resolution. No Go regex approximation is
+used for the reference results. This is a highlighter comparison, not a terminal
+screenshot comparison or a test of automatic filetype detection. The sample
+filenames deliberately do not guess extensions: select the filetype shown in
+the report when opening them in an editor (`set filetype dockerfile`, for example).
+
+All samples have non-default groups in both engines. Equal results mean equal
+per-character syntax groups **for these samples only**, not full language parity.
+The HTML is a viewing aid, not a substitute for the group-level comparison.
+
+## Generate the comparison HTML and reports
+
+Install Go 1.24 or newer and Bun with scoped RegExp modifier support. Start in
+this directory. Point `MICRO_SOURCE` at an existing upstream micro checkout:
+
+```sh
+export MICRO_SOURCE=/absolute/path/to/go-micro
+export PARITY_BUILD_DIR="$(mktemp -d)"
+cp go.mod "$PARITY_BUILD_DIR/oracle.mod"
+go mod edit -modfile="$PARITY_BUILD_DIR/oracle.mod" \
+  -replace="github.com/micro-editor/micro/v2=$MICRO_SOURCE"
+go build -mod=mod -modfile="$PARITY_BUILD_DIR/oracle.mod" \
+  -o "$PARITY_BUILD_DIR/oracle" oracle.go
+bun compare.mjs "$MICRO_SOURCE" "$PARITY_BUILD_DIR/oracle" --upstream-syntax
+```
+
+The final `bun compare.mjs` command writes `comparison.html`, `REPORT.md` and
+`report.json` into this directory. Open `comparison.html` in a browser and expand
+a language to compare micro (left) with jmi (right). Once the oracle executable
+has been built, rerun just that final command to regenerate the page. The HTML
+is intentionally not checked in and has no Git ignore rule; remove it after
+viewing if you do not want it included in a later commit.
+
+Go downloads test-only dependencies to its module cache. It is not a jsmdcui
+runtime dependency; the checkout's module file and upstream sources remain
+unchanged. The reference commit is recorded in the report. Re-running rewrites
+only the reports; it keeps the checked-in samples. `--generate` regenerates
+lexical witnesses and overwrites hand-written samples, so normal comparisons
+should omit it.
+
+Differences are reported rather than hidden or automatically fixed. In
+particular, a regex compiling successfully does not establish equivalence of
+nested region handling, skip masking, or rule ordering.
+
+## Scanner safety checks
+
+Run `bun test tests/highlight-regions.test.js tests/go-regexp.test.js` from the
+jsmdcui root. These cover zero-width cycles, Unicode empty matches, NUL, isolated
+surrogates, cyclic definitions/state, and a 200,003-character line whose final
+keyword must still be highlighted. Region transitions are iterative; there is
+no input line-length cutoff. Malformed definitions are limited to 128 nested
+rule levels. A repeated state at the same position advances by one Unicode
+character and continues highlighting. If malformed states keep creating fresh
+objects, 256 transitions without advancing reset the unfinished region for that
+line instead of carrying it forward.
+
+A manual stability run also processed Vue 3.5.13's `vue.global.prod.js`
+(157,924 bytes, longest line 141,082 UTF-16 units) in about 87 ms on the test
+host, without a stack overflow or truncation. This is not proof of exact Vue
+syntax parity. No downloaded Vue bundle is included in the repository.
+
+These guards prevent scanner loops and stack exhaustion. They cannot interrupt
+a single pathological native JavaScript RegExp execution; arbitrary untrusted
+regexes would need worker/process isolation for a hard execution-time limit.