shirobai 2026.0709.0000 → 2026.0713.0000

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1be87c75d6dee4b98605941e80e3244bac21b501cd78f0fcc6d16b5182897e63
-  data.tar.gz: 466974572bc21d590b8cc0a8fb6fe6c6844c45c17ffe58cc98f71c766b7b9c42
+  metadata.gz: ac67035ad74e8850bceadc1fe62046339127dba699f7976ce45bbea34646273f
+  data.tar.gz: 3453f727a31a8fdf6d12376e6af577a09f503d80b7e0dfe2e8c9df6a10af3260
 SHA512:
-  metadata.gz: 0a10cbada94bc6ac95afdf33d793f0b5b9698eacb4bd5742fb4bb0a9296dff75d21ffc97be68741ceff51caeaa15274a25a6a8ba6db727567354fd0d31d15d2b
-  data.tar.gz: cf0574176bdbc245ad66d3ddd6291cd65ecf3ba7c4d3e6a48004825873612fa7824f2682b3f983cbb1dfec7b2bcce0bcb029fcb4481927e8825e95cdb6c84619
+  metadata.gz: ebe11c9d5e1638871f75f24b5139605694acb3af3387eb442a5c96c1a09aba51afa21c09c5097a23979ca4bb887897fdb4f8ac0005d4cde5efb6ae30d3f64299
+  data.tar.gz: 8807b3a11cd85ba0366c723076db229d96966eda367a0a0413e5a01908975ac8af4102e5b05acc498a204e62be38fc3ab8396ce9e09ef5e80bc66b812a0975d5

data/Cargo.lock

@@ -356,7 +356,7 @@ checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
 
 [[package]]
 name = "shirobai"
-version = "0.1.0"
+version = "2026.713.0"
 dependencies = [
  "magnus",
  "shirobai-core",
@@ -364,10 +364,11 @@ dependencies = [
 
 [[package]]
 name = "shirobai-core"
-version = "0.1.0"
+version = "2026.713.0"
 dependencies = [
  "libc",
  "ruby-prism",
+ "ruby-prism-sys",
  "self_cell",
  "unicode-width",
 ]

data/Cargo.toml

@@ -2,6 +2,13 @@
 members = ["ext/shirobai", "crates/shirobai-core"]
 resolver = "3"
 
+# Single source of the crate version. bump-version.yml rewrites this line to
+# the semver form of the gem CalVer (leading zeros stripped: gem 2026.0709.0000
+# is crate 2026.709.0 -- cargo rejects leading zeros in numeric segments, and
+# Gem::Version normalizes to the same integers, so the two stay value-equal).
+[workspace.package]
+version = "2026.713.0"
+
 # Release tuning. The hot path crosses crate boundaries (ext cdylib -> core
 # check_* -> ruby-prism); without LTO the cross-crate calls are not inlined and
 # codegen-units=16 (the release default) further fragments optimization. Fat LTO

data/crates/shirobai-core/Cargo.toml

@@ -1,10 +1,14 @@
 [package]
 name = "shirobai-core"
-version = "0.1.0"
+version.workspace = true
 edition = "2024"
 
 [dependencies]
 ruby-prism = "1.9.0"
+# Direct dep on the -sys crate to reach pm_parser_init / pm_parse / lex_callback
+# from pm_lex. ruby-prism 1.9.0 already depends on this exact version, so cargo
+# unifies it into a single build (no second prism compile).
+ruby-prism-sys = "=1.9.0"
 self_cell = "1"
 unicode-width = "0.2"
 
@@ -18,3 +22,8 @@ path = "src/bin/walk_profile.rs"
 [[bin]]
 name = "analysis_profile"
 path = "src/bin/analysis_profile.rs"
+
+# pm_lex probe: token dump / canary / profile harness.
+[[bin]]
+name = "pm_lex_probe"
+path = "src/bin/pm_lex_probe.rs"

data/crates/shirobai-core/src/bin/bundle_profile.rs

@@ -297,6 +297,7 @@ fn run_perrule(sources: &[Vec<u8>], cfg: &BundleConfig, iterations: usize) {
                 s,
                 cfg.abc_size_max_floor,
                 cfg.abc_size_discount_repeated,
+                cfg.abc_size_it_is_send,
             ));
         }),
         e!("indentation_consistency", |s: &[u8]| {

data/crates/shirobai-core/src/bin/pm_lex_probe.rs

@@ -0,0 +1,246 @@
+//! pm_lex probe harness (measurement / verification only). Three subcommands:
+//!
+//!   dump-tokens <corpus> <out_file>
+//!       Per .rb file: `# <path>` then one `name\tstart\tlen\tstate` line per
+//!       token. Diff against the Ruby `Prism.lex` oracle (oracle_lex.rb).
+//!
+//!   canary <corpus>
+//!       For every .rb file compare the AST reached via the transmute path
+//!       (parse_with_lex) against a normal ruby_prism::parse(). Reports files,
+//!       AST fingerprint mismatches, and comment-count mismatches. This is the
+//!       ad-hoc, whole-corpus version of the tests/pm_lex_canary.rs CI gate.
+//!
+//!   profile <parse|lex|lex_reuse|reparse> <corpus> [iters]
+//!       CLOCK_PROCESS_CPUTIME_ID median over iters (default 5). Modes:
+//!         parse     - ruby_prism::parse only (AST, no tokens)
+//!         lex       - parse_with_lex (AST + a fresh token Vec per file)
+//!         lex_reuse - parse_with_lex_into (AST + one reused token buffer)
+//!         reparse   - two full parses (reproduces the pm_serialize_lex tax:
+//!                     AST from one parse, tokens from a second full parse)
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use ruby_prism::{Node, Visit};
+use shirobai_core::pm_lex::{RawToken, parse_with_lex, parse_with_lex_into};
+
+#[cfg(unix)]
+fn cpu_time_secs() -> f64 {
+    let mut ts = libc::timespec {
+        tv_sec: 0,
+        tv_nsec: 0,
+    };
+    // SAFETY: ts is a valid timespec.
+    unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) };
+    ts.tv_sec as f64 + ts.tv_nsec as f64 / 1e9
+}
+
+fn collect_rb_files(dir: &Path, out: &mut Vec<PathBuf>) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if path.is_dir() {
+            collect_rb_files(&path, out);
+        } else if path.extension().is_some_and(|e| e == "rb") {
+            out.push(path);
+        }
+    }
+}
+
+fn sorted_rb_files(corpus: &Path) -> Vec<PathBuf> {
+    let mut files = Vec::new();
+    collect_rb_files(corpus, &mut files);
+    files.sort();
+    files
+}
+
+// Pre-order (start,end) offset fingerprint of an AST via the Visit trait.
+struct Fingerprint(Vec<(usize, usize)>);
+impl<'pr> Visit<'pr> for Fingerprint {
+    fn visit_branch_node_enter(&mut self, node: Node<'pr>) {
+        let l = node.location();
+        self.0.push((l.start_offset(), l.end_offset()));
+    }
+    fn visit_leaf_node_enter(&mut self, node: Node<'pr>) {
+        let l = node.location();
+        self.0.push((l.start_offset(), l.end_offset()));
+    }
+}
+fn fingerprint(root: &Node<'_>) -> Vec<(usize, usize)> {
+    let mut fp = Fingerprint(Vec::new());
+    fp.visit(root);
+    fp.0
+}
+
+fn dump_tokens(corpus: &Path, out_file: &Path) {
+    let files = sorted_rb_files(corpus);
+    let mut out = std::io::BufWriter::new(std::fs::File::create(out_file).expect("create out"));
+    for path in &files {
+        let Ok(src) = std::fs::read(path) else {
+            continue;
+        };
+        let rel = path.strip_prefix(corpus).unwrap_or(path);
+        writeln!(out, "# {}", rel.display()).unwrap();
+        let (_result, tokens) = parse_with_lex(&src);
+        for t in &tokens {
+            writeln!(
+                out,
+                "{}\t{}\t{}\t{}",
+                t.type_name(),
+                t.start_offset,
+                t.length,
+                t.lex_state
+            )
+            .unwrap();
+        }
+    }
+    eprintln!("dumped {} files -> {}", files.len(), out_file.display());
+}
+
+fn canary(corpus: &Path) {
+    let files = sorted_rb_files(corpus);
+    let mut n_ok = 0usize;
+    let mut ast_mismatch = 0usize;
+    let mut comment_mismatch = 0usize;
+    let mut first_bad: Vec<String> = Vec::new();
+    for path in &files {
+        let Ok(src) = std::fs::read(path) else {
+            continue;
+        };
+        let normal = ruby_prism::parse(&src);
+        let fp_normal = fingerprint(&normal.node());
+        let n_comments_normal = normal.comments().count();
+        drop(normal);
+
+        let (spiked, _tokens) = parse_with_lex(&src);
+        let fp_spiked = fingerprint(&spiked.node());
+        let n_comments_spiked = spiked.comments().count();
+        drop(spiked); // exercise Drop of a transmuted ParseResult.
+
+        let ast_ok = fp_normal == fp_spiked;
+        let com_ok = n_comments_normal == n_comments_spiked;
+        if ast_ok && com_ok {
+            n_ok += 1;
+        } else {
+            if !ast_ok {
+                ast_mismatch += 1;
+            }
+            if !com_ok {
+                comment_mismatch += 1;
+            }
+            if first_bad.len() < 10 {
+                first_bad.push(format!(
+                    "{} ast_ok={} nodes {}/{} comments {}/{}",
+                    path.display(),
+                    ast_ok,
+                    fp_normal.len(),
+                    fp_spiked.len(),
+                    n_comments_normal,
+                    n_comments_spiked
+                ));
+            }
+        }
+    }
+    println!("canary files={}", files.len());
+    println!("  parity_ok           = {n_ok}");
+    println!("  ast_fingerprint_bad = {ast_mismatch}");
+    println!("  comment_count_bad   = {comment_mismatch}");
+    for b in &first_bad {
+        println!("  BAD {b}");
+    }
+    let parity = if files.is_empty() {
+        0.0
+    } else {
+        100.0 * n_ok as f64 / files.len() as f64
+    };
+    println!("  parity = {parity:.2}%");
+}
+
+fn profile(mode: &str, corpus: &Path, iters: usize) {
+    let files = sorted_rb_files(corpus);
+    let sources: Vec<Vec<u8>> = files.iter().filter_map(|p| std::fs::read(p).ok()).collect();
+    eprintln!("files: {}", sources.len());
+
+    let mut buf: Vec<RawToken> = Vec::new();
+    let run = |mode: &str, buf: &mut Vec<RawToken>| -> usize {
+        let mut sink = 0usize;
+        for s in &sources {
+            match mode {
+                "parse" => {
+                    let r = ruby_prism::parse(s);
+                    sink = sink.wrapping_add(r.node().location().end_offset());
+                }
+                "lex" => {
+                    let (r, tokens) = parse_with_lex(s);
+                    sink = sink.wrapping_add(r.node().location().end_offset());
+                    sink = sink.wrapping_add(tokens.len());
+                }
+                "lex_reuse" => {
+                    // Reuse one token buffer across files (retains capacity):
+                    // isolates pure callback+push cost from per-file heap alloc.
+                    let r = parse_with_lex_into(s, buf);
+                    sink = sink.wrapping_add(r.node().location().end_offset());
+                    sink = sink.wrapping_add(buf.len());
+                }
+                "reparse" => {
+                    // AST from one parse; tokens would come from a *second*
+                    // full parse (the pm_serialize_lex tax being reproduced).
+                    let r = ruby_prism::parse(s);
+                    sink = sink.wrapping_add(r.node().location().end_offset());
+                    let r2 = ruby_prism::parse(s);
+                    sink = sink.wrapping_add(r2.node().location().end_offset());
+                }
+                other => {
+                    eprintln!("unknown profile mode: {other}");
+                    std::process::exit(2);
+                }
+            }
+        }
+        sink
+    };
+
+    // Warm-up (untimed).
+    let warm = run(mode, &mut buf);
+    let mut times: Vec<f64> = (0..iters)
+        .map(|_| {
+            let t0 = cpu_time_secs();
+            let sink = run(mode, &mut buf);
+            let dt = cpu_time_secs() - t0;
+            eprintln!("  iter {mode}: {dt:.4}s (sink {sink})");
+            dt
+        })
+        .collect();
+    times.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    let median = times[times.len() / 2];
+    eprintln!("warm sink {warm}");
+    println!("{mode}\t{median:.4}");
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    match args.get(1).map(String::as_str) {
+        Some("dump-tokens") => {
+            let corpus = PathBuf::from(args.get(2).expect("corpus dir"));
+            let out = PathBuf::from(args.get(3).expect("out file"));
+            dump_tokens(&corpus, &out);
+        }
+        Some("canary") => {
+            let corpus = PathBuf::from(args.get(2).expect("corpus dir"));
+            canary(&corpus);
+        }
+        Some("profile") => {
+            let mode = args.get(2).expect("mode").clone();
+            let corpus = PathBuf::from(args.get(3).expect("corpus dir"));
+            let iters: usize = args.get(4).map_or(5, |v| v.parse().expect("iters"));
+            profile(&mode, &corpus, iters);
+        }
+        _ => {
+            eprintln!(
+                "usage:\n  pm_lex_probe dump-tokens <corpus> <out_file>\n  pm_lex_probe canary <corpus>\n  pm_lex_probe profile <parse|lex|lex_reuse|reparse> <corpus> [iters]"
+            );
+            std::process::exit(2);
+        }
+    }
+}

data/crates/shirobai-core/src/lib.rs

@@ -1 +1,6 @@
 pub mod rules;
+
+// Single-pass parse + lex: one prism parse that yields both the AST and the
+// token stream. See `pm_lex` for the transmute contract.
+pub mod pm_lex;
+mod pm_lex_token_names;

data/crates/shirobai-core/src/pm_lex.rs

@@ -0,0 +1,300 @@
+//! Single-pass parse + lex: raw prism parse that also collects the token stream.
+//!
+//! Goal: get the same in-memory AST that `ruby_prism::parse()` produces AND the
+//! token stream from the *same single parse*, without prism's `pm_serialize_lex`
+//! tax (a second full parse just to get tokens).
+//!
+//! How: `ruby_prism::parse()` hard-wires `pm_parser_init` -> `pm_parse` with no
+//! seam to install a `lex_callback`. We replicate that body ourselves with the
+//! raw `ruby_prism_sys` symbols, set `parser.lex_callback` to a token-collecting
+//! C callback in between, then hand the result back to safe `ruby_prism` code by
+//! `transmute`-ing a layout-identical look-alike into `ruby_prism::ParseResult`
+//! (whose fields are private and whose only constructor, `parse()`, cannot take
+//! a callback).
+//!
+//! The transmute is a load-bearing bet on `ParseResult`'s field layout. Two
+//! things guard it: [`assert_parse_result_layout`] is a `const` static assert on
+//! size + alignment (a compile error if either drifts), and the canary tests
+//! (here and in `tests/pm_lex_canary.rs`) compare the transmute-path AST against
+//! a normal `parse()` on a large corpus, catching any residual field-order
+//! change on a rustc update. A constructor that takes a `lex_callback` is being
+//! proposed upstream to prism; once it lands this module can drop the transmute.
+
+use std::mem::MaybeUninit;
+use std::os::raw::c_void;
+use std::ptr::NonNull;
+
+use ruby_prism::ParseResult;
+use ruby_prism_sys::{
+    pm_lex_callback_t, pm_node_t, pm_parse, pm_parser_init, pm_parser_t, pm_token_t,
+};
+
+pub use crate::pm_lex_token_names::PM_TOKEN_NAMES;
+
+/// One lexer token, collected in-line during the parse.
+/// Byte offsets are from the start of the source (prism is byte-oriented).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RawToken {
+    /// prism `pm_token_type` integer. Name via [`RawToken::type_name`].
+    pub token_type: u32,
+    pub start_offset: usize,
+    pub length: usize,
+    /// `parser.lex_state` sampled at callback time (matches `Prism.lex`).
+    pub lex_state: u32,
+}
+
+impl RawToken {
+    /// Ruby `Prism::Token#type` symbol name for this token type.
+    #[must_use]
+    pub fn type_name(&self) -> &'static str {
+        PM_TOKEN_NAMES
+            .get(self.token_type as usize)
+            .copied()
+            .unwrap_or("")
+    }
+}
+
+/// A look-alike of `ruby_prism::ParseResult<'pr>`.
+///
+/// `ParseResult`'s fields are private, so we cannot name them; we mirror their
+/// declared order and types here and `transmute`. `ParseResult` is `repr(Rust)`
+/// (no `repr(C)`), so field order is not guaranteed by the language — this only
+/// works because two structs with identical field types get the same layout
+/// under the current rustc, and we pin size+alignment with a static assert. If a
+/// future rustc reorders fields, the canary tests break loudly.
+///
+/// Fields are only ever read through the `transmute` into `ParseResult`, so the
+/// compiler cannot see the reads — hence `dead_code` is allowed here.
+#[repr(Rust)]
+#[allow(dead_code)]
+struct ParseResultLookAlike<'pr> {
+    source: &'pr [u8],
+    parser: NonNull<pm_parser_t>,
+    node: NonNull<pm_node_t>,
+}
+
+/// Compile-time guard: our look-alike must match `ParseResult` byte-for-byte in
+/// size and alignment. (Field *order* is the residual, canary-tested bet.)
+pub const fn assert_parse_result_layout() {
+    use std::mem::{align_of, size_of};
+    assert!(
+        size_of::<ParseResultLookAlike<'_>>() == size_of::<ParseResult<'_>>(),
+        "ParseResult size drifted from look-alike; transmute is unsafe"
+    );
+    assert!(
+        align_of::<ParseResultLookAlike<'_>>() == align_of::<ParseResult<'_>>(),
+        "ParseResult alignment drifted from look-alike; transmute is unsafe"
+    );
+}
+const _: () = assert_parse_result_layout();
+
+/// The C callback prism invokes for every lexed token. `data` is a
+/// `*mut Vec<RawToken>`. Reads `parser.lex_state` at call time, mirroring how
+/// `Prism.lex` records lex state.
+///
+/// # Safety
+/// `data` must point to a live `Vec<RawToken>`; `parser`/`token` are valid for
+/// the duration of the call (prism guarantees this while lexing).
+unsafe extern "C" fn collect_token(
+    data: *mut c_void,
+    parser: *mut pm_parser_t,
+    token: *mut pm_token_t,
+) {
+    // SAFETY: contract documented above; all derefs are of live prism/caller
+    // memory for the span of this call.
+    unsafe {
+        let tokens = &mut *(data.cast::<Vec<RawToken>>());
+        let tok = &*token;
+        let source_start = (*parser).start;
+        let start_offset = tok.start.offset_from(source_start) as usize;
+        let length = tok.end.offset_from(tok.start) as usize;
+        let lex_state = (*parser).lex_state;
+        tokens.push(RawToken {
+            token_type: tok.type_,
+            start_offset,
+            length,
+            lex_state,
+        });
+    }
+}
+
+/// Like [`parse_with_lex`] but appends tokens into a caller-owned buffer
+/// (cleared first, capacity retained). Lets a caller amortize the token Vec
+/// allocation across files, isolating the pure callback+push cost from
+/// per-file heap allocation. Returns the `ParseResult`.
+///
+/// # Safety
+/// Uses raw prism FFI and a `transmute` into `ParseResult`. Sound as long as
+/// `ParseResultLookAlike` matches `ParseResult`'s layout (guarded by the static
+/// assert + canary tests) and prism's C ABI is stable.
+#[must_use]
+pub fn parse_with_lex_into<'s>(
+    source: &'s [u8],
+    out: &mut Vec<RawToken>,
+) -> ParseResult<'s> {
+    out.clear();
+    let mut callback = pm_lex_callback_t {
+        data: (out as *mut Vec<RawToken>).cast::<c_void>(),
+        callback: Some(collect_token),
+    };
+
+    // SAFETY: replicate ruby_prism::parse()'s body verbatim, plus lex_callback.
+    let look_alike = unsafe {
+        let uninit = Box::new(MaybeUninit::<pm_parser_t>::uninit());
+        let uninit = Box::into_raw(uninit);
+
+        pm_parser_init(
+            (*uninit).as_mut_ptr(),
+            source.as_ptr(),
+            source.len(),
+            std::ptr::null(),
+        );
+
+        let parser = (*uninit).assume_init_mut();
+        let parser = NonNull::new_unchecked(parser);
+
+        (*parser.as_ptr()).lex_callback = &mut callback as *mut pm_lex_callback_t;
+
+        let node = pm_parse(parser.as_ptr());
+        let node = NonNull::new_unchecked(node);
+
+        (*parser.as_ptr()).lex_callback = std::ptr::null_mut();
+
+        ParseResultLookAlike {
+            source,
+            parser,
+            node,
+        }
+    };
+
+    // SAFETY: identical layout (static assert on size/align; canary on fields).
+    unsafe { std::mem::transmute(look_alike) }
+}
+
+/// Parse `source` and collect its token stream in the *same* parse.
+///
+/// Returns a real `ruby_prism::ParseResult` (safe to walk / drop like any other)
+/// plus the collected tokens. Mirrors `ruby_prism::parse()` exactly, only adding
+/// the `lex_callback` wiring in between init and parse. Allocates a fresh token
+/// Vec per call; use [`parse_with_lex_into`] to reuse one buffer across files.
+///
+/// # Safety
+/// Same contract as [`parse_with_lex_into`].
+#[must_use]
+pub fn parse_with_lex(source: &[u8]) -> (ParseResult<'_>, Vec<RawToken>) {
+    // Tokens Vec lives on the heap in a stable slot that outlives `pm_parse`.
+    let mut tokens: Box<Vec<RawToken>> = Box::default();
+    let mut callback = pm_lex_callback_t {
+        data: (&mut *tokens as *mut Vec<RawToken>).cast::<c_void>(),
+        callback: Some(collect_token),
+    };
+
+    // SAFETY: replicate ruby_prism::parse()'s body verbatim, plus lex_callback.
+    let look_alike = unsafe {
+        let uninit = Box::new(MaybeUninit::<pm_parser_t>::uninit());
+        let uninit = Box::into_raw(uninit);
+
+        pm_parser_init(
+            (*uninit).as_mut_ptr(),
+            source.as_ptr(),
+            source.len(),
+            std::ptr::null(),
+        );
+
+        let parser = (*uninit).assume_init_mut();
+        let parser = NonNull::new_unchecked(parser);
+
+        // Install the collecting callback before parsing.
+        (*parser.as_ptr()).lex_callback = &mut callback as *mut pm_lex_callback_t;
+
+        let node = pm_parse(parser.as_ptr());
+        let node = NonNull::new_unchecked(node);
+
+        // Detach the callback: nothing reads it post-parse, and `callback`
+        // (and `tokens`) get moved/dropped independently of the parser.
+        (*parser.as_ptr()).lex_callback = std::ptr::null_mut();
+
+        ParseResultLookAlike {
+            source,
+            parser,
+            node,
+        }
+    };
+
+    // The load-bearing transmute. Look-alike -> real ParseResult.
+    // SAFETY: identical layout (static assert on size/align; canary on fields).
+    let result: ParseResult<'_> = unsafe { std::mem::transmute(look_alike) };
+
+    (result, *tokens)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ruby_prism::{Node, Visit};
+
+    // Pre-order (start,end) offset fingerprint of an AST via the Visit trait.
+    struct Fingerprint(Vec<(usize, usize)>);
+    impl<'pr> Visit<'pr> for Fingerprint {
+        fn visit_branch_node_enter(&mut self, node: Node<'pr>) {
+            let l = node.location();
+            self.0.push((l.start_offset(), l.end_offset()));
+        }
+        fn visit_leaf_node_enter(&mut self, node: Node<'pr>) {
+            let l = node.location();
+            self.0.push((l.start_offset(), l.end_offset()));
+        }
+    }
+    fn fingerprint(root: &Node<'_>) -> Vec<(usize, usize)> {
+        let mut fp = Fingerprint(Vec::new());
+        fp.visit(root);
+        fp.0
+    }
+
+    #[test]
+    fn layout_static_assert_holds() {
+        assert_parse_result_layout();
+    }
+
+    #[test]
+    fn collects_tokens_matching_a_known_snippet() {
+        let src = b"def foo(a)\n  a + 1\nend\n";
+        let (_result, tokens) = parse_with_lex(src);
+        let named: Vec<(&str, usize, usize, u32)> = tokens
+            .iter()
+            .map(|t| (t.type_name(), t.start_offset, t.length, t.lex_state))
+            .collect();
+        // From `Prism.lex` on the same source.
+        assert_eq!(named[0], ("KEYWORD_DEF", 0, 3, 128));
+        assert_eq!(named[1], ("IDENTIFIER", 4, 3, 8));
+        assert_eq!(named[2], ("PARENTHESIS_LEFT", 7, 1, 1025));
+        // Last token is always EOF.
+        assert_eq!(tokens.last().unwrap().type_name(), "EOF");
+    }
+
+    #[test]
+    fn transmute_path_ast_matches_normal_parse() {
+        let srcs: &[&[u8]] = &[
+            b"x = 1\n",
+            b"def m(a, b)\n  a.foo { |c| c + b }\nend\n",
+            b"class C < D\n  attr_reader :x\n  # comment\nend\n",
+            b"[1, 2, 3].map { _1 * 2 }.select(&:even?)\n",
+            "\u{3042} = 'multibyte'\n".as_bytes(), // non-ASCII
+        ];
+        for src in srcs {
+            let normal = ruby_prism::parse(src);
+            let (spiked, _tokens) = parse_with_lex(src);
+            assert_eq!(
+                fingerprint(&normal.node()),
+                fingerprint(&spiked.node()),
+                "AST fingerprint diverged for {:?}",
+                String::from_utf8_lossy(src)
+            );
+            // Comments must also survive the transmute path.
+            let n_comments_normal = normal.comments().count();
+            let n_comments_spiked = spiked.comments().count();
+            assert_eq!(n_comments_normal, n_comments_spiked);
+            // Drop of `spiked` (a transmuted ParseResult) must not crash.
+        }
+    }
+}