wasmtime 24.0.0 → 25.0.0

data/ext/cargo-vendor/wast-217.0.0/src/lexer.rs

@@ -0,0 +1,1573 @@
+//! Definition of a lexer for the WebAssembly text format.
+//!
+//! This module provides a [`Lexer`][] type which is an iterate over the raw
+//! tokens of a WebAssembly text file. A [`Lexer`][] accounts for every single
+//! byte in a WebAssembly text field, returning tokens even for comments and
+//! whitespace. Typically you'll ignore comments and whitespace, however.
+//!
+//! If you'd like to iterate over the tokens in a file you can do so via:
+//!
+//! ```
+//! # fn foo() -> Result<(), wast::Error> {
+//! use wast::lexer::Lexer;
+//!
+//! let wat = "(module (func $foo))";
+//! for token in Lexer::new(wat).iter(0) {
+//!     println!("{:?}", token?);
+//! }
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! Note that you'll typically not use this module but will rather use
+//! [`ParseBuffer`](crate::parser::ParseBuffer) instead.
+//!
+//! [`Lexer`]: crate::lexer::Lexer
+
+use crate::token::Span;
+use crate::Error;
+use std::borrow::Cow;
+use std::char;
+use std::fmt;
+use std::slice;
+use std::str;
+use std::str::Utf8Error;
+
+/// A structure used to lex the s-expression syntax of WAT files.
+///
+/// This structure is used to generate [`Token`] items, which should account for
+/// every single byte of the input as we iterate over it. A [`LexError`] is
+/// returned for any non-lexable text.
+#[derive(Clone)]
+pub struct Lexer<'a> {
+    input: &'a str,
+    allow_confusing_unicode: bool,
+}
+
+/// A single token parsed from a `Lexer`.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct Token {
+    /// The kind of token this represents, such as whether it's whitespace, a
+    /// keyword, etc.
+    pub kind: TokenKind,
+    /// The byte offset within the original source for where this token came
+    /// from.
+    pub offset: usize,
+    /// The byte length of this token as it resides in the original source.
+    //
+    // NB: this is `u32` to enable packing `Token` into two pointers of size.
+    // This does limit a single token to being at most 4G large, but that seems
+    // probably ok.
+    pub len: u32,
+}
+
+#[test]
+fn token_is_not_too_big() {
+    assert!(std::mem::size_of::<Token>() <= std::mem::size_of::<u64>() * 2);
+}
+
+/// Classification of what was parsed from the input stream.
+///
+/// This enumeration contains all kinds of fragments, including comments and
+/// whitespace.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum TokenKind {
+    /// A line comment, preceded with `;;`
+    LineComment,
+
+    /// A block comment, surrounded by `(;` and `;)`. Note that these can be
+    /// nested.
+    BlockComment,
+
+    /// A fragment of source that represents whitespace.
+    Whitespace,
+
+    /// A left-parenthesis, including the source text for where it comes from.
+    LParen,
+    /// A right-parenthesis, including the source text for where it comes from.
+    RParen,
+
+    /// A string literal, which is actually a list of bytes.
+    String,
+
+    /// An identifier (like `$foo`).
+    ///
+    /// All identifiers start with `$` and the payload here is the original
+    /// source text.
+    Id,
+
+    /// A keyword, or something that starts with an alphabetic character.
+    ///
+    /// The payload here is the original source text.
+    Keyword,
+
+    /// An annotation (like `@foo`).
+    ///
+    /// All annotations start with `@` and the payload will be the name of the
+    /// annotation.
+    Annotation,
+
+    /// A reserved series of `idchar` symbols. Unknown what this is meant to be
+    /// used for, you'll probably generate an error about an unexpected token.
+    Reserved,
+
+    /// An integer.
+    Integer(IntegerKind),
+
+    /// A float.
+    Float(FloatKind),
+}
+
+/// Description of the parsed integer from the source.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct IntegerKind {
+    sign: Option<SignToken>,
+    has_underscores: bool,
+    hex: bool,
+}
+
+/// Description of a parsed float from the source.
+#[allow(missing_docs)]
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum FloatKind {
+    #[doc(hidden)]
+    Inf { negative: bool },
+    #[doc(hidden)]
+    Nan { negative: bool },
+    #[doc(hidden)]
+    NanVal {
+        negative: bool,
+        has_underscores: bool,
+    },
+    #[doc(hidden)]
+    Normal { has_underscores: bool, hex: bool },
+}
+
+enum ReservedKind {
+    /// "..."
+    String,
+    /// anything that's just a sequence of `idchars!()`
+    Idchars,
+    /// $"..."
+    IdString,
+    /// @"..."
+    AnnotationString,
+    /// everything else (a conglomeration of strings, idchars, etc)
+    Reserved,
+}
+
+/// Errors that can be generated while lexing.
+///
+/// All lexing errors have line/colum/position information as well as a
+/// `LexError` indicating what kind of error happened while lexing.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum LexError {
+    /// A dangling block comment was found with an unbalanced `(;` which was
+    /// never terminated in the file.
+    DanglingBlockComment,
+
+    /// An unexpected character was encountered when generally parsing and
+    /// looking for something else.
+    Unexpected(char),
+
+    /// An invalid `char` in a string literal was found.
+    InvalidStringElement(char),
+
+    /// An invalid string escape letter was found (the thing after the `\` in
+    /// string literals)
+    InvalidStringEscape(char),
+
+    /// An invalid hexadecimal digit was found.
+    InvalidHexDigit(char),
+
+    /// An invalid base-10 digit was found.
+    InvalidDigit(char),
+
+    /// Parsing expected `wanted` but ended up finding `found` instead where the
+    /// two characters aren't the same.
+    Expected {
+        /// The character that was expected to be found
+        wanted: char,
+        /// The character that was actually found
+        found: char,
+    },
+
+    /// We needed to parse more but EOF (or end of the string) was encountered.
+    UnexpectedEof,
+
+    /// A number failed to parse because it was too big to fit within the target
+    /// type.
+    NumberTooBig,
+
+    /// An invalid unicode value was found in a `\u{...}` escape in a string,
+    /// only valid unicode scalars can be escaped that way.
+    InvalidUnicodeValue(u32),
+
+    /// A lone underscore was found when parsing a number, since underscores
+    /// should always be preceded and succeeded with a digit of some form.
+    LoneUnderscore,
+
+    /// A "confusing" unicode character is present in a comment or a string
+    /// literal, such as a character that changes the direction text is
+    /// typically displayed in editors. This could cause the human-read
+    /// version to behave differently than the compiler-visible version, so
+    /// these are simply rejected for now.
+    ConfusingUnicode(char),
+
+    /// An invalid utf-8 sequence was found in a quoted identifier, such as
+    /// `$"\ff"`.
+    InvalidUtf8Id(Utf8Error),
+
+    /// An empty identifier was found, or a lone `$`.
+    EmptyId,
+
+    /// An empty identifier was found, or a lone `@`.
+    EmptyAnnotation,
+}
+
+/// A sign token for an integer.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum SignToken {
+    /// Plus sign: "+",
+    Plus,
+    /// Minus sign: "-",
+    Minus,
+}
+
+/// A fully parsed integer from a source string with a payload ready to parse
+/// into an integral type.
+#[derive(Debug, PartialEq)]
+pub struct Integer<'a> {
+    sign: Option<SignToken>,
+    val: Cow<'a, str>,
+    hex: bool,
+}
+
+/// Possible parsed float values
+#[derive(Debug, PartialEq, Eq)]
+pub enum Float<'a> {
+    /// A float `NaN` representation
+    Nan {
+        /// The specific bits to encode for this float, optionally
+        val: Option<Cow<'a, str>>,
+        /// Whether or not this is a negative `NaN` or not.
+        negative: bool,
+    },
+    /// An float infinite representation,
+    Inf {
+        #[allow(missing_docs)]
+        negative: bool,
+    },
+    /// A parsed and separated floating point value
+    Val {
+        /// Whether or not the `integral` and `decimal` are specified in hex
+        hex: bool,
+        /// The float parts before the `.`
+        integral: Cow<'a, str>,
+        /// The float parts after the `.`
+        decimal: Option<Cow<'a, str>>,
+        /// The exponent to multiple this `integral.decimal` portion of the
+        /// float by. If `hex` is true this is `2^exponent` and otherwise it's
+        /// `10^exponent`
+        exponent: Option<Cow<'a, str>>,
+    },
+}
+
+// https://webassembly.github.io/spec/core/text/values.html#text-idchar
+macro_rules! idchars {
+    () => {
+        b'0'..=b'9'
+        | b'A'..=b'Z'
+        | b'a'..=b'z'
+        | b'!'
+        | b'#'
+        | b'$'
+        | b'%'
+        | b'&'
+        | b'\''
+        | b'*'
+        | b'+'
+        | b'-'
+        | b'.'
+        | b'/'
+        | b':'
+        | b'<'
+        | b'='
+        | b'>'
+        | b'?'
+        | b'@'
+        | b'\\'
+        | b'^'
+        | b'_'
+        | b'`'
+        | b'|'
+        | b'~'
+    }
+}
+
+impl<'a> Lexer<'a> {
+    /// Creates a new lexer which will lex the `input` source string.
+    pub fn new(input: &str) -> Lexer<'_> {
+        Lexer {
+            input,
+            allow_confusing_unicode: false,
+        }
+    }
+
+    /// Returns the original source input that we're lexing.
+    pub fn input(&self) -> &'a str {
+        self.input
+    }
+
+    /// Configures whether "confusing" unicode characters are allowed while
+    /// lexing.
+    ///
+    /// If allowed then no error will happen if these characters are found, but
+    /// otherwise if disallowed a lex error will be produced when these
+    /// characters are found. Confusing characters are denied by default.
+    ///
+    /// For now "confusing characters" are primarily related to the "trojan
+    /// source" problem where it refers to characters which cause humans to read
+    /// text differently than this lexer, such as characters that alter the
+    /// left-to-right display of the source code.
+    pub fn allow_confusing_unicode(&mut self, allow: bool) -> &mut Self {
+        self.allow_confusing_unicode = allow;
+        self
+    }
+
+    /// Lexes the next at the byte position `pos` in the input.
+    ///
+    /// Returns `Some` if a token is found or `None` if we're at EOF.
+    ///
+    /// The `pos` argument will be updated to point to the next token on a
+    /// successful parse.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the input is malformed.
+    pub fn parse(&self, pos: &mut usize) -> Result<Option<Token>, Error> {
+        let offset = *pos;
+        Ok(match self.parse_kind(pos)? {
+            Some(kind) => Some(Token {
+                kind,
+                offset,
+                len: (*pos - offset).try_into().unwrap(),
+            }),
+            None => None,
+        })
+    }
+
+    fn parse_kind(&self, pos: &mut usize) -> Result<Option<TokenKind>, Error> {
+        let start = *pos;
+        // This `match` generally parses the grammar specified at
+        //
+        // https://webassembly.github.io/spec/core/text/lexical.html#text-token
+        let remaining = &self.input.as_bytes()[start..];
+        let byte = match remaining.first() {
+            Some(b) => b,
+            None => return Ok(None),
+        };
+
+        match byte {
+            // Open-parens check the next character to see if this is the start
+            // of a block comment, otherwise it's just a bland left-paren
+            // token.
+            b'(' => match remaining.get(1) {
+                Some(b';') => {
+                    let mut level = 1;
+                    // Note that we're doing a byte-level search here for the
+                    // close-delimiter of `;)`. The actual source text is utf-8
+                    // encode in `remaining` but due to how utf-8 works we
+                    // can safely search for an ASCII byte since it'll never
+                    // otherwise appear in the middle of a codepoint and if we
+                    // find it then it's guaranteed to be the right byte.
+                    //
+                    // Mainly we're avoiding the overhead of decoding utf-8
+                    // characters into a Rust `char` since it's otherwise
+                    // unnecessary work.
+                    let mut iter = remaining[2..].iter();
+                    while let Some(ch) = iter.next() {
+                        match ch {
+                            b'(' => {
+                                if let Some(b';') = iter.as_slice().first() {
+                                    level += 1;
+                                    iter.next();
+                                }
+                            }
+                            b';' => {
+                                if let Some(b')') = iter.as_slice().first() {
+                                    level -= 1;
+                                    iter.next();
+                                    if level == 0 {
+                                        let len = remaining.len() - iter.as_slice().len();
+                                        let comment = &self.input[start..][..len];
+                                        *pos += len;
+                                        self.check_confusing_comment(*pos, comment)?;
+                                        return Ok(Some(TokenKind::BlockComment));
+                                    }
+                                }
+                            }
+                            _ => {}
+                        }
+                    }
+                    Err(self.error(start, LexError::DanglingBlockComment))
+                }
+                _ => {
+                    *pos += 1;
+
+                    Ok(Some(TokenKind::LParen))
+                }
+            },
+
+            b')' => {
+                *pos += 1;
+                Ok(Some(TokenKind::RParen))
+            }
+
+            // https://webassembly.github.io/spec/core/text/lexical.html#white-space
+            b' ' | b'\n' | b'\r' | b'\t' => {
+                self.skip_ws(pos);
+                Ok(Some(TokenKind::Whitespace))
+            }
+
+            c @ (idchars!() | b'"') => {
+                let (kind, src) = self.parse_reserved(pos)?;
+                match kind {
+                    // If the reserved token was simply a single string then
+                    // that is converted to a standalone string token
+                    ReservedKind::String => return Ok(Some(TokenKind::String)),
+
+                    // If only idchars were consumed then this could be a
+                    // specific kind of standalone token we're interested in.
+                    ReservedKind::Idchars => {
+                        // https://webassembly.github.io/spec/core/text/values.html#integers
+                        if let Some(ret) = self.classify_number(src) {
+                            return Ok(Some(ret));
+                        // https://webassembly.github.io/spec/core/text/values.html#text-id
+                        } else if *c == b'$' {
+                            return Ok(Some(TokenKind::Id));
+                        // part of the WebAssembly/annotations proposal
+                        // (no online url yet)
+                        } else if *c == b'@' {
+                            return Ok(Some(TokenKind::Annotation));
+                        // https://webassembly.github.io/spec/core/text/lexical.html#text-keyword
+                        } else if b'a' <= *c && *c <= b'z' {
+                            return Ok(Some(TokenKind::Keyword));
+                        }
+                    }
+
+                    ReservedKind::IdString => return Ok(Some(TokenKind::Id)),
+                    ReservedKind::AnnotationString => return Ok(Some(TokenKind::Annotation)),
+
+                    // ... otherwise this was a conglomeration of idchars,
+                    // strings, or just idchars that don't match a prior rule,
+                    // meaning this falls through to the fallback `Reserved`
+                    // token.
+                    ReservedKind::Reserved => {}
+                }
+
+                Ok(Some(TokenKind::Reserved))
+            }
+
+            // This could be a line comment, otherwise `;` is a reserved token.
+            // The second byte is checked to see if it's a `;;` line comment
+            //
+            // Note that this character being considered as part of a
+            // `reserved` token is part of the annotations proposal.
+            b';' => match remaining.get(1) {
+                Some(b';') => {
+                    let remaining = &self.input[*pos..];
+                    let byte_pos = memchr::memchr2(b'\n', b'\r', remaining.as_bytes())
+                        .unwrap_or(remaining.len());
+                    *pos += byte_pos;
+                    let comment = &remaining[..byte_pos];
+                    self.check_confusing_comment(*pos, comment)?;
+                    Ok(Some(TokenKind::LineComment))
+                }
+                _ => {
+                    *pos += 1;
+                    Ok(Some(TokenKind::Reserved))
+                }
+            },
+
+            // Other known reserved tokens other than `;`
+            //
+            // Note that these characters being considered as part of a
+            // `reserved` token is part of the annotations proposal.
+            b',' | b'[' | b']' | b'{' | b'}' => {
+                *pos += 1;
+                Ok(Some(TokenKind::Reserved))
+            }
+
+            _ => {
+                let ch = self.input[start..].chars().next().unwrap();
+                Err(self.error(*pos, LexError::Unexpected(ch)))
+            }
+        }
+    }
+
+    fn skip_ws(&self, pos: &mut usize) {
+        // This table is a byte lookup table to determine whether a byte is a
+        // whitespace byte. There are only 4 whitespace bytes for the `*.wat`
+        // format right now which are ' ', '\t', '\r', and '\n'. These 4 bytes
+        // have a '1' in the table below.
+        //
+        // Due to how utf-8 works (our input is guaranteed to be utf-8) it is
+        // known that if these bytes are found they're guaranteed to be the
+        // whitespace byte, so they can be safely skipped and we don't have to
+        // do full utf-8 decoding. This means that the goal of this function is
+        // to find the first non-whitespace byte in `remaining`.
+        //
+        // For now this lookup table seems to be the fastest, but projects like
+        // https://github.com/lemire/despacer show other simd algorithms which
+        // can possibly accelerate this even more. Note that `*.wat` files often
+        // have a lot of whitespace so this function is typically quite hot when
+        // parsing inputs.
+        #[rustfmt::skip]
+        const WS: [u8; 256] = [
+            //                                   \t \n       \r
+            /* 0x00 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0,
+            /* 0x10 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            //        ' '
+            /* 0x20 */ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x30 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x40 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x50 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x60 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x70 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x80 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0x90 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0xa0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0xb0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0xc0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0xd0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0xe0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            /* 0xf0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        ];
+        let remaining = &self.input[*pos..];
+        let non_ws_pos = remaining
+            .as_bytes()
+            .iter()
+            .position(|b| WS[*b as usize] != 1)
+            .unwrap_or(remaining.len());
+        *pos += non_ws_pos;
+    }
+
+    /// Splits off a "reserved" token which is then further processed later on
+    /// to figure out which kind of token it is `depending on `ReservedKind`.
+    ///
+    /// For more information on this method see the clarification at
+    /// <https://github.com/WebAssembly/spec/pull/1499> but the general gist is
+    /// that this is parsing the grammar:
+    ///
+    /// ```text
+    /// reserved := (idchar | string)+
+    /// ```
+    ///
+    /// which means that it is eating any number of adjacent string/idchar
+    /// tokens (e.g. `a"b"c`) and returning the classification of what was
+    /// eaten. The classification assists in determining what the actual token
+    /// here eaten looks like.
+    fn parse_reserved(&self, pos: &mut usize) -> Result<(ReservedKind, &'a str), Error> {
+        let mut idchars = 0u32;
+        let mut strings = 0u32;
+        let start = *pos;
+        while let Some(byte) = self.input.as_bytes().get(*pos) {
+            match byte {
+                // Normal `idchars` production which appends to the reserved
+                // token that's being produced.
+                idchars!() => {
+                    idchars += 1;
+                    *pos += 1;
+                }
+
+                // https://webassembly.github.io/spec/core/text/values.html#text-string
+                b'"' => {
+                    strings += 1;
+                    *pos += 1;
+                    let mut it = self.input[*pos..].chars();
+                    let result = Lexer::parse_str(&mut it, self.allow_confusing_unicode);
+                    *pos = self.input.len() - it.as_str().len();
+                    match result {
+                        Ok(_) => {}
+                        Err(e) => {
+                            let err_pos = match &e {
+                                LexError::UnexpectedEof => self.input.len(),
+                                _ => self.input[..*pos].char_indices().next_back().unwrap().0,
+                            };
+                            return Err(self.error(err_pos, e));
+                        }
+                    }
+                }
+
+                // Nothing else is considered part of a reserved token
+                _ => break,
+            }
+        }
+        let ret = &self.input[start..*pos];
+        Ok(match (idchars, strings) {
+            (0, 0) => unreachable!(),
+            (0, 1) => (ReservedKind::String, ret),
+            (_, 0) => (ReservedKind::Idchars, ret),
+            // Pattern match `@"..."` and `$"..."` for string-based
+            // identifiers and annotations.
+            (1, 1) if ret.starts_with("$") => (ReservedKind::IdString, ret),
+            (1, 1) if ret.starts_with("@") => (ReservedKind::AnnotationString, ret),
+            _ => (ReservedKind::Reserved, ret),
+        })
+    }
+
+    fn classify_number(&self, src: &str) -> Option<TokenKind> {
+        let (sign, num) = if let Some(stripped) = src.strip_prefix('+') {
+            (Some(SignToken::Plus), stripped)
+        } else if let Some(stripped) = src.strip_prefix('-') {
+            (Some(SignToken::Minus), stripped)
+        } else {
+            (None, src)
+        };
+
+        let negative = sign == Some(SignToken::Minus);
+
+        // Handle `inf` and `nan` which are special numbers here
+        if num == "inf" {
+            return Some(TokenKind::Float(FloatKind::Inf { negative }));
+        } else if num == "nan" {
+            return Some(TokenKind::Float(FloatKind::Nan { negative }));
+        } else if let Some(stripped) = num.strip_prefix("nan:0x") {
+            let mut it = stripped.as_bytes().iter();
+            let has_underscores = skip_underscores(&mut it, |x| char::from(x).is_ascii_hexdigit())?;
+            if it.next().is_some() {
+                return None;
+            }
+            return Some(TokenKind::Float(FloatKind::NanVal {
+                negative,
+                has_underscores,
+            }));
+        }
+
+        // Figure out if we're a hex number or not
+        let test_valid: fn(u8) -> bool;
+        let (mut it, hex) = if let Some(stripped) = num.strip_prefix("0x") {
+            test_valid = |x: u8| char::from(x).is_ascii_hexdigit();
+            (stripped.as_bytes().iter(), true)
+        } else {
+            test_valid = |x: u8| char::from(x).is_ascii_digit();
+            (num.as_bytes().iter(), false)
+        };
+
+        // Evaluate the first part, moving out all underscores
+        let mut has_underscores = skip_underscores(&mut it, test_valid)?;
+
+        match it.clone().next() {
+            // If we're followed by something this may be a float so keep going.
+            Some(_) => {}
+
+            // Otherwise this is a valid integer literal!
+            None => {
+                return Some(TokenKind::Integer(IntegerKind {
+                    has_underscores,
+                    sign,
+                    hex,
+                }))
+            }
+        }
+
+        // A number can optionally be after the decimal so only actually try to
+        // parse one if it's there.
+        if it.clone().next() == Some(&b'.') {
+            it.next();
+            match it.clone().next() {
+                Some(c) if test_valid(*c) => {
+                    if skip_underscores(&mut it, test_valid)? {
+                        has_underscores = true;
+                    }
+                }
+                Some(_) | None => {}
+            }
+        };
+
+        // Figure out if there's an exponential part here to make a float, and
+        // if so parse it but defer its actual calculation until later.
+        match (hex, it.next()) {
+            (true, Some(b'p')) | (true, Some(b'P')) | (false, Some(b'e')) | (false, Some(b'E')) => {
+                match it.clone().next() {
+                    Some(b'-') => {
+                        it.next();
+                    }
+                    Some(b'+') => {
+                        it.next();
+                    }
+                    _ => {}
+                }
+                if skip_underscores(&mut it, |x| char::from(x).is_ascii_digit())? {
+                    has_underscores = true;
+                }
+            }
+            (_, None) => {}
+            _ => return None,
+        }
+
+        // We should have eaten everything by now, if not then this is surely
+        // not a float or integer literal.
+        if it.next().is_some() {
+            return None;
+        }
+
+        return Some(TokenKind::Float(FloatKind::Normal {
+            has_underscores,
+            hex,
+        }));
+
+        fn skip_underscores<'a>(
+            it: &mut slice::Iter<'_, u8>,
+            good: fn(u8) -> bool,
+        ) -> Option<bool> {
+            let mut last_underscore = false;
+            let mut has_underscores = false;
+            let first = *it.next()?;
+            if !good(first) {
+                return None;
+            }
+            while let Some(c) = it.clone().next() {
+                if *c == b'_' && !last_underscore {
+                    has_underscores = true;
+                    it.next();
+                    last_underscore = true;
+                    continue;
+                }
+                if !good(*c) {
+                    break;
+                }
+                last_underscore = false;
+                it.next();
+            }
+            if last_underscore {
+                return None;
+            }
+            Some(has_underscores)
+        }
+    }
+
+    /// Verifies that `comment`, which is about to be returned, has a "confusing
+    /// unicode character" in it and should instead be transformed into an
+    /// error.
+    fn check_confusing_comment(&self, end: usize, comment: &str) -> Result<(), Error> {
+        if self.allow_confusing_unicode {
+            return Ok(());
+        }
+
+        // In an effort to avoid utf-8 decoding the entire `comment` the search
+        // here is a bit more optimized. This checks for the `0xe2` byte because
+        // in the utf-8 encoding that's the leading encoding byte for all
+        // "confusing characters". Each instance of 0xe2 is checked to see if it
+        // starts a confusing character, and if so that's returned.
+        //
+        // Also note that 0xe2 will never be found in the middle of a codepoint,
+        // it's always the start of a codepoint. This means that if our special
+        // characters show up they're guaranteed to start with 0xe2 bytes.
+        let bytes = comment.as_bytes();
+        for pos in memchr::Memchr::new(0xe2, bytes) {
+            if let Some(c) = comment[pos..].chars().next() {
+                if is_confusing_unicode(c) {
+                    // Note that `self.cur()` accounts for already having
+                    // parsed `comment`, so we move backwards to where
+                    // `comment` started and then add the index within
+                    // `comment`.
+                    let pos = end - comment.len() + pos;
+                    return Err(self.error(pos, LexError::ConfusingUnicode(c)));
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn parse_str(
+        it: &mut str::Chars<'a>,
+        allow_confusing_unicode: bool,
+    ) -> Result<Cow<'a, [u8]>, LexError> {
+        enum State {
+            Start,
+            String(Vec<u8>),
+        }
+        let orig = it.as_str();
+        let mut state = State::Start;
+        loop {
+            match it.next().ok_or(LexError::UnexpectedEof)? {
+                '"' => break,
+                '\\' => {
+                    match state {
+                        State::String(_) => {}
+                        State::Start => {
+                            let pos = orig.len() - it.as_str().len() - 1;
+                            state = State::String(orig[..pos].as_bytes().to_vec());
+                        }
+                    }
+                    let buf = match &mut state {
+                        State::String(b) => b,
+                        State::Start => unreachable!(),
+                    };
+                    match it.next().ok_or(LexError::UnexpectedEof)? {
+                        '"' => buf.push(b'"'),
+                        '\'' => buf.push(b'\''),
+                        't' => buf.push(b'\t'),
+                        'n' => buf.push(b'\n'),
+                        'r' => buf.push(b'\r'),
+                        '\\' => buf.push(b'\\'),
+                        'u' => {
+                            Lexer::must_eat_char(it, '{')?;
+                            let n = Lexer::hexnum(it)?;
+                            let c = char::from_u32(n).ok_or(LexError::InvalidUnicodeValue(n))?;
+                            buf.extend(c.encode_utf8(&mut [0; 4]).as_bytes());
+                            Lexer::must_eat_char(it, '}')?;
+                        }
+                        c1 if c1.is_ascii_hexdigit() => {
+                            let c2 = Lexer::hexdigit(it)?;
+                            buf.push(to_hex(c1) * 16 + c2);
+                        }
+                        c => return Err(LexError::InvalidStringEscape(c)),
+                    }
+                }
+                c if (c as u32) < 0x20 || c as u32 == 0x7f => {
+                    return Err(LexError::InvalidStringElement(c))
+                }
+                c if !allow_confusing_unicode && is_confusing_unicode(c) => {
+                    return Err(LexError::ConfusingUnicode(c))
+                }
+                c => match &mut state {
+                    State::Start => {}
+                    State::String(v) => {
+                        v.extend(c.encode_utf8(&mut [0; 4]).as_bytes());
+                    }
+                },
+            }
+        }
+        match state {
+            State::Start => Ok(orig[..orig.len() - it.as_str().len() - 1].as_bytes().into()),
+            State::String(s) => Ok(s.into()),
+        }
+    }
+
+    /// Parses an id-or-string-based name from `it`.
+    ///
+    /// Note that `it` should already have been lexed and this is just
+    /// extracting the value. If the token lexed was `@a` then this should point
+    /// to `a`.
+    ///
+    /// This will automatically detect quoted syntax such as `@"..."` and the
+    /// byte string will be parsed and validated as utf-8.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if a quoted byte string is found and contains invalid
+    /// utf-8.
+    fn parse_name(it: &mut str::Chars<'a>) -> Result<Cow<'a, str>, LexError> {
+        if it.clone().next() == Some('"') {
+            it.next();
+            match Lexer::parse_str(it, true)? {
+                Cow::Borrowed(bytes) => match std::str::from_utf8(bytes) {
+                    Ok(s) => Ok(Cow::Borrowed(s)),
+                    Err(e) => Err(LexError::InvalidUtf8Id(e)),
+                },
+                Cow::Owned(bytes) => match String::from_utf8(bytes) {
+                    Ok(s) => Ok(Cow::Owned(s)),
+                    Err(e) => Err(LexError::InvalidUtf8Id(e.utf8_error())),
+                },
+            }
+        } else {
+            Ok(Cow::Borrowed(it.as_str()))
+        }
+    }
+
+    fn hexnum(it: &mut str::Chars<'_>) -> Result<u32, LexError> {
+        let n = Lexer::hexdigit(it)?;
+        let mut last_underscore = false;
+        let mut n = n as u32;
+        while let Some(c) = it.clone().next() {
+            if c == '_' {
+                it.next();
+                last_underscore = true;
+                continue;
+            }
+            if !c.is_ascii_hexdigit() {
+                break;
+            }
+            last_underscore = false;
+            it.next();
+            n = n
+                .checked_mul(16)
+                .and_then(|n| n.checked_add(to_hex(c) as u32))
+                .ok_or(LexError::NumberTooBig)?;
+        }
+        if last_underscore {
+            return Err(LexError::LoneUnderscore);
+        }
+        Ok(n)
+    }
+
+    /// Reads a hexidecimal digit from the input stream, returning where it's
+    /// defined and the hex value. Returns an error on EOF or an invalid hex
+    /// digit.
+    fn hexdigit(it: &mut str::Chars<'_>) -> Result<u8, LexError> {
+        let ch = Lexer::must_char(it)?;
+        if ch.is_ascii_hexdigit() {
+            Ok(to_hex(ch))
+        } else {
+            Err(LexError::InvalidHexDigit(ch))
+        }
+    }
+
+    /// Reads the next character from the input string and where it's located,
+    /// returning an error if the input stream is empty.
+    fn must_char(it: &mut str::Chars<'_>) -> Result<char, LexError> {
+        it.next().ok_or(LexError::UnexpectedEof)
+    }
+
+    /// Expects that a specific character must be read next
+    fn must_eat_char(it: &mut str::Chars<'_>, wanted: char) -> Result<(), LexError> {
+        let found = Lexer::must_char(it)?;
+        if wanted == found {
+            Ok(())
+        } else {
+            Err(LexError::Expected { wanted, found })
+        }
+    }
+
+    /// Creates an error at `pos` with the specified `kind`
+    fn error(&self, pos: usize, kind: LexError) -> Error {
+        Error::lex(Span { offset: pos }, self.input, kind)
+    }
+
+    /// Returns an iterator over all tokens in the original source string
+    /// starting at the `pos` specified.
+    pub fn iter(&self, mut pos: usize) -> impl Iterator<Item = Result<Token, Error>> + '_ {
+        std::iter::from_fn(move || self.parse(&mut pos).transpose())
+    }
+
+    /// Returns whether an annotation is present at `pos`. If it is present then
+    /// `Ok(Some(token))` is returned corresponding to the token, otherwise
+    /// `Ok(None)` is returned. If the next token cannot be parsed then an error
+    /// is returned.
+    pub fn annotation(&self, mut pos: usize) -> Result<Option<Token>, Error> {
+        let bytes = self.input.as_bytes();
+        // Quickly reject anything that for sure isn't an annotation since this
+        // method is used every time an lparen is parsed.
+        if bytes.get(pos) != Some(&b'@') {
+            return Ok(None);
+        }
+        match self.parse(&mut pos)? {
+            Some(token) => match token.kind {
+                TokenKind::Annotation => Ok(Some(token)),
+                _ => Ok(None),
+            },
+            None => Ok(None),
+        }
+    }
+}
+
+impl Token {
+    /// Returns the original source text for this token.
+    pub fn src<'a>(&self, s: &'a str) -> &'a str {
+        &s[self.offset..][..self.len.try_into().unwrap()]
+    }
+
+    /// Returns the identifier, without the leading `$` symbol, that this token
+    /// represents.
+    ///
+    /// Note that this method returns the contents of the identifier. With a
+    /// string-based identifier this means that escapes have been resolved to
+    /// their string-based equivalent.
+    ///
+    /// Should only be used with `TokenKind::Id`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if this is a string-based identifier (e.g. `$"..."`)
+    /// which is invalid utf-8.
+    pub fn id<'a>(&self, s: &'a str) -> Result<Cow<'a, str>, Error> {
+        let mut ch = self.src(s).chars();
+        let dollar = ch.next();
+        debug_assert_eq!(dollar, Some('$'));
+        let id = Lexer::parse_name(&mut ch).map_err(|e| self.error(s, e))?;
+        if id.is_empty() {
+            return Err(self.error(s, LexError::EmptyId));
+        }
+        Ok(id)
+    }
+
+    /// Returns the annotation, without the leading `@` symbol, that this token
+    /// represents.
+    ///
+    /// Note that this method returns the contents of the identifier. With a
+    /// string-based identifier this means that escapes have been resolved to
+    /// their string-based equivalent.
+    ///
+    /// Should only be used with `TokenKind::Annotation`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if this is a string-based identifier (e.g. `$"..."`)
+    /// which is invalid utf-8.
+    pub fn annotation<'a>(&self, s: &'a str) -> Result<Cow<'a, str>, Error> {
+        let mut ch = self.src(s).chars();
+        let at = ch.next();
+        debug_assert_eq!(at, Some('@'));
+        let id = Lexer::parse_name(&mut ch).map_err(|e| self.error(s, e))?;
+        if id.is_empty() {
+            return Err(self.error(s, LexError::EmptyAnnotation));
+        }
+        Ok(id)
+    }
+
+    /// Returns the keyword this token represents.
+    ///
+    /// Should only be used with [`TokenKind::Keyword`].
+    pub fn keyword<'a>(&self, s: &'a str) -> &'a str {
+        self.src(s)
+    }
+
+    /// Returns the reserved string this token represents.
+    ///
+    /// Should only be used with [`TokenKind::Reserved`].
+    pub fn reserved<'a>(&self, s: &'a str) -> &'a str {
+        self.src(s)
+    }
+
+    /// Returns the parsed string that this token represents.
+    ///
+    /// This returns either a raw byte slice into the source if that's possible
+    /// or an owned representation to handle escaped characters and such.
+    ///
+    /// Should only be used with [`TokenKind::String`].
+    pub fn string<'a>(&self, s: &'a str) -> Cow<'a, [u8]> {
+        let mut ch = self.src(s).chars();
+        ch.next().unwrap();
+        Lexer::parse_str(&mut ch, true).unwrap()
+    }
+
+    /// Returns the decomposed float token that this represents.
+    ///
+    /// This will slice up the float token into its component parts and return a
+    /// description of the float token in the source.
+    ///
+    /// Should only be used with [`TokenKind::Float`].
+    pub fn float<'a>(&self, s: &'a str, kind: FloatKind) -> Float<'a> {
+        match kind {
+            FloatKind::Inf { negative } => Float::Inf { negative },
+            FloatKind::Nan { negative } => Float::Nan {
+                val: None,
+                negative,
+            },
+            FloatKind::NanVal {
+                negative,
+                has_underscores,
+            } => {
+                let src = self.src(s);
+                let src = if src.starts_with("n") { src } else { &src[1..] };
+                let mut val = Cow::Borrowed(src.strip_prefix("nan:0x").unwrap());
+                if has_underscores {
+                    *val.to_mut() = val.replace("_", "");
+                }
+                Float::Nan {
+                    val: Some(val),
+                    negative,
+                }
+            }
+            FloatKind::Normal {
+                has_underscores,
+                hex,
+            } => {
+                let src = self.src(s);
+                let (integral, decimal, exponent) = match src.find('.') {
+                    Some(i) => {
+                        let integral = &src[..i];
+                        let rest = &src[i + 1..];
+                        let exponent = if hex {
+                            rest.find('p').or_else(|| rest.find('P'))
+                        } else {
+                            rest.find('e').or_else(|| rest.find('E'))
+                        };
+                        match exponent {
+                            Some(i) => (integral, Some(&rest[..i]), Some(&rest[i + 1..])),
+                            None => (integral, Some(rest), None),
+                        }
+                    }
+                    None => {
+                        let exponent = if hex {
+                            src.find('p').or_else(|| src.find('P'))
+                        } else {
+                            src.find('e').or_else(|| src.find('E'))
+                        };
+                        match exponent {
+                            Some(i) => (&src[..i], None, Some(&src[i + 1..])),
+                            None => (src, None, None),
+                        }
+                    }
+                };
+                let mut integral = Cow::Borrowed(integral.strip_prefix('+').unwrap_or(integral));
+                let mut decimal = decimal.and_then(|s| {
+                    if s.is_empty() {
+                        None
+                    } else {
+                        Some(Cow::Borrowed(s))
+                    }
+                });
+                let mut exponent =
+                    exponent.map(|s| Cow::Borrowed(s.strip_prefix('+').unwrap_or(s)));
+                if has_underscores {
+                    *integral.to_mut() = integral.replace("_", "");
+                    if let Some(decimal) = &mut decimal {
+                        *decimal.to_mut() = decimal.replace("_", "");
+                    }
+                    if let Some(exponent) = &mut exponent {
+                        *exponent.to_mut() = exponent.replace("_", "");
+                    }
+                }
+                if hex {
+                    *integral.to_mut() = integral.replace("0x", "");
+                }
+                Float::Val {
+                    hex,
+                    integral,
+                    decimal,
+                    exponent,
+                }
+            }
+        }
+    }
+
+    /// Returns the decomposed integer token that this represents.
+    ///
+    /// This will slice up the integer token into its component parts and
+    /// return a description of the integer token in the source.
+    ///
+    /// Should only be used with [`TokenKind::Integer`].
+    pub fn integer<'a>(&self, s: &'a str, kind: IntegerKind) -> Integer<'a> {
+        let src = self.src(s);
+        let val = match kind.sign {
+            Some(SignToken::Plus) => src.strip_prefix('+').unwrap(),
+            Some(SignToken::Minus) => src,
+            None => src,
+        };
+        let mut val = Cow::Borrowed(val);
+        if kind.has_underscores {
+            *val.to_mut() = val.replace("_", "");
+        }
+        if kind.hex {
+            *val.to_mut() = val.replace("0x", "");
+        }
+        Integer {
+            sign: kind.sign,
+            hex: kind.hex,
+            val,
+        }
+    }
+
+    fn error(&self, src: &str, err: LexError) -> Error {
+        Error::lex(
+            Span {
+                offset: self.offset,
+            },
+            src,
+            err,
+        )
+    }
+}
+
+impl<'a> Integer<'a> {
+    /// Returns the sign token for this integer.
+    pub fn sign(&self) -> Option<SignToken> {
+        self.sign
+    }
+
+    /// Returns the value string that can be parsed for this integer, as well
+    /// as the base that it should be parsed in
+    pub fn val(&self) -> (&str, u32) {
+        (&self.val, if self.hex { 16 } else { 10 })
+    }
+}
+
+fn to_hex(c: char) -> u8 {
+    match c {
+        'a'..='f' => c as u8 - b'a' + 10,
+        'A'..='F' => c as u8 - b'A' + 10,
+        _ => c as u8 - b'0',
+    }
+}
+
+impl fmt::Display for LexError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use LexError::*;
+        match self {
+            DanglingBlockComment => f.write_str("unterminated block comment")?,
+            Unexpected(c) => write!(f, "unexpected character '{}'", escape_char(*c))?,
+            InvalidStringElement(c) => {
+                write!(f, "invalid character in string '{}'", escape_char(*c))?
+            }
+            InvalidStringEscape(c) => write!(f, "invalid string escape '{}'", escape_char(*c))?,
+            InvalidHexDigit(c) => write!(f, "invalid hex digit '{}'", escape_char(*c))?,
+            InvalidDigit(c) => write!(f, "invalid decimal digit '{}'", escape_char(*c))?,
+            Expected { wanted, found } => write!(
+                f,
+                "expected '{}' but found '{}'",
+                escape_char(*wanted),
+                escape_char(*found)
+            )?,
+            UnexpectedEof => write!(f, "unexpected end-of-file")?,
+            NumberTooBig => f.write_str("number is too big to parse")?,
+            InvalidUnicodeValue(c) => write!(f, "invalid unicode scalar value 0x{:x}", c)?,
+            LoneUnderscore => write!(f, "bare underscore in numeric literal")?,
+            ConfusingUnicode(c) => write!(f, "likely-confusing unicode character found {:?}", c)?,
+            InvalidUtf8Id(_) => write!(f, "malformed UTF-8 encoding of string-based id")?,
+            EmptyId => write!(f, "empty identifier")?,
+            EmptyAnnotation => write!(f, "empty annotation id")?,
+        }
+        Ok(())
+    }
+}
+
+fn escape_char(c: char) -> String {
+    match c {
+        '\t' => String::from("\\t"),
+        '\r' => String::from("\\r"),
+        '\n' => String::from("\\n"),
+        '\\' => String::from("\\\\"),
+        '\'' => String::from("\\\'"),
+        '\"' => String::from("\""),
+        '\x20'..='\x7e' => String::from(c),
+        _ => c.escape_unicode().to_string(),
+    }
+}
+
+/// This is an attempt to protect agains the "trojan source" [1] problem where
+/// unicode characters can cause editors to render source code differently
+/// for humans than the compiler itself sees.
+///
+/// To mitigate this issue, and because it's relatively rare in practice,
+/// this simply rejects characters of that form.
+///
+/// [1]: https://www.trojansource.codes/
+fn is_confusing_unicode(ch: char) -> bool {
+    matches!(
+        ch,
+        '\u{202a}'
+            | '\u{202b}'
+            | '\u{202d}'
+            | '\u{202e}'
+            | '\u{2066}'
+            | '\u{2067}'
+            | '\u{2068}'
+            | '\u{206c}'
+            | '\u{2069}'
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ws_smoke() {
+        fn get_whitespace(input: &str) -> &str {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Whitespace => token.src(input),
+                other => panic!("unexpected {:?}", other),
+            }
+        }
+        assert_eq!(get_whitespace(" "), " ");
+        assert_eq!(get_whitespace("  "), "  ");
+        assert_eq!(get_whitespace("  \n "), "  \n ");
+        assert_eq!(get_whitespace("  x"), "  ");
+        assert_eq!(get_whitespace("  ;"), "  ");
+    }
+
+    #[test]
+    fn line_comment_smoke() {
+        fn get_line_comment(input: &str) -> &str {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::LineComment => token.src(input),
+                other => panic!("unexpected {:?}", other),
+            }
+        }
+        assert_eq!(get_line_comment(";;"), ";;");
+        assert_eq!(get_line_comment(";; xyz"), ";; xyz");
+        assert_eq!(get_line_comment(";; xyz\nabc"), ";; xyz");
+        assert_eq!(get_line_comment(";;\nabc"), ";;");
+        assert_eq!(get_line_comment(";;   \nabc"), ";;   ");
+        assert_eq!(get_line_comment(";;   \rabc"), ";;   ");
+        assert_eq!(get_line_comment(";;   \r\nabc"), ";;   ");
+    }
+
+    #[test]
+    fn block_comment_smoke() {
+        fn get_block_comment(input: &str) -> &str {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::BlockComment => token.src(input),
+                other => panic!("unexpected {:?}", other),
+            }
+        }
+        assert_eq!(get_block_comment("(;;)"), "(;;)");
+        assert_eq!(get_block_comment("(; ;)"), "(; ;)");
+        assert_eq!(get_block_comment("(; (;;) ;)"), "(; (;;) ;)");
+    }
+
+    fn get_token(input: &str) -> Token {
+        Lexer::new(input)
+            .parse(&mut 0)
+            .expect("no first token")
+            .expect("no token")
+    }
+
+    #[test]
+    fn lparen() {
+        assert_eq!(get_token("((").kind, TokenKind::LParen);
+    }
+
+    #[test]
+    fn rparen() {
+        assert_eq!(get_token(")(").kind, TokenKind::RParen);
+    }
+
+    #[test]
+    fn strings() {
+        fn get_string(input: &str) -> Vec<u8> {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::String => token.string(input).to_vec(),
+                other => panic!("not keyword {:?}", other),
+            }
+        }
+        assert_eq!(&*get_string("\"\""), b"");
+        assert_eq!(&*get_string("\"a\""), b"a");
+        assert_eq!(&*get_string("\"a b c d\""), b"a b c d");
+        assert_eq!(&*get_string("\"\\\"\""), b"\"");
+        assert_eq!(&*get_string("\"\\'\""), b"'");
+        assert_eq!(&*get_string("\"\\n\""), b"\n");
+        assert_eq!(&*get_string("\"\\t\""), b"\t");
+        assert_eq!(&*get_string("\"\\r\""), b"\r");
+        assert_eq!(&*get_string("\"\\\\\""), b"\\");
+        assert_eq!(&*get_string("\"\\01\""), &[1]);
+        assert_eq!(&*get_string("\"\\u{1}\""), &[1]);
+        assert_eq!(
+            &*get_string("\"\\u{0f3}\""),
+            '\u{0f3}'.encode_utf8(&mut [0; 4]).as_bytes()
+        );
+        assert_eq!(
+            &*get_string("\"\\u{0_f_3}\""),
+            '\u{0f3}'.encode_utf8(&mut [0; 4]).as_bytes()
+        );
+
+        for i in 0..=255i32 {
+            let s = format!("\"\\{:02x}\"", i);
+            assert_eq!(&*get_string(&s), &[i as u8]);
+        }
+    }
+
+    #[test]
+    fn id() {
+        fn get_id(input: &str) -> String {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Id => token.id(input).unwrap().to_string(),
+                other => panic!("not id {:?}", other),
+            }
+        }
+        assert_eq!(get_id("$x"), "x");
+        assert_eq!(get_id("$xyz"), "xyz");
+        assert_eq!(get_id("$x_z"), "x_z");
+        assert_eq!(get_id("$0^"), "0^");
+        assert_eq!(get_id("$0^;;"), "0^");
+        assert_eq!(get_id("$0^ ;;"), "0^");
+        assert_eq!(get_id("$\"x\" ;;"), "x");
+    }
+
+    #[test]
+    fn annotation() {
+        fn get_annotation(input: &str) -> String {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Annotation => token.annotation(input).unwrap().to_string(),
+                other => panic!("not annotation {:?}", other),
+            }
+        }
+        assert_eq!(get_annotation("@foo"), "foo");
+        assert_eq!(get_annotation("@foo "), "foo");
+        assert_eq!(get_annotation("@f "), "f");
+        assert_eq!(get_annotation("@\"x\" "), "x");
+        assert_eq!(get_annotation("@0 "), "0");
+    }
+
+    #[test]
+    fn keyword() {
+        fn get_keyword(input: &str) -> &str {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Keyword => token.keyword(input),
+                other => panic!("not keyword {:?}", other),
+            }
+        }
+        assert_eq!(get_keyword("x"), "x");
+        assert_eq!(get_keyword("xyz"), "xyz");
+        assert_eq!(get_keyword("x_z"), "x_z");
+        assert_eq!(get_keyword("x_z "), "x_z");
+        assert_eq!(get_keyword("x_z "), "x_z");
+    }
+
+    #[test]
+    fn reserved() {
+        fn get_reserved(input: &str) -> &str {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Reserved => token.reserved(input),
+                other => panic!("not reserved {:?}", other),
+            }
+        }
+        assert_eq!(get_reserved("^_x "), "^_x");
+    }
+
+    #[test]
+    fn integer() {
+        fn get_integer(input: &str) -> String {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Integer(i) => token.integer(input, i).val.to_string(),
+                other => panic!("not integer {:?}", other),
+            }
+        }
+        assert_eq!(get_integer("1"), "1");
+        assert_eq!(get_integer("0"), "0");
+        assert_eq!(get_integer("-1"), "-1");
+        assert_eq!(get_integer("+1"), "1");
+        assert_eq!(get_integer("+1_000"), "1000");
+        assert_eq!(get_integer("+1_0_0_0"), "1000");
+        assert_eq!(get_integer("+0x10"), "10");
+        assert_eq!(get_integer("-0x10"), "-10");
+        assert_eq!(get_integer("0x10"), "10");
+    }
+
+    #[test]
+    fn float() {
+        fn get_float(input: &str) -> Float<'_> {
+            let token = get_token(input);
+            match token.kind {
+                TokenKind::Float(f) => token.float(input, f),
+                other => panic!("not float {:?}", other),
+            }
+        }
+        assert_eq!(
+            get_float("nan"),
+            Float::Nan {
+                val: None,
+                negative: false
+            },
+        );
+        assert_eq!(
+            get_float("-nan"),
+            Float::Nan {
+                val: None,
+                negative: true,
+            },
+        );
+        assert_eq!(
+            get_float("+nan"),
+            Float::Nan {
+                val: None,
+                negative: false,
+            },
+        );
+        assert_eq!(
+            get_float("+nan:0x1"),
+            Float::Nan {
+                val: Some("1".into()),
+                negative: false,
+            },
+        );
+        assert_eq!(
+            get_float("nan:0x7f_ffff"),
+            Float::Nan {
+                val: Some("7fffff".into()),
+                negative: false,
+            },
+        );
+        assert_eq!(get_float("inf"), Float::Inf { negative: false });
+        assert_eq!(get_float("-inf"), Float::Inf { negative: true });
+        assert_eq!(get_float("+inf"), Float::Inf { negative: false });
+
+        assert_eq!(
+            get_float("1.2"),
+            Float::Val {
+                integral: "1".into(),
+                decimal: Some("2".into()),
+                exponent: None,
+                hex: false,
+            },
+        );
+        assert_eq!(
+            get_float("1.2e3"),
+            Float::Val {
+                integral: "1".into(),
+                decimal: Some("2".into()),
+                exponent: Some("3".into()),
+                hex: false,
+            },
+        );
+        assert_eq!(
+            get_float("-1_2.1_1E+0_1"),
+            Float::Val {
+                integral: "-12".into(),
+                decimal: Some("11".into()),
+                exponent: Some("01".into()),
+                hex: false,
+            },
+        );
+        assert_eq!(
+            get_float("+1_2.1_1E-0_1"),
+            Float::Val {
+                integral: "12".into(),
+                decimal: Some("11".into()),
+                exponent: Some("-01".into()),
+                hex: false,
+            },
+        );
+        assert_eq!(
+            get_float("0x1_2.3_4p5_6"),
+            Float::Val {
+                integral: "12".into(),
+                decimal: Some("34".into()),
+                exponent: Some("56".into()),
+                hex: true,
+            },
+        );
+        assert_eq!(
+            get_float("+0x1_2.3_4P-5_6"),
+            Float::Val {
+                integral: "12".into(),
+                decimal: Some("34".into()),
+                exponent: Some("-56".into()),
+                hex: true,
+            },
+        );
+        assert_eq!(
+            get_float("1."),
+            Float::Val {
+                integral: "1".into(),
+                decimal: None,
+                exponent: None,
+                hex: false,
+            },
+        );
+        assert_eq!(
+            get_float("0x1p-24"),
+            Float::Val {
+                integral: "1".into(),
+                decimal: None,
+                exponent: Some("-24".into()),
+                hex: true,
+            },
+        );
+    }
+}