calcsheet-native 0.1.0

data/ext/calcsheet_native/src/xlsx.rs

@@ -0,0 +1,814 @@
+//! The cells of an XLSX sheet part, read from its XML as calcsheet's `Xlsx::SheetParser` reads them through Ox, for
+//! XML that keeps to a plain subset: elements, attributes and text, a leading XML declaration, well formed. Anything
+//! else (a comment, CDATA, a DOCTYPE, a reference in an attribute, an odd name, a value calcsheet would refuse...)
+//! makes the sheet `Unsupported`: the Ruby side then reads it with calcsheet's own parser, which gives its exact
+//! result or error. The state machine follows `SheetParser`'s callbacks one for one.
+//!
+//! The texts of the cells are copied once, into `out`, which the rows of a batch point into.
+
+use std::sync::Arc;
+
+use memchr::memchr;
+
+use crate::xml::{Item, Lexer, Result, Unsupported, decode, line_ends, local, utf8};
+
+/// The rows and columns of calcsheet's grid.
+const MAX_ROWS: u32 = 1_048_576;
+const MAX_COLUMNS: u32 = 16_384;
+const LAST_COLUMN_INDEX: u32 = MAX_COLUMNS - 1;
+/// Cast::MAX_NUMBER_BYTES: the longest value of a number, date, boolean, error or shared string index.
+const MAX_NUMBER_BYTES: usize = 1_024;
+
+/// The type of a cell, as `SheetParser#finish_cell` dispatches it.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum Kind {
+    Number = 0,
+    Shared = 1,
+    Inline = 2,
+    Boolean = 3,
+    Error = 4,
+    Date = 5,
+    Other = 6,
+}
+
+/// A cell with a text: its column, kind (for `Shared`, the text is the shared string), the type text for `Other`, its style (`None` without one, `Some(-1)` for
+/// one that is not one to ten digits), its text in `out` and whether that text still needs its line ends and
+/// references read (a single piece holding "\r" or "&").
+pub struct Cell {
+    pub column: u32,
+    pub kind: Kind,
+    pub type_text: Option<Vec<u8>>,
+    pub style: Option<i64>,
+    pub start: usize,
+    pub end: usize,
+    pub raw: bool,
+}
+
+pub struct Row {
+    pub number: u32,
+    pub cells: Vec<Cell>,
+}
+
+pub enum Stop {
+    NeedMore,
+    /// The end of sheetData, or of a document without it.
+    End,
+    /// The Ruby parser must read this sheet.
+    Unsupported,
+    /// A cell needs the shared strings, which calcsheet loads when the first cell needs them: hand them over with
+    /// #set_strings and ask again.
+    NeedStrings,
+    Full,
+}
+
+/// The shared strings table as calcsheet keeps it: the bytes of all strings, and the end of each (offset `i + 1`)
+/// after a leading 0. One copy serves the sheet readers and the Ruby side.
+pub struct Strings {
+    pub buffer: Vec<u8>,
+    pub offsets: Vec<u64>,
+}
+
+impl Strings {
+    pub fn size(&self) -> usize {
+        self.offsets.len() - 1
+    }
+
+    /// The string at `index`, 0 to `size - 1`.
+    pub fn get(&self, index: usize) -> &[u8] {
+        &self.buffer[self.offsets[index] as usize..self.offsets[index + 1] as usize]
+    }
+}
+
+/// The elements SheetParser reads, by local name.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Element {
+    C,
+    V,
+    T,
+    Row,
+    Is,
+    RPh,
+    SheetData,
+    Other,
+}
+
+/// The `t` of a cell.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum CellType {
+    Absent,
+    N,
+    S,
+    Inline,
+    B,
+    E,
+    D,
+    Other,
+}
+
+pub struct SheetReader {
+    max_cell_bytes: Option<usize>,
+    lexer: Lexer,
+    done: bool,
+    unsupported: bool,
+    // The last item was a start tag that is not empty: Ox hands over an empty text when its end tag follows.
+    just_started: bool,
+    strings: Option<Arc<Strings>>,
+    need_strings: bool,
+    // The texts of the rows handed out, and the bytes of them the next batch drops.
+    out: Vec<u8>,
+    consumed: usize,
+    row_start: usize,
+    // SheetParser's state.
+    last_number: u32,
+    in_row: bool,
+    row_number: Option<u32>,
+    next_column: u32,
+    in_cell: bool,
+    column: Option<u32>,
+    cell_type: CellType,
+    other_type: Vec<u8>,
+    style: Option<i64>,
+    text: Option<usize>,
+    pieces: usize,
+    special: bool,
+    capture: bool,
+    inline: bool,
+    phonetic: u32,
+    text_limit: Option<usize>,
+    cells: Vec<Cell>,
+}
+
+impl SheetReader {
+    pub fn new(max_cell_bytes: Option<usize>) -> Self {
+        SheetReader {
+            max_cell_bytes,
+            lexer: Lexer::new(),
+            done: false,
+            unsupported: false,
+            just_started: false,
+            strings: None,
+            need_strings: false,
+            out: Vec::new(),
+            consumed: 0,
+            row_start: 0,
+            last_number: 0,
+            in_row: false,
+            row_number: None,
+            next_column: 0,
+            in_cell: false,
+            column: None,
+            cell_type: CellType::Absent,
+            other_type: Vec::new(),
+            style: None,
+            text: None,
+            pieces: 0,
+            special: false,
+            capture: false,
+            inline: false,
+            phonetic: 0,
+            text_limit: None,
+            cells: Vec::new(),
+        }
+    }
+
+    pub fn feed(&mut self, chunk: &[u8]) {
+        self.lexer.feed(chunk);
+    }
+
+    pub fn finish(&mut self) {
+        self.lexer.eof = true;
+    }
+
+    /// The shared strings table: `offsets` holds native-endian 64-bit ends after a leading 0, as calcsheet's
+    /// SharedStrings packs them.
+    pub fn set_strings(&mut self, buffer: &[u8], offsets: &[u8]) {
+        let offsets = offsets
+            .as_chunks::<8>()
+            .0
+            .iter()
+            .map(|bytes| u64::from_ne_bytes(*bytes))
+            .collect();
+        self.set_table(Arc::new(Strings {
+            buffer: buffer.to_vec(),
+            offsets,
+        }));
+    }
+
+    /// The shared strings table read by a StringsReader, shared.
+    pub fn set_table(&mut self, strings: Arc<Strings>) {
+        self.strings = Some(strings);
+        self.need_strings = false;
+    }
+
+    /// The text of a cell of the last batch.
+    pub fn text(&self, cell: &Cell) -> &[u8] {
+        &self.out[cell.start..cell.end]
+    }
+
+    /// Reads items until `max` rows are complete, the text fed so far ends, or the sheet ends.
+    pub fn rows(&mut self, max: usize) -> (Vec<Row>, Stop) {
+        self.drop_consumed();
+        let mut rows = Vec::new();
+        let stop = if self.unsupported {
+            Stop::Unsupported
+        } else if self.done {
+            Stop::End
+        } else {
+            self.read(max, &mut rows)
+        };
+        if matches!(stop, Stop::Unsupported) {
+            self.unsupported = true;
+        }
+        self.consumed = if self.in_row {
+            self.row_start
+        } else {
+            self.out.len()
+        };
+        (rows, stop)
+    }
+
+    fn read(&mut self, max: usize, rows: &mut Vec<Row>) -> Stop {
+        while rows.len() < max {
+            let result = match self.lexer.item() {
+                Ok(Some(item)) => self.handle(item, rows),
+                Ok(None) if self.lexer.eof => {
+                    // The text ends: a complete document without sheetData ends as Ox's parse does.
+                    if self.lexer.complete() {
+                        self.done = true;
+                        return Stop::End;
+                    }
+                    return Stop::Unsupported;
+                }
+                Ok(None) => return Stop::NeedMore,
+                Err(Unsupported) => return Stop::Unsupported,
+            };
+            match result {
+                Ok(true) => {
+                    self.done = true;
+                    return Stop::End;
+                }
+                Ok(false) if self.need_strings => return Stop::NeedStrings,
+                Ok(false) => {}
+                Err(Unsupported) => return Stop::Unsupported,
+            }
+        }
+        Stop::Full
+    }
+
+    /// Drops the texts of the rows handed out, keeping those of the row being read.
+    fn drop_consumed(&mut self) {
+        let consumed = self.consumed;
+        if consumed == 0 {
+            return;
+        }
+        self.out.drain(..consumed);
+        for cell in &mut self.cells {
+            cell.start -= consumed;
+            cell.end -= consumed;
+        }
+        if let Some(text) = self.text.as_mut() {
+            *text -= consumed;
+        }
+        self.row_start = self.row_start.saturating_sub(consumed);
+        self.consumed = 0;
+    }
+
+    // ---- SheetParser's callbacks ----
+
+    /// Handles an item; true at the end of sheetData.
+    fn handle(&mut self, item: Item, rows: &mut Vec<Row>) -> Result<bool> {
+        let just_started = std::mem::replace(&mut self.just_started, false);
+        match item {
+            Item::Text(start, end) => {
+                if self.capture {
+                    self.text_piece(start, end)?;
+                }
+                Ok(false)
+            }
+            Item::Start(name_start, name_end, empty) => {
+                let element = element(local(&self.lexer.buf[name_start..name_end]));
+                self.start(element)?;
+                if empty {
+                    self.end(element, rows)
+                } else {
+                    self.just_started = true;
+                    Ok(false)
+                }
+            }
+            Item::End(name_start, name_end) => {
+                let element = element(local(&self.lexer.buf[name_start..name_end]));
+                // calcsheet loads the shared strings at the first cell that needs them: the end tag is read again
+                // once they are here.
+                if element == Element::C
+                    && self.in_cell
+                    && self.cell_type == CellType::S
+                    && self.text.is_some()
+                    && self.strings.is_none()
+                {
+                    self.lexer.unread_end();
+                    self.just_started = just_started;
+                    self.need_strings = true;
+                    return Ok(false);
+                }
+                // "<v></v>": Ox hands over an empty text ("<v/>" has none).
+                if just_started && self.capture {
+                    self.text_piece(self.lexer.pos, self.lexer.pos)?;
+                }
+                self.end(element, rows)
+            }
+        }
+    }
+
+    fn start(&mut self, element: Element) -> Result<()> {
+        // An element inside a captured text would split it in pieces that Ox may hand over differently.
+        if self.capture {
+            return Err(Unsupported);
+        }
+        match element {
+            Element::C => {
+                if self.in_cell || !self.in_row {
+                    return Err(Unsupported);
+                }
+                self.in_cell = true;
+                self.column = None;
+                self.cell_type = CellType::Absent;
+                self.style = None;
+                self.text = None;
+                self.pieces = 0;
+                self.special = false;
+                self.inline = false;
+                self.check_plain()?;
+                for index in 0..self.lexer.attrs.len() {
+                    let (name_start, name_end, value_start, value_end) = self.lexer.attrs[index];
+                    let value = &self.lexer.buf[value_start..value_end];
+                    match &self.lexer.buf[name_start..name_end] {
+                        b"r" => self.column = column_of(value)?,
+                        b"t" => {
+                            utf8(value)?;
+                            self.cell_type = match value {
+                                b"n" => CellType::N,
+                                b"s" => CellType::S,
+                                b"inlineStr" | b"str" => CellType::Inline,
+                                b"b" => CellType::B,
+                                b"e" => CellType::E,
+                                b"d" => CellType::D,
+                                other => {
+                                    self.other_type.clear();
+                                    self.other_type.extend_from_slice(other);
+                                    CellType::Other
+                                }
+                            };
+                        }
+                        b"s" => self.style = Some(style_index(value)),
+                        _ => {}
+                    }
+                }
+            }
+            Element::V => {
+                self.capture = self.in_cell;
+                let bounded = matches!(
+                    self.cell_type,
+                    CellType::Absent
+                        | CellType::N
+                        | CellType::S
+                        | CellType::B
+                        | CellType::E
+                        | CellType::D
+                );
+                self.text_limit = if bounded {
+                    Some(MAX_NUMBER_BYTES)
+                } else {
+                    self.max_cell_bytes
+                };
+            }
+            Element::T => {
+                self.capture = self.inline && self.phonetic == 0;
+                self.text_limit = self.max_cell_bytes;
+            }
+            Element::Row => {
+                if self.in_row {
+                    return Err(Unsupported);
+                }
+                self.in_row = true;
+                self.row_number = None;
+                self.next_column = 0;
+                self.row_start = self.out.len();
+                self.check_plain()?;
+                for index in 0..self.lexer.attrs.len() {
+                    let (name_start, name_end, value_start, value_end) = self.lexer.attrs[index];
+                    if &self.lexer.buf[name_start..name_end] == b"r" {
+                        self.row_number = Some(self.row_number_of(value_start, value_end)?);
+                    }
+                }
+            }
+            Element::Is => self.inline = self.in_cell,
+            Element::RPh => self.phonetic += 1,
+            Element::SheetData | Element::Other => {}
+        }
+        Ok(())
+    }
+
+    fn end(&mut self, element: Element, rows: &mut Vec<Row>) -> Result<bool> {
+        match element {
+            Element::V | Element::T => self.capture = false,
+            Element::C => {
+                if self.in_cell {
+                    self.finish_cell()?;
+                }
+                self.text = None;
+                self.capture = false;
+                self.inline = false;
+            }
+            Element::Row => {
+                if self.in_row {
+                    self.finish_row(rows)?;
+                }
+            }
+            Element::Is => self.inline = false,
+            Element::RPh => self.phonetic = self.phonetic.saturating_sub(1),
+            Element::SheetData => return Ok(true),
+            Element::Other => {}
+        }
+        Ok(false)
+    }
+
+    /// The attributes of a cell or row, refused when one is prefixed or repeated (Ox would read them its own way).
+    fn check_plain(&self) -> Result<()> {
+        let lexer = &self.lexer;
+        if lexer
+            .attrs
+            .iter()
+            .any(|&(start, end, _, _)| memchr(b':', &lexer.buf[start..end]).is_some())
+        {
+            return Err(Unsupported);
+        }
+        lexer.check_unique()
+    }
+
+    /// A piece of the cell's text. A text of several pieces is joined here only when no piece needs its line ends or
+    /// references read, which calcsheet does per piece; a text too long, or not UTF-8, is left to calcsheet.
+    fn text_piece(&mut self, start: usize, end: usize) -> Result<()> {
+        let piece = &self.lexer.buf[start..end];
+        let special = memchr(b'&', piece).is_some() || memchr(b'\r', piece).is_some();
+        self.pieces += 1;
+        if self.pieces > 1 && (special || self.special) {
+            return Err(Unsupported);
+        }
+        self.special |= special;
+        let text_start = *self.text.get_or_insert(self.out.len());
+        let length = self.out.len() - text_start + piece.len();
+        if self.text_limit.is_some_and(|limit| length > limit) {
+            return Err(Unsupported);
+        }
+        utf8(piece)?;
+        self.out.extend_from_slice(piece);
+        Ok(())
+    }
+
+    fn finish_cell(&mut self) -> Result<()> {
+        self.in_cell = false;
+        let column = self.column.unwrap_or(self.next_column);
+        if column > LAST_COLUMN_INDEX {
+            return Err(Unsupported);
+        }
+        self.next_column = column + 1;
+        let Some(start) = self.text.take() else {
+            return Ok(());
+        };
+        let (kind, type_text) = match self.cell_type {
+            CellType::Absent | CellType::N => (Kind::Number, None),
+            CellType::S => {
+                self.shared_string(start)?;
+                (Kind::Shared, None)
+            }
+            CellType::Inline => (Kind::Inline, None),
+            CellType::B => (Kind::Boolean, None),
+            CellType::E => (Kind::Error, None),
+            CellType::D => (Kind::Date, None),
+            CellType::Other => (Kind::Other, Some(self.other_type.clone())),
+        };
+        self.cells.push(Cell {
+            column,
+            kind,
+            type_text,
+            style: self.style,
+            start,
+            end: self.out.len(),
+            raw: self.special,
+        });
+        Ok(())
+    }
+
+    /// SheetParser#shared_string: the text from `start` in `out` is the index, replaced by the string it names. An
+    /// index that is not plain digits, outside the table, and a string that is not UTF-8 are left to calcsheet.
+    fn shared_string(&mut self, start: usize) -> Result<()> {
+        let index = &self.out[start..];
+        if self.special
+            || index.is_empty()
+            || index.len() > 18
+            || !index.iter().all(u8::is_ascii_digit)
+        {
+            return Err(Unsupported);
+        }
+        let index = index.iter().fold(0usize, |index, &digit| {
+            index * 10 + usize::from(digit - b'0')
+        });
+        let strings = self.strings.as_ref().ok_or(Unsupported)?;
+        if index >= strings.size() {
+            return Err(Unsupported);
+        }
+        let text = strings.get(index);
+        utf8(text)?;
+        self.out.truncate(start);
+        self.out.extend_from_slice(text);
+        Ok(())
+    }
+
+    fn finish_row(&mut self, rows: &mut Vec<Row>) -> Result<()> {
+        self.in_row = false;
+        let number = match self.row_number {
+            Some(number) => number,
+            None if self.last_number < MAX_ROWS => self.last_number + 1,
+            None => return Err(Unsupported),
+        };
+        self.last_number = number;
+        if !self.cells.is_empty() {
+            let cells = std::mem::take(&mut self.cells);
+            rows.push(Row { number, cells });
+        }
+        Ok(())
+    }
+
+    /// SheetParser#row_number: digits without a leading zero, at most seven, 1 to MAX_ROWS, after the last row.
+    fn row_number_of(&self, start: usize, end: usize) -> Result<u32> {
+        let value = &self.lexer.buf[start..end];
+        let canonical = (1..=7).contains(&value.len())
+            && value[0] != b'0'
+            && value.iter().all(u8::is_ascii_digit);
+        if !canonical {
+            return Err(Unsupported);
+        }
+        let number = value
+            .iter()
+            .fold(0u32, |number, &digit| number * 10 + u32::from(digit - b'0'));
+        if number > MAX_ROWS || number <= self.last_number {
+            return Err(Unsupported);
+        }
+        Ok(number)
+    }
+}
+
+fn element(name: &[u8]) -> Element {
+    match name {
+        b"c" => Element::C,
+        b"v" => Element::V,
+        b"t" => Element::T,
+        b"row" => Element::Row,
+        b"is" => Element::Is,
+        b"rPh" => Element::RPh,
+        b"sheetData" => Element::SheetData,
+        _ => Element::Other,
+    }
+}
+
+/// SheetParser#column_of: the 0-based column of the letters of a reference, `None` without letters.
+fn column_of(value: &[u8]) -> Result<Option<u32>> {
+    utf8(value)?;
+    // The common case: one capital letter and the row digits.
+    if let (Some(&first), Some(&second)) = (value.first(), value.get(1))
+        && second < b'A'
+        && first.is_ascii_uppercase()
+    {
+        return Ok(Some(u32::from(first - b'A')));
+    }
+    let mut index: u32 = 0;
+    let mut position = 0;
+    for &byte in value {
+        let byte = if byte > b'Z' { byte - 32 } else { byte };
+        if !byte.is_ascii_uppercase() {
+            break;
+        }
+        if position == 3 {
+            return Err(Unsupported);
+        }
+        index = index * 26 + u32::from(byte - 64);
+        position += 1;
+    }
+    if position == 0 {
+        return Ok(None);
+    }
+    if index > MAX_COLUMNS {
+        return Err(Unsupported);
+    }
+    Ok(Some(index - 1))
+}
+
+/// A style index of one to ten digits; -1 for any other text, which calcsheet refuses where the style is read.
+fn style_index(value: &[u8]) -> i64 {
+    if (1..=10).contains(&value.len()) && value.iter().all(u8::is_ascii_digit) {
+        value
+            .iter()
+            .fold(0i64, |index, &digit| index * 10 + i64::from(digit - b'0'))
+    } else {
+        -1
+    }
+}
+
+/// The shared strings of an XLSX file (`xl/sharedStrings.xml`), read as calcsheet's `SharedStrings::Builder` reads them
+/// through Ox: rich text runs joined, phonetic runs skipped, line ends and references read per text, the `_xHHHH_`
+/// escapes decoded. Anything else is `Unsupported`, and calcsheet loads the table itself.
+pub struct StringsReader {
+    lexer: Lexer,
+    max_cell_bytes: Option<usize>,
+    item: Option<Vec<u8>>,
+    in_text: bool,
+    phonetic: u32,
+    any_element: bool,
+    pub buffer: Vec<u8>,
+    pub offsets: Vec<u64>,
+}
+
+pub enum StringsStop {
+    NeedMore,
+    Done,
+    Unsupported,
+}
+
+impl StringsReader {
+    pub fn new(max_cell_bytes: Option<usize>) -> Self {
+        StringsReader {
+            lexer: Lexer::new(),
+            max_cell_bytes,
+            item: None,
+            in_text: false,
+            phonetic: 0,
+            any_element: false,
+            buffer: Vec::new(),
+            offsets: vec![0],
+        }
+    }
+
+    pub fn feed(&mut self, chunk: &[u8]) {
+        self.lexer.feed(chunk);
+    }
+
+    pub fn finish(&mut self) {
+        self.lexer.eof = true;
+    }
+
+    pub fn read(&mut self) -> StringsStop {
+        match self.items() {
+            Ok(stop) => stop,
+            Err(Unsupported) => StringsStop::Unsupported,
+        }
+    }
+
+    fn items(&mut self) -> Result<StringsStop> {
+        loop {
+            let Some(item) = self.lexer.item()? else {
+                if !self.lexer.eof {
+                    return Ok(StringsStop::NeedMore);
+                }
+                return if self.lexer.complete() && self.any_element {
+                    Ok(StringsStop::Done)
+                } else {
+                    Err(Unsupported)
+                };
+            };
+            match item {
+                Item::Start(name_start, name_end, empty) => {
+                    // An element inside a text would let Ox drop white space that is kept here.
+                    if self.in_text {
+                        return Err(Unsupported);
+                    }
+                    self.any_element = true;
+                    let name = local(&self.lexer.buf[name_start..name_end]).to_vec();
+                    self.start(&name);
+                    if empty {
+                        self.end(&name)?;
+                    }
+                }
+                Item::End(name_start, name_end) => {
+                    let name = local(&self.lexer.buf[name_start..name_end]).to_vec();
+                    self.end(&name)?;
+                }
+                Item::Text(start, end) => {
+                    if self.in_text {
+                        self.text(start, end)?;
+                    }
+                }
+            }
+        }
+    }
+
+    fn start(&mut self, name: &[u8]) {
+        match name {
+            b"si" => self.item = Some(Vec::new()),
+            b"rPh" if self.item.is_some() => self.phonetic += 1,
+            b"t" if self.item.is_some() && self.phonetic == 0 => self.in_text = true,
+            _ => {}
+        }
+    }
+
+    fn end(&mut self, name: &[u8]) -> Result<()> {
+        match name {
+            b"si" => {
+                if let Some(item) = self.item.take() {
+                    self.finish_item(item)?;
+                }
+            }
+            b"rPh" if self.item.is_some() && self.phonetic > 0 => self.phonetic -= 1,
+            b"t" => self.in_text = false,
+            _ => {}
+        }
+        Ok(())
+    }
+
+    fn text(&mut self, start: usize, end: usize) -> Result<()> {
+        let mut piece = self.lexer.buf[start..end].to_vec();
+        if memchr(b'\r', &piece).is_some() {
+            piece = line_ends(&piece);
+        }
+        if memchr(b'&', &piece).is_some() {
+            piece = decode(&piece)?;
+        }
+        let item = self.item.as_mut().ok_or(Unsupported)?;
+        if self
+            .max_cell_bytes
+            .is_some_and(|max| item.len() + piece.len() > max)
+        {
+            return Err(Unsupported);
+        }
+        item.extend_from_slice(&piece);
+        Ok(())
+    }
+
+    fn finish_item(&mut self, item: Vec<u8>) -> Result<()> {
+        let item = if memchr::memmem::find(&item, b"_x").is_some() {
+            escapes(utf8(&item)?)
+        } else {
+            item
+        };
+        self.buffer.extend_from_slice(&item);
+        self.offsets.push(self.buffer.len() as u64);
+        self.in_text = false;
+        self.phonetic = 0;
+        Ok(())
+    }
+}
+
+/// Escapes.decode!: the `_xHHHH_` escapes of OOXML, a surrogate pair of them as one character; a lone surrogate stays
+/// as text.
+fn escapes(text: &[u8]) -> Vec<u8> {
+    let hex = |bytes: &[u8]| -> Option<u32> {
+        if bytes.iter().all(u8::is_ascii_hexdigit) {
+            u32::from_str_radix(std::str::from_utf8(bytes).ok()?, 16).ok()
+        } else {
+            None
+        }
+    };
+    let mut decoded = Vec::with_capacity(text.len());
+    let mut at = 0;
+    while at < text.len() {
+        let rest = &text[at..];
+        // _xD[89AB]hh__xD[C-F]hh_
+        if rest.len() >= 14
+            && rest.starts_with(b"_xD")
+            && matches!(rest[3], b'8' | b'9' | b'A' | b'B')
+            && rest[6..9] == *b"__x"
+            && rest[9] == b'D'
+            && matches!(rest[10], b'C'..=b'F')
+            && rest[13] == b'_'
+            && let (Some(high), Some(low)) = (hex(&rest[2..6]), hex(&rest[9..13]))
+        {
+            let code = (high - 0xD800) * 0x400 + (low - 0xDC00) + 0x10000;
+            let mut bytes = [0; 4];
+            decoded.extend_from_slice(
+                char::from_u32(code)
+                    .expect("a pair")
+                    .encode_utf8(&mut bytes)
+                    .as_bytes(),
+            );
+            at += 14;
+            continue;
+        }
+        if rest.len() >= 7
+            && rest.starts_with(b"_x")
+            && rest[6] == b'_'
+            && let Some(code) = hex(&rest[2..6])
+        {
+            match char::from_u32(code) {
+                Some(character) => {
+                    let mut bytes = [0; 4];
+                    decoded.extend_from_slice(character.encode_utf8(&mut bytes).as_bytes());
+                }
+                None => decoded.extend_from_slice(&rest[..7]),
+            }
+            at += 7;
+            continue;
+        }
+        decoded.push(text[at]);
+        at += 1;
+    }
+    decoded
+}