yerba 0.8.1-aarch64-linux-gnu → 0.9.1-aarch64-linux-gnu

data/rust/src/document/style.rs

@@ -7,6 +7,13 @@ pub struct StyleEnforcement {
   pub value_style: Option<QuoteStyle>,
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct BlankLineOptions {
+  pub before: Vec<String>,
+  pub after: Vec<String>,
+  pub skip_empty: bool,
+}
+
 impl Document {
   pub fn enforce_styles(&mut self, enforcement: &StyleEnforcement) -> Result<(), YerbaError> {
     let source = self.source_text();
@@ -588,6 +595,10 @@ impl Document {
   }
 
   pub fn enforce_blank_lines(&mut self, dot_path: &str, blank_lines: usize) -> Result<(), YerbaError> {
+    self.enforce_blank_lines_with(dot_path, blank_lines, &BlankLineOptions::default())
+  }
+
+  pub fn enforce_blank_lines_with(&mut self, dot_path: &str, blank_lines: usize, options: &BlankLineOptions) -> Result<(), YerbaError> {
     let nodes = if dot_path.contains('[') {
       self.navigate_all_compact(dot_path)
     } else {
@@ -597,14 +608,44 @@ impl Document {
     let mut edits: Vec<(TextRange, String)> = Vec::new();
 
     for current_node in &nodes {
-      let entry_nodes: Vec<SyntaxNode> = match first_collection(current_node) {
-        Some(FirstCollection::Sequence(sequence)) => sequence.entries().map(|entry| entry.syntax().clone()).collect(),
-        Some(FirstCollection::Map(map)) => map.entries().map(|entry| entry.syntax().clone()).collect(),
+      let entries: Vec<BlankLineEntry> = match first_collection(current_node) {
+        Some(FirstCollection::Sequence(sequence)) => sequence
+          .entries()
+          .map(|entry| BlankLineEntry {
+            is_empty: is_empty_value(&node_to_yaml_value(entry.syntax())),
+            node: entry.syntax().clone(),
+            key: None,
+          })
+          .collect(),
+
+        Some(FirstCollection::Map(map)) => map
+          .entries()
+          .map(|entry| {
+            let value = entry
+              .value()
+              .map(|value_node| node_to_yaml_value(value_node.syntax()))
+              .unwrap_or(yaml_serde::Value::Null);
+
+            BlankLineEntry {
+              key: entry.key().and_then(|key_node| extract_scalar_text(key_node.syntax())),
+              is_empty: is_empty_value(&value),
+              node: entry.syntax().clone(),
+            }
+          })
+          .collect(),
+
         None => continue,
       };
 
-      for entry_node in entry_nodes.iter().skip(1) {
-        collect_blank_line_edits(entry_node, blank_lines, &mut edits);
+      for index in 1..entries.len() {
+        let entry = &entries[index];
+        let previous = &entries[index - 1];
+
+        if !should_adjust_gap(entry, previous, options) {
+          continue;
+        }
+
+        collect_blank_line_edits(&entry.node, blank_lines, &mut edits);
       }
     }
 
@@ -738,6 +779,41 @@ impl Document {
     self.enforce_quotes_at(style, None)
   }
 
+  pub fn decode_escapes(&mut self, dot_path: Option<&str>) -> Result<usize, YerbaError> {
+    let scope_ranges: Vec<TextRange> = match dot_path {
+      Some(path) if !path.is_empty() => self.navigate_all_compact(path).iter().map(|node| node.text_range()).collect(),
+      _ => vec![self.root.text_range()],
+    };
+
+    let mut edits: Vec<(TextRange, String)> = Vec::new();
+
+    for element in self.root.descendants_with_tokens() {
+      let Some(token) = element.into_token() else { continue };
+
+      if token.kind() != SyntaxKind::DOUBLE_QUOTED_SCALAR {
+        continue;
+      }
+
+      if !scope_ranges.iter().any(|range| range.contains_range(token.text_range())) {
+        continue;
+      }
+
+      let Some(value) = raw_scalar_value(&token) else { continue };
+
+      let rewritten = format_scalar_value(&value, SyntaxKind::DOUBLE_QUOTED_SCALAR);
+
+      if rewritten != token.text() {
+        edits.push((token.text_range(), rewritten));
+      }
+    }
+
+    let count = edits.len();
+
+    self.apply_edits(edits)?;
+
+    Ok(count)
+  }
+
   pub fn enforce_quotes_at(&mut self, style: &QuoteStyle, dot_path: Option<&str>) -> Result<Vec<String>, YerbaError> {
     let source = self.source_text();
 
@@ -819,11 +895,7 @@ impl Document {
               let is_multiline = trimmed.contains('\n');
 
               let new_text = match style {
-                QuoteStyle::Double => {
-                  let escaped = trimmed.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n");
-
-                  format!("\"{}\"", escaped)
-                }
+                QuoteStyle::Double => format_scalar_value(trimmed, SyntaxKind::DOUBLE_QUOTED_SCALAR),
 
                 QuoteStyle::Single => {
                   if is_multiline {
@@ -879,7 +951,7 @@ impl Document {
 
           let offset: usize = token.text_range().start().into();
           let line_prefix = &source[line_start_at(&source, offset)..offset];
-          let indent = line_prefix.len() - line_prefix.trim_start().len() + 2;
+          let indent = block_scalar_indent(line_prefix);
           let indent_str = " ".repeat(indent);
           let header = style.block_header();
 
@@ -932,6 +1004,54 @@ impl Document {
   }
 }
 
+struct BlankLineEntry {
+  node: SyntaxNode,
+  key: Option<String>,
+  is_empty: bool,
+}
+
+fn should_adjust_gap(entry: &BlankLineEntry, previous: &BlankLineEntry, options: &BlankLineOptions) -> bool {
+  let usable = |candidate: &BlankLineEntry| !options.skip_empty || !candidate.is_empty;
+
+  if options.before.is_empty() && options.after.is_empty() {
+    return usable(entry);
+  }
+
+  (matches_key(entry.key.as_deref(), &options.before) && usable(entry)) || (matches_key(previous.key.as_deref(), &options.after) && usable(previous))
+}
+
+fn matches_key(key: Option<&str>, keys: &[String]) -> bool {
+  match key {
+    Some(key) => keys.iter().any(|wanted| wanted == key),
+    None => false,
+  }
+}
+
+fn is_empty_value(value: &yaml_serde::Value) -> bool {
+  match value {
+    yaml_serde::Value::Null => true,
+    yaml_serde::Value::String(text) => text.is_empty(),
+    yaml_serde::Value::Sequence(entries) => entries.is_empty(),
+    yaml_serde::Value::Mapping(entries) => entries.is_empty(),
+    _ => false,
+  }
+}
+
+fn block_scalar_indent(line_prefix: &str) -> usize {
+  let mut column = 0;
+  let bytes = line_prefix.as_bytes();
+
+  while column < bytes.len() {
+    match bytes[column] {
+      b' ' => column += 1,
+      b'-' if bytes.get(column + 1) == Some(&b' ') => column += 2,
+      _ => break,
+    }
+  }
+
+  column + 2
+}
+
 fn inline_quote_replacement(raw_value: &str, current_kind: SyntaxKind, style: &QuoteStyle) -> Option<String> {
   match style {
     QuoteStyle::Double => {

data/rust/src/error.rs

@@ -10,6 +10,7 @@ pub enum YerbaError {
   IndexOutOfBounds(usize, usize),
   UnknownKeys(Vec<String>),
   DuplicateValues(Vec<crate::DuplicateInfo>),
+  ControlCharacters(Vec<crate::ControlCharacter>),
   #[cfg(feature = "schema")]
   SchemaValidation(Vec<crate::schema::ValidationError>),
   DuplicateKey {
@@ -75,6 +76,19 @@ impl std::fmt::Display for YerbaError {
         write!(f, "found {} {}: {}", duplicates.len(), noun, details.join(", "))
       }
 
+      YerbaError::ControlCharacters(found) => {
+        let noun = if found.len() == 1 { "character" } else { "characters" };
+        let details: Vec<String> = found.iter().map(|control| format!("    {}", control)).collect();
+
+        write!(
+          f,
+          "found {} control {} that YAML parsers reject:\n{}\n\n  Control characters must be removed, or escaped as \\xNN inside a double-quoted value.\n",
+          found.len(),
+          noun,
+          details.join("\n")
+        )
+      }
+
       YerbaError::IndexOutOfBounds(index, length) => {
         write!(f, "index {} out of bounds (length {})", index, length)
       }
@@ -150,6 +164,16 @@ impl GitHubAnnotations for YerbaError {
         })
         .collect(),
 
+      YerbaError::ControlCharacters(found) => found
+        .iter()
+        .map(|control| GitHubAnnotation {
+          level: "error ",
+          file: file.to_string(),
+          line: Some(control.line),
+          message: format!("control character U+{:04X} at column {}", control.character as u32, control.column),
+        })
+        .collect(),
+
       YerbaError::DuplicateKey { key, duplicate_line, .. } => vec![GitHubAnnotation {
         level: "error ",
         file: file.to_string(),

data/rust/src/ffi.rs

@@ -217,6 +217,11 @@ const EMPTY_LOCATION: YerbaLocation = YerbaLocation {
   end_column: 0,
 };
 
+#[no_mangle]
+pub unsafe extern "C" fn yerba_document_generation(document: *const Document) -> u64 {
+  (*document).generation()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn yerba_document_free(document: *mut Document) {
   if !document.is_null() {
@@ -485,25 +490,34 @@ pub unsafe extern "C" fn yerba_document_find(document: *const Document, path: *c
   CString::new(json).unwrap_or_default().into_raw()
 }
 
+unsafe fn borrow_value<'a>(value: *const c_char, length: usize) -> &'a str {
+  if value.is_null() {
+    return "";
+  }
+
+  std::str::from_utf8(std::slice::from_raw_parts(value as *const u8, length)).unwrap_or("")
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn yerba_document_set(
   document: *mut Document,
   path: *const c_char,
   value: *const c_char,
+  value_length: usize,
   value_type: YerbaValueType,
+  plain: bool,
   all: bool,
 ) -> YerbaResult {
   let document = &mut *document;
   let selector_string = CStr::from_ptr(path).to_str().unwrap_or("");
-  let value_string = CStr::from_ptr(value).to_str().unwrap_or("");
-
-  let result = if all {
-    document.set_all(selector_string, value_string)
-  } else {
-    match value_type {
-      YerbaValueType::String => document.set(selector_string, value_string),
-      _ => document.set_plain(selector_string, value_string),
-    }
+  let value_string = borrow_value(value, value_length);
+  let verbatim = plain || value_type != YerbaValueType::String;
+
+  let result = match (all, verbatim) {
+    (true, false) => document.set_all(selector_string, value_string),
+    (true, true) => document.set_all_plain(selector_string, value_string),
+    (false, false) => document.set(selector_string, value_string),
+    (false, true) => document.set_plain(selector_string, value_string),
   };
 
   match result {
@@ -517,17 +531,25 @@ pub unsafe extern "C" fn yerba_document_insert(
   document: *mut Document,
   path: *const c_char,
   value: *const c_char,
+  value_length: usize,
   value_type: YerbaValueType,
+  plain: bool,
+  style: *const c_char,
   before: *const c_char,
   after: *const c_char,
   at: i64,
 ) -> YerbaResult {
   let document = &mut *document;
   let selector_string = CStr::from_ptr(path).to_str().unwrap_or("");
-  let raw_value = CStr::from_ptr(value).to_str().unwrap_or("");
+  let raw_value = borrow_value(value, value_length);
 
   let value_string = match value_type {
-    YerbaValueType::String => crate::syntax::quote_if_needed(raw_value),
+    YerbaValueType::String if !plain => {
+      let style = if style.is_null() { None } else { CStr::from_ptr(style).to_str().ok() };
+
+      crate::syntax::quote_scalar_styled(raw_value, style)
+    }
+
     _ => raw_value.to_string(),
   };
 
@@ -543,7 +565,13 @@ pub unsafe extern "C" fn yerba_document_insert(
     InsertPosition::Last
   };
 
-  match document.insert_into(selector_string, &value_string, position) {
+  let result = if plain {
+    document.insert_fragment_into(selector_string, &value_string, position)
+  } else {
+    document.insert_into(selector_string, &value_string, position)
+  };
+
+  match result {
     Ok(()) => YerbaResult::ok(),
     Err(e) => YerbaResult::err(&e.to_string()),
   }
@@ -843,6 +871,21 @@ pub unsafe extern "C" fn yerba_document_to_string(document: *const Document) ->
   CString::new(content).unwrap_or_default().into_raw()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn yerba_quote_scalar(value: *const c_char, length: usize, style: *const c_char) -> *mut c_char {
+  let bytes = std::slice::from_raw_parts(value as *const u8, length);
+
+  let Ok(value) = std::str::from_utf8(bytes) else {
+    return std::ptr::null_mut();
+  };
+
+  let style = if style.is_null() { None } else { CStr::from_ptr(style).to_str().ok() };
+
+  CString::new(crate::syntax::quote_scalar_styled(value, style))
+    .map(|string| string.into_raw())
+    .unwrap_or(std::ptr::null_mut())
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn yerba_string_free(s: *mut c_char) {
   if !s.is_null() {

data/rust/src/lib.rs

@@ -8,11 +8,12 @@ mod quote_style;
 pub mod schema;
 pub mod selector;
 mod syntax;
+mod validation;
 mod yaml_writer;
 #[cfg(feature = "cli")]
 pub mod yerbafile;
 
-pub use document::style::StyleEnforcement;
+pub use document::style::{BlankLineOptions, StyleEnforcement};
 pub use document::{
   collect_selectors, validate_condition, validate_item_condition, Document, DuplicateInfo, InsertPosition, LocatedNode, Location, NodeInfo, NodeType, SortField,
 };
@@ -20,9 +21,10 @@ pub use error::YerbaError;
 pub use quote_style::{KeyStyle, QuoteStyle};
 pub use selector::Selector;
 pub use syntax::{
-  detect_yaml_type, is_flow_collection, is_inline_scalar_safe, is_plain_safe, is_plain_safe_in_flow, is_quoted_scalar, is_raw_yaml_text, is_valid_inline_value,
-  needs_quoting, needs_quoting_in_flow, quote_if_needed, ScalarValue, YerbaValueType,
+  detect_yaml_type, escape_double_quoted, is_control_character, is_flow_collection, is_inline_scalar_safe, is_plain_safe, is_plain_safe_in_flow,
+  is_quoted_scalar, is_single_quotable, needs_quoting, needs_quoting_in_flow, quote_scalar, quote_scalar_styled, ScalarValue, YerbaValueType,
 };
+pub use validation::{find_control_characters, ControlCharacter};
 pub use yaml_writer::json_to_yaml_text;
 #[cfg(feature = "cli")]
 pub use yerbafile::Yerbafile;

data/rust/src/syntax.rs

@@ -33,7 +33,7 @@ pub fn detect_yaml_type(scalar: &ScalarValue) -> YerbaValueType {
 
 pub fn raw_scalar_value(token: &SyntaxToken) -> Option<String> {
   match token.kind() {
-    SyntaxKind::PLAIN_SCALAR => Some(token.text().to_string()),
+    SyntaxKind::PLAIN_SCALAR => Some(fold_flow_scalar(token.text())),
 
     SyntaxKind::DOUBLE_QUOTED_SCALAR => {
       let text = token.text();
@@ -65,8 +65,16 @@ pub fn extract_scalar(node: &SyntaxNode) -> Option<ScalarValue> {
     .filter_map(|element| element.into_token())
     .find(|token| token.kind() == SyntaxKind::BLOCK_SCALAR_TEXT)?;
 
+  let dedented = dedent_block_scalar(block_token.text());
+
+  let folded = node
+    .descendants()
+    .find(|descendant| descendant.kind() == SyntaxKind::BLOCK_SCALAR)
+    .map(|scalar| scalar.text().to_string().trim_start().starts_with('>'))
+    .unwrap_or(false);
+
   Some(ScalarValue {
-    text: dedent_block_scalar(block_token.text()),
+    text: if folded { fold_flow_scalar(&dedented) } else { dedented },
     kind: SyntaxKind::BLOCK_SCALAR_TEXT,
     file_path: None,
     selector: None,
@@ -168,10 +176,7 @@ pub fn find_scalar_token(node: &SyntaxNode) -> Option<SyntaxToken> {
 
 pub fn format_scalar_value(value: &str, kind: SyntaxKind) -> String {
   match kind {
-    SyntaxKind::DOUBLE_QUOTED_SCALAR => {
-      let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
-      format!("\"{}\"", escaped)
-    }
+    SyntaxKind::DOUBLE_QUOTED_SCALAR => format!("\"{}\"", escape_double_quoted(value)),
 
     SyntaxKind::SINGLE_QUOTED_SCALAR => {
       let escaped = value.replace('\'', "''");
@@ -182,6 +187,32 @@ pub fn format_scalar_value(value: &str, kind: SyntaxKind) -> String {
   }
 }
 
+pub fn is_control_character(character: char) -> bool {
+  matches!(character as u32, 0x00..=0x08 | 0x0b | 0x0c | 0x0e..=0x1f | 0x7f | 0x80..=0x84 | 0x86..=0x9f)
+}
+
+pub fn is_single_quotable(value: &str) -> bool {
+  !value.contains('\n') && !value.contains('\r') && !value.chars().any(is_control_character)
+}
+
+pub fn escape_double_quoted(value: &str) -> String {
+  let mut result = String::with_capacity(value.len());
+
+  for character in value.chars() {
+    match character {
+      '\\' => result.push_str("\\\\"),
+      '"' => result.push_str("\\\""),
+      '\n' => result.push_str("\\n"),
+      '\r' => result.push_str("\\r"),
+      '\t' => result.push_str("\\t"),
+      _ if is_control_character(character) => result.push_str(&format!("\\x{:02x}", character as u32)),
+      _ => result.push(character),
+    }
+  }
+
+  result
+}
+
 const LEADING_INDICATORS: [char; 16] = ['#', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`', ',', '[', ']', '{', '}'];
 const FLOW_INDICATORS: [char; 5] = [',', '[', ']', '{', '}'];
 
@@ -194,6 +225,10 @@ pub fn is_plain_safe(value: &str) -> bool {
     return false;
   }
 
+  if value.chars().any(is_control_character) {
+    return false;
+  }
+
   if value.contains(": ") || value.ends_with(':') {
     return false;
   }
@@ -277,18 +312,6 @@ pub fn is_flow_collection(value: &str) -> bool {
   (value.starts_with('[') && value.ends_with(']')) || (value.starts_with('{') && value.ends_with('}'))
 }
 
-pub fn is_raw_yaml_text(value: &str) -> bool {
-  value.contains('\n') || value.starts_with("- ") || is_quoted_scalar(value) || is_flow_collection(value)
-}
-
-pub fn is_valid_inline_value(value: &str) -> bool {
-  if value.is_empty() {
-    return true;
-  }
-
-  is_raw_yaml_text(value) || is_plain_safe(value)
-}
-
 pub fn is_inline_scalar_safe(value: &str) -> bool {
   if value.is_empty() {
     return true;
@@ -309,11 +332,7 @@ pub fn needs_quoting_in_flow(value: &str) -> bool {
   is_yaml_non_string(value) || !is_plain_safe_in_flow(value)
 }
 
-pub fn quote_if_needed(value: &str) -> String {
-  if is_raw_yaml_text(value) {
-    return value.to_string();
-  }
-
+pub fn quote_scalar(value: &str) -> String {
   if needs_quoting(value) {
     format_scalar_value(value, SyntaxKind::DOUBLE_QUOTED_SCALAR)
   } else {
@@ -321,6 +340,15 @@ pub fn quote_if_needed(value: &str) -> String {
   }
 }
 
+pub fn quote_scalar_styled(value: &str, style: Option<&str>) -> String {
+  match style {
+    Some("double") => format_scalar_value(value, SyntaxKind::DOUBLE_QUOTED_SCALAR),
+    Some("single") if is_single_quotable(value) => format_scalar_value(value, SyntaxKind::SINGLE_QUOTED_SCALAR),
+
+    _ => quote_scalar(value),
+  }
+}
+
 pub fn extract_scalar_text(node: &SyntaxNode) -> Option<String> {
   extract_scalar(node).map(|scalar| scalar.text)
 }
@@ -343,11 +371,114 @@ pub fn dedent_block_scalar(text: &str) -> String {
   dedented.trim().to_string()
 }
 
+fn push_hex_escape(result: &mut String, characters: &mut Scan<'_>, digits: usize) {
+  let escape: String = characters.clone().take(digits).collect();
+
+  let decoded = (escape.len() == digits && escape.chars().all(|character| character.is_ascii_hexdigit()))
+    .then(|| u32::from_str_radix(&escape, 16).ok().and_then(char::from_u32))
+    .flatten();
+
+  match decoded {
+    Some(character) => {
+      for _ in 0..digits {
+        characters.next();
+      }
+
+      result.push(character);
+    }
+
+    None => {
+      result.push('\\');
+      result.push(match digits {
+        2 => 'x',
+        4 => 'u',
+        _ => 'U',
+      });
+    }
+  }
+}
+
+type Scan<'a> = std::iter::Peekable<std::str::Chars<'a>>;
+
+fn skip_spaces(characters: &mut Scan<'_>) {
+  while matches!(characters.peek(), Some(' ' | '\t')) {
+    characters.next();
+  }
+}
+
+fn fold_break(result: &mut String, characters: &mut Scan<'_>) {
+  while result.ends_with(' ') || result.ends_with('\t') {
+    result.pop();
+  }
+
+  let mut breaks = 1;
+
+  loop {
+    skip_spaces(characters);
+
+    match characters.peek() {
+      Some('\n') => {
+        characters.next();
+        breaks += 1;
+      }
+
+      Some('\r') => {
+        characters.next();
+        characters.next_if_eq(&'\n');
+        breaks += 1;
+      }
+
+      _ => break,
+    }
+  }
+
+  if breaks == 1 {
+    result.push(' ');
+  } else {
+    result.extend(std::iter::repeat_n('\n', breaks - 1));
+  }
+}
+
+pub fn fold_flow_scalar(text: &str) -> String {
+  if !text.contains('\n') && !text.contains('\r') {
+    return text.to_string();
+  }
+
+  let mut result = String::with_capacity(text.len());
+  let mut characters = text.chars().peekable();
+
+  while let Some(character) = characters.next() {
+    match character {
+      '\n' => fold_break(&mut result, &mut characters),
+
+      '\r' => {
+        characters.next_if_eq(&'\n');
+        fold_break(&mut result, &mut characters);
+      }
+
+      _ => result.push(character),
+    }
+  }
+
+  result
+}
+
 pub fn unescape_double_quoted(text: &str) -> String {
   let mut result = String::with_capacity(text.len());
-  let mut chars = text.chars();
+  let mut chars = text.chars().peekable();
 
   while let Some(character) = chars.next() {
+    if character == '\n' {
+      fold_break(&mut result, &mut chars);
+      continue;
+    }
+
+    if character == '\r' {
+      chars.next_if_eq(&'\n');
+      fold_break(&mut result, &mut chars);
+      continue;
+    }
+
     if character == '\\' {
       match chars.next() {
         Some('n') => result.push('\n'),
@@ -363,7 +494,19 @@ pub fn unescape_double_quoted(text: &str) -> String {
         Some('v') => result.push('\u{0b}'),
         Some(' ') => result.push(' '),
         Some('_') => result.push('\u{a0}'),
-        Some('\n') => {} // line continuation: skip newline and leading whitespace
+        Some('N') => result.push('\u{85}'),
+        Some('L') => result.push('\u{2028}'),
+        Some('P') => result.push('\u{2029}'),
+        Some('x') => push_hex_escape(&mut result, &mut chars, 2),
+        Some('u') => push_hex_escape(&mut result, &mut chars, 4),
+        Some('U') => push_hex_escape(&mut result, &mut chars, 8),
+        Some('\n') => skip_spaces(&mut chars),
+
+        Some('\r') => {
+          chars.next_if_eq(&'\n');
+          skip_spaces(&mut chars);
+        }
+
         Some(other) => {
           result.push('\\');
           result.push(other);
@@ -379,7 +522,7 @@ pub fn unescape_double_quoted(text: &str) -> String {
 }
 
 pub fn unescape_single_quoted(text: &str) -> String {
-  text.replace("''", "'")
+  fold_flow_scalar(&text.replace("''", "'"))
 }
 
 pub fn line_at(source: &str, offset: usize) -> usize {

data/rust/src/validation.rs

@@ -0,0 +1,30 @@
+use crate::syntax::is_control_character;
+
+#[derive(Debug, Clone)]
+pub struct ControlCharacter {
+  pub character: char,
+  pub line: usize,
+  pub column: usize,
+}
+
+impl std::fmt::Display for ControlCharacter {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "U+{:04X} at line {}, column {}", self.character as u32, self.line, self.column)
+  }
+}
+
+pub fn find_control_characters(source: &str) -> Vec<ControlCharacter> {
+  source
+    .lines()
+    .enumerate()
+    .flat_map(|(index, line)| {
+      line.chars().enumerate().filter_map(move |(column, character)| {
+        is_control_character(character).then_some(ControlCharacter {
+          character,
+          line: index + 1,
+          column: column + 1,
+        })
+      })
+    })
+    .collect()
+}