yerba 0.8.1 → 0.9.1

data/rust/src/error.rs

@@ -10,6 +10,7 @@ pub enum YerbaError {
   IndexOutOfBounds(usize, usize),
   UnknownKeys(Vec<String>),
   DuplicateValues(Vec<crate::DuplicateInfo>),
+  ControlCharacters(Vec<crate::ControlCharacter>),
   #[cfg(feature = "schema")]
   SchemaValidation(Vec<crate::schema::ValidationError>),
   DuplicateKey {
@@ -75,6 +76,19 @@ impl std::fmt::Display for YerbaError {
         write!(f, "found {} {}: {}", duplicates.len(), noun, details.join(", "))
       }
 
+      YerbaError::ControlCharacters(found) => {
+        let noun = if found.len() == 1 { "character" } else { "characters" };
+        let details: Vec<String> = found.iter().map(|control| format!("    {}", control)).collect();
+
+        write!(
+          f,
+          "found {} control {} that YAML parsers reject:\n{}\n\n  Control characters must be removed, or escaped as \\xNN inside a double-quoted value.\n",
+          found.len(),
+          noun,
+          details.join("\n")
+        )
+      }
+
       YerbaError::IndexOutOfBounds(index, length) => {
         write!(f, "index {} out of bounds (length {})", index, length)
       }
@@ -150,6 +164,16 @@ impl GitHubAnnotations for YerbaError {
         })
         .collect(),
 
+      YerbaError::ControlCharacters(found) => found
+        .iter()
+        .map(|control| GitHubAnnotation {
+          level: "error ",
+          file: file.to_string(),
+          line: Some(control.line),
+          message: format!("control character U+{:04X} at column {}", control.character as u32, control.column),
+        })
+        .collect(),
+
       YerbaError::DuplicateKey { key, duplicate_line, .. } => vec![GitHubAnnotation {
         level: "error ",
         file: file.to_string(),

data/rust/src/ffi.rs

@@ -217,6 +217,11 @@ const EMPTY_LOCATION: YerbaLocation = YerbaLocation {
   end_column: 0,
 };
 
+#[no_mangle]
+pub unsafe extern "C" fn yerba_document_generation(document: *const Document) -> u64 {
+  (*document).generation()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn yerba_document_free(document: *mut Document) {
   if !document.is_null() {
@@ -485,25 +490,34 @@ pub unsafe extern "C" fn yerba_document_find(document: *const Document, path: *c
   CString::new(json).unwrap_or_default().into_raw()
 }
 
+unsafe fn borrow_value<'a>(value: *const c_char, length: usize) -> &'a str {
+  if value.is_null() {
+    return "";
+  }
+
+  std::str::from_utf8(std::slice::from_raw_parts(value as *const u8, length)).unwrap_or("")
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn yerba_document_set(
   document: *mut Document,
   path: *const c_char,
   value: *const c_char,
+  value_length: usize,
   value_type: YerbaValueType,
+  plain: bool,
   all: bool,
 ) -> YerbaResult {
   let document = &mut *document;
   let selector_string = CStr::from_ptr(path).to_str().unwrap_or("");
-  let value_string = CStr::from_ptr(value).to_str().unwrap_or("");
-
-  let result = if all {
-    document.set_all(selector_string, value_string)
-  } else {
-    match value_type {
-      YerbaValueType::String => document.set(selector_string, value_string),
-      _ => document.set_plain(selector_string, value_string),
-    }
+  let value_string = borrow_value(value, value_length);
+  let verbatim = plain || value_type != YerbaValueType::String;
+
+  let result = match (all, verbatim) {
+    (true, false) => document.set_all(selector_string, value_string),
+    (true, true) => document.set_all_plain(selector_string, value_string),
+    (false, false) => document.set(selector_string, value_string),
+    (false, true) => document.set_plain(selector_string, value_string),
   };
 
   match result {
@@ -517,17 +531,25 @@ pub unsafe extern "C" fn yerba_document_insert(
   document: *mut Document,
   path: *const c_char,
   value: *const c_char,
+  value_length: usize,
   value_type: YerbaValueType,
+  plain: bool,
+  style: *const c_char,
   before: *const c_char,
   after: *const c_char,
   at: i64,
 ) -> YerbaResult {
   let document = &mut *document;
   let selector_string = CStr::from_ptr(path).to_str().unwrap_or("");
-  let raw_value = CStr::from_ptr(value).to_str().unwrap_or("");
+  let raw_value = borrow_value(value, value_length);
 
   let value_string = match value_type {
-    YerbaValueType::String => crate::syntax::quote_if_needed(raw_value),
+    YerbaValueType::String if !plain => {
+      let style = if style.is_null() { None } else { CStr::from_ptr(style).to_str().ok() };
+
+      crate::syntax::quote_scalar_styled(raw_value, style)
+    }
+
     _ => raw_value.to_string(),
   };
 
@@ -543,7 +565,13 @@ pub unsafe extern "C" fn yerba_document_insert(
     InsertPosition::Last
   };
 
-  match document.insert_into(selector_string, &value_string, position) {
+  let result = if plain {
+    document.insert_fragment_into(selector_string, &value_string, position)
+  } else {
+    document.insert_into(selector_string, &value_string, position)
+  };
+
+  match result {
     Ok(()) => YerbaResult::ok(),
     Err(e) => YerbaResult::err(&e.to_string()),
   }
@@ -843,6 +871,21 @@ pub unsafe extern "C" fn yerba_document_to_string(document: *const Document) ->
   CString::new(content).unwrap_or_default().into_raw()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn yerba_quote_scalar(value: *const c_char, length: usize, style: *const c_char) -> *mut c_char {
+  let bytes = std::slice::from_raw_parts(value as *const u8, length);
+
+  let Ok(value) = std::str::from_utf8(bytes) else {
+    return std::ptr::null_mut();
+  };
+
+  let style = if style.is_null() { None } else { CStr::from_ptr(style).to_str().ok() };
+
+  CString::new(crate::syntax::quote_scalar_styled(value, style))
+    .map(|string| string.into_raw())
+    .unwrap_or(std::ptr::null_mut())
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn yerba_string_free(s: *mut c_char) {
   if !s.is_null() {

data/rust/src/lib.rs

@@ -8,11 +8,12 @@ mod quote_style;
 pub mod schema;
 pub mod selector;
 mod syntax;
+mod validation;
 mod yaml_writer;
 #[cfg(feature = "cli")]
 pub mod yerbafile;
 
-pub use document::style::StyleEnforcement;
+pub use document::style::{BlankLineOptions, StyleEnforcement};
 pub use document::{
   collect_selectors, validate_condition, validate_item_condition, Document, DuplicateInfo, InsertPosition, LocatedNode, Location, NodeInfo, NodeType, SortField,
 };
@@ -20,9 +21,10 @@ pub use error::YerbaError;
 pub use quote_style::{KeyStyle, QuoteStyle};
 pub use selector::Selector;
 pub use syntax::{
-  detect_yaml_type, is_flow_collection, is_inline_scalar_safe, is_plain_safe, is_plain_safe_in_flow, is_quoted_scalar, is_raw_yaml_text, is_valid_inline_value,
-  needs_quoting, needs_quoting_in_flow, quote_if_needed, ScalarValue, YerbaValueType,
+  detect_yaml_type, escape_double_quoted, is_control_character, is_flow_collection, is_inline_scalar_safe, is_plain_safe, is_plain_safe_in_flow,
+  is_quoted_scalar, is_single_quotable, needs_quoting, needs_quoting_in_flow, quote_scalar, quote_scalar_styled, ScalarValue, YerbaValueType,
 };
+pub use validation::{find_control_characters, ControlCharacter};
 pub use yaml_writer::json_to_yaml_text;
 #[cfg(feature = "cli")]
 pub use yerbafile::Yerbafile;

data/rust/src/syntax.rs

@@ -33,7 +33,7 @@ pub fn detect_yaml_type(scalar: &ScalarValue) -> YerbaValueType {
 
 pub fn raw_scalar_value(token: &SyntaxToken) -> Option<String> {
   match token.kind() {
-    SyntaxKind::PLAIN_SCALAR => Some(token.text().to_string()),
+    SyntaxKind::PLAIN_SCALAR => Some(fold_flow_scalar(token.text())),
 
     SyntaxKind::DOUBLE_QUOTED_SCALAR => {
       let text = token.text();
@@ -65,8 +65,16 @@ pub fn extract_scalar(node: &SyntaxNode) -> Option<ScalarValue> {
     .filter_map(|element| element.into_token())
     .find(|token| token.kind() == SyntaxKind::BLOCK_SCALAR_TEXT)?;
 
+  let dedented = dedent_block_scalar(block_token.text());
+
+  let folded = node
+    .descendants()
+    .find(|descendant| descendant.kind() == SyntaxKind::BLOCK_SCALAR)
+    .map(|scalar| scalar.text().to_string().trim_start().starts_with('>'))
+    .unwrap_or(false);
+
   Some(ScalarValue {
-    text: dedent_block_scalar(block_token.text()),
+    text: if folded { fold_flow_scalar(&dedented) } else { dedented },
     kind: SyntaxKind::BLOCK_SCALAR_TEXT,
     file_path: None,
     selector: None,
@@ -168,10 +176,7 @@ pub fn find_scalar_token(node: &SyntaxNode) -> Option<SyntaxToken> {
 
 pub fn format_scalar_value(value: &str, kind: SyntaxKind) -> String {
   match kind {
-    SyntaxKind::DOUBLE_QUOTED_SCALAR => {
-      let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
-      format!("\"{}\"", escaped)
-    }
+    SyntaxKind::DOUBLE_QUOTED_SCALAR => format!("\"{}\"", escape_double_quoted(value)),
 
     SyntaxKind::SINGLE_QUOTED_SCALAR => {
       let escaped = value.replace('\'', "''");
@@ -182,6 +187,32 @@ pub fn format_scalar_value(value: &str, kind: SyntaxKind) -> String {
   }
 }
 
+pub fn is_control_character(character: char) -> bool {
+  matches!(character as u32, 0x00..=0x08 | 0x0b | 0x0c | 0x0e..=0x1f | 0x7f | 0x80..=0x84 | 0x86..=0x9f)
+}
+
+pub fn is_single_quotable(value: &str) -> bool {
+  !value.contains('\n') && !value.contains('\r') && !value.chars().any(is_control_character)
+}
+
+pub fn escape_double_quoted(value: &str) -> String {
+  let mut result = String::with_capacity(value.len());
+
+  for character in value.chars() {
+    match character {
+      '\\' => result.push_str("\\\\"),
+      '"' => result.push_str("\\\""),
+      '\n' => result.push_str("\\n"),
+      '\r' => result.push_str("\\r"),
+      '\t' => result.push_str("\\t"),
+      _ if is_control_character(character) => result.push_str(&format!("\\x{:02x}", character as u32)),
+      _ => result.push(character),
+    }
+  }
+
+  result
+}
+
 const LEADING_INDICATORS: [char; 16] = ['#', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`', ',', '[', ']', '{', '}'];
 const FLOW_INDICATORS: [char; 5] = [',', '[', ']', '{', '}'];
 
@@ -194,6 +225,10 @@ pub fn is_plain_safe(value: &str) -> bool {
     return false;
   }
 
+  if value.chars().any(is_control_character) {
+    return false;
+  }
+
   if value.contains(": ") || value.ends_with(':') {
     return false;
   }
@@ -277,18 +312,6 @@ pub fn is_flow_collection(value: &str) -> bool {
   (value.starts_with('[') && value.ends_with(']')) || (value.starts_with('{') && value.ends_with('}'))
 }
 
-pub fn is_raw_yaml_text(value: &str) -> bool {
-  value.contains('\n') || value.starts_with("- ") || is_quoted_scalar(value) || is_flow_collection(value)
-}
-
-pub fn is_valid_inline_value(value: &str) -> bool {
-  if value.is_empty() {
-    return true;
-  }
-
-  is_raw_yaml_text(value) || is_plain_safe(value)
-}
-
 pub fn is_inline_scalar_safe(value: &str) -> bool {
   if value.is_empty() {
     return true;
@@ -309,11 +332,7 @@ pub fn needs_quoting_in_flow(value: &str) -> bool {
   is_yaml_non_string(value) || !is_plain_safe_in_flow(value)
 }
 
-pub fn quote_if_needed(value: &str) -> String {
-  if is_raw_yaml_text(value) {
-    return value.to_string();
-  }
-
+pub fn quote_scalar(value: &str) -> String {
   if needs_quoting(value) {
     format_scalar_value(value, SyntaxKind::DOUBLE_QUOTED_SCALAR)
   } else {
@@ -321,6 +340,15 @@ pub fn quote_if_needed(value: &str) -> String {
   }
 }
 
+pub fn quote_scalar_styled(value: &str, style: Option<&str>) -> String {
+  match style {
+    Some("double") => format_scalar_value(value, SyntaxKind::DOUBLE_QUOTED_SCALAR),
+    Some("single") if is_single_quotable(value) => format_scalar_value(value, SyntaxKind::SINGLE_QUOTED_SCALAR),
+
+    _ => quote_scalar(value),
+  }
+}
+
 pub fn extract_scalar_text(node: &SyntaxNode) -> Option<String> {
   extract_scalar(node).map(|scalar| scalar.text)
 }
@@ -343,11 +371,114 @@ pub fn dedent_block_scalar(text: &str) -> String {
   dedented.trim().to_string()
 }
 
+fn push_hex_escape(result: &mut String, characters: &mut Scan<'_>, digits: usize) {
+  let escape: String = characters.clone().take(digits).collect();
+
+  let decoded = (escape.len() == digits && escape.chars().all(|character| character.is_ascii_hexdigit()))
+    .then(|| u32::from_str_radix(&escape, 16).ok().and_then(char::from_u32))
+    .flatten();
+
+  match decoded {
+    Some(character) => {
+      for _ in 0..digits {
+        characters.next();
+      }
+
+      result.push(character);
+    }
+
+    None => {
+      result.push('\\');
+      result.push(match digits {
+        2 => 'x',
+        4 => 'u',
+        _ => 'U',
+      });
+    }
+  }
+}
+
+type Scan<'a> = std::iter::Peekable<std::str::Chars<'a>>;
+
+fn skip_spaces(characters: &mut Scan<'_>) {
+  while matches!(characters.peek(), Some(' ' | '\t')) {
+    characters.next();
+  }
+}
+
+fn fold_break(result: &mut String, characters: &mut Scan<'_>) {
+  while result.ends_with(' ') || result.ends_with('\t') {
+    result.pop();
+  }
+
+  let mut breaks = 1;
+
+  loop {
+    skip_spaces(characters);
+
+    match characters.peek() {
+      Some('\n') => {
+        characters.next();
+        breaks += 1;
+      }
+
+      Some('\r') => {
+        characters.next();
+        characters.next_if_eq(&'\n');
+        breaks += 1;
+      }
+
+      _ => break,
+    }
+  }
+
+  if breaks == 1 {
+    result.push(' ');
+  } else {
+    result.extend(std::iter::repeat_n('\n', breaks - 1));
+  }
+}
+
+pub fn fold_flow_scalar(text: &str) -> String {
+  if !text.contains('\n') && !text.contains('\r') {
+    return text.to_string();
+  }
+
+  let mut result = String::with_capacity(text.len());
+  let mut characters = text.chars().peekable();
+
+  while let Some(character) = characters.next() {
+    match character {
+      '\n' => fold_break(&mut result, &mut characters),
+
+      '\r' => {
+        characters.next_if_eq(&'\n');
+        fold_break(&mut result, &mut characters);
+      }
+
+      _ => result.push(character),
+    }
+  }
+
+  result
+}
+
 pub fn unescape_double_quoted(text: &str) -> String {
   let mut result = String::with_capacity(text.len());
-  let mut chars = text.chars();
+  let mut chars = text.chars().peekable();
 
   while let Some(character) = chars.next() {
+    if character == '\n' {
+      fold_break(&mut result, &mut chars);
+      continue;
+    }
+
+    if character == '\r' {
+      chars.next_if_eq(&'\n');
+      fold_break(&mut result, &mut chars);
+      continue;
+    }
+
     if character == '\\' {
       match chars.next() {
         Some('n') => result.push('\n'),
@@ -363,7 +494,19 @@ pub fn unescape_double_quoted(text: &str) -> String {
         Some('v') => result.push('\u{0b}'),
         Some(' ') => result.push(' '),
         Some('_') => result.push('\u{a0}'),
-        Some('\n') => {} // line continuation: skip newline and leading whitespace
+        Some('N') => result.push('\u{85}'),
+        Some('L') => result.push('\u{2028}'),
+        Some('P') => result.push('\u{2029}'),
+        Some('x') => push_hex_escape(&mut result, &mut chars, 2),
+        Some('u') => push_hex_escape(&mut result, &mut chars, 4),
+        Some('U') => push_hex_escape(&mut result, &mut chars, 8),
+        Some('\n') => skip_spaces(&mut chars),
+
+        Some('\r') => {
+          chars.next_if_eq(&'\n');
+          skip_spaces(&mut chars);
+        }
+
         Some(other) => {
           result.push('\\');
           result.push(other);
@@ -379,7 +522,7 @@ pub fn unescape_double_quoted(text: &str) -> String {
 }
 
 pub fn unescape_single_quoted(text: &str) -> String {
-  text.replace("''", "'")
+  fold_flow_scalar(&text.replace("''", "'"))
 }
 
 pub fn line_at(source: &str, offset: usize) -> usize {

data/rust/src/validation.rs

@@ -0,0 +1,30 @@
+use crate::syntax::is_control_character;
+
+#[derive(Debug, Clone)]
+pub struct ControlCharacter {
+  pub character: char,
+  pub line: usize,
+  pub column: usize,
+}
+
+impl std::fmt::Display for ControlCharacter {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "U+{:04X} at line {}, column {}", self.character as u32, self.line, self.column)
+  }
+}
+
+pub fn find_control_characters(source: &str) -> Vec<ControlCharacter> {
+  source
+    .lines()
+    .enumerate()
+    .flat_map(|(index, line)| {
+      line.chars().enumerate().filter_map(move |(column, character)| {
+        is_control_character(character).then_some(ControlCharacter {
+          character,
+          line: index + 1,
+          column: column + 1,
+        })
+      })
+    })
+    .collect()
+}

data/rust/src/yaml_writer.rs

@@ -10,6 +10,8 @@ pub fn json_to_yaml_text(value: &Value, quote_style: &QuoteStyle, indent: usize)
       map
         .iter()
         .map(|(k, v)| match v {
+          Value::Array(arr) if arr.is_empty() => format!("{}{}: []", prefix, k),
+
           Value::Array(arr) => {
             let items: Vec<String> = arr
               .iter()
@@ -26,6 +28,8 @@ pub fn json_to_yaml_text(value: &Value, quote_style: &QuoteStyle, indent: usize)
             format!("{}{}:\n{}", prefix, k, items.join("\n"))
           }
 
+          Value::Object(inner_map) if inner_map.is_empty() => format!("{}{}: {{}}", prefix, k),
+
           Value::Object(_) => {
             let inner = json_to_yaml_text(v, quote_style, indent + 2);
             format!("{}{}:\n{}", prefix, k, inner)
@@ -149,31 +153,15 @@ fn format_yaml_scalar(value: &Value, quote_style: &QuoteStyle) -> String {
     Value::Bool(boolean) => boolean.to_string(),
     Value::Number(number) => number.to_string(),
     Value::String(string) => match quote_style {
-      QuoteStyle::Double => {
-        let escaped = string.replace('\\', "\\\\").replace('"', "\\\"");
-
-        format!("\"{}\"", escaped)
-      }
-
-      QuoteStyle::Single => {
+      QuoteStyle::Single if crate::syntax::is_single_quotable(string) => {
         let escaped = string.replace('\'', "''");
 
         format!("'{}'", escaped)
       }
 
-      QuoteStyle::Plain => {
-        if crate::syntax::needs_quoting(string) {
-          crate::syntax::format_scalar_value(string, yaml_parser::SyntaxKind::DOUBLE_QUOTED_SCALAR)
-        } else {
-          string.clone()
-        }
-      }
+      QuoteStyle::Plain if !crate::syntax::needs_quoting(string) => string.clone(),
 
-      _ => {
-        let escaped = string.replace('\\', "\\\\").replace('"', "\\\"");
-
-        format!("\"{}\"", escaped)
-      }
+      _ => crate::syntax::format_scalar_value(string, yaml_parser::SyntaxKind::DOUBLE_QUOTED_SCALAR),
     },
 
     Value::Array(array) => {