html-to-markdown 2.24.6 → 2.25.0

data/rust-vendor/markup5ever_rcdom/tests/xml-tokenizer.rs

@@ -1,374 +0,0 @@
-// Copyright 2014-2017 The html5ever Project Developers. See the
-// COPYRIGHT file at the top-level directory of this distribution.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-use serde_json::{Map, Value};
-use std::borrow::Cow::Borrowed;
-use std::cell::RefCell;
-use std::ffi::OsStr;
-use std::io::Read;
-use std::path::Path;
-use xml5ever::tokenizer::ProcessResult;
-
-use util::find_tests::foreach_xml5lib_test;
-use util::runner::{run_all, Test};
-
-use markup5ever::buffer_queue::BufferQueue;
-use xml5ever::tendril::{SliceExt, StrTendril};
-use xml5ever::tokenizer::{
-    Doctype, EmptyTag, EndTag, Pi, ShortTag, StartTag, Tag, Token, TokenSink, XmlTokenizer,
-    XmlTokenizerOpts,
-};
-use xml5ever::{ns, Attribute, LocalName, QualName};
-
-mod util {
-    pub mod find_tests;
-    pub mod runner;
-}
-
-// Return all ways of splitting the string into at most n
-// possibly-empty pieces.
-fn splits(s: &str, n: usize) -> Vec<Vec<StrTendril>> {
-    if n == 1 {
-        return vec![vec![s.to_tendril()]];
-    }
-
-    let mut points: Vec<usize> = s.char_indices().map(|(n, _)| n).collect();
-    points.push(s.len());
-
-    // do this with iterators?
-    let mut out = vec![];
-    for p in points.into_iter() {
-        let y = &s[p..];
-        for mut x in splits(&s[..p], n - 1).into_iter() {
-            x.push(y.to_tendril());
-            out.push(x);
-        }
-    }
-
-    out.extend(splits(s, n - 1));
-    out
-}
-
-struct TokenLogger {
-    tokens: RefCell<Vec<Token>>,
-    current_str: RefCell<StrTendril>,
-    exact_errors: bool,
-}
-
-impl TokenLogger {
-    fn new(exact_errors: bool) -> TokenLogger {
-        TokenLogger {
-            tokens: RefCell::new(vec![]),
-            current_str: RefCell::new(StrTendril::new()),
-            exact_errors,
-        }
-    }
-
-    // Push anything other than character tokens
-    fn push(&self, token: Token) {
-        self.finish_str();
-        self.tokens.borrow_mut().push(token);
-    }
-
-    fn finish_str(&self) {
-        if !self.current_str.borrow().is_empty() {
-            let s = self.current_str.take();
-            self.tokens.borrow_mut().push(Token::Characters(s));
-        }
-    }
-
-    fn get_tokens(self) -> Vec<Token> {
-        self.finish_str();
-        self.tokens.take()
-    }
-}
-
-impl TokenSink for TokenLogger {
-    type Handle = ();
-
-    fn process_token(&self, token: Token) -> ProcessResult<()> {
-        match token {
-            Token::Characters(characters) => {
-                self.current_str.borrow_mut().push_slice(&characters);
-            },
-
-            Token::NullCharacter => {
-                self.current_str.borrow_mut().push_char('\0');
-            },
-
-            Token::ParseError(_) => {
-                if self.exact_errors {
-                    self.push(Token::ParseError(Borrowed("")));
-                }
-            },
-            Token::Tag(mut t) => {
-                // The spec seems to indicate that one can emit
-                // erroneous end tags with attrs, but the test
-                // cases don't contain them.
-                match t.kind {
-                    EndTag => {
-                        t.attrs = vec![];
-                    },
-                    _ => t.attrs.sort_by(|a1, a2| a1.name.cmp(&a2.name)),
-                }
-                self.push(Token::Tag(t));
-            },
-            Token::EndOfFile => (),
-            _ => self.push(token),
-        };
-        ProcessResult::Continue
-    }
-}
-
-fn tokenize_xml(input: Vec<StrTendril>, opts: XmlTokenizerOpts) -> Vec<Token> {
-    let sink = TokenLogger::new(opts.exact_errors);
-    let tok = XmlTokenizer::new(sink, opts);
-    let buf = BufferQueue::default();
-
-    for chunk in input.into_iter() {
-        buf.push_back(chunk);
-        let _ = tok.feed(&buf);
-    }
-    let _ = tok.feed(&buf);
-    tok.end();
-    tok.sink.get_tokens()
-}
-
-#[allow(dead_code)]
-trait JsonExt: Sized {
-    fn get_str(&self) -> String;
-    fn get_tendril(&self) -> StrTendril;
-    fn get_nullable_tendril(&self) -> Option<StrTendril>;
-    fn get_bool(&self) -> bool;
-    fn get_obj(&self) -> &Map<String, Self>;
-    fn get_list(&self) -> &Vec<Self>;
-    fn find(&self, key: &str) -> &Self;
-}
-
-impl JsonExt for Value {
-    fn get_str(&self) -> String {
-        match *self {
-            Value::String(ref s) => s.to_string(),
-            _ => panic!("Value::get_str: not a String"),
-        }
-    }
-
-    fn get_tendril(&self) -> StrTendril {
-        match *self {
-            Value::String(ref s) => s.to_tendril(),
-            _ => panic!("Value::get_tendril: not a String"),
-        }
-    }
-
-    fn get_nullable_tendril(&self) -> Option<StrTendril> {
-        match *self {
-            Value::Null => None,
-            Value::String(ref s) => Some(s.to_tendril()),
-            _ => panic!("Value::get_nullable_tendril: not a String"),
-        }
-    }
-
-    fn get_bool(&self) -> bool {
-        match *self {
-            Value::Bool(b) => b,
-            _ => panic!("Value::get_bool: not a Boolean"),
-        }
-    }
-
-    fn get_obj(&self) -> &Map<String, Value> {
-        match self {
-            Value::Object(m) => m,
-            _ => panic!("Value::get_obj: not an Object"),
-        }
-    }
-
-    fn get_list(&self) -> &Vec<Value> {
-        match self {
-            Value::Array(m) => m,
-            _ => panic!("Value::get_list: not an Array"),
-        }
-    }
-
-    fn find(&self, key: &str) -> &Value {
-        self.get_obj().get(&key.to_string()).unwrap()
-    }
-}
-
-// Parse a JSON object (other than "ParseError") to a token.
-fn json_to_token(js: &Value) -> Token {
-    let parts = js.as_array().unwrap();
-    // Collect refs here so we don't have to use "ref" in all the patterns below.
-    let args: Vec<&Value> = parts[1..].iter().collect();
-    match &*parts[0].get_str() {
-        "StartTag" => Token::Tag(Tag {
-            kind: StartTag,
-            name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())),
-            attrs: args[1]
-                .get_obj()
-                .iter()
-                .map(|(k, v)| Attribute {
-                    name: QualName::new(None, ns!(), LocalName::from(&**k)),
-                    value: v.get_tendril(),
-                })
-                .collect(),
-        }),
-
-        "EndTag" => Token::Tag(Tag {
-            kind: EndTag,
-            name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())),
-            attrs: vec![],
-        }),
-
-        "ShortTag" => Token::Tag(Tag {
-            kind: ShortTag,
-            name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())),
-            attrs: vec![],
-        }),
-
-        "EmptyTag" => Token::Tag(Tag {
-            kind: EmptyTag,
-            name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())),
-            attrs: args[1]
-                .get_obj()
-                .iter()
-                .map(|(k, v)| Attribute {
-                    name: QualName::new(None, ns!(), LocalName::from(&**k)),
-                    value: v.get_tendril(),
-                })
-                .collect(),
-        }),
-
-        "Comment" => Token::Comment(args[0].get_tendril()),
-
-        "Character" => Token::Characters(args[0].get_tendril()),
-
-        "PI" => Token::ProcessingInstruction(Pi {
-            target: args[0].get_tendril(),
-            data: args[1].get_tendril(),
-        }),
-
-        "DOCTYPE" => Token::Doctype(Doctype {
-            name: args[0].get_nullable_tendril(),
-            public_id: args[1].get_nullable_tendril(),
-            system_id: args[2].get_nullable_tendril(),
-        }),
-
-        // We don't need to produce NullCharacterToken because
-        // the TokenLogger will convert them to CharacterTokens.
-        _ => panic!("don't understand token {parts:?}"),
-    }
-}
-
-// Parse the "output" field of the test case into a vector of tokens.
-fn json_to_tokens(js: &Value, exact_errors: bool) -> Vec<Token> {
-    // Use a TokenLogger so that we combine character tokens separated
-    // by an ignored error.
-    let sink = TokenLogger::new(exact_errors);
-    for tok in js.as_array().unwrap().iter() {
-        match *tok {
-            Value::String(ref s) if &s[..] == "ParseError" => {
-                let _ = sink.process_token(Token::ParseError(Borrowed("")));
-            },
-            _ => {
-                let _ = sink.process_token(json_to_token(tok));
-            },
-        }
-    }
-    sink.get_tokens()
-}
-
-fn mk_xml_test(name: String, input: String, expect: Value, opts: XmlTokenizerOpts) -> Test {
-    Test {
-        name,
-        skip: false,
-        test: Box::new(move || {
-            // Split up the input at different points to test incremental tokenization.
-            let insplits = splits(&input, 3);
-            for input in insplits.into_iter() {
-                // Clone 'input' so we have it for the failure message.
-                // Also clone opts.  If we don't, we get the wrong
-                // result but the compiler doesn't catch it!
-                // Possibly mozilla/rust#12223.
-                let output = tokenize_xml(input.clone(), opts);
-                let expect = json_to_tokens(&expect, opts.exact_errors);
-                if output != expect {
-                    panic!("\ninput: {input:?}\ngot: {output:?}\nexpected: {expect:?}");
-                }
-            }
-        }),
-    }
-}
-
-fn mk_xml_tests(tests: &mut Vec<Test>, filename: &str, js: &Value) {
-    let input: &str = &js.find("input").get_str();
-    let expect = js.find("output");
-    let desc = format!("tok: {}: {}", filename, js.find("description").get_str());
-
-    // Some tests want to start in a state other than Data.
-    let state_overrides = vec![None];
-
-    // Build the tests.
-    for state in state_overrides.into_iter() {
-        for &exact_errors in [false, true].iter() {
-            let mut newdesc = desc.clone();
-            if let Some(s) = state {
-                newdesc = format!("{newdesc} (in state {s:?})")
-            };
-            if exact_errors {
-                newdesc = format!("{newdesc} (exact errors)");
-            }
-
-            tests.push(mk_xml_test(
-                newdesc,
-                String::from(input),
-                expect.clone(),
-                XmlTokenizerOpts {
-                    exact_errors,
-                    initial_state: state,
-
-                    // Not discarding a BOM is what the test suite expects; see
-                    // https://github.com/html5lib/html5lib-tests/issues/2
-                    discard_bom: false,
-
-                    ..Default::default()
-                },
-            ));
-        }
-    }
-}
-
-fn tests(src_dir: &Path) -> Vec<Test> {
-    let mut tests = vec![];
-    foreach_xml5lib_test(
-        src_dir,
-        "tokenizer",
-        OsStr::new("test"),
-        |path, mut file| {
-            let mut s = String::new();
-            file.read_to_string(&mut s).expect("file reading error");
-            let js: Value = serde_json::from_str(&s).expect("json parse error");
-
-            if let Value::Array(ref lst) = js["tests"] {
-                for test in lst.iter() {
-                    mk_xml_tests(
-                        &mut tests,
-                        path.file_name().unwrap().to_str().unwrap(),
-                        test,
-                    );
-                }
-            }
-        },
-    );
-
-    tests
-}
-
-fn main() {
-    run_all(tests(Path::new("./")));
-}

data/rust-vendor/markup5ever_rcdom/tests/xml-tree-builder.rs

@@ -1,237 +0,0 @@
-// Copyright 2014-2017 The html5ever Project Developers. See the
-// COPYRIGHT file at the top-level directory of this distribution.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-use markup5ever::ns;
-use markup5ever_rcdom::*;
-use std::collections::{HashMap, HashSet};
-use std::ffi::OsStr;
-use std::io::BufRead;
-use std::path::Path;
-use std::{fs, io, iter, mem};
-use util::find_tests::foreach_xml5lib_test;
-use util::runner::{run_all, Test};
-use xml5ever::driver::parse_document;
-use xml5ever::tendril::TendrilSink;
-
-mod util {
-    pub mod find_tests;
-    pub mod runner;
-}
-
-fn parse_tests<It: Iterator<Item = String>>(mut lines: It) -> Vec<HashMap<String, String>> {
-    let mut tests = vec![];
-    let mut test = HashMap::new();
-    let mut key: Option<String> = None;
-    let mut val = String::new();
-
-    macro_rules! finish_val ( () => (
-        match key.take() {
-            None => (),
-            Some(key) => {
-                assert!(test.insert(key, mem::take(&mut val)).is_none());
-            }
-        }
-    ));
-
-    macro_rules! finish_test ( () => (
-        if !test.is_empty() {
-            tests.push(mem::take(&mut test));
-        }
-    ));
-
-    loop {
-        match lines.next() {
-            None => break,
-            Some(line) => {
-                if let Some(rest) = line.strip_prefix('#') {
-                    finish_val!();
-                    if line == "#data" {
-                        finish_test!();
-                    }
-                    key = Some(rest.to_string());
-                } else {
-                    val.push_str(&line);
-                    val.push('\n');
-                }
-            },
-        }
-    }
-
-    finish_val!();
-    finish_test!();
-    tests
-}
-
-fn serialize(buf: &mut String, indent: usize, handle: Handle) {
-    buf.push('|');
-    buf.extend(iter::repeat_n(" ", indent));
-
-    let node = handle;
-    match &node.data {
-        NodeData::Document => panic!("should not reach Document"),
-
-        NodeData::Doctype {
-            name,
-            public_id,
-            system_id,
-        } => {
-            buf.push_str("<!DOCTYPE ");
-            buf.push_str(name);
-            if !public_id.is_empty() || !system_id.is_empty() {
-                buf.push_str(&format!(" \"{public_id}\" \"{system_id}\""));
-            }
-            buf.push_str(">\n");
-        },
-
-        NodeData::Text { contents } => {
-            buf.push('"');
-            buf.push_str(&contents.borrow());
-            buf.push_str("\"\n");
-        },
-
-        NodeData::ProcessingInstruction { target, contents } => {
-            buf.push_str("<?");
-            buf.push_str(target);
-            buf.push(' ');
-            buf.push_str(contents);
-            buf.push_str("?>\n");
-        },
-
-        NodeData::Comment { contents } => {
-            buf.push_str("<!-- ");
-            buf.push_str(contents);
-            buf.push_str(" -->\n");
-        },
-
-        NodeData::Element { name, attrs, .. } => {
-            buf.push('<');
-
-            if name.ns != ns!() {
-                buf.push('{');
-                buf.push_str(&name.ns);
-                buf.push('}');
-            };
-
-            if let Some(prefix) = &name.prefix {
-                buf.push_str(prefix);
-                buf.push(':');
-            }
-
-            buf.push_str(&name.local);
-            buf.push_str(">\n");
-
-            let mut attrs = attrs.borrow().clone();
-            attrs.sort_by(|x, y| x.name.local.cmp(&y.name.local));
-            // FIXME: sort by UTF-16 code unit
-
-            for attr in attrs.into_iter() {
-                buf.push('|');
-                buf.extend(iter::repeat_n(" ", indent + 2));
-
-                if !attr.name.ns.is_empty() {
-                    buf.push('{');
-                    buf.push_str(&attr.name.ns);
-                    buf.push('}');
-                }
-
-                if let Some(attr_prefix) = attr.name.prefix {
-                    buf.push_str(&attr_prefix);
-                    buf.push(':');
-                }
-
-                buf.push_str(&format!("{}=\"{}\"\n", attr.name.local, attr.value));
-            }
-        },
-    }
-
-    for child in node.children.borrow().iter() {
-        serialize(buf, indent + 2, child.clone());
-    }
-}
-
-// Ignore tests containing these strings; we don't support these features yet.
-static IGNORE_SUBSTRS: &[&str] = &["<template"];
-
-fn make_xml_test(
-    tests: &mut Vec<Test>,
-    ignores: &HashSet<String>,
-    filename: &str,
-    idx: usize,
-    fields: HashMap<String, String>,
-) {
-    let get_field = |key| {
-        let field = fields.get(key).expect("missing field");
-        field.trim_end_matches('\n').to_string()
-    };
-
-    let data = get_field("data");
-    let expected = get_field("document");
-    let name = format!("tb: {filename}-{idx}");
-    let skip = ignores.contains(&name) || IGNORE_SUBSTRS.iter().any(|&ig| data.contains(ig));
-
-    tests.push(Test {
-        name,
-        skip,
-        test: Box::new(move || {
-            let mut result = String::new();
-
-            let dom = parse_document(RcDom::default(), Default::default()).one(data.clone());
-            for child in dom.document.children.borrow().iter() {
-                serialize(&mut result, 1, child.clone());
-            }
-
-            let len = result.len();
-            result.truncate(len - 1); // drop the trailing newline
-
-            if result != expected {
-                panic!("\ninput: {data}\ngot:\n{result}\nexpected:\n{expected}\n");
-            }
-        }),
-    });
-}
-
-fn tests(src_dir: &Path, ignores: &HashSet<String>) -> Vec<Test> {
-    let mut tests = vec![];
-
-    foreach_xml5lib_test(
-        src_dir,
-        "tree-construction",
-        OsStr::new("dat"),
-        |path, file| {
-            let buf = io::BufReader::new(file);
-            let lines = buf.lines().map(|res| res.expect("couldn't read"));
-            let data = parse_tests(lines);
-
-            for (i, test) in data.into_iter().enumerate() {
-                make_xml_test(
-                    &mut tests,
-                    ignores,
-                    path.file_name().unwrap().to_str().unwrap(),
-                    i,
-                    test,
-                );
-            }
-        },
-    );
-
-    tests
-}
-
-fn main() {
-    let src_dir = Path::new("./");
-    let mut ignores = HashSet::new();
-    if let Ok(f) = fs::File::open(src_dir.join("data/test/ignore")) {
-        let r = io::BufReader::new(f);
-        for ln in r.lines() {
-            ignores.insert(ln.unwrap().trim_end().to_string());
-        }
-    }
-
-    run_all(tests(src_dir, &ignores));
-}

data/rust-vendor/markup5ever_rcdom/xml5lib-tests/AUTHORS.rst

@@ -1,9 +0,0 @@
-Credits
-=======
-
-The ``xml5lib`` test data is maintained by:
-
-- Daniel Fath
-
-Contributors
-------------

data/rust-vendor/markup5ever_rcdom/xml5lib-tests/LICENSE

@@ -1,21 +0,0 @@
-Copyright (c) 2006-2013 James Graham, Geoffrey Sneddon and
-other contributors
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.