apex-ruby 1.0.6 → 1.0.7

data/ext/apex_ext/apex_src/src/apex.c

@@ -1,12 +1,11 @@
 #include "apex/apex.h"
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <stdio.h>
 #include <ctype.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#include <libgen.h>
-#include <time.h>
 #include <sys/time.h>
 
 /* cmark-gfm headers */
@@ -42,6 +41,11 @@
 #include "extensions/fenced_divs.h"
 #include "extensions/syntax_highlight.h"
 #include "plugins.h"
+#include "ast_json.h"
+#include "apex/ast_markdown.h"
+#include "apex/ast_terminal.h"
+#include "apex/ast_man.h"
+#include "filters_ast.h"
 
 /* Custom renderer */
 #include "html_renderer.h"
@@ -70,6 +74,185 @@ static char *apex_encode_hex_entities(const char *text, size_t len) {
     return out;
 }
 
+/**
+ * Escape string for safe HTML attribute usage.
+ */
+static char *apex_escape_html_attr(const char *input) {
+    if (!input) return strdup("");
+
+    size_t len = strlen(input);
+    size_t max_len = len * 6 + 1; /* Worst case for &quot; */
+    char *out = malloc(max_len);
+    if (!out) return NULL;
+
+    char *w = out;
+    for (const char *p = input; *p; p++) {
+        switch (*p) {
+            case '&':
+                memcpy(w, "&amp;", 5);
+                w += 5;
+                break;
+            case '<':
+                memcpy(w, "&lt;", 4);
+                w += 4;
+                break;
+            case '>':
+                memcpy(w, "&gt;", 4);
+                w += 4;
+                break;
+            case '"':
+                memcpy(w, "&quot;", 6);
+                w += 6;
+                break;
+            case '\'':
+                memcpy(w, "&#39;", 5);
+                w += 5;
+                break;
+            default:
+                *w++ = *p;
+                break;
+        }
+    }
+    *w = '\0';
+    return out;
+}
+
+/**
+ * Normalize metadata key by removing spaces and lowercasing.
+ */
+static char *apex_normalize_meta_key(const char *key) {
+    if (!key) return NULL;
+    size_t len = strlen(key);
+    char *normalized = malloc(len + 1);
+    if (!normalized) return NULL;
+
+    char *out = normalized;
+    for (const char *in = key; *in; in++) {
+        if (!isspace((unsigned char)*in)) {
+            *out++ = (char)tolower((unsigned char)*in);
+        }
+    }
+    *out = '\0';
+    return normalized;
+}
+
+/**
+ * Keys handled elsewhere (title/lang/css/html header/footer/etc.) should not
+ * be emitted as generic <meta name="..."> tags.
+ */
+static bool apex_skip_generic_meta_key(const char *key) {
+    char *normalized = apex_normalize_meta_key(key);
+    if (!normalized) return false;
+
+    static const char *skip_keys[] = {
+        "title",
+        "css",
+        "language",
+        "htmlheader",
+        "htmlfooter",
+        "htmlheaderlevel",
+        "baseheaderlevel",
+        "quoteslanguage",
+        NULL
+    };
+
+    bool skip = false;
+    for (int i = 0; skip_keys[i]; i++) {
+        if (strcmp(normalized, skip_keys[i]) == 0) {
+            skip = true;
+            break;
+        }
+    }
+    free(normalized);
+    return skip;
+}
+
+/**
+ * Render metadata list to newline-separated generic HTML meta tags.
+ */
+static char *apex_render_generic_meta_tags(apex_metadata_item *metadata) {
+    if (!metadata) return NULL;
+
+    size_t capacity = 256;
+    size_t used = 0;
+    char *out = malloc(capacity);
+    if (!out) return NULL;
+    out[0] = '\0';
+
+    /* Metadata entries are prepended during parsing; reverse iteration restores
+     * source declaration order in generated head tags. */
+    size_t item_count = 0;
+    for (apex_metadata_item *it = metadata; it; it = it->next) item_count++;
+    if (item_count == 0) {
+        free(out);
+        return NULL;
+    }
+    apex_metadata_item **items = malloc(item_count * sizeof(apex_metadata_item *));
+    if (!items) {
+        free(out);
+        return NULL;
+    }
+    size_t item_index = 0;
+    for (apex_metadata_item *it = metadata; it; it = it->next) {
+        items[item_index++] = it;
+    }
+
+    for (size_t i = item_count; i > 0; i--) {
+        apex_metadata_item *item = items[i - 1];
+        if (!item->key || !item->value || apex_skip_generic_meta_key(item->key)) {
+            continue;
+        }
+
+        char *escaped_key = apex_escape_html_attr(item->key);
+        char *escaped_value = apex_escape_html_attr(item->value);
+        if (!escaped_key || !escaped_value) {
+            if (escaped_key) free(escaped_key);
+            if (escaped_value) free(escaped_value);
+            free(items);
+            free(out);
+            return NULL;
+        }
+
+        size_t needed = strlen(escaped_key) + strlen(escaped_value) + 36;
+        if (used + needed + 1 > capacity) {
+            size_t new_capacity = capacity * 2;
+            while (used + needed + 1 > new_capacity) {
+                new_capacity *= 2;
+            }
+            char *new_out = realloc(out, new_capacity);
+            if (!new_out) {
+                free(escaped_key);
+                free(escaped_value);
+                free(items);
+                free(out);
+                return NULL;
+            }
+            out = new_out;
+            capacity = new_capacity;
+        }
+
+        int written = snprintf(out + used, capacity - used,
+                               "  <meta name=\"%s\" content=\"%s\"/>\n",
+                               escaped_key, escaped_value);
+        free(escaped_key);
+        free(escaped_value);
+        if (written < 0) {
+            free(items);
+            free(out);
+            return NULL;
+        }
+        used += (size_t)written;
+    }
+    free(items);
+
+    if (used == 0) {
+        free(out);
+        return NULL;
+    }
+
+    return out;
+}
+
 /**
  * Base64 encode binary data
  * Caller must free the returned buffer.
@@ -526,9 +709,36 @@ static char *apex_preprocess_autolinks(const char *text, const apex_options *opt
             continue;
         }
 
-        /* Check if we're at the start of a reference link definition: [id]: URL */
+        /* At start of line: handle reference definitions and indented code blocks */
         if (r == text || r[-1] == '\n') {
             const char *line_start = r;
+
+            /* First: skip indented code blocks (4+ spaces or a leading tab) entirely */
+            int indent_spaces = 0;
+            while (*line_start == ' ' && indent_spaces < 4) {
+                line_start++;
+                indent_spaces++;
+            }
+            if (indent_spaces == 4 || *line_start == '\t') {
+                const char *line_end = strchr(r, '\n');
+                if (!line_end) line_end = r + strlen(r);
+                size_t line_len = line_end - r;
+                if ((size_t)(w - out) + line_len + 1 > cap) {
+                    size_t used = (size_t)(w - out);
+                    cap = (used + line_len + 1) * 2;
+                    char *new_out = realloc(out, cap);
+                    if (!new_out) { free(out); return NULL; }
+                    out = new_out;
+                    w = out + used;
+                }
+                memcpy(w, r, line_len);
+                w += line_len;
+                r = line_end;
+                continue;
+            }
+
+            /* Then: check for reference link definitions: [id]: URL */
+            line_start = r;
             /* Skip leading whitespace */
             while (*line_start == ' ' || *line_start == '\t') {
                 line_start++;
@@ -1293,19 +1503,49 @@ static char *apex_preprocess_table_captions(const char *text) {
                     }
                     table_check++;
                 }
-                /* Check for : Caption format (Pandoc-style, only after tables) */
-                /* Skip up to 3 leading spaces (matching definition list rules) */
+                /* Check for : Caption format (Pandoc-style, only in table context)
+                 * Require prev_line_was_table_row or in_table_section - NOT prev_line_was_blank alone.
+                 * prev_line_was_blank alone would wrongly convert "Term\n\n: definition 1" (def list) to caption. */
+                if ((prev_line_was_table_row || in_table_section) &&
+                    !is_pandoc_caption_line) {
+                    const char *check = p;
+                    int spaces = 0;
+                    while (spaces < 3 && check < line_end && *check == ' ') {
+                        spaces++;
+                        check++;
+                    }
+                    if (check < line_end && *check == ':' &&
+                        (check + 1) < line_end &&
+                        (check[1] == ' ' || check[1] == '\t')) {
+                        is_colon_caption_line = true;
+                    }
+                }
+            } else {
+                /* Check for : Caption BEFORE table (next non-blank line is a table row) */
                 const char *check = p;
                 int spaces = 0;
                 while (spaces < 3 && check < line_end && *check == ' ') {
                     spaces++;
                     check++;
                 }
-                /* Must start with : followed by space or tab */
                 if (check < line_end && *check == ':' &&
                     (check + 1) < line_end &&
                     (check[1] == ' ' || check[1] == '\t')) {
-                    is_colon_caption_line = true;
+                    /* Peek ahead: is next non-blank line a table row? */
+                    const char *next = line_end;
+                    if (next < text + len && *next == '\n') next++;
+                    if (next < text + len && *next == '\r') next++;
+                    while (next < text + len && (*next == '\n' || *next == '\r' || *next == ' ' || *next == '\t')) {
+                        if (*next == '\n' || *next == '\r') {
+                            next++;
+                            if (next < text + len && next[-1] == '\r' && *next == '\n') next++;
+                        } else {
+                            next++;
+                        }
+                    }
+                    if (next < text + len && *next == '|') {
+                        is_colon_caption_line = true;
+                    }
                 }
             }
         }
@@ -1427,10 +1667,9 @@ static char *apex_preprocess_table_captions(const char *text) {
                     *write++ = '\n';
                 }
             }
-        } else if (!in_code_block &&
-                   prev_line_was_table_row &&
-                   is_colon_caption_line) {
-            /* Case 3: Pandoc-style ': Caption {#id .class}' -> convert to '[Caption {#id .class}]' */
+        } else if (!in_code_block && is_colon_caption_line) {
+            /* Case 3: Pandoc-style ': Caption {#id .class}' -> convert to '[Caption {#id .class}]'
+             * Handles both: (a) after table, (b) before table (next line is | table row) */
             /* Skip leading whitespace (up to 3 spaces) */
             const char *caption_start = p;
             int spaces = 0;
@@ -2474,11 +2713,14 @@ apex_options apex_options_default(void) {
     opts.base_directory = NULL;
 
     /* Output options */
+    opts.output_format = APEX_OUTPUT_HTML;  /* Default: HTML output */
     opts.unsafe = true;
     opts.validate_utf8 = true;
     opts.github_pre_lang = true;
     opts.standalone = false;
     opts.pretty = false;
+    opts.xhtml = false;
+    opts.strict_xhtml = false;
     opts.stylesheet_paths = NULL;
     opts.stylesheet_count = 0;
     opts.document_title = NULL;
@@ -2552,9 +2794,10 @@ apex_options apex_options_default(void) {
     opts.enable_emoji_autocorrect = true;  /* Enabled by default in unified mode */
 
     /* Syntax highlighting options */
-    opts.code_highlighter = NULL;   /* Default: no external syntax highlighting */
-    opts.code_line_numbers = false; /* Default: no line numbers */
-    opts.highlight_language_only = false; /* Default: highlight all code blocks */
+    opts.code_highlighter = NULL;          /* Default: no external syntax highlighting */
+    opts.code_line_numbers = false;        /* Default: no line numbers */
+    opts.highlight_language_only = false;  /* Default: highlight all code blocks */
+    opts.code_highlight_theme = NULL;      /* Default: no explicit theme */
 
     /* Marked / integration-specific options (unified defaults) */
     opts.enable_widont = false;
@@ -2571,10 +2814,25 @@ apex_options apex_options_default(void) {
     /* Source file information (used by plugins via APEX_FILE_PATH) */
     opts.input_file_path = NULL;
 
+    /* AST filter options (Pandoc-style JSON filters) */
+    opts.ast_filter_commands = NULL;
+    opts.ast_filter_count = 0;
+    opts.ast_filter_strict = true; /* Default: fail fast on filter errors */
+
     /* Progress reporting */
     opts.progress_callback = NULL;
     opts.progress_user_data = NULL;
 
+    /* Custom cmark extension callback */
+    opts.cmark_init = NULL;
+    opts.cmark_done = NULL;
+    opts.cmark_user_data = NULL;
+
+    /* Terminal theme and width (for -t terminal/terminal256) */
+    opts.theme_name = NULL;
+    opts.terminal_width = 0;
+    opts.paginate = false;
+
     return opts;
 }
 
@@ -2860,13 +3118,7 @@ static void apex_register_extensions(cmark_parser *parser, const apex_options *o
         }
     }
 
-    /* Definition lists (Kramdown/PHP Extra style) */
-    if (options->enable_definition_lists) {
-        cmark_syntax_extension *deflist_ext = create_definition_list_extension();
-        if (deflist_ext) {
-            cmark_parser_attach_syntax_extension(parser, deflist_ext);
-        }
-    }
+    /* Definition lists (one-line format: Term :: Definition) - handled by preprocessing only */
 
     /* Advanced footnotes (block-level content support) */
     if (options->enable_footnotes) {
@@ -3913,6 +4165,190 @@ static char *apex_apply_widont_to_headings(const char *html) {
     return output;
 }
 
+static const char *apex_find_unquoted_gt(const char *p, const char *end) {
+    int quote = 0;
+    while (p < end) {
+        if (quote) {
+            if (*p == quote) quote = 0;
+            p++;
+            continue;
+        }
+        if (*p == '"' || *p == '\'') {
+            quote = *p;
+            p++;
+            continue;
+        }
+        if (*p == '>') return p;
+        p++;
+    }
+    return NULL;
+}
+
+static bool apex_html_void_element_name(const char *name, size_t nlen) {
+    static const char *void_tags[] = {
+        "area", "base", "br", "col", "embed", "hr", "img", "input",
+        "link", "meta", "param", "source", "track", "wbr"
+    };
+    for (size_t t = 0; t < sizeof(void_tags) / sizeof(void_tags[0]); t++) {
+        const char *v = void_tags[t];
+        size_t vl = strlen(v);
+        if (vl != nlen) continue;
+        size_t j;
+        for (j = 0; j < nlen; j++) {
+            if (tolower((unsigned char)name[j]) != tolower((unsigned char)v[j])) break;
+        }
+        if (j == nlen) return true;
+    }
+    return false;
+}
+
+static int apex_html_buf_append(char **outp, size_t *cap, size_t *olen, const char *s, size_t n) {
+    while (*olen + n + 1 > *cap) {
+        size_t new_cap = *cap ? *cap * 2 : 8192;
+        char *nbuf = realloc(*outp, new_cap);
+        if (!nbuf) return -1;
+        *outp = nbuf;
+        *cap = new_cap;
+    }
+    memcpy(*outp + *olen, s, n);
+    *olen += n;
+    (*outp)[*olen] = '\0';
+    return 0;
+}
+
+/**
+ * Rewrite HTML void/empty elements to XML self-closing form (e.g. <br> -> <br />).
+ * Skips contents of script, style, and HTML comments. Returns newly allocated string or NULL.
+ */
+static char *apex_html_apply_xhtml_void_tags(const char *html) {
+    if (!html) return NULL;
+
+    size_t cap = strlen(html) * 2 + 256;
+    if (cap < 8192) cap = 8192;
+    char *out = malloc(cap);
+    if (!out) return NULL;
+    size_t olen = 0;
+
+    const char *r = html;
+    const char *end = html + strlen(html);
+
+    while (r < end) {
+        if (*r != '<') {
+            if (apex_html_buf_append(&out, &cap, &olen, r, 1) != 0) goto fail;
+            r++;
+            continue;
+        }
+
+        /* Comment */
+        if (r + 4 <= end && strncmp(r, "<!--", 4) == 0) {
+            const char *ce = strstr(r + 4, "-->");
+            if (!ce) {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+                break;
+            }
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(ce + 3 - r)) != 0) goto fail;
+            r = ce + 3;
+            continue;
+        }
+
+        /* CDATA */
+        if (r + 9 <= end && strncmp(r, "<![CDATA[", 9) == 0) {
+            const char *ce = strstr(r + 9, "]]>");
+            if (!ce) {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+                break;
+            }
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(ce + 3 - r)) != 0) goto fail;
+            r = ce + 3;
+            continue;
+        }
+
+        /* script */
+        if (r + 7 <= end && strncasecmp(r, "<script", 7) == 0) {
+            const char *close = strcasestr(r + 7, "</script>");
+            if (!close) {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+                break;
+            }
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(close + 9 - r)) != 0) goto fail;
+            r = close + 9;
+            continue;
+        }
+
+        /* style */
+        if (r + 6 <= end && strncasecmp(r, "<style", 6) == 0) {
+            const char *close = strcasestr(r + 6, "</style>");
+            if (!close) {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+                break;
+            }
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(close + 8 - r)) != 0) goto fail;
+            r = close + 8;
+            continue;
+        }
+
+        /* Declaration <!...> */
+        if (r + 1 < end && r[1] == '!') {
+            const char *gt = apex_find_unquoted_gt(r, end);
+            if (!gt) {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+                break;
+            }
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(gt + 1 - r)) != 0) goto fail;
+            r = gt + 1;
+            continue;
+        }
+
+        /* Closing tag */
+        if (r + 1 < end && r[1] == '/') {
+            const char *gt = apex_find_unquoted_gt(r, end);
+            if (!gt) {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+                break;
+            }
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(gt + 1 - r)) != 0) goto fail;
+            r = gt + 1;
+            continue;
+        }
+
+        /* Opening tag: extract name */
+        const char *name_start = r + 1;
+        while (name_start < end && isspace((unsigned char)*name_start)) name_start++;
+        const char *name_end = name_start;
+        while (name_end < end && (isalnum((unsigned char)*name_end) || *name_end == '-' || *name_end == '_' || *name_end == ':')) {
+            name_end++;
+        }
+        size_t name_len = (size_t)(name_end - name_start);
+
+        const char *gt = apex_find_unquoted_gt(r, end);
+        if (!gt) {
+            if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(end - r)) != 0) goto fail;
+            break;
+        }
+
+        if (name_len > 0 && apex_html_void_element_name(name_start, name_len)) {
+            const char *slash = gt - 1;
+            while (slash > r && isspace((unsigned char)*slash)) slash--;
+            if (slash >= r && *slash == '/') {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(gt + 1 - r)) != 0) goto fail;
+            } else {
+                if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(gt - r)) != 0) goto fail;
+                if (apex_html_buf_append(&out, &cap, &olen, " />", 3) != 0) goto fail;
+            }
+            r = gt + 1;
+            continue;
+        }
+
+        if (apex_html_buf_append(&out, &cap, &olen, r, (size_t)(gt + 1 - r)) != 0) goto fail;
+        r = gt + 1;
+    }
+
+    return out;
+fail:
+    free(out);
+    return NULL;
+}
+
 char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options *options) {
     if (!markdown || len == 0) {
         char *empty = malloc(1);
@@ -3934,6 +4370,15 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
     /* Use local_opts for rest of function (mutable) - shadow the const parameter */
     #define options (&local_opts)
 
+    if (local_opts.strict_xhtml) {
+        local_opts.xhtml = true;
+    }
+
+    /* Man/man-html output: force disable smart typography so option names (e.g. --to) stay as literal -- */
+    if (options->output_format == APEX_OUTPUT_MAN || options->output_format == APEX_OUTPUT_MAN_HTML) {
+        local_opts.enable_smart_typography = false;
+    }
+
     /* Extract metadata if enabled (preprocessing step) */
     /* Safety check: ensure len doesn't exceed actual string length */
     size_t actual_len = strlen(markdown);
@@ -3985,12 +4430,15 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
                 len, text_ptr, len > 200 ? "..." : "");
     }
 
+    /* Create deflist debug log as soon as conversion starts (so it exists even if we exit early or deflists are disabled) */
+    apex_deflist_debug_touch(options->enable_definition_lists);
+
     if (options->mode == APEX_MODE_MULTIMARKDOWN ||
         options->mode == APEX_MODE_KRAMDOWN ||
         options->mode == APEX_MODE_UNIFIED) {
         /* Extract metadata FIRST */
         PROFILE_START(metadata);
-        metadata = apex_extract_metadata(&text_ptr);
+        metadata = apex_extract_metadata_for_mode(&text_ptr, options->mode);
         PROFILE_END(metadata);
         if (getenv("APEX_DEBUG_PIPELINE")) {
             size_t len = strlen(text_ptr);
@@ -4949,12 +5397,16 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
     /* Register extensions based on mode and options */
     apex_register_extensions(parser, options);
 
+    if (options->cmark_init) {
+        options->cmark_init(parser, options, cmark_opts, options->cmark_user_data);
+    }
+
     /* Feed normalized text to parser */
     if (getenv("APEX_DEBUG_PIPELINE")) {
         fprintf(stderr, "[APEX_DEBUG] markdown to parse (len=%zu): %.350s%s\n",
                 text_len, text_ptr, text_len > 350 ? "..." : "");
     }
-    cmark_parser_feed(parser, text_ptr, text_len);
+    cmark_parser_feed(parser, text_len ? text_ptr : "", text_len);
     cmark_node *document = cmark_parser_finish(parser);
     PROFILE_END(parsing);
 
@@ -4964,12 +5416,62 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
     }
 
     if (!document) {
+        if (options->cmark_done) {
+            options->cmark_done(parser, options, cmark_opts, options->cmark_user_data);
+        }
         cmark_parser_free(parser);
         free(working_text);
         apex_free_metadata(metadata);
         return NULL;
     }
 
+    /* If output format is JSON, emit JSON right after parsing (before AST filters) */
+    if (options->output_format == APEX_OUTPUT_JSON) {
+        char *json = apex_cmark_to_pandoc_json(document, options);
+        cmark_node_free(document);
+        cmark_parser_free(parser);
+        free(working_text);
+        apex_free_metadata(metadata);
+        /* Note: Preprocessing buffers are conditionally allocated and may not be in scope here.
+         * This is acceptable as JSON output is typically used for debugging/inspection. */
+        return json;
+    }
+
+    /* Run AST-level filters (Pandoc-style JSON filters) before any */
+    /* AST post-processing or rendering. */
+    if (options->ast_filter_commands && options->ast_filter_count > 0) {
+        /* Determine target format string for filters based on output format */
+        const char *target_format = "html";
+        if (options->output_format == APEX_OUTPUT_JSON ||
+            options->output_format == APEX_OUTPUT_JSON_FILTERED) {
+            target_format = "json";
+        } else if (options->output_format == APEX_OUTPUT_MARKDOWN ||
+                   options->output_format == APEX_OUTPUT_MMD ||
+                   options->output_format == APEX_OUTPUT_COMMONMARK ||
+                   options->output_format == APEX_OUTPUT_KRAMDOWN ||
+                   options->output_format == APEX_OUTPUT_GFM) {
+            target_format = "markdown";
+        } else if (options->output_format == APEX_OUTPUT_TERMINAL ||
+                   options->output_format == APEX_OUTPUT_TERMINAL256) {
+            target_format = "terminal";
+        }
+        cmark_node *filtered = apex_run_ast_filters(document, options, target_format);
+        if (!filtered && options->ast_filter_strict) {
+            cmark_node_free(document);
+            if (options->cmark_done) {
+                options->cmark_done(parser, options, cmark_opts, options->cmark_user_data);
+            }
+            cmark_parser_free(parser);
+            free(working_text);
+            apex_free_metadata(metadata);
+            return NULL;
+        }
+        if (filtered && filtered != document) {
+            cmark_node_free(document);
+            document = filtered;
+        }
+    }
+
     /* Postprocess wiki links if enabled */
     if (options->enable_wiki_links) {
         /* Fast path: skip AST walk if no wiki link markers present */
@@ -4990,20 +5492,10 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
         apex_process_callouts_in_tree(document);
     }
 
-    /* Process manual header IDs (MMD [id] and Kramdown {#id}) */
-    if (options->generate_header_ids) {
-        cmark_iter *iter = cmark_iter_new(document);
-        cmark_event_type event;
-        while ((event = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
-            cmark_node *node = cmark_iter_get_node(iter);
-            if (event == CMARK_EVENT_ENTER && cmark_node_get_type(node) == CMARK_NODE_HEADING) {
-                apex_process_manual_header_id(node);
-            }
-        }
-        cmark_iter_free(iter);
-    }
-
-    /* Process IAL (Inline Attribute Lists) if in Kramdown or Unified mode */
+    /* Process IAL (Inline Attribute Lists) BEFORE manual header IDs.
+       IAL handles {: #id}, {#id}, and {.class} - running first ensures these
+       are extracted and removed from heading text before manual header ID
+       looks for MMD [id] or Kramdown {#id}. Avoids duplicate handling. */
     if (alds || options->mode == APEX_MODE_KRAMDOWN || options->mode == APEX_MODE_UNIFIED) {
         /* Fast path: skip AST walk if no IAL markers present */
         /* Check for both Kramdown-style ({:) and Pandoc-style ({# or {.) IALs */
@@ -5016,6 +5508,21 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
         }
     }
 
+    /* Process manual header IDs (MMD [id] and Kramdown {#id}) - after IAL
+       so IAL's {#id} handling doesn't conflict; manual ID handles [id] and
+       any {#id} IAL might have missed (e.g. in multi-child headings) */
+    if (options->generate_header_ids) {
+        cmark_iter *iter = cmark_iter_new(document);
+        cmark_event_type event;
+        while ((event = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
+            cmark_node *node = cmark_iter_get_node(iter);
+            if (event == CMARK_EVENT_ENTER && cmark_node_get_type(node) == CMARK_NODE_HEADING) {
+                apex_process_manual_header_id(node);
+            }
+        }
+        cmark_iter_free(iter);
+    }
+
     /* Apply image attributes to image nodes */
     if (img_attrs) {
         PROFILE_START(image_attrs);
@@ -5030,6 +5537,56 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
 
     /* Note: Critic Markup is now handled via preprocessing (before parsing) */
 
+    /* If output format is JSON (after filters), serialize AST to JSON and return */
+    if (options->output_format == APEX_OUTPUT_JSON_FILTERED) {
+        char *json = apex_cmark_to_pandoc_json(document, options);
+        /* Note: Cleanup happens at end of function - document and other resources
+         * will be freed there. We return the JSON string here. */
+        return json;
+    }
+
+    /* If output format is Markdown, serialize AST to Markdown and return */
+    if (options->output_format == APEX_OUTPUT_MARKDOWN ||
+        options->output_format == APEX_OUTPUT_MMD ||
+        options->output_format == APEX_OUTPUT_COMMONMARK ||
+        options->output_format == APEX_OUTPUT_KRAMDOWN ||
+        options->output_format == APEX_OUTPUT_GFM) {
+        apex_markdown_dialect_t dialect;
+        if (options->output_format == APEX_OUTPUT_MARKDOWN) {
+            dialect = APEX_MD_DIALECT_UNIFIED;
+        } else if (options->output_format == APEX_OUTPUT_MMD) {
+            dialect = APEX_MD_DIALECT_MMD;
+        } else if (options->output_format == APEX_OUTPUT_COMMONMARK) {
+            dialect = APEX_MD_DIALECT_COMMONMARK;
+        } else if (options->output_format == APEX_OUTPUT_KRAMDOWN) {
+            dialect = APEX_MD_DIALECT_KRAMDOWN;
+        } else { /* APEX_OUTPUT_GFM */
+            dialect = APEX_MD_DIALECT_GFM;
+        }
+        char *markdown = apex_cmark_to_markdown(document, options, dialect);
+        /* Note: Cleanup happens at end of function - document and other resources
+         * will be freed there. We return the markdown string here. */
+        return markdown;
+    }
+
+    /* If output format is terminal/terminal256, serialize AST to ANSI terminal and return */
+    if (options->output_format == APEX_OUTPUT_TERMINAL ||
+        options->output_format == APEX_OUTPUT_TERMINAL256) {
+        bool use_256 = (options->output_format == APEX_OUTPUT_TERMINAL256);
+        char *tty = apex_cmark_to_terminal(document, options, use_256);
+        return tty;
+    }
+
+    /* If output format is man (roff) or man-html, serialize AST and return */
+    if (options->output_format == APEX_OUTPUT_MAN) {
+        char *roff = apex_cmark_to_man_roff(document, options);
+        return roff ? roff : strdup(".TH stub 1 \"\" \"\"\n");
+    }
+    if (options->output_format == APEX_OUTPUT_MAN_HTML) {
+        char *man_html = apex_cmark_to_man_html(document, options);
+        return man_html ? man_html : strdup("<!DOCTYPE html><html><body><p>stub</p></body></html>");
+    }
+
     /* Render to HTML
      * Use custom renderer when we have attributes (IAL, ALDs, or image attributes)
      * Otherwise use standard renderer
@@ -5167,6 +5724,7 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
     char *html_footer_metadata = NULL;
     char *language_metadata = NULL;
     char *quotes_lang_metadata = NULL;
+    char *generic_meta_tags = NULL;
     int base_header_level = 1;  /* Default is 1 */
 
     if (metadata) {
@@ -5215,6 +5773,9 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
                 base_header_level = (int)level;
             }
         }
+
+        /* Collect remaining metadata as generic head meta tags. */
+        generic_meta_tags = apex_render_generic_meta_tags(metadata);
     }
 
     /* Adjust header levels and quote language based on metadata */
@@ -5240,6 +5801,21 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
         }
     }
 
+    /* Expand auto media (discover formats from filesystem for img with auto attribute).
+     * Use base_directory when set (e.g. from file path or metadata); otherwise use "."
+     * so auto expansion runs when piping stdin (images resolved relative to cwd). */
+    if (html && strstr(html, "data-apex-replace-auto=1")) {
+        PROFILE_START(expand_auto_media);
+        const char *base = options->base_directory && options->base_directory[0]
+            ? options->base_directory : ".";
+        char *expanded = apex_expand_auto_media(html, base);
+        PROFILE_END(expand_auto_media);
+        if (expanded) {
+            free(html);
+            html = expanded;
+        }
+    }
+
     /* Convert images to figures with captions (caption="..." always wraps; otherwise when enable_image_captions) */
     if (html) {
         PROFILE_START(image_captions);
@@ -5251,6 +5827,24 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
         }
     }
 
+    /* Strip redundant <p> around single <img> inside <figure> (e.g. from ::: >figure with "< ![Image](...)") */
+    if (html) {
+        char *stripped = apex_strip_figure_paragraph_wrapper(html);
+        if (stripped) {
+            free(html);
+            html = stripped;
+        }
+    }
+
+    /* Strip <p> that wraps only a single block element (figure, video, picture) - invalid HTML5 */
+    if (html) {
+        char *stripped = apex_strip_block_paragraph_wrapper(html);
+        if (stripped) {
+            free(html);
+            html = stripped;
+        }
+    }
+
     /* Inject header IDs if enabled */
     if (options->generate_header_ids && html) {
         PROFILE_START(header_ids);
@@ -5325,7 +5919,13 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
     /* Apply external syntax highlighting if requested */
     if (options->code_highlighter && html) {
         PROFILE_START(syntax_highlight);
-        char *highlighted = apex_apply_syntax_highlighting(html, options->code_highlighter, options->code_line_numbers, options->highlight_language_only);
+        bool ansi_out = (options->output_format == APEX_OUTPUT_TERMINAL || options->output_format == APEX_OUTPUT_TERMINAL256);
+        char *highlighted = apex_apply_syntax_highlighting(html,
+                                                           options->code_highlighter,
+                                                           options->code_line_numbers,
+                                                           options->highlight_language_only,
+                                                           ansi_out,
+                                                           options->code_highlight_theme);
         PROFILE_END(syntax_highlight);
         if (highlighted && highlighted != html) {
             free(html);
@@ -5480,6 +6080,9 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
 
     /* Clean up */
     cmark_node_free(document);
+    if (options->cmark_done) {
+        options->cmark_done(parser, options, cmark_opts, options->cmark_user_data);
+    }
     cmark_parser_free(parser);
     free(working_text);
     if (ial_preprocessed) free(ial_preprocessed);
@@ -5627,10 +6230,36 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
 
         const char *footer_to_use = footer_with_scripts ? footer_with_scripts : html_footer_metadata;
 
+        /* Combine generated generic meta tags with any explicit HTML Header metadata. */
+        char *combined_head_metadata = NULL;
+        const char *head_to_use = html_header_metadata;
+        if (generic_meta_tags || html_header_metadata) {
+            size_t generic_len = generic_meta_tags ? strlen(generic_meta_tags) : 0;
+            size_t header_len = html_header_metadata ? strlen(html_header_metadata) : 0;
+            size_t newline_len = (generic_len > 0 && header_len > 0) ? 1 : 0;
+            combined_head_metadata = malloc(generic_len + newline_len + header_len + 1);
+            if (combined_head_metadata) {
+                size_t pos = 0;
+                if (generic_len > 0) {
+                    memcpy(combined_head_metadata + pos, generic_meta_tags, generic_len);
+                    pos += generic_len;
+                }
+                if (newline_len) {
+                    combined_head_metadata[pos++] = '\n';
+                }
+                if (header_len > 0) {
+                    memcpy(combined_head_metadata + pos, html_header_metadata, header_len);
+                    pos += header_len;
+                }
+                combined_head_metadata[pos] = '\0';
+                head_to_use = combined_head_metadata;
+            }
+        }
+
         PROFILE_START(standalone_wrap);
         char *document = apex_wrap_html_document(html, local_opts.document_title, css_paths, css_count,
-                                                 local_opts.code_highlighter, html_header_metadata, footer_to_use,
-                                                 language_metadata);
+                                                 local_opts.code_highlighter, head_to_use, footer_to_use,
+                                                 language_metadata, local_opts.strict_xhtml);
         PROFILE_END(standalone_wrap);
 
         /* Free temporary metadata stylesheet array if we allocated it */
@@ -5645,6 +6274,9 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
         if (footer_with_scripts) {
             free(footer_with_scripts);
         }
+        if (combined_head_metadata) {
+            free(combined_head_metadata);
+        }
 
         /* If requested, replace stylesheet links with embedded CSS contents */
         if (html && css_paths && css_count > 0 && local_opts.embed_stylesheet) {
@@ -5773,6 +6405,7 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
     if (html_footer_metadata) free(html_footer_metadata);
     if (language_metadata) free(language_metadata);
     if (quotes_lang_metadata) free(quotes_lang_metadata);
+    if (generic_meta_tags) free(generic_meta_tags);
     if (h1_title) free(h1_title);
 
     /* Remove blank lines within tables (applies to both pretty and non-pretty) */
@@ -5810,6 +6443,17 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
         }
     }
 
+    /* XHTML-style void elements (--xhtml / --strict-xhtml); run after pretty-print (HTML only) */
+    if (local_opts.xhtml && html && options->output_format == APEX_OUTPUT_HTML) {
+        PROFILE_START(xhtml_void_tags);
+        char *xhtml_out = apex_html_apply_xhtml_void_tags(html);
+        PROFILE_END(xhtml_void_tags);
+        if (xhtml_out) {
+            free(html);
+            html = xhtml_out;
+        }
+    }
+
     PROFILE_END(total);
 
     if (profiling_enabled()) {
@@ -5822,7 +6466,7 @@ char *apex_markdown_to_html(const char *markdown, size_t len, const apex_options
 /**
  * Wrap HTML content in complete HTML5 document structure
  */
-char *apex_wrap_html_document(const char *content, const char *title, const char **stylesheet_paths, size_t stylesheet_count, const char *code_highlighter, const char *html_header, const char *html_footer, const char *language) {
+char *apex_wrap_html_document(const char *content, const char *title, const char **stylesheet_paths, size_t stylesheet_count, const char *code_highlighter, const char *html_header, const char *html_footer, const char *language, bool strict_xhtml) {
     if (!content) return NULL;
 
     const char *doc_title = title ? title : "Document";
@@ -5906,10 +6550,19 @@ char *apex_wrap_html_document(const char *content, const char *title, const char
     const char *version_str = APEX_VERSION_STRING;
     if (!version_str) version_str = "unknown";
 
-    /* HTML5 doctype and opening */
+    /* HTML5 doctype and opening (polyglot XHTML when strict_xhtml) */
     /* Add body class if code highlighting is enabled */
     const char *body_class = code_highlighter ? " class=\"code-highlighted\"" : "";
-    int n = snprintf(write, remaining, "<!DOCTYPE html>\n<html lang=\"%s\">\n<head>\n", lang);
+    int n;
+    if (strict_xhtml) {
+        n = snprintf(write, remaining,
+                     "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                     "<!DOCTYPE html>\n"
+                     "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"%s\" lang=\"%s\">\n<head>\n",
+                     lang, lang);
+    } else {
+        n = snprintf(write, remaining, "<!DOCTYPE html>\n<html lang=\"%s\">\n<head>\n", lang);
+    }
     if (n < 0 || (size_t)n >= remaining) {
         free(output);
         return strdup(content);
@@ -5918,7 +6571,12 @@ char *apex_wrap_html_document(const char *content, const char *title, const char
     remaining -= n;
 
     /* Meta tags */
-    n = snprintf(write, remaining, "  <meta charset=\"UTF-8\">\n");
+    if (strict_xhtml) {
+        n = snprintf(write, remaining,
+                     "  <meta http-equiv=\"Content-Type\" content=\"application/xhtml+xml; charset=UTF-8\" />\n");
+    } else {
+        n = snprintf(write, remaining, "  <meta charset=\"UTF-8\">\n");
+    }
     if (n < 0 || (size_t)n >= remaining) {
         free(output);
         return strdup(content);