@nataliapc/mcp-openmsx 1.2.5 → 1.2.6

package/resources/sdcc/lyx2md.py

@@ -0,0 +1,745 @@
+#!/usr/bin/env python3
+"""Convert a LyX file to Markdown, preserving math and formatting."""
+
+import re
+import sys
+
+
+def parse_lyx(filepath):
+    """Read a LyX file and return list of lines."""
+    with open(filepath, "r", encoding="utf-8", errors="replace") as f:
+        return f.readlines()
+
+
+class LyxToMarkdown:
+    def __init__(self):
+        self.output = []
+        self.lines = []
+        self.pos = 0
+        self.in_body = False
+        self.enumerate_counter = 0
+        self.label_map = {}  # label name -> section title
+        self.current_section = ""
+        # Stack of (layout_context, accumulated_text) for nested layouts
+        self.layout_stack = []
+        self.current_text = ""
+        self._layout_context = "unknown"
+
+    def convert(self, filepath):
+        self.lines = parse_lyx(filepath)
+        self.pos = 0
+
+        # First pass: collect labels and map them to section titles
+        self._collect_labels()
+
+        # Second pass: convert
+        self.pos = 0
+        self.in_body = False
+        while self.pos < len(self.lines):
+            line = self.lines[self.pos].rstrip("\n")
+            self._process_line(line)
+            self.pos += 1
+
+        return "\n".join(self.output)
+
+    def _collect_labels(self):
+        """First pass to map label names to nearby section titles."""
+        # We'll look for label names and associate them with the most recent
+        # section/chapter heading text
+        current_heading = ""
+        for i, line in enumerate(self.lines):
+            line = line.strip()
+            # Detect heading layouts
+            m = re.match(
+                r"\\begin_layout (Chapter\*?|Section|Subsection|Subsubsection)",
+                line,
+            )
+            if m:
+                # Next non-blank, non-directive line is the title
+                for j in range(i + 1, min(i + 5, len(self.lines))):
+                    candidate = self.lines[j].strip()
+                    if (
+                        candidate
+                        and not candidate.startswith("\\")
+                        and not candidate.startswith("<")
+                    ):
+                        current_heading = candidate
+                        break
+            # Detect label names
+            if line.startswith('name "') and line.endswith('"'):
+                label_name = line[6:-1]
+                if label_name and current_heading:
+                    self.label_map[label_name] = current_heading
+            # Also handle Float/Table captions for label association
+            m2 = re.match(r"\\begin_inset Float", line)
+            if m2:
+                # Look ahead for caption text
+                for j in range(i + 1, min(i + 50, len(self.lines))):
+                    cline = self.lines[j].strip()
+                    if cline == "\\begin_inset Caption":
+                        for k in range(j + 1, min(j + 10, len(self.lines))):
+                            capline = self.lines[k].strip()
+                            if (
+                                capline
+                                and not capline.startswith("\\")
+                                and not capline.startswith("<")
+                            ):
+                                current_heading = capline
+                                break
+                        break
+
+    def _process_line(self, line):
+        stripped = line.strip()
+
+        if stripped == "\\begin_body":
+            self.in_body = True
+            return
+        if not self.in_body:
+            return
+        if stripped == "\\end_body":
+            self.in_body = False
+            return
+
+        # Layout begin
+        if stripped.startswith("\\begin_layout "):
+            layout_type = stripped[len("\\begin_layout ") :]
+            self._handle_layout_begin(layout_type)
+            return
+
+        if stripped == "\\end_layout":
+            self._handle_layout_end()
+            return
+
+        # Inset begin
+        if stripped.startswith("\\begin_inset "):
+            inset_spec = stripped[len("\\begin_inset ") :]
+            self._handle_inset_begin(inset_spec)
+            return
+
+        if stripped == "\\end_inset":
+            # Most end_insets are no-ops for content accumulation
+            return
+
+        # Skip status/alignment/formatting directives
+        if stripped in ("status open", "status collapsed", "status inlined"):
+            return
+        if stripped.startswith("\\align "):
+            return
+        if stripped.startswith("\\noindent"):
+            return
+        if stripped.startswith("\\paragraph_spacing"):
+            return
+        if stripped.startswith("wide ") or stripped.startswith("sideways "):
+            return
+
+        # Formatting commands - inline markers
+        # We use \x01 as a "soft space" after closing markers - it becomes a space
+        # if the next character is a word char, or is removed before punctuation.
+        if stripped.startswith("\\emph "):
+            val = stripped.split()[1]
+            if val == "on":
+                if self.current_text and not self.current_text.endswith(
+                    (" ", "\n", "(", "[", "\u201c", "\x01")
+                ):
+                    self.current_text += " "
+                self.current_text += "*"
+            elif val in ("default", "off"):
+                if self.current_text.endswith(" "):
+                    self.current_text = self.current_text[:-1]
+                self.current_text += "*\x01"
+            return
+
+        if stripped.startswith("\\series "):
+            val = stripped.split()[1]
+            if val == "bold":
+                if self.current_text and not self.current_text.endswith(
+                    (" ", "\n", "(", "[", "\u201c", "\x01")
+                ):
+                    self.current_text += " "
+                self.current_text += "**"
+            elif val in ("default", "medium"):
+                if self.current_text.endswith(" "):
+                    self.current_text = self.current_text[:-1]
+                self.current_text += "**\x01"
+            return
+
+        if stripped.startswith("\\shape "):
+            val = stripped.split()[1]
+            if val == "italic":
+                if self.current_text and not self.current_text.endswith(
+                    (" ", "\n", "(", "[", "\u201c", "\x01")
+                ):
+                    self.current_text += " "
+                self.current_text += "*"
+            elif val in ("default", "up"):
+                if self.current_text.endswith(" "):
+                    self.current_text = self.current_text[:-1]
+                self.current_text += "*\x01"
+            return
+
+        # Skip other formatting directives
+        for prefix in (
+            "\\noun ",
+            "\\bar ",
+            "\\color ",
+            "\\size ",
+            "\\family ",
+            "\\strikeout",
+        ):
+            if stripped.startswith(prefix):
+                return
+
+        # Backslash escape
+        if stripped == "\\backslash":
+            self.current_text += "\\"
+            return
+
+        # Skip deeper/end_deeper
+        if stripped in ("\\begin_deeper", "\\end_deeper"):
+            return
+
+        # Skip various LyX preamble/header directives that might leak
+        skip_prefixes = (
+            "\\lyxformat",
+            "\\papersize",
+            "\\use_",
+            "\\font_",
+            "\\graphics",
+            "\\paper",
+            "\\spacing",
+            "\\cite_",
+            "\\secnumdepth",
+            "\\tocdepth",
+            "\\paragraph_",
+            "\\defskip",
+            "\\quotes_",
+            "\\tracking_",
+            "\\output_",
+            "\\author",
+            "\\textclass",
+            "\\language",
+            "\\inputencoding",
+            "\\default_",
+            "\\maintain_",
+        )
+        if stripped.startswith("\\"):
+            for prefix in skip_prefixes:
+                if stripped.startswith(prefix):
+                    return
+            # Other backslash commands we don't recognize - skip silently
+            if not stripped.startswith("\\begin") and not stripped.startswith("\\end"):
+                return
+
+        # Regular text content
+        if stripped and not stripped.startswith("<"):
+            # Handle soft space marker: convert to real space before words,
+            # remove before punctuation
+            if self.current_text.endswith("\x01"):
+                self.current_text = self.current_text[:-1]  # remove marker
+                if stripped[0] not in ".,;:!?)-]":
+                    self.current_text += " "
+            # Add space before text if current_text doesn't end with whitespace/markers
+            # and the text doesn't start with punctuation
+            elif (
+                self.current_text
+                and not self.current_text.endswith((" ", "\n", "(", "[", "*"))
+                and stripped[0] not in ".,;:!?)-]"
+            ):
+                self.current_text += " "
+            self.current_text += stripped
+        # Skip XML-like tags
+
+    def _handle_layout_begin(self, layout_type):
+        """Handle the beginning of a layout, pushing state onto the stack."""
+        # Push current state
+        self.layout_stack.append((self._layout_context, self.current_text))
+
+        # Determine new context
+        if layout_type == "Title":
+            ctx = "title"
+        elif layout_type == "Author":
+            ctx = "author"
+        elif layout_type.startswith("Chapter"):
+            ctx = "chapter"
+        elif layout_type == "Section":
+            ctx = "section"
+        elif layout_type == "Subsection":
+            ctx = "subsection"
+        elif layout_type == "Subsubsection":
+            ctx = "subsubsection"
+        elif layout_type == "Enumerate":
+            ctx = "enumerate"
+            self.enumerate_counter += 1
+        elif layout_type == "Itemize":
+            ctx = "itemize"
+        elif layout_type == "Standard":
+            ctx = "standard"
+        elif layout_type == "Plain" or layout_type.startswith("Plain "):
+            ctx = "plain"
+        else:
+            ctx = "unknown"
+
+        self._layout_context = ctx
+        self.current_text = ""
+
+    def _handle_layout_end(self):
+        """Handle end of layout, popping state from the stack."""
+        text = self._clean_text(self.current_text)
+        ctx = self._layout_context
+
+        # Pop parent state
+        if self.layout_stack:
+            parent_ctx, parent_text = self.layout_stack.pop()
+        else:
+            parent_ctx, parent_text = "unknown", ""
+
+        # Emit content based on context
+        if ctx == "title":
+            self.output.append(f"# {text}")
+            self.output.append("")
+        elif ctx == "author":
+            self.output.append(f"*{text}*")
+            self.output.append("")
+        elif ctx == "chapter":
+            self.output.append(f"## {text}")
+            self.output.append("")
+            self.current_section = text
+            self.enumerate_counter = 0
+        elif ctx == "section":
+            self.output.append(f"### {text}")
+            self.output.append("")
+            self.current_section = text
+            self.enumerate_counter = 0
+        elif ctx == "subsection":
+            self.output.append(f"#### {text}")
+            self.output.append("")
+            self.current_section = text
+            self.enumerate_counter = 0
+        elif ctx == "subsubsection":
+            self.output.append(f"##### {text}")
+            self.output.append("")
+            self.current_section = text
+            self.enumerate_counter = 0
+        elif ctx == "enumerate":
+            self.output.append(f"{self.enumerate_counter}. {text}")
+        elif ctx == "itemize":
+            self.output.append(f"- {text}")
+        elif ctx == "standard":
+            if text.strip():
+                self.output.append(text)
+                self.output.append("")
+        elif ctx == "plain":
+            # Plain layout inside insets - push text back to parent
+            if text.strip():
+                parent_text += " " + text
+        elif ctx == "unknown":
+            if text.strip():
+                self.output.append(text)
+                self.output.append("")
+
+        # Restore parent state
+        self._layout_context = parent_ctx
+        self.current_text = parent_text
+
+    def _handle_inset_begin(self, inset_spec):
+        if inset_spec.startswith("Formula"):
+            self._handle_formula(inset_spec)
+        elif inset_spec.startswith("Quotes"):
+            self._handle_quotes(inset_spec)
+        elif inset_spec.startswith("Newline"):
+            self.current_text += "  \n"
+        elif inset_spec.startswith("Newpage"):
+            pass
+        elif inset_spec.startswith("CommandInset label"):
+            self._handle_label()
+        elif inset_spec.startswith("CommandInset ref"):
+            self._handle_ref()
+        elif inset_spec.startswith("Graphics"):
+            self._handle_graphics()
+        elif inset_spec.startswith("Float"):
+            pass  # Float is just a container; nested layouts handle content
+        elif inset_spec.startswith("Caption"):
+            # Mark that we're in a caption so the plain layout can format it
+            self.current_text += "\n\n*Caption:* "
+        elif inset_spec.startswith("Tabular"):
+            self._handle_tabular()
+        elif inset_spec.startswith("Box"):
+            self._handle_box()
+        elif inset_spec.startswith("Text"):
+            pass  # Text inset is a container
+        elif inset_spec.startswith("ERT"):
+            self._handle_ert()
+
+    def _handle_box(self):
+        """Consume Box inset parameters (position, hor_pos, etc.)."""
+        # Box parameters appear on separate lines until 'status open/collapsed'
+        # We just skip them; the content inside will be handled by nested layouts
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            nline = self.lines[self.pos].strip()
+            if nline.startswith("status "):
+                return  # done consuming parameters
+            if nline.startswith("\\begin_layout") or nline == "\\end_inset":
+                # Oops, went too far - back up
+                self.pos -= 1
+                return
+
+    def _handle_formula(self, inset_spec):
+        """Handle Formula insets - both inline and display math."""
+        formula_start = inset_spec[len("Formula ") :]
+
+        if formula_start.startswith("$"):
+            # Inline math
+            content = formula_start.strip()
+            while self.pos + 1 < len(self.lines):
+                self.pos += 1
+                nline = self.lines[self.pos].strip()
+                if nline == "\\end_inset":
+                    break
+                if nline:
+                    content += nline
+            # Handle soft space marker before math
+            if self.current_text.endswith("\x01"):
+                self.current_text = self.current_text[:-1]
+                self.current_text += " "
+            # Ensure space before inline math if needed
+            elif (
+                self.current_text
+                and not self.current_text.endswith((" ", "\n", "(", "[", "*"))
+            ):
+                self.current_text += " "
+            self.current_text += content
+            # Peek ahead to add space after if next char is text
+            # (handled by the text accumulation logic)
+
+        elif formula_start.startswith("\\["):
+            # Display math: \[...\]
+            math_content = formula_start.strip()
+            while self.pos + 1 < len(self.lines):
+                self.pos += 1
+                nline = self.lines[self.pos].strip()
+                if nline == "\\end_inset":
+                    break
+                if nline:
+                    math_content += nline
+            # Strip \[ and \]
+            math_content = math_content.strip()
+            if math_content.startswith("\\["):
+                math_content = math_content[2:]
+            if math_content.endswith("\\]"):
+                math_content = math_content[:-2]
+            self.current_text += "\n\n$$\n" + math_content.strip() + "\n$$\n\n"
+
+        elif formula_start.startswith("\\begin{"):
+            # LaTeX environment
+            math_content = formula_start.strip()
+            while self.pos + 1 < len(self.lines):
+                self.pos += 1
+                nline = self.lines[self.pos].strip()
+                if nline == "\\end_inset":
+                    break
+                if nline:
+                    math_content += "\n" + nline
+            math_content = self._convert_latex_env(math_content)
+            self.current_text += "\n\n$$\n" + math_content.strip() + "\n$$\n\n"
+
+        else:
+            # Unknown formula type
+            math_content = formula_start
+            while self.pos + 1 < len(self.lines):
+                self.pos += 1
+                nline = self.lines[self.pos].strip()
+                if nline == "\\end_inset":
+                    break
+                if nline:
+                    math_content += nline
+            self.current_text += math_content
+
+    def _convert_latex_env(self, content):
+        """Convert LaTeX environments to simpler display math."""
+        content = re.sub(r"\\begin\{eqnarray\*?\}", "", content)
+        content = re.sub(r"\\end\{eqnarray\*?\}", "", content)
+        content = re.sub(r"\\begin\{align\*?\}", "", content)
+        content = re.sub(r"\\end\{align\*?\}", "", content)
+        content = re.sub(r"\\begin\{equation\*?\}", "", content)
+        content = re.sub(r"\\end\{equation\*?\}", "", content)
+        content = re.sub(r"\\begin\{gathered\}", "", content)
+        content = re.sub(r"\\end\{gathered\}", "", content)
+        content = re.sub(r"\\begin\{array\}\{[^}]*\}", "", content)
+        content = re.sub(r"\\end\{array\}", "", content)
+        # Remove \label{...} from display math
+        content = re.sub(r"\\label\{[^}]*\}", "", content)
+        return content.strip()
+
+    def _handle_quotes(self, inset_spec):
+        """Handle quote insets."""
+        if "eld" in inset_spec:
+            self.current_text += "\u201c"
+        elif "erd" in inset_spec:
+            self.current_text += "\u201d"
+        elif "els" in inset_spec:
+            self.current_text += "\u2018"
+        elif "ers" in inset_spec:
+            self.current_text += "\u2019"
+        else:
+            self.current_text += '"'
+        # Consume until end_inset
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            if self.lines[self.pos].strip() == "\\end_inset":
+                return
+
+    def _handle_label(self):
+        """Handle CommandInset label - skip it."""
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            if self.lines[self.pos].strip() == "\\end_inset":
+                return
+
+    def _handle_ref(self):
+        """Handle CommandInset ref - cross-references."""
+        ref_name = ""
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            nline = self.lines[self.pos].strip()
+            if nline.startswith("reference "):
+                ref_name = nline.split('"')[1] if '"' in nline else ""
+            if nline == "\\end_inset":
+                break
+        if ref_name and ref_name in self.label_map:
+            resolved = self.label_map[ref_name]
+            # Ensure space before ref text
+            if self.current_text and not self.current_text.endswith(
+                (" ", "\n", "(", "[")
+            ):
+                self.current_text += " "
+            self.current_text += resolved
+        # If unresolved, just skip
+
+    def _handle_graphics(self):
+        """Handle Graphics inset."""
+        filename = ""
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            nline = self.lines[self.pos].strip()
+            if nline.startswith("filename "):
+                filename = nline[len("filename ") :]
+            if nline == "\\end_inset":
+                break
+        if filename:
+            self.current_text += f"\n\n![{filename}]({filename})\n\n"
+
+    def _handle_tabular(self):
+        """Handle Tabular inset - parse lyxtabular format."""
+        rows = []
+        current_row = []
+        current_cell_text = ""
+        in_cell = False
+        depth = 1
+
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            nline = self.lines[self.pos].strip()
+
+            if nline == "\\end_inset":
+                depth -= 1
+                if depth <= 0:
+                    break
+                continue
+
+            if nline.startswith("\\begin_inset "):
+                depth += 1
+                if nline.startswith("\\begin_inset Formula"):
+                    formula_spec = nline[len("\\begin_inset ") :]
+                    formula_text = formula_spec[len("Formula ") :]
+                    if formula_text.startswith("$"):
+                        current_cell_text += formula_text.strip()
+                        while self.pos + 1 < len(self.lines):
+                            self.pos += 1
+                            fl = self.lines[self.pos].strip()
+                            if fl == "\\end_inset":
+                                depth -= 1
+                                break
+                            if fl:
+                                current_cell_text += fl
+                    else:
+                        math = formula_text
+                        while self.pos + 1 < len(self.lines):
+                            self.pos += 1
+                            fl = self.lines[self.pos].strip()
+                            if fl == "\\end_inset":
+                                depth -= 1
+                                break
+                            if fl:
+                                math += " " + fl
+                        current_cell_text += math.strip()
+                elif nline.startswith("\\begin_inset Text"):
+                    in_cell = True
+                    current_cell_text = ""
+                elif nline.startswith("\\begin_inset ERT"):
+                    # Skip ERT (usually \hline) inside tables
+                    ert_depth = 1
+                    while self.pos + 1 < len(self.lines):
+                        self.pos += 1
+                        el = self.lines[self.pos].strip()
+                        if el.startswith("\\begin_inset"):
+                            ert_depth += 1
+                        if el == "\\end_inset":
+                            ert_depth -= 1
+                            if ert_depth <= 0:
+                                depth -= 1  # balance the depth++ above
+                                break
+                continue
+
+            if nline.startswith("<row"):
+                current_row = []
+            elif nline.startswith("</row"):
+                if current_row:
+                    rows.append(current_row)
+            elif nline.startswith("<cell"):
+                current_cell_text = ""
+                in_cell = True
+            elif nline.startswith("</cell"):
+                current_row.append(current_cell_text.strip())
+                in_cell = False
+            elif nline.startswith("</lyxtabular"):
+                if current_row:
+                    rows.append(current_row)
+                break
+            elif nline.startswith("<"):
+                continue
+            elif in_cell and nline and not nline.startswith("\\"):
+                if current_cell_text and not current_cell_text.endswith(" "):
+                    current_cell_text += " "
+                current_cell_text += nline
+
+        if rows:
+            self._emit_markdown_table(rows)
+
+    def _emit_markdown_table(self, rows):
+        """Convert rows to markdown table format."""
+        if not rows:
+            return
+        num_cols = max(len(row) for row in rows)
+        for row in rows:
+            while len(row) < num_cols:
+                row.append("")
+
+        self.current_text += "\n\n"
+        self.current_text += "| " + " | ".join(rows[0]) + " |\n"
+        self.current_text += "| " + " | ".join(["---"] * num_cols) + " |\n"
+        for row in rows[1:]:
+            self.current_text += "| " + " | ".join(row) + " |\n"
+        self.current_text += "\n"
+
+    def _handle_ert(self):
+        """Handle ERT (Evil Red Text = raw LaTeX) insets - skip."""
+        depth = 1
+        while self.pos + 1 < len(self.lines):
+            self.pos += 1
+            nline = self.lines[self.pos].strip()
+            if nline.startswith("\\begin_inset"):
+                depth += 1
+            if nline == "\\end_inset":
+                depth -= 1
+                if depth <= 0:
+                    return
+
+    def _clean_text(self, text):
+        """Clean up accumulated text."""
+        # Remove any leftover soft space markers
+        text = text.replace("\x01", " ")
+        text = text.strip()
+        # Collapse multiple spaces (but preserve newlines for display math)
+        lines = text.split("\n")
+        lines = [re.sub(r"[ \t]+", " ", line) for line in lines]
+        text = "\n".join(lines)
+        # Fix spaces before punctuation (but not inside math)
+        # Only fix spaces before punctuation outside of $ delimiters
+        text = re.sub(r" ([.,;:!?)\]])", r"\1", text)
+        # Fix space after opening paren/bracket
+        text = re.sub(r"([\[(]) ", r"\1", text)
+        # Fix double spaces
+        text = re.sub(r"  +", " ", text)
+        return text
+
+
+def clean_markdown(text):
+    """Post-process the markdown for cleanliness."""
+    lines = text.split("\n")
+    result = []
+    prev_blank = False
+
+    for line in lines:
+        is_blank = line.strip() == ""
+
+        if is_blank:
+            if prev_blank:
+                continue
+            prev_blank = True
+        else:
+            prev_blank = False
+
+        result.append(line)
+
+    text = "\n".join(result)
+
+    # Clean up [?...?] comment markers -> just content
+    text = re.sub(r"\[\?", "", text)
+    text = re.sub(r"\?\]", "", text)
+
+    # Clean up stray formatting
+    text = re.sub(r"\n{3,}", "\n\n", text)
+
+    # Remove spurious "LatexCommand label" or "LatexCommand ref" remnants
+    text = re.sub(r"\(LatexCommand label\)", "", text)
+    text = re.sub(r"\(LatexCommand ref\)", "", text)
+    text = re.sub(r"LatexCommand label", "", text)
+    text = re.sub(r"LatexCommand ref", "", text)
+
+    # Fix space after closing inline math before word chars
+    # Match: content$word but NOT $$word (display math)
+    # Use lookbehind for non-$ char, then $ not followed by $, then word char
+    # This catches closing $ only (opening $ is preceded by space or start)
+    # Actually this is too error-prone - skip it and rely on the converter
+
+    # Remove empty emphasis markers
+    text = re.sub(r"\*\* \*\*", "", text)
+    text = re.sub(r"\* \*", "", text)
+    text = text.replace("****", "")
+
+    # Ensure blank line before headings
+    text = re.sub(r"([^\n])\n(#{2,} )", r"\1\n\n\2", text)
+
+    # Ensure no trailing whitespace on lines
+    lines = text.split("\n")
+    lines = [line.rstrip() for line in lines]
+    text = "\n".join(lines)
+
+    return text.strip() + "\n"
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: lyx2md.py <input.lyx> [output.md]")
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    if len(sys.argv) > 2:
+        output_file = sys.argv[2]
+    else:
+        output_file = input_file.rsplit(".", 1)[0] + ".md"
+
+    converter = LyxToMarkdown()
+    result = converter.convert(input_file)
+    result = clean_markdown(result)
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        f.write(result)
+
+    print(f"Converted {input_file} -> {output_file}")
+    print(f"Output: {len(result)} characters, {result.count(chr(10))} lines")
+
+
+if __name__ == "__main__":
+    main()