@zosmaai/pi-llm-wiki 0.2.2 → 0.3.0

package/extensions/llm-wiki/lib/source-packet.ts

@@ -206,12 +206,20 @@ export async function captureFile(
   mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
 
-  const isPdf = filePath.toLowerCase().endsWith(".pdf");
+  const lowerPath = filePath.toLowerCase();
+  const isPdf = lowerPath.endsWith(".pdf");
+  const isXml = lowerPath.endsWith(".xml");
   const content = isPdf ? "" : readText(filePath);
   const fileName = filePath.split("/").pop() || "unknown";
 
-  // Try MarkItDown for PDFs.
   let extracted = content;
+
+  // Convert XML to markdown
+  if (isXml && content) {
+    extracted = xmlToMarkdown(content);
+  }
+
+  // Try MarkItDown for PDFs.
   if (isPdf) {
     const markitdown = await exec(
       pi,
@@ -363,12 +371,58 @@ status: skeleton
 `;
 }
 
+/** Basic XML to markdown conversion: strip tags while preserving text structure. */
+function xmlToMarkdown(xml: string): string {
+  // Extract title from first <title> or root element
+  let title = "";
+  const titleMatch = xml.match(/<title[^>]*>([^<]*)<\/title>/i);
+  if (titleMatch) title = titleMatch[1].trim();
+
+  // Strip XML declaration and doctype
+  let text = xml.replace(/<\?xml[^>]*\?>\s*/gi, "");
+  text = text.replace(/<!DOCTYPE[^>]*>\s*/gi, "");
+
+  // Replace block-level tags with newlines
+  text = text.replace(/<\/(p|div|section|article|li|h\d|tr|blockquote|pre)>/gi, "\n");
+  text = text.replace(/<br\s*\/?>/gi, "\n");
+
+  // Strip remaining tags — match < followed by tag name characters to >
+  // Using a loop to handle malformed/broken tags that lack a closing >
+  let prev = "";
+  while (prev !== text) {
+    prev = text;
+    text = text.replace(/<[a-zA-Z\/!?][^>]*>/g, "");
+  }
+  // Remove any stray < that didn't form a complete tag
+  text = text.replace(/</g, "");
+
+  // Decode XML entities in a single pass to avoid double-unescaping
+  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
+    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
+    const lower = entity.toLowerCase();
+    if (map[lower]) return map[lower];
+    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
+    return entity;
+  });
+
+  // Clean up excessive blank lines
+  text = text.replace(/\n{3,}/g, "\n\n").trim();
+
+  if (!text) return xml; // fallback: return raw if stripping produced nothing
+
+  const lines = [];
+  if (title) lines.push(`# ${title}\n`);
+  lines.push(text);
+  return lines.join("\n\n");
+}
+
 function guessFormat(filePath: string): string {
   const lower = filePath.toLowerCase();
   if (lower.endsWith(".pdf")) return "pdf";
   if (lower.endsWith(".md")) return "markdown";
   if (lower.endsWith(".txt")) return "text";
   if (lower.endsWith(".html") || lower.endsWith(".htm")) return "html";
+  if (lower.endsWith(".xml")) return "xml";
   if (lower.endsWith(".docx")) return "docx";
   return "file";
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
   "keywords": [
     "pi-package",

package/test/llm-wiki.test.ts

@@ -388,6 +388,56 @@ describe("source packet capture", () => {
     // Original file should be preserved
     expect(existsSync(join(result.packetPath, "original", "notes.md"))).toBe(true);
   });
+
+  it("should convert XML files to readable markdown in extracted.md", async () => {
+    const paths = makePaths();
+    const xmlContent = `<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <title>Project Report</title>
+  <section>
+    <heading>Findings</heading>
+    <p>The analysis revealed several key insights.</p>
+    <list>
+      <item>First finding</item>
+      <item>Second finding</item>
+    </list>
+  </section>
+</document>`;
+    const xmlPath = join(tempDir, "report.xml");
+    writeFileSync(xmlPath, xmlContent, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, xmlPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    // Should have extracted title
+    expect(extracted).toContain("Project Report");
+    // Should have extracted text content
+    expect(extracted).toContain("The analysis revealed several key insights.");
+    expect(extracted).toContain("First finding");
+    expect(extracted).toContain("Second finding");
+    // Should NOT contain raw XML tags
+    expect(extracted).not.toContain("<?xml");
+    expect(extracted).not.toContain("<document>");
+    expect(extracted).not.toContain("</document>");
+
+    // Original file should be preserved
+    expect(existsSync(join(result.packetPath, "original", "report.xml"))).toBe(true);
+  });
+
+  it("should fall back to raw XML content when tag stripping produces nothing", async () => {
+    const paths = makePaths();
+    const xmlContent = `<?xml version="1.0"?><data><![CDATA[Hello]]></data>`;
+    const xmlPath = join(tempDir, "minimal.xml");
+    writeFileSync(xmlPath, xmlContent, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, xmlPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    // Should have the text content at minimum
+    expect(extracted).toContain("Hello");
+  });
 });
 
 describe("wiki directory structure", () => {