nx-json-parser 1.0.0 → 1.2.0

package/src/parser.ts

@@ -2,18 +2,42 @@ import { unified } from 'unified';
 import remarkParse from 'remark-parse';
 import remarkGfm from 'remark-gfm';
 import { toString } from 'mdast-util-to-string';
-import { MarkdownSection } from './types.js';
-import { remarkBulletSections } from './plugins/bullet-sections.js';
+import { MarkdownSection, BulletMode, ParserOptions } from './types.js';
+import { detectBulletMode } from './plugins/detect-bullet-mode.js';
 import { toCamelCase } from 'nx-helpers';
 
 export class RemarkParser {
     private processor = unified()
         .use(remarkParse)
-        .use(remarkGfm)
-        .use(remarkBulletSections);
+        .use(remarkGfm);
+
+    private options: Required<ParserOptions>;
+
+    constructor(options: ParserOptions = {}) {
+        this.options = {
+            bulletMode: options.bulletMode ?? BulletMode.AUTO,
+            debug: options.debug ?? false
+        };
+    }
 
     parse(markdown: string): MarkdownSection[] {
         const tree = this.processor.runSync(this.processor.parse(markdown));
+
+        // Detect bullet mode if AUTO
+        let bulletMode = this.options.bulletMode;
+        if (bulletMode === BulletMode.AUTO) {
+            const detection = detectBulletMode(tree);
+            bulletMode = detection.mode === 'sections' ? BulletMode.SECTIONS : BulletMode.ARRAY;
+
+            if (this.options.debug) {
+                console.log('🔍 Bullet Mode Detection:');
+                console.log(`  Mode: ${detection.mode}`);
+                console.log(`  Confidence: ${(detection.confidence * 100).toFixed(0)}%`);
+                console.log(`  Reasons:`);
+                detection.reasons.forEach(r => console.log(`    - ${r}`));
+            }
+        }
+
         const sections: MarkdownSection[] = [];
         let currentSection: MarkdownSection | null = null;
         let currentNodes: any[] = [];
@@ -22,9 +46,9 @@ export class RemarkParser {
 
         for (const node of rootChildren) {
             if (node.type === 'heading') {
+                // Standard heading section (### Section)
                 if (currentSection) {
-                    currentSection.content = this.processContent(currentNodes);
-                    sections.push(currentSection);
+                    this.mergeAndPushSection(sections, currentSection, currentNodes, bulletMode);
                 }
 
                 currentSection = {
@@ -34,18 +58,95 @@ export class RemarkParser {
                     format: 'heading'
                 };
                 currentNodes = [];
+
+            } else if (node.type === 'list' && !node.ordered && bulletMode === BulletMode.SECTIONS) {
+                // Bullet-style sections mode: each root-level bullet is a section
+                if (currentSection) {
+                    this.mergeAndPushSection(sections, currentSection, currentNodes, bulletMode);
+                    currentSection = null;
+                    currentNodes = [];
+                }
+
+                // Process each list item as a section
+                const newSections: MarkdownSection[] = [];
+
+                // If we have a current section pending content, and this list is just text items, maybe it belongs to currentSection?
+                // But detect-bullet-mode says SECTIONS. So we expect sections.
+
+                for (const listItem of node.children) {
+                    newSections.push(...this.processBulletAsSection(listItem));
+                }
+
+                if (newSections.length > 0) {
+                    // Refined Sibling Logic:
+                    // If a section has empty content, and is followed by sections that look like simple items...
+                    // But processBulletAsSection transforms every item into a section.
+                    // We need to merge them if they are actually content.
+
+                    // Post-processing merge:
+                    const mergedSections: MarkdownSection[] = [];
+                    let activeSection: MarkdownSection | null = null;
+
+                    for (const sec of newSections) {
+                        // Heuristic: Is this really a section or just an item?
+                        // If detect-bullet-mode said SECTIONS, usually we trust it. 
+                        // But for mixed lists: "- Assumptions" (sec) "- Item 1" (sec?) "- Item 2" (sec?)
+
+                        // Check if 'sec' looks like a property or a value.
+                        // Ideally strictly, everything is a property.
+                        // But in the test case "- Assumptions \n - Val 1 \n - Val 2", 
+                        // they are siblings. 
+
+                        if (activeSection && this.shouldMergeAsContent(activeSection, sec)) {
+                            // Merge content
+                            if (!activeSection.content) {
+                                activeSection.content = [sec.heading]; // Use heading as value
+                            } else if (Array.isArray(activeSection.content)) {
+                                activeSection.content.push(sec.heading);
+                            } else {
+                                // Convert existing string content to array? Or mix?
+                                activeSection.content = [activeSection.content, sec.heading];
+                            }
+                            // APPEND any content of sec as well?
+                            if (sec.content) {
+                                if (Array.isArray(sec.content)) {
+                                    (activeSection.content as any[]).push(...sec.content);
+                                } else {
+                                    (activeSection.content as any[]).push(sec.content);
+                                }
+                            }
+                        } else {
+                            activeSection = sec;
+                            mergedSections.push(sec);
+                        }
+                    }
+
+                    // Push all except the last one as complete sections
+                    if (mergedSections.length > 0) {
+                        for (let i = 0; i < mergedSections.length - 1; i++) {
+                            sections.push(mergedSections[i]!);
+                        }
+                        // Keep the last one as currentSection to capture subsequent loose content
+                        currentSection = mergedSections[mergedSections.length - 1]!;
+                    }
+                }
+
+
+                // Original simple loop removed in favor of refined logic above
+
             } else {
                 currentNodes.push(node);
             }
         }
 
+        // Save last section
         if (currentSection) {
-            currentSection.content = this.processContent(currentNodes);
-            sections.push(currentSection);
+            this.mergeAndPushSection(sections, currentSection, currentNodes, bulletMode);
         } else if (currentNodes.length > 0) {
+            // No sections found, put everything in root
             sections.push({
                 heading: 'Root',
-                content: this.processContent(currentNodes),
+                content: this.processContent(currentNodes, bulletMode),
                 level: 0,
                 format: 'text'
             });
@@ -54,31 +155,139 @@ export class RemarkParser {
         return sections;
     }
 
-    private processContent(nodes: any[]): any {
+    /**
+     * Process content nodes into appropriate format
+     */
+    private processContent(nodes: any[], bulletMode: BulletMode): any {
         if (nodes.length === 0) return '';
 
+        // Single table - return as array of objects
         if (nodes.length === 1 && nodes[0].type === 'table') {
             return this.tableToArray(nodes[0]);
         }
 
+        // Single list - behavior depends on mode
         if (nodes.length === 1 && nodes[0].type === 'list') {
-            return nodes[0].children.map((item: any) => toString(item).trim());
+            if (bulletMode === BulletMode.ARRAY) {
+                // Mode 1: Return as simple array of strings
+                return this.listToArray(nodes[0]);
+            } else {
+                // Mode 2: In sections mode, nested lists are still arrays
+                return this.listToArray(nodes[0]);
+            }
         }
 
+        // Multiple nodes or mixed content
         return nodes.map(node => {
             if (node.type === 'table') {
-                return JSON.stringify(this.tableToArray(node));
+                return this.tableToArray(node);
+            }
+            if (node.type === 'list') {
+                return this.listToArray(node);
             }
             return toString(node);
         }).join('\n\n').trim();
     }
 
+    /**
+     * Process a bullet list item as a section (Mode 2: SECTIONS)
+     */
+    private processBulletAsSection(listItem: any): MarkdownSection[] {
+        const sections: MarkdownSection[] = [];
+
+        // First child is the bullet text (section heading)
+        const firstChild = listItem.children[0];
+        if (!firstChild) return sections;
+
+        const rawText = toString(firstChild).trim();
+        const lines = rawText.split('\n');
+        const heading = lines[0]?.trim() || '';
+        const sameNodeContent = lines.slice(1).join('\n').trim();
+
+        // Clean heading: remove trailing colon if present
+        const cleanHeading = heading.replace(/:$/, '');
+
+        // Rest of the children are the content
+        const contentNodes = listItem.children.slice(1);
+
+        let content: any;
+
+        if (contentNodes.length === 0) {
+            // No extra nodes, but maybe same-node content?
+            content = sameNodeContent;
+        } else {
+            // If we have sameNodeContent, we should prepend it to the text content?
+            // This gets complicated if contentNodes are mixed.
+            // Simplified: If sameNodeContent exists, assume it's part of the text.
+            // Check for nested list
+            const nestedList = contentNodes.find((node: any) => node.type === 'list');
+
+            if (nestedList && contentNodes.length === 1) {
+                // ONLY a nested list - convert to array
+                content = this.listToArray(nestedList);
+            } else if (nestedList) {
+                // Mixed content: paragraphs + nested list
+                // For now, convert nested list to array and combine
+                const paragraphs = contentNodes
+                    .filter((node: any) => node.type !== 'list')
+                    .map((node: any) => toString(node))
+                    .join('\n\n')
+                    .trim();
+
+                const fullText = sameNodeContent ? sameNodeContent + '\n\n' + paragraphs : paragraphs;
+
+                const nestedArray = this.listToArray(nestedList);
+
+                // Combine: return object with text and items
+                content = {
+                    text: fullText,
+                    items: nestedArray
+                };
+            } else {
+                // Flatten all other children into text
+                const nodeText = contentNodes
+                    .map((node: any) => this.nodeToString(node))
+                    .join('\n\n')
+                    .trim();
+                content = sameNodeContent ? sameNodeContent + '\n\n' + nodeText : nodeText;
+            }
+        }
+
+        sections.push({
+            heading: cleanHeading,
+            content,
+            level: 1,
+            format: 'bullet'
+        });
+
+        return sections;
+    }
+
+    /**
+     * Convert list to simple array of strings (Mode 1: ARRAY)
+     */
+    private listToArray(listNode: any): string[] {
+        return listNode.children.map((item: any) => {
+            // Get just the first paragraph/text, ignore nested content
+            const firstChild = item.children[0];
+            return toString(firstChild).trim();
+        });
+    }
+
+    /**
+     * Convert table to array of objects
+     */
     private tableToArray(tableNode: any): any[] {
-        const headers = tableNode.children[0].children.map((cell: any) =>
+        const rows = tableNode.children;
+        if (rows.length === 0) return [];
+
+        // First row = headers
+        const headers = rows[0].children.map((cell: any) =>
             toCamelCase(toString(cell).trim())
         );
 
-        return tableNode.children.slice(1).map((row: any) => {
+        // Remaining rows = data
+        return rows.slice(1).map((row: any) => {
             const obj: any = {};
             row.children.forEach((cell: any, i: number) => {
                 const key = headers[i] || `column${i}`;
@@ -87,4 +296,82 @@ export class RemarkParser {
             return obj;
         });
     }
-}
+
+    /**
+     * Convert sections to object (utility method)
+     */
+    sectionsToObject(sections: MarkdownSection[]): Record<string, any> {
+        const result: Record<string, any> = {};
+
+        for (const section of sections) {
+            const key = toCamelCase(section.heading);
+            result[key] = section.content;
+        }
+
+        return result;
+    }
+
+    private mergeAndPushSection(sections: MarkdownSection[], section: MarkdownSection, nodes: any[], bulletMode: BulletMode) {
+        // ... (existing implementation)
+        const newContent = this.processContent(nodes, bulletMode);
+
+        if (newContent) {
+            if (!section.content) {
+                section.content = newContent;
+            } else if (typeof section.content === 'string' && typeof newContent === 'string') {
+                section.content += '\n\n' + newContent;
+            } else {
+                if (Array.isArray(section.content)) {
+                    section.content.push(newContent);
+                } else {
+                    section.content = [section.content, newContent];
+                }
+            }
+        }
+
+        sections.push(section);
+    }
+
+    private nodeToString(node: any): string {
+        if (node.type === 'table') {
+            return JSON.stringify(this.tableToArray(node));
+        }
+        if (node.type === 'list') {
+            return this.listToArray(node).map(item => `- ${item}`).join('\n');
+        }
+        return toString(node);
+    }
+    private shouldMergeAsContent(parent: MarkdownSection, child: MarkdownSection): boolean {
+        // Decide if 'child' should be merged into 'parent' content instead of being a new section.
+
+        // 1. If child has no content and is short?
+        // 2. If parent matches known strictly-list sections (Assumptions, Evidence, Unknowns)
+
+        // List of keys that usually contain lists of items as siblings in badly formatted markdown
+        const LIST_SECTIONS = [
+            'assumptions',
+            'unknowns',
+            'evidence',
+            'examples',
+            'notes',
+            'analysis',
+            'findings',
+            'recommendations',
+            'considerations'
+        ];
+        const parentKey = toCamelCase(parent.heading);
+        const childKey = toCamelCase(child.heading);
+
+        // If the child is itself a known list section, do NOT merge it.
+        // It should be a new section.
+        if (LIST_SECTIONS.includes(childKey)) {
+            return false;
+        }
+
+        if (LIST_SECTIONS.includes(parentKey)) {
+            return true;
+        }
+
+        return false;
+    }
+}

package/src/plugins/bullet-sections.ts

@@ -1,86 +1,61 @@
-import { toString } from 'mdast-util-to-string';
-
-export function remarkBulletSections() {
-    return (tree: any) => {
-        const children = tree.children;
-        if (!children) return;
-
-        for (let i = 0; i < children.length; i++) {
-            const node = children[i];
-            if (node.type === 'list' && node.ordered === false) {
-                const items = node.children;
-                if (items.length === 0) continue;
-
-                const newRootNodes: any[] = [];
-                let currentListItems: any[] = [];
-
-                items.forEach((item: any, idx: number) => {
-                    const firstChild = item.children[0];
-                    const text = firstChild ? toString(firstChild) : '';
-                    const lines = text.trim().split('\n');
-                    const firstLine = lines[0]?.trim() || '';
 
-                    const isShort = firstLine.length > 0 && firstLine.length < 150;
-                    const hasMoreContent = lines.length > 1 || item.children.length > 1;
-
-                    // Section detection:
-                    // 1. It's short and has more content.
-                    // 2. OR it's short and is followed by a non-short item? (Hard to check here)
-                    // 3. OR it's one of the "known" section keywords.
-                    const keywords = ['answer', 'assumptions', 'unknowns', 'evidence', 'protection', 'control', 'management', 'design', 'logging', 'monitoring', 'backups', 'compliance', 'governance', 'modeling', 'incident', 'vendor', 'changes'];
-                    const isSectionKeyword = keywords.some(k => firstLine.toLowerCase().includes(k));
-
-                    if (isShort && (hasMoreContent || isSectionKeyword)) {
-                        // Flush existing list items if any
-                        if (currentListItems.length > 0) {
-                            newRootNodes.push({
-                                type: 'list',
-                                ordered: false,
-                                children: [...currentListItems]
-                            });
-                            currentListItems = [];
-                        }
-
-                        // Add as heading
-                        newRootNodes.push({
-                            type: 'heading',
-                            depth: 2,
-                            children: [{ type: 'text', value: firstLine }]
-                        });
-
-                        // Add content
-                        if (lines.length > 1) {
-                            newRootNodes.push({
-                                type: 'paragraph',
-                                children: [{ type: 'text', value: lines.slice(1).join('\n').trim() }]
-                            });
-                        }
-                        if (item.children.length > 1) {
-                            newRootNodes.push(...item.children.slice(1));
-                        }
-                    } else {
-                        currentListItems.push(item);
-                    }
-                });
+/**
+ * Remark plugin to handle bullet-style sections
+ * Converts bullets that look like sections into heading nodes
+ */
+import { visit } from 'unist-util-visit';
+import { toString } from 'mdast-util-to-string';
 
-                // Flush remaining
-                if (currentListItems.length > 0) {
-                    newRootNodes.push({
-                        type: 'list',
-                        ordered: false,
-                        children: [...currentListItems]
-                    });
-                }
+const SECTION_KEYWORDS = [
+  'answer', 'summary', 'introduction', 'conclusion', 'overview',
+  'assumptions', 'unknowns', 'evidence', 'notes', 'details',
+  'description', 'background', 'analysis', 'findings', 'recommendations',
+  'data', 'identity', 'network', 'security', 'monitoring', 'governance',
+  'availability', 'backup', 'patch', 'operational', 'provider'
+];
 
-                if (newRootNodes.length > 0) {
-                    // If we performed any transformation (i.e., we found at least one heading)
-                    const hasHeadings = newRootNodes.some(n => n.type === 'heading');
-                    if (hasHeadings) {
-                        children.splice(i, 1, ...newRootNodes);
-                        i += newRootNodes.length - 1;
-                    }
-                }
-            }
-        }
-    };
-}
+export function remarkBulletSections() {
+  return (tree: any) => {
+    const transformations: Array<{
+      parent: any;
+      index: number;
+      listItem: any;
+    }> = [];
+
+    // First pass: identify bullets that should become headings
+ visit(tree, 'list', (node: any, index?: number, parent?: any) => {
+  if (!parent || node.ordered) return; // Only unordered lists
+
+    // Don't transform if there are no section bullets
+    // This preserves normal list behavior
+    if (transformations.length === 0) {
+      return tree;
+    }
+
+    // For now, just mark them for detection
+    // The actual transformation happens in the parser
+    return tree;
+  });
+}}
+
+function isSectionBullet(listItem: any): boolean {
+  if (!listItem.children || listItem.children.length === 0) {
+    return false;
+  }
+
+  const firstChild = listItem.children[0];
+  const text = toString(firstChild).toLowerCase();
+
+  // Check for section indicators
+  const hasKeyword = SECTION_KEYWORDS.some(kw => text.includes(kw));
+  const hasColon = text.includes(':');
+  const hasContent = listItem.children.length > 1;
+  const isCapitalized = /^[A-Z]/.test(text);
+  const isLong = text.length > 30;
+
+  // It's likely a section if it has multiple indicators
+  const indicators = [hasKeyword, hasColon, hasContent, isCapitalized, isLong];
+  const score = indicators.filter(Boolean).length;
+
+  return score >= 2;
+}

package/src/plugins/bullet-sections.ts.old

@@ -0,0 +1,86 @@
+import { toString } from 'mdast-util-to-string';
+
+export function remarkBulletSections() {
+    return (tree: any) => {
+        const children = tree.children;
+        if (!children) return;
+
+        for (let i = 0; i < children.length; i++) {
+            const node = children[i];
+            if (node.type === 'list' && node.ordered === false) {
+                const items = node.children;
+                if (items.length === 0) continue;
+
+                const newRootNodes: any[] = [];
+                let currentListItems: any[] = [];
+
+                items.forEach((item: any, idx: number) => {
+                    const firstChild = item.children[0];
+                    const text = firstChild ? toString(firstChild) : '';
+                    const lines = text.trim().split('\n');
+                    const firstLine = lines[0]?.trim() || '';
+
+                    const isShort = firstLine.length > 0 && firstLine.length < 150;
+                    const hasMoreContent = lines.length > 1 || item.children.length > 1;
+
+                    // Section detection:
+                    // 1. It's short and has more content.
+                    // 2. OR it's short and is followed by a non-short item? (Hard to check here)
+                    // 3. OR it's one of the "known" section keywords.
+                    const keywords = ['answer', 'assumptions', 'unknowns', 'evidence', 'protection', 'control', 'management', 'design', 'logging', 'monitoring', 'backups', 'compliance', 'governance', 'modeling', 'incident', 'vendor', 'changes'];
+                    const isSectionKeyword = keywords.some(k => firstLine.toLowerCase().includes(k));
+
+                    if (isShort && (hasMoreContent || isSectionKeyword)) {
+                        // Flush existing list items if any
+                        if (currentListItems.length > 0) {
+                            newRootNodes.push({
+                                type: 'list',
+                                ordered: false,
+                                children: [...currentListItems]
+                            });
+                            currentListItems = [];
+                        }
+
+                        // Add as heading
+                        newRootNodes.push({
+                            type: 'heading',
+                            depth: 2,
+                            children: [{ type: 'text', value: firstLine }]
+                        });
+
+                        // Add content
+                        if (lines.length > 1) {
+                            newRootNodes.push({
+                                type: 'paragraph',
+                                children: [{ type: 'text', value: lines.slice(1).join('\n').trim() }]
+                            });
+                        }
+                        if (item.children.length > 1) {
+                            newRootNodes.push(...item.children.slice(1));
+                        }
+                    } else {
+                        currentListItems.push(item);
+                    }
+                });
+
+                // Flush remaining
+                if (currentListItems.length > 0) {
+                    newRootNodes.push({
+                        type: 'list',
+                        ordered: false,
+                        children: [...currentListItems]
+                    });
+                }
+
+                if (newRootNodes.length > 0) {
+                    // If we performed any transformation (i.e., we found at least one heading)
+                    const hasHeadings = newRootNodes.some(n => n.type === 'heading');
+                    if (hasHeadings) {
+                        children.splice(i, 1, ...newRootNodes);
+                        i += newRootNodes.length - 1;
+                    }
+                }
+            }
+        }
+    };
+}