nx-json-parser 1.0.0 → 1.1.0

package/src/parser.ts

@@ -2,18 +2,42 @@ import { unified } from 'unified';
 import remarkParse from 'remark-parse';
 import remarkGfm from 'remark-gfm';
 import { toString } from 'mdast-util-to-string';
-import { MarkdownSection } from './types.js';
-import { remarkBulletSections } from './plugins/bullet-sections.js';
+import { MarkdownSection, BulletMode, ParserOptions } from './types.js';
+import { detectBulletMode } from './plugins/detect-bullet-mode.js';
 import { toCamelCase } from 'nx-helpers';
 
 export class RemarkParser {
     private processor = unified()
         .use(remarkParse)
-        .use(remarkGfm)
-        .use(remarkBulletSections);
+        .use(remarkGfm);
+
+    private options: Required<ParserOptions>;
+
+    constructor(options: ParserOptions = {}) {
+        this.options = {
+            bulletMode: options.bulletMode ?? BulletMode.AUTO,
+            debug: options.debug ?? false
+        };
+    }
 
     parse(markdown: string): MarkdownSection[] {
         const tree = this.processor.runSync(this.processor.parse(markdown));
+
+        // Detect bullet mode if AUTO
+        let bulletMode = this.options.bulletMode;
+        if (bulletMode === BulletMode.AUTO) {
+            const detection = detectBulletMode(tree);
+            bulletMode = detection.mode === 'sections' ? BulletMode.SECTIONS : BulletMode.ARRAY;
+
+            if (this.options.debug) {
+                console.log('🔍 Bullet Mode Detection:');
+                console.log(`  Mode: ${detection.mode}`);
+                console.log(`  Confidence: ${(detection.confidence * 100).toFixed(0)}%`);
+                console.log(`  Reasons:`);
+                detection.reasons.forEach(r => console.log(`    - ${r}`));
+            }
+        }
+
         const sections: MarkdownSection[] = [];
         let currentSection: MarkdownSection | null = null;
         let currentNodes: any[] = [];
@@ -22,9 +46,9 @@ export class RemarkParser {
 
         for (const node of rootChildren) {
             if (node.type === 'heading') {
+                // Standard heading section (### Section)
                 if (currentSection) {
-                    currentSection.content = this.processContent(currentNodes);
-                    sections.push(currentSection);
+                    this.mergeAndPushSection(sections, currentSection, currentNodes, bulletMode);
                 }
 
                 currentSection = {
@@ -34,18 +58,43 @@ export class RemarkParser {
                     format: 'heading'
                 };
                 currentNodes = [];
+
+            } else if (node.type === 'list' && bulletMode === BulletMode.SECTIONS) {
+                // Bullet-style sections mode: each root-level bullet is a section
+                if (currentSection) {
+                    this.mergeAndPushSection(sections, currentSection, currentNodes, bulletMode);
+                    currentSection = null;
+                    currentNodes = [];
+                }
+
+                // Process each list item as a section
+                const newSections: MarkdownSection[] = [];
+                for (const listItem of node.children) {
+                    newSections.push(...this.processBulletAsSection(listItem));
+                }
+
+                if (newSections.length > 0) {
+                    // Push all except the last one as complete sections
+                    for (let i = 0; i < newSections.length - 1; i++) {
+                        sections.push(newSections[i]!);
+                    }
+                    // Keep the last one as currentSection to capture subsequent loose content
+                    currentSection = newSections[newSections.length - 1]!;
+                }
+
             } else {
                 currentNodes.push(node);
             }
         }
 
+        // Save last section
         if (currentSection) {
-            currentSection.content = this.processContent(currentNodes);
-            sections.push(currentSection);
+            this.mergeAndPushSection(sections, currentSection, currentNodes, bulletMode);
         } else if (currentNodes.length > 0) {
+            // No sections found, put everything in root
             sections.push({
                 heading: 'Root',
-                content: this.processContent(currentNodes),
+                content: this.processContent(currentNodes, bulletMode),
                 level: 0,
                 format: 'text'
             });
@@ -54,31 +103,139 @@ export class RemarkParser {
         return sections;
     }
 
-    private processContent(nodes: any[]): any {
+    /**
+     * Process content nodes into appropriate format
+     */
+    private processContent(nodes: any[], bulletMode: BulletMode): any {
         if (nodes.length === 0) return '';
 
+        // Single table - return as array of objects
         if (nodes.length === 1 && nodes[0].type === 'table') {
             return this.tableToArray(nodes[0]);
         }
 
+        // Single list - behavior depends on mode
         if (nodes.length === 1 && nodes[0].type === 'list') {
-            return nodes[0].children.map((item: any) => toString(item).trim());
+            if (bulletMode === BulletMode.ARRAY) {
+                // Mode 1: Return as simple array of strings
+                return this.listToArray(nodes[0]);
+            } else {
+                // Mode 2: In sections mode, nested lists are still arrays
+                return this.listToArray(nodes[0]);
+            }
         }
 
+        // Multiple nodes or mixed content
         return nodes.map(node => {
             if (node.type === 'table') {
-                return JSON.stringify(this.tableToArray(node));
+                return this.tableToArray(node);
+            }
+            if (node.type === 'list') {
+                return this.listToArray(node);
             }
             return toString(node);
         }).join('\n\n').trim();
     }
 
+    /**
+     * Process a bullet list item as a section (Mode 2: SECTIONS)
+     */
+    private processBulletAsSection(listItem: any): MarkdownSection[] {
+        const sections: MarkdownSection[] = [];
+
+        // First child is the bullet text (section heading)
+        const firstChild = listItem.children[0];
+        if (!firstChild) return sections;
+
+        const rawText = toString(firstChild).trim();
+        const lines = rawText.split('\n');
+        const heading = lines[0]?.trim() || '';
+        const sameNodeContent = lines.slice(1).join('\n').trim();
+
+        // Clean heading: remove trailing colon if present
+        const cleanHeading = heading.replace(/:$/, '');
+
+        // Rest of the children are the content
+        const contentNodes = listItem.children.slice(1);
+
+        let content: any;
+
+        if (contentNodes.length === 0) {
+            // No extra nodes, but maybe same-node content?
+            content = sameNodeContent;
+        } else {
+            // If we have sameNodeContent, we should prepend it to the text content?
+            // This gets complicated if contentNodes are mixed.
+            // Simplified: If sameNodeContent exists, assume it's part of the text.
+            // Check for nested list
+            const nestedList = contentNodes.find((node: any) => node.type === 'list');
+
+            if (nestedList && contentNodes.length === 1) {
+                // ONLY a nested list - convert to array
+                content = this.listToArray(nestedList);
+            } else if (nestedList) {
+                // Mixed content: paragraphs + nested list
+                // For now, convert nested list to array and combine
+                const paragraphs = contentNodes
+                    .filter((node: any) => node.type !== 'list')
+                    .map((node: any) => toString(node))
+                    .join('\n\n')
+                    .trim();
+
+                const fullText = sameNodeContent ? sameNodeContent + '\n\n' + paragraphs : paragraphs;
+
+                const nestedArray = this.listToArray(nestedList);
+
+                // Combine: return object with text and items
+                content = {
+                    text: fullText,
+                    items: nestedArray
+                };
+            } else {
+                // Only paragraphs/text content
+                const nodeText = contentNodes
+                    .map((node: any) => toString(node))
+                    .join('\n\n')
+                    .trim();
+                content = sameNodeContent ? sameNodeContent + '\n\n' + nodeText : nodeText;
+            }
+        }
+
+        sections.push({
+            heading: cleanHeading,
+            content,
+            level: 1,
+            format: 'bullet'
+        });
+
+        return sections;
+    }
+
+    /**
+     * Convert list to simple array of strings (Mode 1: ARRAY)
+     */
+    private listToArray(listNode: any): string[] {
+        return listNode.children.map((item: any) => {
+            // Get just the first paragraph/text, ignore nested content
+            const firstChild = item.children[0];
+            return toString(firstChild).trim();
+        });
+    }
+
+    /**
+     * Convert table to array of objects
+     */
     private tableToArray(tableNode: any): any[] {
-        const headers = tableNode.children[0].children.map((cell: any) =>
+        const rows = tableNode.children;
+        if (rows.length === 0) return [];
+
+        // First row = headers
+        const headers = rows[0].children.map((cell: any) =>
             toCamelCase(toString(cell).trim())
         );
 
-        return tableNode.children.slice(1).map((row: any) => {
+        // Remaining rows = data
+        return rows.slice(1).map((row: any) => {
             const obj: any = {};
             row.children.forEach((cell: any, i: number) => {
                 const key = headers[i] || `column${i}`;
@@ -87,4 +244,45 @@ export class RemarkParser {
             return obj;
         });
     }
-}
+
+    /**
+     * Convert sections to object (utility method)
+     */
+    sectionsToObject(sections: MarkdownSection[]): Record<string, any> {
+        const result: Record<string, any> = {};
+
+        for (const section of sections) {
+            const key = toCamelCase(section.heading);
+            result[key] = section.content;
+        }
+
+        return result;
+    }
+
+    private mergeAndPushSection(sections: MarkdownSection[], section: MarkdownSection, nodes: any[], bulletMode: BulletMode) {
+        // If the section is already in the list (because we pushed it in the loop but kept a reference),
+        // we shouldn't push it again?
+        // Wait, my logic implementation in loop:
+        // "newSections.push(...); if (newSections.length > 0) { ... loop push n-1 ... currentSection = nth }"
+        // So currentSection is NOT in sections yet.
+        // It's safe to push.
+
+        const newContent = this.processContent(nodes, bulletMode);
+
+        if (newContent) {
+            if (!section.content) {
+                section.content = newContent;
+            } else if (typeof section.content === 'string' && typeof newContent === 'string') {
+                section.content += '\n\n' + newContent;
+            } else {
+                if (Array.isArray(section.content)) {
+                    section.content.push(newContent);
+                } else {
+                    section.content = [section.content, newContent];
+                }
+            }
+        }
+
+        sections.push(section);
+    }
+}

package/src/plugins/bullet-sections.ts

@@ -1,86 +1,61 @@
-import { toString } from 'mdast-util-to-string';
-
-export function remarkBulletSections() {
-    return (tree: any) => {
-        const children = tree.children;
-        if (!children) return;
-
-        for (let i = 0; i < children.length; i++) {
-            const node = children[i];
-            if (node.type === 'list' && node.ordered === false) {
-                const items = node.children;
-                if (items.length === 0) continue;
-
-                const newRootNodes: any[] = [];
-                let currentListItems: any[] = [];
-
-                items.forEach((item: any, idx: number) => {
-                    const firstChild = item.children[0];
-                    const text = firstChild ? toString(firstChild) : '';
-                    const lines = text.trim().split('\n');
-                    const firstLine = lines[0]?.trim() || '';
 
-                    const isShort = firstLine.length > 0 && firstLine.length < 150;
-                    const hasMoreContent = lines.length > 1 || item.children.length > 1;
-
-                    // Section detection:
-                    // 1. It's short and has more content.
-                    // 2. OR it's short and is followed by a non-short item? (Hard to check here)
-                    // 3. OR it's one of the "known" section keywords.
-                    const keywords = ['answer', 'assumptions', 'unknowns', 'evidence', 'protection', 'control', 'management', 'design', 'logging', 'monitoring', 'backups', 'compliance', 'governance', 'modeling', 'incident', 'vendor', 'changes'];
-                    const isSectionKeyword = keywords.some(k => firstLine.toLowerCase().includes(k));
-
-                    if (isShort && (hasMoreContent || isSectionKeyword)) {
-                        // Flush existing list items if any
-                        if (currentListItems.length > 0) {
-                            newRootNodes.push({
-                                type: 'list',
-                                ordered: false,
-                                children: [...currentListItems]
-                            });
-                            currentListItems = [];
-                        }
-
-                        // Add as heading
-                        newRootNodes.push({
-                            type: 'heading',
-                            depth: 2,
-                            children: [{ type: 'text', value: firstLine }]
-                        });
-
-                        // Add content
-                        if (lines.length > 1) {
-                            newRootNodes.push({
-                                type: 'paragraph',
-                                children: [{ type: 'text', value: lines.slice(1).join('\n').trim() }]
-                            });
-                        }
-                        if (item.children.length > 1) {
-                            newRootNodes.push(...item.children.slice(1));
-                        }
-                    } else {
-                        currentListItems.push(item);
-                    }
-                });
+/**
+ * Remark plugin to handle bullet-style sections
+ * Converts bullets that look like sections into heading nodes
+ */
+import { visit } from 'unist-util-visit';
+import { toString } from 'mdast-util-to-string';
 
-                // Flush remaining
-                if (currentListItems.length > 0) {
-                    newRootNodes.push({
-                        type: 'list',
-                        ordered: false,
-                        children: [...currentListItems]
-                    });
-                }
+const SECTION_KEYWORDS = [
+  'answer', 'summary', 'introduction', 'conclusion', 'overview',
+  'assumptions', 'unknowns', 'evidence', 'notes', 'details',
+  'description', 'background', 'analysis', 'findings', 'recommendations',
+  'data', 'identity', 'network', 'security', 'monitoring', 'governance',
+  'availability', 'backup', 'patch', 'operational', 'provider'
+];
 
-                if (newRootNodes.length > 0) {
-                    // If we performed any transformation (i.e., we found at least one heading)
-                    const hasHeadings = newRootNodes.some(n => n.type === 'heading');
-                    if (hasHeadings) {
-                        children.splice(i, 1, ...newRootNodes);
-                        i += newRootNodes.length - 1;
-                    }
-                }
-            }
-        }
-    };
-}
+export function remarkBulletSections() {
+  return (tree: any) => {
+    const transformations: Array<{
+      parent: any;
+      index: number;
+      listItem: any;
+    }> = [];
+
+    // First pass: identify bullets that should become headings
+ visit(tree, 'list', (node: any, index?: number, parent?: any) => {
+  if (!parent || node.ordered) return; // Only unordered lists
+
+    // Don't transform if there are no section bullets
+    // This preserves normal list behavior
+    if (transformations.length === 0) {
+      return tree;
+    }
+
+    // For now, just mark them for detection
+    // The actual transformation happens in the parser
+    return tree;
+  });
+}}
+
+function isSectionBullet(listItem: any): boolean {
+  if (!listItem.children || listItem.children.length === 0) {
+    return false;
+  }
+
+  const firstChild = listItem.children[0];
+  const text = toString(firstChild).toLowerCase();
+
+  // Check for section indicators
+  const hasKeyword = SECTION_KEYWORDS.some(kw => text.includes(kw));
+  const hasColon = text.includes(':');
+  const hasContent = listItem.children.length > 1;
+  const isCapitalized = /^[A-Z]/.test(text);
+  const isLong = text.length > 30;
+
+  // It's likely a section if it has multiple indicators
+  const indicators = [hasKeyword, hasColon, hasContent, isCapitalized, isLong];
+  const score = indicators.filter(Boolean).length;
+
+  return score >= 2;
+}

package/src/plugins/bullet-sections.ts.old

@@ -0,0 +1,86 @@
+import { toString } from 'mdast-util-to-string';
+
+export function remarkBulletSections() {
+    return (tree: any) => {
+        const children = tree.children;
+        if (!children) return;
+
+        for (let i = 0; i < children.length; i++) {
+            const node = children[i];
+            if (node.type === 'list' && node.ordered === false) {
+                const items = node.children;
+                if (items.length === 0) continue;
+
+                const newRootNodes: any[] = [];
+                let currentListItems: any[] = [];
+
+                items.forEach((item: any, idx: number) => {
+                    const firstChild = item.children[0];
+                    const text = firstChild ? toString(firstChild) : '';
+                    const lines = text.trim().split('\n');
+                    const firstLine = lines[0]?.trim() || '';
+
+                    const isShort = firstLine.length > 0 && firstLine.length < 150;
+                    const hasMoreContent = lines.length > 1 || item.children.length > 1;
+
+                    // Section detection:
+                    // 1. It's short and has more content.
+                    // 2. OR it's short and is followed by a non-short item? (Hard to check here)
+                    // 3. OR it's one of the "known" section keywords.
+                    const keywords = ['answer', 'assumptions', 'unknowns', 'evidence', 'protection', 'control', 'management', 'design', 'logging', 'monitoring', 'backups', 'compliance', 'governance', 'modeling', 'incident', 'vendor', 'changes'];
+                    const isSectionKeyword = keywords.some(k => firstLine.toLowerCase().includes(k));
+
+                    if (isShort && (hasMoreContent || isSectionKeyword)) {
+                        // Flush existing list items if any
+                        if (currentListItems.length > 0) {
+                            newRootNodes.push({
+                                type: 'list',
+                                ordered: false,
+                                children: [...currentListItems]
+                            });
+                            currentListItems = [];
+                        }
+
+                        // Add as heading
+                        newRootNodes.push({
+                            type: 'heading',
+                            depth: 2,
+                            children: [{ type: 'text', value: firstLine }]
+                        });
+
+                        // Add content
+                        if (lines.length > 1) {
+                            newRootNodes.push({
+                                type: 'paragraph',
+                                children: [{ type: 'text', value: lines.slice(1).join('\n').trim() }]
+                            });
+                        }
+                        if (item.children.length > 1) {
+                            newRootNodes.push(...item.children.slice(1));
+                        }
+                    } else {
+                        currentListItems.push(item);
+                    }
+                });
+
+                // Flush remaining
+                if (currentListItems.length > 0) {
+                    newRootNodes.push({
+                        type: 'list',
+                        ordered: false,
+                        children: [...currentListItems]
+                    });
+                }
+
+                if (newRootNodes.length > 0) {
+                    // If we performed any transformation (i.e., we found at least one heading)
+                    const hasHeadings = newRootNodes.some(n => n.type === 'heading');
+                    if (hasHeadings) {
+                        children.splice(i, 1, ...newRootNodes);
+                        i += newRootNodes.length - 1;
+                    }
+                }
+            }
+        }
+    };
+}

package/src/plugins/detect-bullet-mode.ts

@@ -0,0 +1,161 @@
+/**
+ * Detect if bullets should be treated as array items or sections
+ */
+import { visit } from 'unist-util-visit';
+import { toString } from 'mdast-util-to-string';
+
+export interface BulletModeResult {
+  mode: 'array' | 'sections';
+  confidence: number;
+  reasons: string[];
+}
+
+const SECTION_KEYWORDS = [
+  'answer', 'summary', 'introduction', 'conclusion', 'overview',
+  'assumptions', 'unknowns', 'evidence', 'notes', 'details',
+  'description', 'background', 'analysis', 'findings', 'recommendations',
+  'data', 'identity', 'network', 'security', 'monitoring', 'governance',
+  'availability', 'backup', 'patch', 'operational', 'provider'
+];
+
+export function detectBulletMode(tree: any): BulletModeResult {
+  const reasons: string[] = [];
+  let sectionScore = 0;
+  let arrayScore = 0;
+
+  let bulletCount = 0;
+  let bulletsWithContent = 0;
+  let bulletsWithColons = 0;
+  let bulletsWithNestedLists = 0;
+  let bulletsWithKeywords = 0;
+  let totalBulletLength = 0;
+  let rootLevelBullets = 0;
+
+  // Analyze the tree structure
+  visit(tree, 'list', (listNode: any, index?: number, parent?: any) => {
+    // Only analyze root-level lists or lists directly under root
+    const isRootLevel = parent?.type === 'root';
+    
+    if (!isRootLevel) return;
+
+    rootLevelBullets += listNode.children.length;
+
+    for (const listItem of listNode.children) {
+      bulletCount++;
+
+      // Get the text of this list item (first paragraph/text only)
+      const firstChild = listItem.children[0];
+      const itemText = firstChild ? toString(firstChild) : '';
+      totalBulletLength += itemText.length;
+
+      // 1. Check for colons (title pattern: "Short answer:" or "Data protection:")
+      if (itemText.includes(':')) {
+        bulletsWithColons++;
+      }
+
+      // 2. Check for content after the bullet (paragraphs, nested items)
+      if (listItem.children.length > 1) {
+        bulletsWithContent++;
+      }
+
+      // 3. Check for nested lists
+      const hasNestedList = listItem.children.some((child: any) => child.type === 'list');
+      if (hasNestedList) {
+        bulletsWithNestedLists++;
+      }
+
+      // 4. Check for section keywords
+      const lowerText = itemText.toLowerCase();
+      if (SECTION_KEYWORDS.some(kw => lowerText.includes(kw))) {
+        bulletsWithKeywords++;
+      }
+    }
+  });
+
+  if (bulletCount === 0) {
+    return {
+      mode: 'array',
+      confidence: 0,
+      reasons: ['No bullets found']
+    };
+  }
+
+  const avgLength = totalBulletLength / bulletCount;
+
+  // === SCORING FOR SECTIONS MODE ===
+
+  if (bulletsWithColons > 0) {
+    const percent = (bulletsWithColons / bulletCount * 100).toFixed(0);
+    sectionScore += 3;
+    reasons.push(`${bulletsWithColons}/${bulletCount} bullets have colons (${percent}%) - strong section indicator`);
+  }
+
+  if (bulletsWithContent > 0) {
+    const percent = (bulletsWithContent / bulletCount * 100).toFixed(0);
+    sectionScore += 3;
+    reasons.push(`${bulletsWithContent}/${bulletCount} bullets have content below (${percent}%) - strong section indicator`);
+  }
+
+  if (bulletsWithNestedLists > 0) {
+    sectionScore += 2;
+    reasons.push(`${bulletsWithNestedLists} bullets have nested lists - section indicator`);
+  }
+
+  if (bulletsWithKeywords > 0) {
+    sectionScore += 2;
+    reasons.push(`${bulletsWithKeywords} bullets contain section keywords`);
+  }
+
+  if (avgLength > 30) {
+    sectionScore += 1;
+    reasons.push(`Long bullet text (avg ${avgLength.toFixed(0)} chars) - suggests titles`);
+  }
+
+  if (bulletCount <= 5 && (bulletsWithContent > 0 || bulletsWithColons > 0)) {
+    sectionScore += 1;
+    reasons.push('Few bullets with rich content suggests sections');
+  }
+
+  // === SCORING FOR ARRAY MODE ===
+
+  if (bulletsWithColons === 0 && bulletsWithContent === 0 && bulletsWithNestedLists === 0) {
+    arrayScore += 4;
+    reasons.push('No bullets have content, colons, or nesting - pure list indicator');
+  }
+
+  if (avgLength < 30) {
+    arrayScore += 2;
+    reasons.push(`Short bullet text (avg ${avgLength.toFixed(0)} chars) - suggests list items`);
+  }
+
+  if (bulletCount >= 3 && bulletsWithContent === 0) {
+    arrayScore += 2;
+    reasons.push(`${bulletCount} simple bullets without content - array pattern`);
+  }
+
+  if (bulletsWithKeywords === 0) {
+    arrayScore += 1;
+    reasons.push('No section keywords found');
+  }
+
+  // Edge case: all bullets are very short and no special features
+  if (avgLength < 20 && bulletsWithContent === 0 && bulletsWithColons === 0) {
+    arrayScore += 2;
+    reasons.push('Very short bullets with no features - definitely array');
+  }
+
+  // === DECISION ===
+
+  const totalScore = sectionScore + arrayScore;
+  const mode = sectionScore > arrayScore ? 'sections' : 'array';
+  const confidence = totalScore > 0 ? Math.abs(sectionScore - arrayScore) / totalScore : 0;
+
+  // Format reasons (top 5)
+  const topReasons = reasons.slice(0, 5);
+
+  return {
+    mode,
+    confidence,
+    reasons: topReasons
+  };
+}