nx-json-parser 1.0.0 → 1.2.0

package/src/plugins/detect-bullet-mode.ts

@@ -0,0 +1,164 @@
+/**
+ * Detect if bullets should be treated as array items or sections
+ */
+import { visit } from 'unist-util-visit';
+import { toString } from 'mdast-util-to-string';
+
+export interface BulletModeResult {
+    mode: 'array' | 'sections';
+    confidence: number;
+    reasons: string[];
+}
+
+const SECTION_KEYWORDS = [
+    'answer', 'summary', 'introduction', 'conclusion', 'overview',
+    'assumptions', 'unknowns', 'evidence', 'notes', 'details',
+    'description', 'background', 'analysis', 'findings', 'recommendations',
+    'data', 'identity', 'network', 'security', 'monitoring', 'governance',
+    'availability', 'backup', 'patch', 'operational', 'provider',
+    'short', 'full', 'long' // Added common answer prefixes
+];
+
+export function detectBulletMode(tree: any): BulletModeResult {
+    const reasons: string[] = [];
+    let sectionScore = 0;
+    let arrayScore = 0;
+
+    let bulletCount = 0;
+    let bulletsWithContent = 0;
+    let bulletsWithColons = 0;
+    let bulletsWithNestedLists = 0;
+    let bulletsWithKeywords = 0;
+    let totalBulletLength = 0;
+    let rootLevelBullets = 0;
+
+    // Analyze the tree structure
+    visit(tree, 'list', (listNode: any, index?: number, parent?: any) => {
+        // Only analyze root-level lists or lists directly under root
+        const isRootLevel = parent?.type === 'root';
+
+        if (!isRootLevel) return;
+
+        rootLevelBullets += listNode.children.length;
+
+        for (const listItem of listNode.children) {
+            bulletCount++;
+
+            // Get the text of this list item (first paragraph/text only)
+            const firstChild = listItem.children[0];
+            const itemText = firstChild ? toString(firstChild) : '';
+            totalBulletLength += itemText.length;
+
+            // 1. Check for colons (title pattern: "Short answer:" or "Data protection:")
+            if (itemText.includes(':')) {
+                bulletsWithColons++;
+            }
+
+            // 2. Check for content after the bullet (paragraphs, nested items)
+            if (listItem.children.length > 1) {
+                bulletsWithContent++;
+            }
+
+            // 3. Check for nested lists
+            const hasNestedList = listItem.children.some((child: any) => child.type === 'list');
+            if (hasNestedList) {
+                bulletsWithNestedLists++;
+            }
+
+            // 4. Check for section keywords
+            const lowerText = itemText.toLowerCase();
+            if (SECTION_KEYWORDS.some(kw => lowerText.includes(kw))) {
+                bulletsWithKeywords++;
+            }
+        }
+    });
+
+    if (bulletCount === 0) {
+        return {
+            mode: 'array',
+            confidence: 0,
+            reasons: ['No bullets found']
+        };
+    }
+
+    const avgLength = totalBulletLength / bulletCount;
+
+    // === SCORING FOR SECTIONS MODE ===
+
+    if (bulletsWithColons > 0) {
+        const percent = (bulletsWithColons / bulletCount * 100).toFixed(0);
+        sectionScore += 3;
+        reasons.push(`${bulletsWithColons}/${bulletCount} bullets have colons (${percent}%) - strong section indicator`);
+    }
+
+    if (bulletsWithContent > 0) {
+        const percent = (bulletsWithContent / bulletCount * 100).toFixed(0);
+        sectionScore += 3;
+        reasons.push(`${bulletsWithContent}/${bulletCount} bullets have content below (${percent}%) - strong section indicator`);
+    }
+
+    if (bulletsWithNestedLists > 0) {
+        sectionScore += 2;
+        reasons.push(`${bulletsWithNestedLists} bullets have nested lists - section indicator`);
+    }
+
+    if (bulletsWithKeywords > 0) {
+        // Huge boost for keywords - these are almost certainly section headers
+        sectionScore += 10;
+        reasons.push(`${bulletsWithKeywords} bullets contain section keywords - STRONG section indicator`);
+    }
+
+    if (avgLength > 30) {
+        sectionScore += 1;
+        reasons.push(`Long bullet text (avg ${avgLength.toFixed(0)} chars) - suggests titles`);
+    }
+
+    if (bulletCount <= 5 && (bulletsWithContent > 0 || bulletsWithColons > 0)) {
+        sectionScore += 1;
+        reasons.push('Few bullets with rich content suggests sections');
+    }
+
+    // === SCORING FOR ARRAY MODE ===
+
+    if (bulletsWithColons === 0 && bulletsWithContent === 0 && bulletsWithNestedLists === 0 && bulletsWithKeywords === 0) {
+        // Only if NO keywords are found
+        arrayScore += 4;
+        reasons.push('No bullets have content, colons, nesting, OR keywords - pure list indicator');
+    }
+
+    if (avgLength < 30) {
+        arrayScore += 2;
+        reasons.push(`Short bullet text (avg ${avgLength.toFixed(0)} chars) - suggests list items`);
+    }
+
+    if (bulletCount >= 3 && bulletsWithContent === 0) {
+        arrayScore += 2;
+        reasons.push(`${bulletCount} simple bullets without content - array pattern`);
+    }
+
+    if (bulletsWithKeywords === 0) {
+        arrayScore += 1;
+        reasons.push('No section keywords found');
+    }
+
+    // Edge case: all bullets are very short and no special features
+    if (avgLength < 20 && bulletsWithContent === 0 && bulletsWithColons === 0) {
+        arrayScore += 2;
+        reasons.push('Very short bullets with no features - definitely array');
+    }
+
+    // === DECISION ===
+
+    const totalScore = sectionScore + arrayScore;
+    const mode = sectionScore > arrayScore ? 'sections' : 'array';
+    const confidence = totalScore > 0 ? Math.abs(sectionScore - arrayScore) / totalScore : 0;
+
+    // Format reasons (top 5)
+    const topReasons = reasons.slice(0, 5);
+
+    return {
+        mode,
+        confidence,
+        reasons: topReasons
+    };
+}

package/src/remark-markdown-parser.ts

@@ -0,0 +1,340 @@
+/**
+ * Remark-based Markdown Parser
+ * Replaces regex-based parsing with AST-based parsing using unified/remark
+ */
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGfm from 'remark-gfm';
+import { visit } from 'unist-util-visit';
+import { toString } from 'mdast-util-to-string';
+import { toCamelCase } from 'nx-helpers';
+
+export interface MarkdownSection {
+  heading: string;
+  content: string;
+  level: number;
+}
+
+export enum BulletMode {
+  ARRAY = 'array',
+  SECTIONS = 'sections',
+  AUTO = 'auto'
+}
+
+export interface RemarkParserOptions {
+  bulletMode?: BulletMode;
+  sectionKeywords?: string[];
+  debug?: boolean;
+}
+
+export class RemarkMarkdownParser {
+  private processor: any;
+  private options: Required<RemarkParserOptions>;
+
+  constructor(options: RemarkParserOptions = {}) {
+    this.processor = unified()
+      .use(remarkParse)
+      .use(remarkGfm);
+
+    this.options = {
+      bulletMode: options.bulletMode || BulletMode.AUTO,
+      sectionKeywords: options.sectionKeywords || this.getDefaultKeywords(),
+      debug: options.debug || false
+    };
+  }
+
+  /**
+   * Parse markdown into sections
+   */
+  parseSections(markdown: string): MarkdownSection[] {
+    const tree = this.processor.parse(markdown);
+    
+    // Detect bullet mode if AUTO
+    const bulletMode = this.options.bulletMode === BulletMode.AUTO
+      ? this.detectBulletMode(tree)
+      : this.options.bulletMode;
+
+    if (this.options.debug) {
+      console.log(`📋 Bullet mode: ${bulletMode}`);
+    }
+
+    if (bulletMode === BulletMode.SECTIONS) {
+      return this.parseSectionsMode(tree, markdown);
+    } else {
+      return this.parseHeadingMode(tree);
+    }
+  }
+
+  /**
+   * Parse content (tables, lists, text)
+   */
+  parseContent(content: string): any {
+    const trimmed = content.trim();
+    if (!trimmed) return '';
+
+    const tree = this.processor.parse(trimmed);
+    
+    // Single node - return specific type
+    if (tree.children.length === 1) {
+      const child = tree.children[0];
+      
+      if (child.type === 'table') {
+        return this.parseTable(child);
+      }
+      
+      if (child.type === 'list') {
+        return this.parseList(child);
+      }
+
+      if (child.type === 'paragraph') {
+        return toString(child);
+      }
+    }
+
+    // Multiple children - combine
+    const results: any[] = [];
+    
+    for (const child of tree.children) {
+      if (child.type === 'table') {
+        return this.parseTable(child); // Single table dominates
+      } else if (child.type === 'list') {
+        return this.parseList(child); // Single list dominates
+      } else if (child.type === 'paragraph') {
+        results.push(toString(child));
+      } else {
+        results.push(toString(child));
+      }
+    }
+
+    // Join text results
+    const text = results.join('\n\n').trim();
+    return text || trimmed;
+  }
+
+  /**
+   * Convert sections to object
+   */
+  sectionsToObject(sections: MarkdownSection[]): Record<string, any> {
+    const result: Record<string, any> = {};
+
+    for (const section of sections) {
+      const key = toCamelCase(section.heading);
+      result[key] = this.parseContent(section.content);
+    }
+
+    return result;
+  }
+
+  // ========================================================================
+  // PRIVATE METHODS - Mode Detection
+  // ========================================================================
+
+  private detectBulletMode(tree: any): BulletMode {
+    let bulletCount = 0;
+    let bulletsWithContent = 0;
+    let bulletsWithColons = 0;
+    let bulletsWithNestedLists = 0;
+    let bulletsWithKeywords = 0;
+    let totalLength = 0;
+
+visit(tree, 'list', (listNode: any, index?: number, parent?: any) => {
+  // Only analyze root-level lists
+  if (!parent || parent.type !== 'root') return;
+
+      for (const listItem of listNode.children) {
+        bulletCount++;
+
+        const firstChild = listItem.children[0];
+        const text = firstChild ? toString(firstChild) : '';
+        totalLength += text.length;
+
+        // Check indicators
+        if (text.includes(':')) bulletsWithColons++;
+        if (listItem.children.length > 1) bulletsWithContent++;
+        
+        const hasNestedList = listItem.children.some((c: any) => c.type === 'list');
+        if (hasNestedList) bulletsWithNestedLists++;
+
+        const lowerText = text.toLowerCase();
+        if (this.options.sectionKeywords.some(kw => lowerText.includes(kw))) {
+          bulletsWithKeywords++;
+        }
+      }
+    });
+
+    if (bulletCount === 0) return BulletMode.ARRAY;
+
+    const avgLength = totalLength / bulletCount;
+
+    // Scoring
+    let sectionScore = 0;
+    let arrayScore = 0;
+
+    if (bulletsWithColons > 0) sectionScore += 3;
+    if (bulletsWithContent > 0) sectionScore += 3;
+    if (bulletsWithNestedLists > 0) sectionScore += 2;
+    if (bulletsWithKeywords > 0) sectionScore += 2;
+    if (avgLength > 30) sectionScore += 1;
+
+    if (bulletsWithContent === 0 && bulletsWithColons === 0) arrayScore += 4;
+    if (avgLength < 30) arrayScore += 2;
+    if (bulletCount >= 3 && bulletsWithContent === 0) arrayScore += 2;
+
+    if (this.options.debug) {
+      console.log(`🔍 Detection scores - Sections: ${sectionScore}, Array: ${arrayScore}`);
+    }
+
+    return sectionScore > arrayScore ? BulletMode.SECTIONS : BulletMode.ARRAY;
+  }
+
+  // ========================================================================
+  // PRIVATE METHODS - Parsing Modes
+  // ========================================================================
+
+  private parseHeadingMode(tree: any): MarkdownSection[] {
+    const sections: MarkdownSection[] = [];
+    let currentSection: MarkdownSection | null = null;
+    let currentContent: string[] = [];
+
+    for (const node of tree.children) {
+      if (node.type === 'heading') {
+        // Save previous section
+        if (currentSection) {
+          currentSection.content = currentContent.join('\n\n').trim();
+          sections.push(currentSection);
+        }
+
+        // Start new section
+        currentSection = {
+          heading: toString(node),
+          content: '',
+          level: node.depth
+        };
+        currentContent = [];
+      } else if (currentSection) {
+        // Add to current section content
+        currentContent.push(this.nodeToString(node));
+      }
+    }
+
+    // Save last section
+    if (currentSection) {
+      currentSection.content = currentContent.join('\n\n').trim();
+      sections.push(currentSection);
+    }
+
+    return sections;
+  }
+
+  private parseSectionsMode(tree: any, markdown: string): MarkdownSection[] {
+    const sections: MarkdownSection[] = [];
+    let inList = false;
+
+    for (const node of tree.children) {
+      if (node.type === 'heading') {
+        // Regular heading
+        const heading = toString(node);
+        const content = ''; // Will be filled by next nodes
+        sections.push({
+          heading,
+          content,
+          level: node.depth
+        });
+      } else if (node.type === 'list') {
+        // Process each list item as a section
+        for (const listItem of node.children) {
+          const section = this.listItemToSection(listItem);
+          if (section) {
+            sections.push(section);
+          }
+        }
+      } else if (sections.length > 0) {
+        // Add content to last section
+        const lastSection = sections[sections.length - 1];
+        const nodeContent = this.nodeToString(node);
+        lastSection.content = lastSection.content
+          ? `${lastSection.content}\n\n${nodeContent}`
+          : nodeContent;
+      }
+    }
+
+    return sections;
+  }
+
+  private listItemToSection(listItem: any): MarkdownSection | null {
+    if (!listItem.children || listItem.children.length === 0) {
+      return null;
+    }
+
+    const firstChild = listItem.children[0];
+    const heading = toString(firstChild).replace(/:$/, ''); // Remove trailing colon
+
+    const contentNodes = listItem.children.slice(1);
+    let content = '';
+
+    if (contentNodes.length > 0) {
+      content = contentNodes.map((node: any) => this.nodeToString(node)).join('\n\n').trim();
+    }
+
+    return {
+      heading,
+      content,
+      level: 1
+    };
+  }
+
+  // ========================================================================
+  // PRIVATE METHODS - Content Parsing
+  // ========================================================================
+
+  private parseTable(tableNode: any): any[] {
+    const rows = tableNode.children;
+    if (rows.length === 0) return [];
+
+    // First row = headers
+    const headerRow = rows[0];
+    const headers = headerRow.children.map((cell: any) =>
+      toCamelCase(toString(cell).trim())
+    );
+
+    // Data rows
+    return rows.slice(1).map((row: any) => {
+      const obj: any = {};
+      row.children.forEach((cell: any, i: number) => {
+        const key = headers[i] || `column${i}`;
+        obj[key] = toString(cell).trim();
+      });
+      return obj;
+    });
+  }
+
+  private parseList(listNode: any): string[] {
+    return listNode.children.map((item: any) => {
+      // Get first child only (ignore nested content for simple lists)
+      const firstChild = item.children[0];
+      return toString(firstChild).trim();
+    });
+  }
+
+  private nodeToString(node: any): string {
+    if (node.type === 'table') {
+      // Return markdown representation or JSON
+      return JSON.stringify(this.parseTable(node));
+    }
+    if (node.type === 'list') {
+      // Return as list
+      return this.parseList(node).map(item => `- ${item}`).join('\n');
+    }
+    return toString(node);
+  }
+
+  private getDefaultKeywords(): string[] {
+    return [
+      'answer', 'summary', 'introduction', 'conclusion', 'overview',
+      'assumptions', 'unknowns', 'evidence', 'notes', 'details',
+      'description', 'background', 'analysis', 'findings', 'recommendations',
+      'data', 'identity', 'network', 'security', 'monitoring', 'governance',
+      'availability', 'backup', 'patch', 'operational', 'provider'
+    ];
+  }
+}

package/src/types.ts

@@ -1,13 +1,29 @@
+export enum BulletMode {
+  ARRAY = 'array',
+  SECTIONS = 'sections',
+  AUTO = 'auto'
+}
+
 export interface MarkdownSection {
   heading: string;
-  content: any; // Can be string, array of objects (for tables), etc.
+  content: any;
   level: number;
-  format: 'heading' | 'list' | 'table' | 'text';
+  format?: 'heading' | 'bullet' | 'text';
 }
 
-export type ParseResult = Record<string, any>;
+export interface ParserOptions {
+  bulletMode?: BulletMode;
+  debug?: boolean;
+}
 
-export interface OutputFormatSpec {
-  // Define if the user wants to enforce a specific schema later
-  [key: string]: any;
+export interface BulletModeResult {
+  mode: 'array' | 'sections';
+  confidence: number;
+  reasons: string[];
 }
+
+// ADD THIS:
+export interface ParseResult {
+  sections?: MarkdownSection[];
+  [key: string]: any;  // Allow any string keys with any values
+}

package/test/bullet-mode.test.ts

@@ -0,0 +1,99 @@
+import { markdownToJson } from '../src/index.js';
+import { RemarkParser } from '../src/parser.js';
+import { BulletMode } from '../src/types.js';
+
+describe('Bullet Mode Detection', () => {
+
+    it('should auto-detect simple arrays', () => {
+        const md = `
+- Item 1
+- Item 2
+- Item 3
+    `;
+        const result = markdownToJson(md);
+        // Should be an array, or if it has no heading, maybe it's just the root content?
+        // In current implementation: "No sections found, put everything in root".
+        // Root content for a list in ARRAY mode is string[].
+
+        // However, JSONTransformer converts sections to an object.
+        // If there is only 'Root' section, markdownToJson returns { root: [...] } ?
+        // Let's check JSONTransformer logic.
+        // keys are camelCase headings. 'Root' -> 'root'.
+
+        expect(result.root).toEqual(['Item 1', 'Item 2', 'Item 3']);
+    });
+
+    it('should auto-detect sections', () => {
+        const md = `
+- Short Answer
+  The sky is blue.
+
+- Evidence
+  1. Look up.
+  2. See blue.
+    `;
+        const result = markdownToJson(md);
+        // Should explode into keys
+        expect(result.shortAnswer).toBe('The sky is blue.');
+        // Evidence has a nested list.
+        // In SECTIONS mode, nested list becomes content.
+        // processBulletAsSection logic: "Check for nested list... ONLY a nested list -> convert to array"
+        // OR "Mixed content -> object { text, items }"
+        // Here we have "1. Look up..." which is an ordered list.
+        // Our logic handles 'list' nodes inside the bullet item.
+
+        // Wait, "1. Look up" is an ordered list. detect-bullet-mode uses 'list' type check.
+        // processBulletAsSection finds `node.type === 'list'`.
+
+        expect(result.evidence).toBeDefined();
+        expect(Array.isArray(result.evidence)).toBe(true);
+        expect(result.evidence[0]).toContain('Look up');
+    });
+
+    it('should respect manual override to ARRAY', () => {
+        // A string that LOOKS like sections but we force ARRAY
+        const md = `
+- Short Answer
+- Evidence
+    `;
+        // If auto, might think it's array (no content). 
+        // Let's make it look like sections:
+        const mdSec = `
+- Section A:
+  Content A
+- Section B:
+  Content B
+    `;
+
+        const parser = new RemarkParser({ bulletMode: BulletMode.ARRAY });
+        // We need to bypass JSONTransformer for a direct parser test, or pass parser to transformer
+        // JSONTransformer can take a parser in options.
+
+        // But markdownToJson doesn't expose options. We'll use the class directly.
+        const sections = parser.parse(mdSec);
+
+        // In ARRAY mode, it should be treated as a single list under 'Root' (or wherever it is)
+        // and the content should be a flattened string array of the bullets text.
+
+        expect(sections.length).toBe(1);
+        expect(sections[0]?.heading).toBe('Root');
+        expect(Array.isArray(sections[0]?.content)).toBe(true);
+        expect(sections[0]?.content[0]).toContain('Section A'); // Just the text
+    });
+
+    it('should respect manual override to SECTIONS', () => {
+        // A string that LOOKS like array
+        const md = `
+- Item 1
+- Item 2
+    `;
+        const parser = new RemarkParser({ bulletMode: BulletMode.SECTIONS });
+        const sections = parser.parse(md);
+
+        // Should try to explode them.
+        // "Item 1" becomes a heading, content empty.
+        expect(sections.length).toBe(2);
+        expect(sections[0]?.heading).toBe('Item 1');
+        expect(sections[1]?.heading).toBe('Item 2');
+    });
+});

package/test/parser.2.test.ts

@@ -1,7 +1,7 @@
 import { markdownToJson } from '../src/index.js';
 
 describe('nx-json-parser - Complex Input', () => {
-    const str = `- Short answer
+  const str = `- Short answer
 Key considerations include data protection (encryption at rest and in transit), strong access control and IAM, secure network design, configuration and patch management, monitoring and logging, backups and disaster recovery, compliance and governance, and a clear shared responsibility model with incident response planning.
 
 - Full answer
@@ -87,17 +87,17 @@ Deploying a database in the cloud shifts part of the security burden to the clou
 11. Data residency considerations: Data localization requirements can influence data storage and transfer controls.  
 12. Vendor/service model considerations: Clear understanding of the provider’s and customer’s security responsibilities and controls is essential for effective security posture.`;
 
-    it('should parse the large security overview correctly', () => {
-        const result = markdownToJson(str);
+  it('should parse the large security overview correctly', () => {
+    const result = markdownToJson(str);
 
-        expect(result.shortAnswer).toBeDefined();
-        expect(result.fullAnswer).toBeDefined();
-        expect(result.dataProtection).toBeDefined();
-        expect(result.accessControlAndIdentityManagement).toBeDefined();
-        expect(result.assumptions).toBeInstanceOf(Array);
-        expect(result.evidence).toBeInstanceOf(Array);
+    expect(result.shortAnswer).toBeDefined();
+    expect(result.fullAnswer).toBeDefined();
+    expect(result.dataProtection).toBeDefined();
+    expect(result.accessControlAndIdentityManagement).toBeDefined();
+    expect(result.assumptions).toBeInstanceOf(Array);
+    expect(result.evidence).toBeInstanceOf(Array);
 
-        expect((result.evidence as string[]).length).toBe(12);
-        expect(result.shortAnswer).toContain('Key considerations include data protection');
-    });
+    expect((result.evidence as string[]).length).toBe(12);
+    expect(result.shortAnswer).toContain('Key considerations include data protection');
+  });
 });