flex-md 1.0.0 → 2.0.0

package/dist/parsers/lists.js

@@ -0,0 +1,36 @@
+/**
+ * Parse nested Markdown lists into a tree structure.
+ * Supports both unordered (-) and ordered (1.) lists.
+ */
+export function parseList(md) {
+    const lines = md.split("\n");
+    const isListLine = (s) => /^\s*(-\s+|\d+\.\s+)/.test(s);
+    const listLines = lines.filter(isListLine);
+    if (!listLines.length)
+        return null;
+    // Determine if ordered by presence of numbered item
+    const ordered = listLines.some(l => /^\s*\d+\.\s+/.test(l));
+    const root = [];
+    const stack = [];
+    for (const line of listLines) {
+        const indent = (line.match(/^\s*/)?.[0].length) ?? 0;
+        const mOrdered = line.match(/^\s*(\d+)\.\s+(.*)$/);
+        const mUn = line.match(/^\s*-\s+(.*)$/);
+        const text = (mOrdered?.[2] ?? mUn?.[1] ?? "").trim();
+        const item = { text, children: [] };
+        if (mOrdered)
+            item.index = Number(mOrdered[1]);
+        // Pop stack until we find parent (lower indent)
+        while (stack.length && stack[stack.length - 1].indent >= indent) {
+            stack.pop();
+        }
+        if (!stack.length) {
+            root.push(item);
+        }
+        else {
+            stack[stack.length - 1].item.children.push(item);
+        }
+        stack.push({ indent, item });
+    }
+    return { kind: "list", ordered, items: root };
+}

package/dist/parsers/tables.d.ts

@@ -0,0 +1,10 @@
+import type { ParsedTable } from "../types.js";
+/**
+ * Parse a single GFM pipe table block.
+ * Returns null if the block is not a valid table.
+ */
+export declare function parsePipeTable(block: string): ParsedTable | null;
+/**
+ * Extract all pipe tables from a Markdown document.
+ */
+export declare function extractAllTables(md: string): ParsedTable[];

package/dist/parsers/tables.js

@@ -0,0 +1,58 @@
+/**
+ * Parse a single GFM pipe table block.
+ * Returns null if the block is not a valid table.
+ */
+export function parsePipeTable(block) {
+    const lines = block.split("\n").map(l => l.trim()).filter(Boolean);
+    if (lines.length < 2)
+        return null;
+    const header = lines[0];
+    const sep = lines[1];
+    if (!header || !header.includes("|"))
+        return null;
+    if (!sep || !sep.match(/^\|?[\s:-]+\|/))
+        return null;
+    const parseRow = (row) => row.replace(/^\|/, "").replace(/\|$/, "").split("|").map(c => c.trim());
+    const columns = parseRow(header);
+    const rows = lines.slice(2).map(parseRow);
+    // Normalize row lengths
+    for (const r of rows) {
+        while (r.length < columns.length)
+            r.push("");
+    }
+    // Detect ordered table (first column is "#")
+    const isOrdered = columns[0] === "#";
+    const kind = isOrdered ? "ordered_table" : "table";
+    return { kind, columns, rows };
+}
+/**
+ * Extract all pipe tables from a Markdown document.
+ */
+export function extractAllTables(md) {
+    const tables = [];
+    const lines = md.split("\n");
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i];
+        // Look for potential table start (contains |)
+        if (line && line.includes("|")) {
+            // Collect consecutive lines that look like table rows
+            const tableLines = [];
+            let j = i;
+            while (j < lines.length && lines[j] && lines[j].includes("|")) {
+                tableLines.push(lines[j]);
+                j++;
+            }
+            if (tableLines.length >= 2) {
+                const table = parsePipeTable(tableLines.join("\n"));
+                if (table) {
+                    tables.push(table);
+                    i = j;
+                    continue;
+                }
+            }
+        }
+        i++;
+    }
+    return tables;
+}

package/dist/test-runner.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test-runner.js

@@ -0,0 +1,328 @@
+import { 
+// Layer A
+parseFlexMd, stringifyFlexMd, validateFlexMd, 
+// Layer B
+parseOutputFormatSpec, stringifyOutputFormatSpec, enrichInstructions, validateOutput, extractOutput, 
+// Outline
+buildOutline, renderOutline, 
+// Parsers
+parseList, parsePipeTable, extractAllTables, 
+// Layer C
+detectObjects, parseAny } from "./index.js";
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+    try {
+        fn();
+        console.log(`✅ ${name}`);
+        passed++;
+    }
+    catch (e) {
+        console.error(`❌ ${name}`);
+        console.error(e);
+        failed++;
+    }
+}
+function assert(condition, message) {
+    if (!condition) {
+        throw new Error(`Assertion failed: ${message}`);
+    }
+}
+console.log("🧪 Running FlexMD v1.1 Tests\n");
+// ============================================================================
+// Layer A Tests
+// ============================================================================
+console.log("📦 Layer A - FlexMD Frames\n");
+test("Parse basic FlexMD frame", () => {
+    const md = `[[message role=user id=m1]]
+@tags: auth, login
+Hello world
+`;
+    const doc = parseFlexMd(md);
+    assert(doc.frames.length === 1, "Should have 1 frame");
+    assert(doc.frames[0].type === "message", "Type should be message");
+    assert(doc.frames[0].role === "user", "Role should be user");
+    assert(Array.isArray(doc.frames[0].meta?.tags), "Tags should be array");
+});
+test("Stringify FlexMD frame", () => {
+    const doc = {
+        frames: [{
+                type: "message",
+                role: "user",
+                id: "m1",
+                meta: { tags: ["auth", "login"] },
+                body_md: "Hello world\n"
+            }]
+    };
+    const md = stringifyFlexMd(doc);
+    assert(md.includes("[[message"), "Should contain frame header");
+    assert(md.includes("@tags:"), "Should contain meta");
+});
+test("Parse with type inference", () => {
+    const md = `[[message]]
+@priority: 5
+@enabled: true
+@value: null
+Body text
+`;
+    const doc = parseFlexMd(md, { metaTypeMode: "infer" });
+    assert(doc.frames[0].meta?.priority === 5, "Should infer number");
+    assert(doc.frames[0].meta?.enabled === true, "Should infer boolean");
+    assert(doc.frames[0].meta?.value === null, "Should infer null");
+});
+test("Validate FlexMD", () => {
+    const valid = `[[message role=user]]
+Hello
+`;
+    const result = validateFlexMd(valid);
+    assert(result.valid === true, "Should be valid");
+});
+// ============================================================================
+// Layer B Tests
+// ============================================================================
+console.log("\n📋 Layer B - Output Format Spec\n");
+test("Parse OFS", () => {
+    const md = `## Output format (Markdown)
+Include these sections somewhere (order does not matter):
+
+- Short answer — prose
+- Reasoning — ordered list
+- Assumptions — list
+`;
+    const spec = parseOutputFormatSpec(md);
+    assert(spec.sections.length === 3, "Should have 3 sections");
+    assert(spec.sections[0].name === "Short answer", "First section name");
+    assert(spec.sections[0].kind === "prose", "First section kind");
+    assert(spec.sections[1].kind === "ordered_list", "Second section kind");
+});
+test("Stringify OFS", () => {
+    const spec = {
+        descriptorType: "output_format_spec",
+        format: "markdown",
+        sectionOrderMatters: false,
+        sections: [
+            { name: "Answer", kind: "prose" }
+        ],
+        tablesOptional: true,
+        tables: [],
+        emptySectionValue: "None"
+    };
+    const md = stringifyOutputFormatSpec(spec);
+    assert(md.includes("Answer — prose"), "Should contain section");
+    assert(md.includes("None"), "Should contain empty value");
+});
+test("Enrich instructions", () => {
+    const spec = {
+        descriptorType: "output_format_spec",
+        format: "markdown",
+        sectionOrderMatters: false,
+        sections: [
+            { name: "Steps", kind: "ordered_list" },
+            { name: "Notes", kind: "list" }
+        ],
+        tablesOptional: true,
+        tables: [],
+        emptySectionValue: "None"
+    };
+    const instructions = enrichInstructions(spec);
+    assert(instructions.includes("numbered items"), "Should mention ordered lists");
+    assert(instructions.includes("bullets"), "Should mention lists");
+    assert(instructions.includes("None"), "Should mention empty value");
+});
+test("Validate output against OFS", () => {
+    const spec = {
+        descriptorType: "output_format_spec",
+        format: "markdown",
+        sectionOrderMatters: false,
+        sections: [
+            { name: "Answer", kind: "prose" }
+        ],
+        tablesOptional: true,
+        tables: []
+    };
+    const md = `## Answer\nThe answer is 42.`;
+    const result = validateOutput(md, spec);
+    assert(result.ok === true, "Should validate successfully");
+});
+test("Extract output from OFS", () => {
+    const spec = {
+        descriptorType: "output_format_spec",
+        format: "markdown",
+        sectionOrderMatters: false,
+        sections: [
+            { name: "Answer", kind: "prose" },
+            { name: "Steps", kind: "ordered_list" }
+        ],
+        tablesOptional: true,
+        tables: []
+    };
+    const md = `## Answer
+The answer is 42.
+
+## Steps
+1. First step
+2. Second step
+`;
+    const extracted = extractOutput(md, spec);
+    assert(extracted.sectionsByName["Answer"].md.includes("42"), "Should extract answer");
+    assert(extracted.sectionsByName["Steps"].list?.ordered === true, "Should parse ordered list");
+    assert(extracted.sectionsByName["Steps"].list?.items.length === 2, "Should have 2 items");
+});
+// ============================================================================
+// Outline Tests
+// ============================================================================
+console.log("\n🌳 Outline & Tree Building\n");
+test("Build outline from headings", () => {
+    const md = `# Title
+Content
+
+## Section 1
+More content
+
+### Subsection 1.1
+Details
+
+## Section 2
+Other content
+`;
+    const outline = buildOutline(md);
+    assert(outline.nodes.length === 1, "Should have 1 root node");
+    assert(outline.nodes[0].title === "Title", "Root title");
+    assert(outline.nodes[0].children.length === 2, "Should have 2 children");
+    assert(outline.nodes[0].children[0].children.length === 1, "First child has 1 child");
+});
+test("Render outline back to Markdown", () => {
+    const outline = {
+        type: "md_outline",
+        nodes: [{
+                title: "Title",
+                level: 1,
+                key: "title",
+                content_md: "Content\n",
+                children: [{
+                        title: "Section",
+                        level: 2,
+                        key: "section",
+                        content_md: "More\n",
+                        children: []
+                    }]
+            }]
+    };
+    const md = renderOutline(outline);
+    assert(md.includes("# Title"), "Should have h1");
+    assert(md.includes("## Section"), "Should have h2");
+    assert(!md.includes("key"), "Should not include internal keys");
+});
+// ============================================================================
+// Parser Tests
+// ============================================================================
+console.log("\n📝 List & Table Parsers\n");
+test("Parse unordered list", () => {
+    const md = `- Item 1
+  - Nested 1.1
+- Item 2
+`;
+    const list = parseList(md);
+    assert(list !== null, "Should parse list");
+    assert(list.ordered === false, "Should be unordered");
+    assert(list.items.length === 2, "Should have 2 items");
+    assert(list.items[0].children.length === 1, "First item has 1 child");
+});
+test("Parse ordered list", () => {
+    const md = `1. First
+   1. Sub-first
+2. Second
+`;
+    const list = parseList(md);
+    assert(list !== null, "Should parse list");
+    assert(list.ordered === true, "Should be ordered");
+    assert(list.items.length === 2, "Should have 2 items");
+    assert(list.items[0].index === 1, "First item index");
+});
+test("Parse pipe table", () => {
+    const md = `| Name | Age |
+|------|-----|
+| Alice | 30 |
+| Bob | 25 |
+`;
+    const table = parsePipeTable(md);
+    assert(table !== null, "Should parse table");
+    assert(table.columns.length === 2, "Should have 2 columns");
+    assert(table.rows.length === 2, "Should have 2 rows");
+    assert(table.kind === "table", "Should be regular table");
+});
+test("Parse ordered table", () => {
+    const md = `| # | Task | Status |
+|---|------|--------|
+| 1 | Do it | Done |
+| 2 | Check | Pending |
+`;
+    const table = parsePipeTable(md);
+    assert(table !== null, "Should parse table");
+    assert(table.kind === "ordered_table", "Should be ordered table");
+    assert(table.columns[0] === "#", "First column should be #");
+});
+test("Extract all tables", () => {
+    const md = `Some text
+
+| A | B |
+|---|---|
+| 1 | 2 |
+
+More text
+
+| C | D |
+|---|---|
+| 3 | 4 |
+`;
+    const tables = extractAllTables(md);
+    assert(tables.length === 2, "Should find 2 tables");
+});
+// ============================================================================
+// Layer C Tests
+// ============================================================================
+console.log("\n🔍 Layer C - Detection & Extraction\n");
+test("Detect flexmd fence", () => {
+    const text = `Some text
+\`\`\`flexmd
+[[message]]
+Hello
+\`\`\`
+More text
+`;
+    const detected = detectObjects(text);
+    assert(detected.length > 0, "Should detect object");
+    assert(detected[0].kind === "flexmd_fence", "Should be flexmd fence");
+    assert(detected[0].confidence === 1.0, "Should have high confidence");
+});
+test("Detect JSON FlexDocument", () => {
+    const text = `\`\`\`json
+{"frames": [{"type": "message", "body_md": "Hello"}]}
+\`\`\`
+`;
+    const detected = detectObjects(text);
+    assert(detected.length > 0, "Should detect object");
+    assert(detected[0].kind === "flexdoc_json_fence", "Should be JSON FlexDoc");
+});
+test("Parse any text", () => {
+    const text = `Random text
+\`\`\`flexmd
+[[message role=user]]
+Hello
+\`\`\`
+More text
+`;
+    const result = parseAny(text);
+    assert(result.flexDocs.length === 1, "Should extract 1 FlexDoc");
+    assert(result.flexDocs[0].frames.length === 1, "Should have 1 frame");
+});
+// ============================================================================
+// Summary
+// ============================================================================
+console.log("\n" + "=".repeat(50));
+console.log(`✅ Passed: ${passed}`);
+console.log(`❌ Failed: ${failed}`);
+console.log("=" + "=".repeat(50));
+if (failed > 0) {
+    throw new Error("Tests failed");
+}

package/dist/types.d.ts

@@ -22,6 +22,8 @@ export interface FlexFrame {
     payloads?: Record<string, FlexPayload>;
 }
 export interface FlexDocument {
+    title?: string;
+    meta?: Record<string, FlexMetaValue>;
     frames: FlexFrame[];
 }
 export interface ParseOptions {
@@ -30,6 +32,18 @@ export interface ParseOptions {
      * Default: ["tags", "refs"]
      */
     arrayMetaKeys?: string[];
+    /**
+     * How to parse metadata values:
+     * - "strings": all values remain strings (default)
+     * - "infer": safely infer boolean, null, number
+     * - "schema": use metaSchema to determine types
+     */
+    metaTypeMode?: "strings" | "infer" | "schema";
+    /**
+     * Schema for metadata value types (used when metaTypeMode="schema")
+     * Example: { priority: "number", enabled: "boolean" }
+     */
+    metaSchema?: Record<string, "string" | "number" | "boolean" | "null">;
 }
 export interface StringifyOptions {
     /**
@@ -47,3 +61,80 @@ export interface StringifyOptions {
      */
     fence?: "```" | "~~~";
 }
+export interface FlexValidationError {
+    line: number;
+    message: string;
+    severity: "error" | "warning";
+}
+export interface FlexValidationResult {
+    valid: boolean;
+    errors: FlexValidationError[];
+}
+export type SectionKind = "prose" | "list" | "ordered_list";
+export interface OfsSection {
+    name: string;
+    kind: SectionKind;
+    hint?: string;
+}
+export type TableKind = "table" | "ordered_table";
+export interface OfsTable {
+    columns: string[];
+    kind: TableKind;
+    by?: string;
+}
+export interface OutputFormatSpec {
+    descriptorType: "output_format_spec";
+    format: "markdown";
+    sectionOrderMatters: false;
+    sections: OfsSection[];
+    tablesOptional: boolean;
+    tables: OfsTable[];
+    emptySectionValue?: string;
+}
+export interface MdNode {
+    title: string;
+    level: number;
+    key: string;
+    id?: string;
+    content_md: string;
+    children: MdNode[];
+}
+export interface MdOutline {
+    type: "md_outline";
+    nodes: MdNode[];
+}
+export interface ListItem {
+    text: string;
+    index?: number;
+    children: ListItem[];
+}
+export interface ParsedList {
+    kind: "list";
+    ordered: boolean;
+    items: ListItem[];
+}
+export interface ParsedTable {
+    kind: "table" | "ordered_table";
+    by?: string;
+    columns: string[];
+    rows: string[][];
+}
+export interface ExtractedResult {
+    outline: MdOutline;
+    sectionsByName: Record<string, {
+        nodeKey: string;
+        nodeLevel: number;
+        md: string;
+        list?: ParsedList;
+    }>;
+    tables: ParsedTable[];
+}
+export type DetectedKind = "flexmd_fence" | "flexdoc_json_fence" | "raw_flexmd" | "markdown_snippet" | "none";
+export interface DetectedObject {
+    kind: DetectedKind;
+    confidence: number;
+    start: number;
+    end: number;
+    raw: string;
+    inner?: string;
+}

package/dist/validator.d.ts

@@ -0,0 +1,2 @@
+import type { FlexValidationResult } from "./types.js";
+export declare function validateFlexMd(input: string): FlexValidationResult;

package/dist/validator.js

@@ -0,0 +1,80 @@
+const HEADER_START = "[[";
+const HEADER_END = "]]";
+const PAYLOAD_DECL_RE = /^@payload:name:\s*(.+)\s*$/;
+const FENCE_RE = /^(```|~~~)/;
+export function validateFlexMd(input) {
+    const lines = input.split("\n");
+    const errors = [];
+    let inFence = null;
+    let pendingPayloadLine = null;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const lineNum = i + 1;
+        // Check for unterminated headers
+        if (line.includes(HEADER_START) && !line.includes(HEADER_END)) {
+            // Small check: if it really looks like a header start at beginning of line
+            if (line.trim().startsWith(HEADER_START)) {
+                errors.push({
+                    line: lineNum,
+                    message: "Possible unterminated frame header (missing ']]')",
+                    severity: "error",
+                });
+            }
+        }
+        // Handle code fences
+        const fenceMatch = line.match(FENCE_RE);
+        if (fenceMatch) {
+            const fence = fenceMatch[1];
+            if (inFence) {
+                if (line.trimEnd() === inFence) {
+                    inFence = null;
+                    pendingPayloadLine = null; // Payload satisfied
+                }
+            }
+            else {
+                inFence = fence;
+            }
+        }
+        else if (pendingPayloadLine !== null && !inFence && line.trim().length > 0) {
+            // We had a @payload:name: but the next non-empty line wasn't a fence
+            errors.push({
+                line: pendingPayloadLine,
+                message: "Payload declaration not followed by a code fence",
+                severity: "warning",
+            });
+            pendingPayloadLine = null;
+        }
+        // Check for payload declarations
+        const payloadMatch = line.match(PAYLOAD_DECL_RE);
+        if (payloadMatch) {
+            if (inFence) {
+                errors.push({
+                    line: lineNum,
+                    message: "Payload declaration inside a code fence",
+                    severity: "warning",
+                });
+            }
+            else {
+                pendingPayloadLine = lineNum;
+            }
+        }
+    }
+    if (inFence) {
+        errors.push({
+            line: lines.length,
+            message: "Unterminated code fence at end of document",
+            severity: "error",
+        });
+    }
+    if (pendingPayloadLine !== null) {
+        errors.push({
+            line: pendingPayloadLine,
+            message: "Dangling payload declaration at end of document",
+            severity: "warning",
+        });
+    }
+    return {
+        valid: !errors.some((e) => e.severity === "error"),
+        errors,
+    };
+}

package/package.json

@@ -1,10 +1,21 @@
 {
   "name": "flex-md",
-  "version": "1.0.0",
-  "description": "Parse and stringify FlexMD Frames (semi-structured Markdown) to/from JSON.",
+  "version": "2.0.0",
+  "description": "Parse and stringify FlexMD: semi-structured Markdown with three powerful layers - Frames, Output Format Spec (OFS), and Detection/Extraction.",
   "license": "MIT",
   "author": "",
-  "keywords": ["markdown", "parser", "serialization", "llm", "prompting", "semi-structured"],
+  "keywords": [
+    "markdown",
+    "parser",
+    "serialization",
+    "llm",
+    "prompting",
+    "semi-structured",
+    "ofs",
+    "output-format",
+    "detection",
+    "extraction"
+  ],
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -16,12 +27,15 @@
       "require": "./dist/index.cjs"
     }
   },
-  "files": ["dist"],
+  "files": [
+    "dist",
+    "SPEC.md"
+  ],
   "scripts": {
     "build": "tsc && node ./scripts/cjs-bridge.mjs",
-    "test": "node ./dist/index.js"
+    "test": "npm run build && node ./dist/test-runner.js"
   },
   "devDependencies": {
     "typescript": "^5.6.3"
   }
-}
+}