flex-md 1.1.0 → 3.0.0

package/dist/__tests__/validate.test.js

@@ -0,0 +1,108 @@
+import { describe, it, expect } from "vitest";
+import { validateMarkdownAgainstOfs } from "../validate/validate.js";
+import { parseIssuesEnvelope } from "../ofs/issuesEnvelope.js";
+const SPEC = {
+    emptySectionValue: "None",
+    sections: [
+        { name: "Short answer", kind: "text", required: true },
+        { name: "Long answer", kind: "text", required: true },
+        { name: "Reasoning", kind: "ordered_list", required: true },
+        { name: "Assumptions", kind: "list", required: true },
+        { name: "Unknowns", kind: "list", required: true },
+    ],
+};
+function mdL1Good() {
+    return [
+        "## Short answer",
+        "Yes.",
+        "",
+        "## Long answer",
+        "More details.",
+        "",
+        "## Reasoning",
+        "1. First",
+        "2. Second",
+        "",
+        "## Assumptions",
+        "- A",
+        "",
+        "## Unknowns",
+        "- U",
+    ].join("\n");
+}
+describe("parseIssuesEnvelope()", () => {
+    it("detects envelope and extracts bullets", () => {
+        const env = [
+            "## Status",
+            "Non-compliant output (cannot be repaired to the required format).",
+            "",
+            "## Issues",
+            "- Missing required section: \"Short answer\"",
+            "",
+            "## Expected",
+            "- Headings ...",
+            "",
+            "## Found",
+            "- Something ...",
+            "",
+            "## How to fix",
+            "- Do X",
+        ].join("\n");
+        const parsed = parseIssuesEnvelope(env);
+        expect(parsed.isIssuesEnvelope).toBe(true);
+        expect(parsed.sections["issues"].bullets[0]).toContain("Missing required section");
+    });
+});
+describe("validateMarkdownAgainstOfs()", () => {
+    it("L0 accepts anything", () => {
+        const r = validateMarkdownAgainstOfs("whatever", SPEC, 0);
+        expect(r.ok).toBe(true);
+    });
+    it("L1 fails when required section missing", () => {
+        const md = [
+            "## Short answer",
+            "Yes.",
+            "",
+            "## Reasoning",
+            "1. A",
+        ].join("\n");
+        const r = validateMarkdownAgainstOfs(md, SPEC, 1);
+        expect(r.ok).toBe(false);
+        expect(r.issues.some(i => i.code === "MISSING_SECTION")).toBe(true);
+    });
+    it("L2 requires a single fenced markdown block", () => {
+        const r = validateMarkdownAgainstOfs(mdL1Good(), SPEC, 2);
+        expect(r.ok).toBe(false);
+        expect(r.issues.some(i => i.code === "CONTAINER_MISSING")).toBe(true);
+    });
+    it("L2 passes with one fence containing valid L1", () => {
+        const md = ["```markdown", mdL1Good(), "```"].join("\n");
+        const r = validateMarkdownAgainstOfs(md, SPEC, 2);
+        expect(r.ok).toBe(true);
+    });
+    it("L3 enforces section kinds (Reasoning must be ordered list)", () => {
+        const bad = [
+            "```markdown",
+            [
+                "## Short answer",
+                "Yes",
+                "",
+                "## Long answer",
+                "Details",
+                "",
+                "## Reasoning",
+                "- bullet but should be ordered",
+                "",
+                "## Assumptions",
+                "- A",
+                "",
+                "## Unknowns",
+                "- U",
+            ].join("\n"),
+            "```",
+        ].join("\n");
+        const r = validateMarkdownAgainstOfs(bad, SPEC, 3);
+        expect(r.ok).toBe(false);
+        expect(r.issues.some(i => i.code === "WRONG_SECTION_KIND")).toBe(true);
+    });
+});

package/dist/detect/json/detectIntent.d.ts

@@ -0,0 +1,2 @@
+import { DetectJsonIntentResult } from "./types.js";
+export declare function detectJsonIntent(text: string): DetectJsonIntentResult;

package/dist/detect/json/detectIntent.js

@@ -0,0 +1,79 @@
+const HARD_PATTERNS = [
+    /\breturn\s+only\s+json\b/i,
+    /\boutput\s+only\s+json\b/i,
+    /\bvalid\s+json\s+(object|array)\b/i,
+    /\bno\s+(prose|text|markdown)\b/i,
+    /\bdo\s+not\s+output\s+anything\s+else\b/i,
+];
+const SOFT_PATTERNS = [
+    /\binclude\s+json\b/i,
+    /\bjson\s+preferred\b/i,
+    /\badd\s+a\s+json\b/i,
+    /\bjson\s+format\b/i,
+];
+const SCHEMA_PATTERNS = [
+    /\bjson\s+schema\b/i,
+    /\bschema\b.*\bjson\b/i,
+    /\bajv\b/i,
+    /\bzod\b/i,
+    /\bopenapi\b/i,
+];
+const TOOLING_PATTERNS = [
+    /\bfunction\s+calling\b/i,
+    /\btools?\b/i,
+    /\bresponse_format\b/i,
+    /\btool\s+call\b/i,
+    /\barguments\b.*\bjson\b/i,
+];
+export function detectJsonIntent(text) {
+    const signals = [];
+    const add = (rx, strength, idx, len) => {
+        signals.push({
+            type: "pattern",
+            value: rx.source,
+            strength,
+            start: idx,
+            end: idx != null && len != null ? idx + len : undefined,
+        });
+    };
+    const scan = (patterns, strength) => {
+        for (const base of patterns) {
+            const rx = new RegExp(base.source, base.flags.includes("g") ? base.flags : base.flags + "g");
+            rx.lastIndex = 0;
+            let m;
+            while ((m = rx.exec(text)) !== null) {
+                add(base, strength, m.index, m[0]?.length ?? 0);
+                if (m[0]?.length === 0)
+                    rx.lastIndex++;
+            }
+        }
+    };
+    scan(HARD_PATTERNS, "hard");
+    scan(TOOLING_PATTERNS, "soft");
+    scan(SCHEMA_PATTERNS, "soft");
+    scan(SOFT_PATTERNS, "soft");
+    const hasHard = signals.some(s => s.strength === "hard");
+    const hasSchema = signals.some(s => SCHEMA_PATTERNS.some(rx => rx.source === s.value));
+    const hasTooling = signals.some(s => TOOLING_PATTERNS.some(rx => rx.source === s.value));
+    const hasSoft = signals.some(s => s.strength === "soft");
+    let intent = "none";
+    if (hasHard)
+        intent = "hard";
+    else if (hasSchema)
+        intent = "schema";
+    else if (hasTooling)
+        intent = "tooling";
+    else if (hasSoft)
+        intent = "soft";
+    const confidence = scoreConfidence(intent, signals);
+    return { intent, signals, confidence };
+}
+function scoreConfidence(intent, signals) {
+    if (intent === "none")
+        return signals.length ? 0.2 : 0.0;
+    const hard = signals.filter(s => s.strength === "hard").length;
+    const soft = signals.filter(s => s.strength === "soft").length;
+    if (intent === "hard")
+        return Math.min(1, 0.7 + hard * 0.15);
+    return Math.min(1, 0.4 + soft * 0.1);
+}

package/dist/detect/json/detectPresence.d.ts

@@ -0,0 +1,6 @@
+import { DetectJsonPresenceResult, DetectJsonContainersResult } from "./types.js";
+export declare function detectJsonContainers(md: string): DetectJsonContainersResult;
+export declare function detectJsonPresence(text: string, opts?: {
+    parse?: boolean;
+    maxParses?: number;
+}): DetectJsonPresenceResult;

package/dist/detect/json/detectPresence.js

@@ -0,0 +1,191 @@
+const RX_JSON_FENCE = /```json\s*\n([\s\S]*?)\n```/gi;
+const RX_JSON_MARKER = /(BEGIN_JSON|BEGIN\s+JSON)\s*\n([\s\S]*?)\n(END_JSON|END\s+JSON)/gi;
+// Very conservative raw JSON: must be only whitespace + object/array + whitespace
+const RX_RAW_JSON = /^\s*(\{[\s\S]*\}|\[[\s\S]*\])\s*$/;
+// Inline candidate finder
+const RX_INLINE_CANDIDATE = /(\{|\[)/g;
+export function detectJsonContainers(md) {
+    const fences = [];
+    const markers = [];
+    RX_JSON_FENCE.lastIndex = 0;
+    RX_JSON_MARKER.lastIndex = 0;
+    let m;
+    while ((m = RX_JSON_FENCE.exec(md)) !== null) {
+        fences.push({
+            start: m.index,
+            end: m.index + m[0].length,
+            kind: "fence",
+            snippet: preview(m[0]),
+        });
+    }
+    while ((m = RX_JSON_MARKER.exec(md)) !== null) {
+        markers.push({
+            start: m.index,
+            end: m.index + m[0].length,
+            kind: "marker",
+            snippet: preview(m[0]),
+        });
+    }
+    return {
+        hasJsonFence: fences.length > 0,
+        hasJsonMarkers: markers.length > 0,
+        fences,
+        markers,
+    };
+}
+export function detectJsonPresence(text, opts) {
+    const spans = [];
+    const kinds = new Set();
+    // 1) fenced json blocks
+    let m;
+    RX_JSON_FENCE.lastIndex = 0;
+    while ((m = RX_JSON_FENCE.exec(text)) !== null) {
+        kinds.add("fenced_json");
+        spans.push({
+            start: m.index,
+            end: m.index + m[0].length,
+            kind: "fence",
+            snippet: preview(m[0]),
+        });
+    }
+    // 2) BEGIN_JSON / END_JSON
+    RX_JSON_MARKER.lastIndex = 0;
+    while ((m = RX_JSON_MARKER.exec(text)) !== null) {
+        kinds.add("fenced_json");
+        spans.push({
+            start: m.index,
+            end: m.index + m[0].length,
+            kind: "marker",
+            snippet: preview(m[0]),
+        });
+    }
+    // 3) raw json (standalone)
+    if (RX_RAW_JSON.test(text) && looksLikeJson(text)) {
+        kinds.add("raw_json");
+        spans.push({ start: 0, end: text.length, kind: "raw", snippet: preview(text) });
+    }
+    else {
+        // 4) inline json-ish occurrences (best effort)
+        const inlineSpans = findInlineJsonishSpans(text);
+        if (inlineSpans.length) {
+            kinds.add("inline_json");
+            for (const s of inlineSpans)
+                spans.push(s);
+        }
+    }
+    const present = kinds.size > 0;
+    // Optional parse
+    const parse = !!opts?.parse;
+    if (!parse || !present) {
+        return { present, kinds: Array.from(kinds), spans };
+    }
+    const maxParses = opts?.maxParses ?? 3;
+    const parsedValues = [];
+    const errors = [];
+    const parseCandidates = spans
+        .map((s, i) => ({ s, i }))
+        .filter(x => x.s.kind === "fence" || x.s.kind === "raw" || x.s.kind === "inline")
+        .slice(0, maxParses);
+    for (const { s, i } of parseCandidates) {
+        const content = extractJsonText(text, s);
+        try {
+            parsedValues.push(JSON.parse(content));
+        }
+        catch (e) {
+            errors.push({ spanIndex: i, message: String(e?.message ?? e) });
+            if (looksLikeJson(content))
+                kinds.add("json_like");
+        }
+    }
+    return {
+        present,
+        kinds: Array.from(kinds),
+        spans,
+        parsed: {
+            ok: errors.length === 0 && parsedValues.length > 0,
+            values: parsedValues,
+            errors: errors.length ? errors : undefined,
+        },
+    };
+}
+function extractJsonText(full, span) {
+    const slice = full.slice(span.start, span.end);
+    if (span.kind === "fence") {
+        const m = slice.match(/^```json\s*\n([\s\S]*?)\n```$/i);
+        if (m)
+            return m[1] ?? "";
+    }
+    if (span.kind === "marker") {
+        const m = slice.match(/(BEGIN_JSON|BEGIN\s+JSON)\s*\n([\s\S]*?)\n(END_JSON|END\s+JSON)/i);
+        if (m)
+            return m[2] ?? "";
+    }
+    return slice;
+}
+function looksLikeJson(s) {
+    const t = s.trim();
+    if (!t)
+        return false;
+    if ((t.startsWith("{") && t.endsWith("}")) || (t.startsWith("[") && t.endsWith("]")))
+        return true;
+    return false;
+}
+function findInlineJsonishSpans(text) {
+    const spans = [];
+    const maxLen = 5000;
+    const masked = maskNonJsonCodeFences(text);
+    let m;
+    RX_INLINE_CANDIDATE.lastIndex = 0;
+    while ((m = RX_INLINE_CANDIDATE.exec(masked)) !== null) {
+        const start = m.index;
+        const open = masked[start];
+        const close = open === "{" ? "}" : "]";
+        let depth = 0;
+        let end = -1;
+        for (let i = start; i < Math.min(masked.length, start + maxLen); i++) {
+            const ch = masked[i];
+            if (ch === open)
+                depth++;
+            else if (ch === close) {
+                depth--;
+                if (depth === 0) {
+                    end = i + 1;
+                    break;
+                }
+            }
+        }
+        if (end > start) {
+            const snippet = text.slice(start, Math.min(end, start + 300));
+            if (snippet.trim().length >= 5) {
+                spans.push({ start, end, kind: "inline", snippet: preview(text.slice(start, end)) });
+            }
+            RX_INLINE_CANDIDATE.lastIndex = end;
+        }
+    }
+    return squashOverlaps(spans);
+}
+function maskNonJsonCodeFences(text) {
+    const rx = /```(\w+)?\s*\n([\s\S]*?)\n```/g;
+    return text.replace(rx, (full, lang) => {
+        const l = String(lang ?? "").toLowerCase();
+        if (l === "json")
+            return full;
+        return " ".repeat(full.length);
+    });
+}
+function squashOverlaps(spans) {
+    spans.sort((a, b) => a.start - b.start || (b.end - b.start) - (a.end - a.start));
+    const out = [];
+    let last = null;
+    for (const s of spans) {
+        if (!last || s.start >= last.end) {
+            out.push(s);
+            last = s;
+        }
+    }
+    return out;
+}
+function preview(s, n = 160) {
+    const t = s.replace(/\s+/g, " ").trim();
+    return t.length > n ? t.slice(0, n - 1) + "…" : t;
+}

package/dist/detect/json/index.d.ts

@@ -0,0 +1,7 @@
+import { DetectJsonAllResult } from "./types.js";
+export * from "./types.js";
+export { detectJsonIntent } from "./detectIntent.js";
+export { detectJsonContainers, detectJsonPresence } from "./detectPresence.js";
+export declare function detectJsonAll(textOrMd: string, opts?: {
+    parseJson?: boolean;
+}): DetectJsonAllResult;

package/dist/detect/json/index.js

@@ -0,0 +1,12 @@
+import { detectJsonIntent } from "./detectIntent.js";
+import { detectJsonContainers, detectJsonPresence } from "./detectPresence.js";
+export * from "./types.js";
+export { detectJsonIntent } from "./detectIntent.js";
+export { detectJsonContainers, detectJsonPresence } from "./detectPresence.js";
+export function detectJsonAll(textOrMd, opts) {
+    return {
+        intent: detectJsonIntent(textOrMd),
+        containers: detectJsonContainers(textOrMd),
+        presence: detectJsonPresence(textOrMd, { parse: !!opts?.parseJson }),
+    };
+}

package/dist/detect/json/types.d.ts

@@ -0,0 +1,43 @@
+export type JsonIntent = "none" | "soft" | "hard" | "schema" | "tooling";
+export type JsonPresenceKind = "none" | "fenced_json" | "raw_json" | "inline_json" | "json_like";
+export interface JsonSpan {
+    start: number;
+    end: number;
+    kind: "fence" | "raw" | "inline" | "marker";
+    snippet: string;
+}
+export interface DetectJsonPresenceResult {
+    present: boolean;
+    kinds: JsonPresenceKind[];
+    spans: JsonSpan[];
+    parsed?: {
+        ok: boolean;
+        values: unknown[];
+        errors?: Array<{
+            spanIndex: number;
+            message: string;
+        }>;
+    };
+}
+export interface DetectJsonContainersResult {
+    hasJsonFence: boolean;
+    hasJsonMarkers: boolean;
+    fences: JsonSpan[];
+    markers: JsonSpan[];
+}
+export interface DetectJsonIntentResult {
+    intent: JsonIntent;
+    signals: Array<{
+        type: "pattern";
+        value: string;
+        strength: "soft" | "hard";
+        start?: number;
+        end?: number;
+    }>;
+    confidence: number;
+}
+export interface DetectJsonAllResult {
+    intent: DetectJsonIntentResult;
+    containers: DetectJsonContainersResult;
+    presence: DetectJsonPresenceResult;
+}

package/dist/detect/json/types.js

@@ -0,0 +1 @@
+export {};

package/dist/detection/detector.d.ts

@@ -0,0 +1,6 @@
+import type { DetectedObject } from "../types.js";
+/**
+ * Detect FlexMD and other structured objects in arbitrary text.
+ * Returns all detected objects with confidence scores and byte ranges.
+ */
+export declare function detectObjects(text: string): DetectedObject[];

package/dist/detection/detector.js

@@ -0,0 +1,104 @@
+/**
+ * Detect FlexMD and other structured objects in arbitrary text.
+ * Returns all detected objects with confidence scores and byte ranges.
+ */
+export function detectObjects(text) {
+    const detected = [];
+    // Tier A: Detect ```flexmd fenced blocks (highest confidence)
+    detected.push(...detectFlexMdFences(text));
+    // Tier B: Detect ```json blocks with FlexDocument shape
+    detected.push(...detectFlexDocJsonFences(text));
+    // Tier C: Detect raw FlexMD markers (best effort)
+    detected.push(...detectRawFlexMd(text));
+    // Sort by start position
+    detected.sort((a, b) => a.start - b.start);
+    return detected;
+}
+/**
+ * Tier A: Detect ```flexmd fenced blocks.
+ */
+function detectFlexMdFences(text) {
+    const detected = [];
+    const regex = /```flexmd\n([\s\S]*?)```/g;
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        detected.push({
+            kind: "flexmd_fence",
+            confidence: 1.0,
+            start: match.index,
+            end: match.index + match[0].length,
+            raw: match[0],
+            inner: match[1]
+        });
+    }
+    return detected;
+}
+/**
+ * Tier B: Detect ```json blocks that match FlexDocument shape.
+ */
+function detectFlexDocJsonFences(text) {
+    const detected = [];
+    const regex = /```json\n([\s\S]*?)```/g;
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        const inner = match[1];
+        // Try to parse as JSON
+        try {
+            const parsed = JSON.parse(inner);
+            // Check if it has the FlexDocument shape (has "frames" array)
+            if (parsed && typeof parsed === "object" && Array.isArray(parsed.frames)) {
+                detected.push({
+                    kind: "flexdoc_json_fence",
+                    confidence: 0.9,
+                    start: match.index,
+                    end: match.index + match[0].length,
+                    raw: match[0],
+                    inner
+                });
+            }
+        }
+        catch {
+            // Not valid JSON, skip
+        }
+    }
+    return detected;
+}
+/**
+ * Tier C: Detect raw FlexMD markers (best effort).
+ * Looks for at least 2 strong markers within first 500 chars:
+ * - [[...]]
+ * - @key:
+ * - @payload:name:
+ */
+function detectRawFlexMd(text) {
+    const detected = [];
+    // Look for frame markers
+    const frameRegex = /\[\[([^\]]+)\]\]/g;
+    const metaRegex = /@[a-zA-Z_][a-zA-Z0-9_]*:/g;
+    const payloadRegex = /@payload:[a-zA-Z_][a-zA-Z0-9_]*:/g;
+    let match;
+    const markers = [];
+    // Collect all marker positions
+    while ((match = frameRegex.exec(text)) !== null) {
+        markers.push(match.index);
+    }
+    while ((match = metaRegex.exec(text)) !== null) {
+        markers.push(match.index);
+    }
+    while ((match = payloadRegex.exec(text)) !== null) {
+        markers.push(match.index);
+    }
+    // If we have at least 2 markers, consider it raw FlexMD
+    if (markers.length >= 2) {
+        const start = Math.min(...markers);
+        const end = text.length;
+        detected.push({
+            kind: "raw_flexmd",
+            confidence: 0.7,
+            start,
+            end,
+            raw: text.substring(start, end)
+        });
+    }
+    return detected;
+}

package/dist/detection/extractor.d.ts

@@ -0,0 +1,10 @@
+import type { FlexDocument } from "../types.js";
+export interface ParseAnyResult {
+    flexDocs: FlexDocument[];
+    markdownSnippets: string[];
+    remainder: string;
+}
+/**
+ * Parse any text and extract all FlexMD documents and Markdown snippets.
+ */
+export declare function parseAny(text: string): ParseAnyResult;

package/dist/detection/extractor.js

@@ -0,0 +1,54 @@
+import { detectObjects } from "./detector.js";
+import { parseFlexMd } from "../parser.js";
+/**
+ * Parse any text and extract all FlexMD documents and Markdown snippets.
+ */
+export function parseAny(text) {
+    const detected = detectObjects(text);
+    const flexDocs = [];
+    const markdownSnippets = [];
+    for (const obj of detected) {
+        if (obj.kind === "flexmd_fence" && obj.inner) {
+            // Parse fenced FlexMD
+            try {
+                const doc = parseFlexMd(obj.inner);
+                flexDocs.push(doc);
+            }
+            catch {
+                // Parse failed, treat as markdown snippet
+                markdownSnippets.push(obj.inner);
+            }
+        }
+        else if (obj.kind === "flexdoc_json_fence" && obj.inner) {
+            // Parse JSON FlexDocument
+            try {
+                const doc = JSON.parse(obj.inner);
+                flexDocs.push(doc);
+            }
+            catch {
+                // Parse failed, skip
+            }
+        }
+        else if (obj.kind === "raw_flexmd") {
+            // Parse raw FlexMD
+            try {
+                const doc = parseFlexMd(obj.raw);
+                flexDocs.push(doc);
+            }
+            catch {
+                // Parse failed, treat as markdown snippet
+                markdownSnippets.push(obj.raw);
+            }
+        }
+    }
+    // Calculate remainder (text not covered by detected objects)
+    let remainder = text;
+    for (const obj of detected) {
+        remainder = remainder.replace(obj.raw, "");
+    }
+    return {
+        flexDocs,
+        markdownSnippets,
+        remainder: remainder.trim()
+    };
+}

package/dist/extract/extract.d.ts

@@ -0,0 +1,5 @@
+import { OutputFormatSpec, ExtractedResult } from "../types.js";
+/**
+ * Extracts sections, lists, and tables from Markdown based on the OFS.
+ */
+export declare function extractFromMarkdown(md: string, spec: OutputFormatSpec): ExtractedResult;