flex-md 1.0.0 → 2.0.0

package/SPEC.md

@@ -0,0 +1,559 @@
+# flex-md — End-to-end Spec (v1.1)
+
+## 0) What this package does
+
+`flex-md` provides **two complementary layers**:
+
+### Layer A — FlexMD Frames (semi-structured Markdown)
+
+A tiny set of anchors (frames, meta, payload binding) on top of Markdown to reliably round-trip to/from JSON.
+
+### Layer B — Plain Markdown "Output Format Spec" (OFS)
+
+A **Markdown-native contract** that any LLM can follow without knowing FlexMD.
+From OFS, the package can:
+
+* generate minimal LLM guidance (enricher)
+* validate the response format
+* extract content to JSON, building a **nested tree** from heading levels
+
+### Layer C — Detection & Extraction from arbitrary text
+
+Find and parse:
+
+* fenced ` ```flexmd ` objects (best)
+* fenced JSON "FlexDocument" objects
+* raw/unframed FlexMD (best-effort)
+* optional generic Markdown snippets (opaque or lightly structured)
+
+---
+
+## 1) Core principles (hard requirements)
+
+1. **Markdown-first**: All guidance is written in plain Markdown concepts (headings, lists, tables, fenced blocks).
+2. **Section order never matters**.
+3. **Order matters only when the content type says it does**:
+   * `ordered list` ⇒ numbered list
+   * `ordered table` ⇒ `#` column with `1..N`
+4. **Structure comes from heading levels**: accept any heading level (`#..######`) and build nested JSON from it.
+5. **Internal keys/ids/paths are never rendered back to Markdown** (unless debug mode).
+
+---
+
+## 2) Data models (TypeScript)
+
+### 2.1 FlexMD Frames (Layer A)
+
+```ts
+export type FlexMetaValue = string | string[] | number | boolean | null;
+
+export interface FlexPayload {
+  lang?: string;     // e.g. "json", "table"
+  raw: string;       // always preserved
+  value: unknown;    // parsed JSON for json; parsed table structure for table; otherwise string
+  parseError?: string;
+}
+
+export interface FlexFrame {
+  type: string;      // e.g. "message", "section", ...
+  role?: string;     // user|assistant|system|tool|...
+  id?: string;
+  ts?: string;
+
+  meta?: Record<string, FlexMetaValue>;
+  title?: string;
+  body_md?: string;
+
+  payloads?: Record<string, FlexPayload>;
+}
+
+export interface FlexDocument {
+  title?: string;
+  meta?: Record<string, FlexMetaValue>;
+  frames: FlexFrame[];
+}
+```
+
+### 2.2 Output Format Spec (OFS) (Layer B)
+
+```ts
+export type SectionKind = "prose" | "list" | "ordered_list";
+
+export interface OfsSection {
+  name: string;       // "Short answer"
+  kind: SectionKind;  // prose/list/ordered_list
+  hint?: string;      // optional text after delimiter, not required
+}
+
+export type TableKind = "table" | "ordered_table";
+
+export interface OfsTable {
+  columns: string[];  // ["property1","property2"]
+  kind: TableKind;
+  by?: string;        // informational dimension, not mandatory sorting
+}
+
+export interface OutputFormatSpec {
+  descriptorType: "output_format_spec";
+  format: "markdown";
+  sectionOrderMatters: false;
+
+  sections: OfsSection[];
+
+  tablesOptional: boolean; // default true
+  tables: OfsTable[];
+
+  emptySectionValue?: string; // default "None"
+}
+```
+
+### 2.3 Markdown Outline Tree (nested headings)
+
+```ts
+export interface MdNode {
+  title: string;     // heading text, cleaned
+  level: number;     // 1..6
+  key: string;       // slugified internal key
+  id?: string;       // optional internal
+  content_md: string;
+  children: MdNode[];
+}
+
+export interface MdOutline {
+  type: "md_outline";
+  nodes: MdNode[];
+}
+```
+
+### 2.4 Extracted Result (sections + structure)
+
+```ts
+export interface ListItem {
+  text: string;
+  index?: number;         // for ordered lists
+  children: ListItem[];
+}
+
+export interface ParsedList {
+  kind: "list";
+  ordered: boolean;
+  items: ListItem[];
+}
+
+export interface ParsedTable {
+  kind: "table" | "ordered_table";
+  by?: string;
+  columns: string[];      // includes "#" first column for ordered_table
+  rows: string[][];
+}
+
+export interface ExtractedResult {
+  outline: MdOutline;
+  sectionsByName: Record<string, {
+    nodeKey: string;
+    nodeLevel: number;
+    md: string;           // raw content markdown
+    list?: ParsedList;    // only if section kind requires list parsing (or enabled)
+  }>;
+  tables: ParsedTable[];
+}
+```
+
+---
+
+## 3) Layer A — FlexMD Frames format
+
+### 3.1 Frame header (two accepted forms)
+
+**Bracket line**
+
+```md
+[[message role=user id=m1 ts=...]]
+```
+
+**Markdown heading with brackets**
+
+```md
+## [[message role=user id=m1]]
+```
+
+### 3.2 Meta lines
+
+Immediately after a frame header (meta block), lines like:
+
+```md
+@tags: a, b
+@priority: high
+```
+
+Default rule: meta is only recognized before the first non-meta body line (unless `metaAnywhere=true`).
+
+### 3.3 Payload binding
+
+````md
+@payload:name: input
+```json
+{"a":1}
+```
+````
+
+- The line `@payload:name: X` binds the **next fenced block** to payload `X`.
+- `json` is parsed; parsing errors go to `parseError`.
+
+---
+
+## 4) Layer B — Output Format Spec (OFS): the LLM-facing descriptor
+
+### 4.1 Canonical OFS block (example)
+
+```md
+## Output format (Markdown)
+Include these sections somewhere (order does not matter):
+
+- Short answer — prose
+- Long answer — prose
+- Reasoning — ordered list
+- Assumptions — list
+- Unknowns — list
+
+Tables (only if needed):
+- (property1, property2, property3 — table)
+- (property1, property2, property3 — ordered table, by property2)
+
+Empty sections:
+- If a section is empty, write `None`.
+```
+
+### 4.2 Meaning of kinds (hard rules)
+
+* `prose`: any Markdown text; lists allowed but not required.
+* `list`: must be `None` OR contain at least one bullet line `- ` (nested allowed).
+* `ordered list`: must be `None` OR contain at least one numbered line `^\d+\.` (nested allowed).
+* `table`: Markdown pipe table with listed columns.
+* `ordered table`: same but includes first column **exactly** `#` and rows numbered `1..N`.
+
+> "by property2" is informational; do **not** enforce sorting unless the task explicitly requests sorting.
+
+---
+
+## 5) Instruction Enricher (feature-driven, minimal)
+
+Given an `OutputFormatSpec`, generate only relevant guidance.
+
+### 5.1 Enricher output template (generated)
+
+Always include (if emptySectionValue is set):
+
+* `If a section is empty, write \`None\`.`
+
+If any `list` sections:
+
+* `List sections must use '-' bullets (nested allowed).`
+
+If any `ordered_list` sections:
+
+* `Ordered-list sections must use numbered items (nested allowed).`
+
+If any `table` declared:
+
+* `Tables must be Markdown pipe tables with the specified columns.`
+
+If any `ordered_table` declared:
+
+* `Ordered tables must add a first column named '#' with rows numbered 1..N.`
+
+### 5.2 Example implementation (TypeScript)
+
+```ts
+export function enrichInstructions(spec: OutputFormatSpec): string {
+  const lines: string[] = [];
+
+  if (spec.emptySectionValue) {
+    lines.push(`- If a section is empty, write \`${spec.emptySectionValue}\`.`);
+  }
+
+  const hasList = spec.sections.some(s => s.kind === "list");
+  const hasOrderedList = spec.sections.some(s => s.kind === "ordered_list");
+  const hasTable = spec.tables.length > 0;
+  const hasOrderedTable = spec.tables.some(t => t.kind === "ordered_table");
+
+  if (hasList) lines.push(`- List sections must use '-' bullets (nested allowed).`);
+  if (hasOrderedList) lines.push(`- Ordered-list sections must use numbered items (nested allowed).`);
+  if (hasTable) lines.push(`- Tables must be Markdown pipe tables with the specified columns.`);
+  if (hasOrderedTable) lines.push(`- Ordered tables must add a first column named '#' with rows numbered 1..N.`);
+
+  return lines.length ? `Rules:\n${lines.map(l => `- ${l.replace(/^- /, "")}`).join("\n")}\n` : "";
+}
+```
+
+---
+
+## 6) Accept any heading level and build nested JSON outline
+
+### 6.1 Outline build algorithm (stack)
+
+**Parsing rule**
+
+* Any heading `#..######` is accepted.
+* Heading levels define parent/child relationships.
+
+**Tree construction**
+
+* Use a stack of nodes.
+* New node at level `L` becomes:
+  * child of the nearest previous node with level `< L`,
+  * otherwise a root node.
+
+### 6.2 Rendering rule (tree → Markdown/FlexMD)
+
+When converting outline back to Markdown:
+
+* Render headings using `level` and `title`
+* Append `content_md`
+* Render children recursively
+* **Do not render** `key`, `id`, `path`, dedup suffixes (internal only)
+
+---
+
+## 7) Nested lists (sub-items) parsing & rendering
+
+### 7.1 Parsing lists into a tree
+
+* Unordered item: `^\s*-\s+`
+* Ordered item: `^\s*\d+\.\s+`
+* Nesting is indentation-based.
+
+**Output structure**
+
+* `items[]` with `children[]`
+
+---
+
+## 8) Tables + ordered tables
+
+### 8.1 Parsing GFM pipe tables (minimum)
+
+* Header row + separator row required
+* Alignment markers optional
+* Cells are strings
+
+### 8.2 Ordered table rule (hard)
+
+If a table is declared as `ordered table`:
+
+* first column header must be `#`
+* rows must have `#` values `1..N`
+
+---
+
+## 9) Datatype handling
+
+### 9.1 Meta values (FlexMD Frames meta and doc meta)
+
+Add parse option:
+
+* `metaTypeMode: "strings" | "infer" | "schema"`
+
+**strings (default)**
+All meta values are strings (except configured array keys like `tags`, `refs`).
+
+**infer**
+Safely infer:
+
+* `true/false` → boolean
+* `null` → null
+* integers/floats → number (avoid leading-zero pitfalls like `"0012"` unless `0` or `0.xxx`)
+
+**schema**
+User provides types per key; schema wins.
+
+### 9.2 Tables
+
+Default: all table cells remain strings.
+Typed tables are optional future feature; keep v1.1 simple unless you explicitly need it.
+
+### 9.3 JSON payloads
+
+If payload fence is `json`:
+
+* parse JSON → native types
+* on parse error: keep raw + parseError
+
+---
+
+## 10) Detection & extraction from arbitrary text
+
+### 10.1 Supported "objects" to detect
+
+1. ` ```flexmd ` fenced blocks (highest confidence)
+2. ` ```json ` fenced blocks that match FlexDocument shape (`{frames:[...]}`)
+3. Raw/unframed FlexMD markers (best effort)
+4. Optional generic Markdown snippets (opaque or lightly structured)
+
+### 10.2 Detection tiers
+
+* Tier A: ` ```flexmd `
+* Tier B: ` ```json ` + shape match
+* Tier C: raw sniff (at least 2 strong markers within first N lines): `[[...]]`, `@key:`, `@payload:name:`
+
+### 10.3 API (spec)
+
+```ts
+export type DetectedKind =
+  | "flexmd_fence"
+  | "flexdoc_json_fence"
+  | "raw_flexmd"
+  | "markdown_snippet"
+  | "none";
+
+export interface DetectedObject {
+  kind: DetectedKind;
+  confidence: number;
+  start: number;
+  end: number;
+  raw: string;
+  inner?: string; // for fenced blocks
+}
+```
+
+---
+
+## 11) End-to-end pipeline (recommended)
+
+### 11.1 Creating a call (you know desired output)
+
+1. Build OFS block based on required sections/tables.
+2. Enrich with only relevant rules (lists/tables/None).
+3. Send to LLM as the "Output format" section.
+
+### 11.2 Receiving a response
+
+1. Detect and extract structured objects:
+   * if ` ```flexmd ` exists → parse FlexMD Frames
+   * else treat response as Markdown and apply OFS validator/extractor
+2. Build outline tree from headings.
+3. Match required sections by name (case-insensitive, ignore `:`).
+4. For each section:
+   * extract `content_md`
+   * if `kind=list|ordered_list` parse nested lists
+5. Extract tables (if needed) and validate ordered tables (`#` column).
+
+### 11.3 Duplicates (safe rule)
+
+If required section title appears multiple times:
+
+* choose the match at the **highest level** (smallest heading level number)
+* if multiple at same highest level: merge content in appearance order
+
+Nested occurrences remain as children in outline.
+
+---
+
+## 12) Practical "public API" exports (what you ship)
+
+### Layer A
+
+* `parseFlexMd(text, options) -> FlexDocument`
+* `stringifyFlexMd(doc, options) -> string`
+
+### Layer B
+
+* `parseOutputFormatSpec(md) -> OutputFormatSpec`
+* `stringifyOutputFormatSpec(spec) -> string`
+* `enrichInstructions(spec) -> string`
+* `buildOutline(md) -> MdOutline`
+* `validateOutput(md, spec) -> { ok: boolean; errors: ...; warnings: ... }`
+* `extractOutput(md, spec, opts) -> ExtractedResult`
+* `renderOutline(outline) -> string`
+
+### Layer C
+
+* `detectObjects(text) -> DetectedObject[]`
+* `parseAny(text) -> { flexDocs: FlexDocument[]; markdownSnippets: string[]; remainder: string }`
+
+---
+
+## Appendix A — A canonical OFS generator (example)
+
+```ts
+export function makeDefaultOfs(): OutputFormatSpec {
+  return {
+    descriptorType: "output_format_spec",
+    format: "markdown",
+    sectionOrderMatters: false,
+    sections: [
+      { name: "Short answer", kind: "prose" },
+      { name: "Long answer", kind: "prose" },
+      { name: "Reasoning", kind: "ordered_list" },
+      { name: "Assumptions", kind: "list" },
+      { name: "Unknowns", kind: "list" },
+    ],
+    tablesOptional: true,
+    tables: [],
+    emptySectionValue: "None",
+  };
+}
+```
+
+---
+
+## Appendix B — Validator essentials (skeleton)
+
+```ts
+export function validateOutput(md: string, spec: OutputFormatSpec) {
+  const outline = buildOutline(md);
+
+  // index nodes by normalized title
+  const matches = collectMatches(outline);
+
+  const errors: string[] = [];
+
+  for (const s of spec.sections) {
+    const key = normalizeTitle(s.name);
+    const nodes = matches.get(key) ?? [];
+    if (nodes.length === 0) {
+      errors.push(`missing_section:${s.name}`);
+      continue;
+    }
+
+    const chosen = chooseBestNode(nodes); // highest-level; merge if needed
+    const body = chosen.content_md.trim();
+
+    if (spec.emptySectionValue && body === "") {
+      errors.push(`empty_section_without_none:${s.name}`);
+    }
+
+    if (spec.emptySectionValue && normalizeNone(body) === true) continue;
+
+    if (s.kind === "list") {
+      if (!/^\s*-\s+/.test(body)) errors.push(`section_not_bullets:${s.name}`);
+    }
+    if (s.kind === "ordered_list") {
+      if (!/^\s*\d+\.\s+/.test(body)) errors.push(`section_not_numbered:${s.name}`);
+    }
+  }
+
+  return { ok: errors.length === 0, errors };
+}
+
+function normalizeTitle(t: string) {
+  return t.trim().replace(/[:\-–—]\s*$/, "").trim().toLowerCase();
+}
+function normalizeNone(body: string) {
+  return body.trim().toLowerCase() === "none";
+}
+```
+
+---
+
+## Appendix C — What we ignore when rendering tree → Markdown
+
+When converting outline JSON back to Markdown/FlexMD:
+
+* ignore: `id`, `key`, `path`, array indexes
+* render:
+  * heading level
+  * title
+  * content
+  * ordered list numbering (when kind requires)
+  * ordered table `#` column (when required)

package/dist/detection/detector.d.ts

@@ -0,0 +1,6 @@
+import type { DetectedObject } from "../types.js";
+/**
+ * Detect FlexMD and other structured objects in arbitrary text.
+ * Returns all detected objects with confidence scores and byte ranges.
+ */
+export declare function detectObjects(text: string): DetectedObject[];

package/dist/detection/detector.js

@@ -0,0 +1,104 @@
+/**
+ * Detect FlexMD and other structured objects in arbitrary text.
+ * Returns all detected objects with confidence scores and byte ranges.
+ */
+export function detectObjects(text) {
+    const detected = [];
+    // Tier A: Detect ```flexmd fenced blocks (highest confidence)
+    detected.push(...detectFlexMdFences(text));
+    // Tier B: Detect ```json blocks with FlexDocument shape
+    detected.push(...detectFlexDocJsonFences(text));
+    // Tier C: Detect raw FlexMD markers (best effort)
+    detected.push(...detectRawFlexMd(text));
+    // Sort by start position
+    detected.sort((a, b) => a.start - b.start);
+    return detected;
+}
+/**
+ * Tier A: Detect ```flexmd fenced blocks.
+ */
+function detectFlexMdFences(text) {
+    const detected = [];
+    const regex = /```flexmd\n([\s\S]*?)```/g;
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        detected.push({
+            kind: "flexmd_fence",
+            confidence: 1.0,
+            start: match.index,
+            end: match.index + match[0].length,
+            raw: match[0],
+            inner: match[1]
+        });
+    }
+    return detected;
+}
+/**
+ * Tier B: Detect ```json blocks that match FlexDocument shape.
+ */
+function detectFlexDocJsonFences(text) {
+    const detected = [];
+    const regex = /```json\n([\s\S]*?)```/g;
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        const inner = match[1];
+        // Try to parse as JSON
+        try {
+            const parsed = JSON.parse(inner);
+            // Check if it has the FlexDocument shape (has "frames" array)
+            if (parsed && typeof parsed === "object" && Array.isArray(parsed.frames)) {
+                detected.push({
+                    kind: "flexdoc_json_fence",
+                    confidence: 0.9,
+                    start: match.index,
+                    end: match.index + match[0].length,
+                    raw: match[0],
+                    inner
+                });
+            }
+        }
+        catch {
+            // Not valid JSON, skip
+        }
+    }
+    return detected;
+}
+/**
+ * Tier C: Detect raw FlexMD markers (best effort).
+ * Looks for at least 2 strong markers within first 500 chars:
+ * - [[...]]
+ * - @key:
+ * - @payload:name:
+ */
+function detectRawFlexMd(text) {
+    const detected = [];
+    // Look for frame markers
+    const frameRegex = /\[\[([^\]]+)\]\]/g;
+    const metaRegex = /@[a-zA-Z_][a-zA-Z0-9_]*:/g;
+    const payloadRegex = /@payload:[a-zA-Z_][a-zA-Z0-9_]*:/g;
+    let match;
+    const markers = [];
+    // Collect all marker positions
+    while ((match = frameRegex.exec(text)) !== null) {
+        markers.push(match.index);
+    }
+    while ((match = metaRegex.exec(text)) !== null) {
+        markers.push(match.index);
+    }
+    while ((match = payloadRegex.exec(text)) !== null) {
+        markers.push(match.index);
+    }
+    // If we have at least 2 markers, consider it raw FlexMD
+    if (markers.length >= 2) {
+        const start = Math.min(...markers);
+        const end = text.length;
+        detected.push({
+            kind: "raw_flexmd",
+            confidence: 0.7,
+            start,
+            end,
+            raw: text.substring(start, end)
+        });
+    }
+    return detected;
+}

package/dist/detection/extractor.d.ts

@@ -0,0 +1,10 @@
+import type { FlexDocument } from "../types.js";
+export interface ParseAnyResult {
+    flexDocs: FlexDocument[];
+    markdownSnippets: string[];
+    remainder: string;
+}
+/**
+ * Parse any text and extract all FlexMD documents and Markdown snippets.
+ */
+export declare function parseAny(text: string): ParseAnyResult;