@uurtech/jdf-cli 0.1.21 → 0.1.23

package/dist/index.js

@@ -5,8 +5,9 @@ import { fileURLToPath } from 'url';
 import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import JSZip from 'jszip';
-import { createHash } from 'crypto';
+import crypto, { createHash } from 'crypto';
 import { readFile } from 'fs/promises';
+import { execFileSync } from 'child_process';
 
 // ../../packages/jdf-core/src/manifest.ts
 var JDFX_MANIFEST_VERSION = "1.0.0";
@@ -240,6 +241,58 @@ function shouldUseJdfx(doc) {
 }
 
 // src/commands/import-md.ts
+function parseInline(text) {
+  const runs = [];
+  let i = 0;
+  let buf = "";
+  const flush = (extra) => {
+    if (buf) {
+      runs.push({ text: buf, ...extra });
+      buf = "";
+    }
+  };
+  while (i < text.length) {
+    const rest = text.slice(i);
+    const link = rest.match(/^\[([^\]]+)\]\(([^)\s]+)[^)]*\)/);
+    if (link) {
+      flush();
+      runs.push({ text: link[1], link: link[2] });
+      i += link[0].length;
+      continue;
+    }
+    const code = rest.match(/^`([^`]+)`/);
+    if (code) {
+      flush();
+      runs.push({ text: code[1], fontFamily: "JetBrains Mono", color: "#be185d" });
+      i += code[0].length;
+      continue;
+    }
+    const bold = rest.match(/^(\*\*|__)([^]+?)\1/);
+    if (bold) {
+      flush();
+      runs.push(...parseInline(bold[2]).map((r) => ({ ...r, bold: true })));
+      i += bold[0].length;
+      continue;
+    }
+    const ital = rest.match(/^(\*|_)([^]+?)\1/);
+    if (ital) {
+      flush();
+      runs.push(...parseInline(ital[2]).map((r) => ({ ...r, italic: true })));
+      i += ital[0].length;
+      continue;
+    }
+    buf += text[i];
+    i++;
+  }
+  flush();
+  return runs.length ? runs : [{ text }];
+}
+function hasFormatting(runs) {
+  return runs.some((r) => r.bold || r.italic || r.link || r.fontFamily || r.color);
+}
+function stripInline(text) {
+  return parseInline(text).map((r) => r.text).join("");
+}
 async function importMarkdown(inputPath, outputPath) {
   const input = path2.resolve(inputPath);
   console.log(`Importing: ${input}`);
@@ -330,11 +383,28 @@ function convertMarkdownToJdf(md, title, baseDir = process.cwd()) {
       i++;
       continue;
     }
-    const imageMatch = line.match(/^\s*!\[([^\]]*)\]\(\s*([^)\s]+)[^)]*\)\s*$/);
-    if (imageMatch) {
-      const alt = imageMatch[1];
-      const src = resolveImageSrc(imageMatch[2], baseDir);
-      const height2 = 60;
+    if (line.match(/^\s*([-*_])(\s*\1){2,}\s*$/)) {
+      if (y + 6 > maxY) {
+        pages.push({ id: `page-${pageNum}`, elements });
+        elements = [];
+        y = 5;
+        pageNum++;
+      }
+      elements.push({ type: "shape", shape: "rect", position: { x: 0, y: y + 2 }, width: contentWidth, height: 0.3, fill: "#cbd5e1" });
+      y += 6;
+      i++;
+      continue;
+    }
+    if (line.includes("|") && i + 1 < lines.length && lines[i + 1].match(/^\s*\|?[\s:|-]+\|?\s*$/) && lines[i + 1].includes("-")) {
+      const splitRow = (r) => r.replace(/^\s*\|/, "").replace(/\|\s*$/, "").split("|").map((c) => stripInline(c.trim()));
+      const headers = splitRow(line);
+      i += 2;
+      const rows = [];
+      while (i < lines.length && lines[i].includes("|") && lines[i].trim() !== "") {
+        rows.push(splitRow(lines[i]));
+        i++;
+      }
+      const height2 = (rows.length + 1) * 7 + 3;
       if (y + height2 > maxY) {
         pages.push({ id: `page-${pageNum}`, elements });
         elements = [];
@@ -342,25 +412,27 @@ function convertMarkdownToJdf(md, title, baseDir = process.cwd()) {
         pageNum++;
       }
       elements.push({
-        type: "image",
-        src,
-        alt,
+        type: "table",
+        headers,
+        rows,
         position: { x: 0, y },
         width: contentWidth,
-        height: height2,
-        fit: "contain"
+        borders: true,
+        headerStyle: { backgroundColor: "#f1f5f9", fontWeight: "bold", fontSize: 10, color: "#0f172a" },
+        rowStyle: { color: "#334155", fontSize: 10 },
+        style: { fontFamily: "Inter", fontSize: 10 }
       });
-      y += height2 + 4;
-      i++;
+      y += height2;
       continue;
     }
-    if (line.match(/^\s*[-*+]\s/)) {
-      const items = [];
-      while (i < lines.length && lines[i].match(/^\s*[-*+]\s/)) {
-        items.push({ content: lines[i].replace(/^\s*[-*+]\s+/, "").trim() });
+    if (line.match(/^\s*>\s?/)) {
+      const quoteLines = [];
+      while (i < lines.length && lines[i].match(/^\s*>\s?/)) {
+        quoteLines.push(lines[i].replace(/^\s*>\s?/, ""));
         i++;
       }
-      const height2 = items.length * 5 + 3;
+      const quote = quoteLines.join(" ").trim();
+      const height2 = Math.ceil(quote.length / 70) * 4.5 + 6;
       if (y + height2 > maxY) {
         pages.push({ id: `page-${pageNum}`, elements });
         elements = [];
@@ -368,23 +440,66 @@ function convertMarkdownToJdf(md, title, baseDir = process.cwd()) {
         pageNum++;
       }
       elements.push({
-        type: "list",
-        listType: "unordered",
+        type: "text",
+        content: quote,
         position: { x: 0, y },
         width: contentWidth,
-        style: { fontFamily: "Inter", fontSize: 10, lineHeight: 1.6, color: "#334155" },
-        items
+        style: { fontFamily: "Inter", fontSize: 10, lineHeight: 1.6, color: "#475569", fontStyle: "italic", backgroundColor: "#f8fafc", padding: 10, borderRadius: 6, marginTop: 4 }
       });
       y += height2;
       continue;
     }
-    if (line.match(/^\s*\d+\.\s/)) {
-      const items = [];
-      while (i < lines.length && lines[i].match(/^\s*\d+\.\s/)) {
-        items.push({ content: lines[i].replace(/^\s*\d+\.\s+/, "").trim() });
+    const imageMatch = line.match(/^\s*!\[([^\]]*)\]\(\s*([^)\s]+)[^)]*\)\s*$/);
+    if (imageMatch) {
+      const alt = imageMatch[1];
+      const src = resolveImageSrc(imageMatch[2], baseDir);
+      const height2 = 60;
+      if (y + height2 > maxY) {
+        pages.push({ id: `page-${pageNum}`, elements });
+        elements = [];
+        y = 5;
+        pageNum++;
+      }
+      elements.push({
+        type: "image",
+        src,
+        alt,
+        position: { x: 0, y },
+        width: contentWidth,
+        height: height2,
+        fit: "contain"
+      });
+      y += height2 + 4;
+      i++;
+      continue;
+    }
+    const listMatch = line.match(/^(\s*)([-*+]|\d+\.)\s/);
+    if (listMatch) {
+      const raw = [];
+      while (i < lines.length) {
+        const m = lines[i].match(/^(\s*)([-*+]|\d+\.)\s+(.*)$/);
+        if (!m) break;
+        raw.push({ indent: m[1].length, ordered: /\d/.test(m[2]), content: m[3].trim() });
         i++;
       }
-      const height2 = items.length * 5 + 3;
+      const rootOrdered = raw[0].ordered;
+      const root = [];
+      const stack = [{ indent: raw[0].indent, items: root }];
+      for (const r of raw) {
+        while (stack.length > 1 && r.indent < stack[stack.length - 1].indent) stack.pop();
+        if (r.indent > stack[stack.length - 1].indent) {
+          const parentItems = stack[stack.length - 1].items;
+          const parent = parentItems[parentItems.length - 1];
+          if (parent) {
+            parent.children = parent.children || [];
+            parent.listType = r.ordered ? "ordered" : "unordered";
+            stack.push({ indent: r.indent, items: parent.children });
+          }
+        }
+        stack[stack.length - 1].items.push({ content: stripInline(r.content) });
+      }
+      const count = raw.length;
+      const height2 = count * 5 + 3;
       if (y + height2 > maxY) {
         pages.push({ id: `page-${pageNum}`, elements });
         elements = [];
@@ -393,11 +508,11 @@ function convertMarkdownToJdf(md, title, baseDir = process.cwd()) {
       }
       elements.push({
         type: "list",
-        listType: "ordered",
+        listType: rootOrdered ? "ordered" : "unordered",
         position: { x: 0, y },
         width: contentWidth,
         style: { fontFamily: "Inter", fontSize: 10, lineHeight: 1.6, color: "#334155" },
-        items
+        items: root
       });
       y += height2;
       continue;
@@ -442,10 +557,15 @@ function convertMarkdownToJdf(md, title, baseDir = process.cwd()) {
       continue;
     }
     let para = "";
-    while (i < lines.length && lines[i].trim() !== "" && !lines[i].match(/^#{1,6}\s/) && !lines[i].match(/^\s*[-*+]\s/) && !lines[i].match(/^\s*\d+\.\s/) && !lines[i].startsWith("```")) {
+    while (i < lines.length && lines[i].trim() !== "" && !lines[i].match(/^#{1,6}\s/) && !lines[i].match(/^\s*([-*+]|\d+\.)\s/) && !lines[i].startsWith("```") && !lines[i].match(/^\s*>\s?/) && !lines[i].match(/^\s*([-*_])(\s*\1){2,}\s*$/) && !lines[i].includes("|")) {
       para += (para ? " " : "") + lines[i].trim();
       i++;
     }
+    if (!para) {
+      para = lines[i].trim();
+      i++;
+      if (!para) continue;
+    }
     const paraLines = Math.ceil(para.length / 80);
     const height = paraLines * 4.5 + 3;
     if (y + height > maxY) {
@@ -454,13 +574,24 @@ function convertMarkdownToJdf(md, title, baseDir = process.cwd()) {
       y = 5;
       pageNum++;
     }
-    elements.push({
-      type: "text",
-      content: para,
-      position: { x: 0, y },
-      width: contentWidth,
-      style: { fontFamily: "Inter", fontSize: 10, lineHeight: 1.6, color: "#334155" }
-    });
+    const runs = parseInline(para);
+    if (hasFormatting(runs)) {
+      elements.push({
+        type: "richtext",
+        runs,
+        position: { x: 0, y },
+        width: contentWidth,
+        style: { fontFamily: "Inter", fontSize: 10, lineHeight: 1.6, color: "#334155" }
+      });
+    } else {
+      elements.push({
+        type: "text",
+        content: para,
+        position: { x: 0, y },
+        width: contentWidth,
+        style: { fontFamily: "Inter", fontSize: 10, lineHeight: 1.6, color: "#334155" }
+      });
+    }
     y += height;
   }
   if (elements.length > 0) {
@@ -582,17 +713,35 @@ async function walkOps(page, OPS, viewport) {
       const y1Local = (va.y - minY) * PT_TO_MM;
       const x2Local = (vb.x - minX) * PT_TO_MM;
       const y2Local = (vb.y - minY) * PT_TO_MM;
-      shapes.push({
-        kind: "path",
-        x: minX * PT_TO_MM,
-        y: minY * PT_TO_MM,
-        width: Math.max(0.05, (maxX - minX) * PT_TO_MM),
-        height: Math.max(0.05, (maxY - minY) * PT_TO_MM),
-        stroke: isStroke ? gs.stroke : void 0,
-        strokeWidth: isStroke ? gs.lineWidth * PT_TO_MM : void 0,
-        opacity: gs.strokeAlpha,
-        path: `M ${x1Local.toFixed(2)} ${y1Local.toFixed(2)} L ${x2Local.toFixed(2)} ${y2Local.toFixed(2)}`
-      });
+      const wLocal = Math.max(0.05, (maxX - minX) * PT_TO_MM);
+      const hLocal = Math.max(0.05, (maxY - minY) * PT_TO_MM);
+      const dx = Math.abs(va.x - vb.x);
+      const dy = Math.abs(va.y - vb.y);
+      const axisAligned = dx < 0.5 || dy < 0.5;
+      if (axisAligned) {
+        shapes.push({
+          kind: "line",
+          x: minX * PT_TO_MM,
+          y: minY * PT_TO_MM,
+          width: wLocal,
+          height: hLocal,
+          stroke: isStroke ? gs.stroke : void 0,
+          strokeWidth: isStroke ? gs.lineWidth * PT_TO_MM : void 0,
+          opacity: gs.strokeAlpha
+        });
+      } else {
+        shapes.push({
+          kind: "path",
+          x: minX * PT_TO_MM,
+          y: minY * PT_TO_MM,
+          width: wLocal,
+          height: hLocal,
+          stroke: isStroke ? gs.stroke : void 0,
+          strokeWidth: isStroke ? gs.lineWidth * PT_TO_MM : void 0,
+          opacity: gs.strokeAlpha,
+          path: `M ${x1Local.toFixed(2)} ${y1Local.toFixed(2)} L ${x2Local.toFixed(2)} ${y2Local.toFixed(2)}`
+        });
+      }
     } else if (pathSegments.length > 0) {
       const vpSegments = pathSegments.map((seg) => {
         if (seg.type === "Z") return seg;
@@ -1443,40 +1592,499 @@ function wrapElements(elements, title, meta) {
     ]
   };
 }
+var DEFAULT_MAX_TOKENS = 512;
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function hashText(text) {
+  return crypto.createHash("sha256").update(text).digest("hex").slice(0, 12);
+}
+function headingLevel(el) {
+  const h = el.heading;
+  if (h === true) return 1;
+  if (typeof h === "number" && h >= 1 && h <= 6) return h;
+  if (typeof el.tocLevel === "number") return el.tocLevel;
+  return null;
+}
+function serializeElement(el) {
+  const e = el;
+  switch (e.type) {
+    case "text":
+      return String(e.content ?? "").trim();
+    case "richtext":
+      return (e.runs || []).map((r) => r.text ?? "").join("").trim();
+    case "list": {
+      const walk = (items, depth = 0) => (items || []).flatMap((it) => {
+        const line = "  ".repeat(depth) + "- " + String(it.content ?? "").trim();
+        return it.children?.length ? [line, ...walk(it.children, depth + 1)] : [line];
+      });
+      return walk(e.items).join("\n");
+    }
+    case "table": {
+      const headers = e.headers ?? e.columns?.map((c) => c.header || "").filter((h) => h) ?? [];
+      const cell = (c) => typeof c === "string" ? c : c?.content ?? "";
+      const rows = e.rows || [];
+      const lines = rows.map(
+        (row) => row.map((c, i) => {
+          const h = headers[i];
+          const v = cell(c).trim();
+          return h ? `${h}: ${v}` : v;
+        }).filter((s) => s !== "").join(" | ")
+      );
+      return lines.join("\n");
+    }
+    case "collapsible": {
+      const title = String(e.title ?? "").trim();
+      const inner = (e.elements || []).map(serializeElement).filter(Boolean).join("\n");
+      return [title, inner].filter(Boolean).join("\n");
+    }
+    case "input":
+    case "textarea":
+    case "select": {
+      const label = e.label ? `${e.label}: ` : "";
+      const val = e.multiple ? (e.values || []).join(", ") : e.value ?? "";
+      return `${label}${val}`.trim();
+    }
+    case "checkbox":
+      return `${e.checked ? "[x]" : "[ ]"} ${e.label ?? ""}`.trim();
+    case "image":
+      return e.alt ? `[image: ${e.alt}]` : "";
+    case "toc":
+    case "shape":
+    case "signature":
+      return "";
+    default:
+      return "";
+  }
+}
+function elementId(el, pageIdx, elIdx) {
+  if (typeof el.id === "string" && el.id.length > 0) return el.id;
+  return `p${pageIdx + 1}e${elIdx}`;
+}
+function flatten(doc) {
+  const out = [];
+  (doc.pages || []).forEach((page, pi) => {
+    (page.elements || []).forEach((el, ei) => {
+      out.push({ el, page: pi + 1, id: elementId(el, pi, ei) });
+    });
+  });
+  return out;
+}
+function makeChunk(group, breadcrumb) {
+  const parts = group.map((g) => serializeElement(g.el)).filter((s) => s.length > 0);
+  const text = parts.join("\n\n").trim();
+  if (!text) return null;
+  const types = Array.from(new Set(group.map((g) => g.el.type)));
+  return {
+    id: group[0].id,
+    text,
+    path: [...breadcrumb],
+    page: group[0].page,
+    types,
+    tokens: estimateTokens(text),
+    hash: hashText(text)
+  };
+}
+function chunkDocument(doc, options = {}) {
+  const strategy = options.strategy ?? "section";
+  const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+  const flat = flatten(doc);
+  const chunks = [];
+  if (strategy === "element") {
+    const crumb2 = [];
+    for (const f of flat) {
+      const lvl = headingLevel(f.el);
+      if (lvl != null) {
+        crumb2.length = Math.max(0, lvl - 1);
+        crumb2[lvl - 1] = serializeElement(f.el);
+      }
+      const c = makeChunk([f], crumb2);
+      if (c) chunks.push(c);
+    }
+    return chunks;
+  }
+  if (strategy === "fixed") {
+    const crumb2 = [];
+    let buf2 = [];
+    let bufTokens = 0;
+    const flush = () => {
+      const c = makeChunk(buf2, crumb2);
+      if (c) chunks.push(c);
+      buf2 = [];
+      bufTokens = 0;
+    };
+    for (const f of flat) {
+      const lvl = headingLevel(f.el);
+      if (lvl != null) {
+        crumb2.length = Math.max(0, lvl - 1);
+        crumb2[lvl - 1] = serializeElement(f.el);
+      }
+      const t = estimateTokens(serializeElement(f.el));
+      if (bufTokens + t > maxTokens && buf2.length > 0) flush();
+      buf2.push(f);
+      bufTokens += t;
+    }
+    flush();
+    return chunks;
+  }
+  const crumb = [];
+  let buf = [];
+  const flushSection = () => {
+    if (buf.length === 0) return;
+    let sub = [];
+    let subTokens = 0;
+    for (const f of buf) {
+      const t = estimateTokens(serializeElement(f.el));
+      if (subTokens + t > maxTokens && sub.length > 0) {
+        const c2 = makeChunk(sub, crumb);
+        if (c2) chunks.push(c2);
+        sub = [];
+        subTokens = 0;
+      }
+      sub.push(f);
+      subTokens += t;
+    }
+    const c = makeChunk(sub, crumb);
+    if (c) chunks.push(c);
+    buf = [];
+  };
+  for (const f of flat) {
+    const lvl = headingLevel(f.el);
+    if (lvl != null) {
+      flushSection();
+      crumb.length = Math.max(0, lvl - 1);
+      crumb[lvl - 1] = serializeElement(f.el);
+    }
+    buf.push(f);
+  }
+  flushSection();
+  return chunks;
+}
+async function loadJdf(filePath) {
+  if (filePath.toLowerCase().endsWith(".jdfx")) {
+    const zip = await JSZip.loadAsync(fs.readFileSync(filePath));
+    const docFile = zip.file(JDFX_DOCUMENT_PATH);
+    if (!docFile) throw new Error(`Bundle missing ${JDFX_DOCUMENT_PATH}`);
+    return JSON.parse(await docFile.async("string"));
+  }
+  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+}
+async function chunkFile(inputPath, opts = {}) {
+  const input = path2.resolve(inputPath);
+  if (!fs.existsSync(input)) throw new Error(`File not found: ${input}`);
+  const doc = await loadJdf(input);
+  const strategy = opts.strategy ?? "section";
+  const chunks = chunkDocument(doc, { strategy, maxTokens: opts.maxTokens });
+  const format = opts.format ?? "jsonl";
+  console.log(`Chunking:  ${input}`);
+  console.log(`Strategy:  ${strategy}${opts.maxTokens ? ` (max ${opts.maxTokens} tokens)` : ""}`);
+  if (format === "inline") {
+    const out = opts.output ? path2.resolve(opts.output) : input.replace(/\.(jdf|jdfx)$/i, ".jdf");
+    const withIndex = { ...doc, index: { chunker: `jdf-${strategy}-v1`, chunks } };
+    fs.writeFileSync(out, JSON.stringify(withIndex, null, 2));
+    console.log(`Output:    ${out}  (${chunks.length} chunks in "index" block)`);
+  } else if (format === "json") {
+    const out = opts.output ? path2.resolve(opts.output) : input.replace(/\.(jdf|jdfx)$/i, ".chunks.json");
+    fs.writeFileSync(out, JSON.stringify(chunks, null, 2));
+    console.log(`Output:    ${out}  (${chunks.length} chunks)`);
+  } else {
+    const out = opts.output ? path2.resolve(opts.output) : input.replace(/\.(jdf|jdfx)$/i, ".chunks.jsonl");
+    fs.writeFileSync(out, chunks.map((c) => JSON.stringify(c)).join("\n") + "\n");
+    console.log(`Output:    ${out}  (${chunks.length} chunks)`);
+  }
+  const totalTokens = chunks.reduce((a, c) => a + c.tokens, 0);
+  console.log(`
+Done! ${chunks.length} chunks, ~${totalTokens} tokens total.`);
+  return chunks;
+}
+var DEFAULT_MODEL = {
+  ollama: "nomic-embed-text",
+  openai: "text-embedding-3-small"
+};
+async function embedBatch(provider, model, inputs) {
+  if (inputs.length === 0) return [];
+  switch (provider) {
+    case "ollama":
+      return embedOllama(model, inputs);
+    case "openai":
+      return embedOpenAI(model, inputs);
+    default:
+      throw new Error(`Unknown embedding provider: ${provider}`);
+  }
+}
+var OLLAMA_HOST = process.env.OLLAMA_HOST || "http://localhost:11434";
+var OLLAMA_CONTAINER = "jdf-ollama";
+async function ollamaUp() {
+  try {
+    const res = await fetch(`${OLLAMA_HOST}/api/tags`, { signal: AbortSignal.timeout(1500) });
+    return res.ok;
+  } catch {
+    return false;
+  }
+}
+function dockerReady() {
+  try {
+    execFileSync("docker", ["info"], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function ensureOllama(model, autoStart) {
+  if (await ollamaUp()) {
+    await ollamaPull(model);
+    return;
+  }
+  const manualHint = `  Option A \u2014 install Ollama natively (https://ollama.com), then:
+      ollama serve   &&   ollama pull ${model}
+  Option B \u2014 start Docker, then re-run this command (the CLI will launch Ollama for you), or run it yourself:
+      docker run -d --name ${OLLAMA_CONTAINER} -p 11434:11434 -v jdf-ollama:/root/.ollama ollama/ollama
+      docker exec ${OLLAMA_CONTAINER} ollama pull ${model}`;
+  if (!autoStart) {
+    throw new Error(`Ollama isn't running at ${OLLAMA_HOST} and --no-auto-start was given.
+${manualHint}`);
+  }
+  if (!dockerReady()) {
+    throw new Error(
+      `Ollama isn't running at ${OLLAMA_HOST}, and the Docker daemon isn't available to auto-start it.
+${manualHint}`
+    );
+  }
+  console.log(`Ollama not reachable \u2014 starting it via Docker (container "${OLLAMA_CONTAINER}")\u2026`);
+  try {
+    execFileSync("docker", ["start", OLLAMA_CONTAINER], { stdio: "ignore" });
+  } catch {
+    try {
+      execFileSync("docker", [
+        "run",
+        "-d",
+        "--name",
+        OLLAMA_CONTAINER,
+        "-p",
+        "11434:11434",
+        "-v",
+        "jdf-ollama:/root/.ollama",
+        "ollama/ollama"
+      ], { stdio: "inherit" });
+    } catch (e) {
+      throw new Error(`Failed to start the Ollama container via Docker.
+${manualHint}`);
+    }
+  }
+  for (let i = 0; i < 30; i++) {
+    if (await ollamaUp()) {
+      console.log("  Ollama is up.");
+      await ollamaPull(model);
+      return;
+    }
+    await new Promise((r) => setTimeout(r, 1e3));
+  }
+  throw new Error("Ollama container started but never became reachable on :11434.");
+}
+async function ollamaPull(model) {
+  try {
+    const show = await fetch(`${OLLAMA_HOST}/api/show`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name: model })
+    });
+    if (show.ok) return;
+  } catch {
+  }
+  console.log(`Pulling embedding model "${model}" into Ollama (first run only)\u2026`);
+  const res = await fetch(`${OLLAMA_HOST}/api/pull`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ name: model, stream: false })
+  });
+  if (!res.ok) throw new Error(`Ollama pull failed (${res.status}) for model "${model}".`);
+  console.log("  Model ready.");
+}
+async function embedOllama(model, inputs) {
+  const out = [];
+  for (const text of inputs) {
+    const res = await fetch(`${OLLAMA_HOST}/api/embeddings`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ model, prompt: text })
+    });
+    if (!res.ok) throw new Error(`Ollama embeddings ${res.status} for model "${model}".`);
+    const json = await res.json();
+    if (!Array.isArray(json.embedding)) throw new Error(`Ollama returned no embedding for model "${model}".`);
+    out.push(json.embedding);
+  }
+  return out;
+}
+async function prompt(question, hidden = false) {
+  if (!process.stdin.isTTY) return "";
+  const readline = await import('readline');
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  if (hidden) {
+    const out = rl;
+    out._writeToOutput = (s) => {
+      if (s.trim().length) out.output.write("*");
+      else out.output.write(s);
+    };
+  }
+  return new Promise((resolve) => rl.question(question, (a) => {
+    rl.close();
+    if (hidden) process.stdout.write("\n");
+    resolve(a.trim());
+  }));
+}
+var openaiCreds = null;
+async function resolveOpenAICreds() {
+  if (openaiCreds) return openaiCreds;
+  let base = process.env.OPENAI_BASE_URL || "";
+  let key = process.env.OPENAI_API_KEY || "";
+  if (!base) base = await prompt("OpenAI API endpoint [https://api.openai.com/v1]: ") || "https://api.openai.com/v1";
+  if (!key) key = await prompt("OpenAI API key: ", true);
+  if (!key) {
+    throw new Error("No OpenAI API key. Set OPENAI_API_KEY (and optionally OPENAI_BASE_URL) or run interactively so the CLI can prompt.");
+  }
+  openaiCreds = { base: base.replace(/\/$/, ""), key };
+  return openaiCreds;
+}
+async function embedOpenAI(model, inputs) {
+  const { base, key } = await resolveOpenAICreds();
+  const res = await fetch(`${base}/embeddings`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
+    body: JSON.stringify({ model, input: inputs })
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`OpenAI embeddings API ${res.status}: ${body.slice(0, 200)}`);
+  }
+  const json = await res.json();
+  return json.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
+}
+async function loadJdf2(filePath) {
+  if (filePath.toLowerCase().endsWith(".jdfx")) {
+    const zip = await JSZip.loadAsync(fs.readFileSync(filePath));
+    const docFile = zip.file(JDFX_DOCUMENT_PATH);
+    if (!docFile) throw new Error(`Bundle missing ${JDFX_DOCUMENT_PATH}`);
+    return JSON.parse(await docFile.async("string"));
+  }
+  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+}
+function loadCache(cachePath) {
+  try {
+    if (!fs.existsSync(cachePath)) return null;
+    return JSON.parse(fs.readFileSync(cachePath, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+function batched(items, size) {
+  const out = [];
+  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
+  return out;
+}
+async function embedFile(inputPath, opts = {}) {
+  const input = path2.resolve(inputPath);
+  if (!fs.existsSync(input)) throw new Error(`File not found: ${input}`);
+  const provider = opts.provider ?? "ollama";
+  const model = opts.model ?? DEFAULT_MODEL[provider];
+  const strategy = opts.strategy ?? "section";
+  const doc = await loadJdf2(input);
+  const chunks = chunkDocument(doc, { strategy, maxTokens: opts.maxTokens });
+  const output = opts.output ? path2.resolve(opts.output) : input.replace(/\.(jdf|jdfx)$/i, ".embeddings.json");
+  const cachePath = opts.cache ? path2.resolve(opts.cache) : output;
+  console.log(`Embedding: ${input}`);
+  console.log(`Provider:  ${provider} / ${model}${provider === "ollama" ? " (local \u2014 no data leaves this machine)" : " (remote API)"}`);
+  console.log(`Strategy:  ${strategy} \u2192 ${chunks.length} chunks`);
+  if (provider === "ollama") {
+    await ensureOllama(model, opts.autoStart !== false);
+  }
+  const cache = opts.incremental ? loadCache(cachePath) : null;
+  const cachedVectors = cache?.model === model ? cache.vectors : void 0;
+  if (opts.incremental && cache && cache.model !== model) {
+    console.log(`  (cache model ${cache.model} \u2260 ${model} \u2014 re-embedding all)`);
+  }
+  const toEmbed = [];
+  const reused = {};
+  for (const c of chunks) {
+    const hit = cachedVectors?.[c.id];
+    if (hit && hit.hash === c.hash) reused[c.id] = hit;
+    else toEmbed.push(c);
+  }
+  if (opts.incremental) {
+    console.log(`  Reused ${Object.keys(reused).length} cached, embedding ${toEmbed.length} changed/new.`);
+  }
+  const vectors = { ...reused };
+  let dims = cache?.dims ?? 0;
+  for (const batch of batched(toEmbed, 96)) {
+    const embs = await embedBatch(provider, model, batch.map((c) => c.text));
+    batch.forEach((c, i) => {
+      vectors[c.id] = { hash: c.hash, vector: embs[i] };
+      if (!dims) dims = embs[i].length;
+    });
+    console.log(`  Embedded ${Object.keys(vectors).length}/${chunks.length}\u2026`);
+  }
+  const sidecar = {
+    model,
+    provider,
+    dims,
+    chunker: `jdf-${strategy}-v1`,
+    vectors
+  };
+  fs.writeFileSync(output, JSON.stringify(sidecar));
+  console.log(`
+Done! ${Object.keys(vectors).length} vectors (${dims}-dim) \u2192 ${output}`);
+  return sidecar;
+}
 
 // src/index.ts
 var HELP = `jdf \u2014 JSON Document Format CLI
 
-The CLI exists for two workflows:
+The CLI exists for these workflows:
   \u2022 PDF \u2192 JDF        legacy documents become a structured JSON tree your
                      RAG / agent / pipeline can read natively.
   \u2022 JSON \u2192 JDF       LLMs and code emit JSON; this command wraps that JSON
                      into a validated .jdf (or .jdfx) you can ship.
+  \u2022 JDF \u2192 chunks     turn a document into retrieval-ready chunks (RAG).
+  \u2022 JDF \u2192 vectors    embed those chunks, incrementally, for a vector store.
 
 Usage:
   jdf validate <file.jdf>
-  jdf import <file.{pdf,json,md}> [-o output.{jdf,jdfx}] [--json]
+  jdf convert  <file.{pdf,json,md}> [-o output.{jdf,jdfx}] [--json]
+  jdf chunk    <file.{jdf,jdfx}> [--strategy section|element|fixed] [--format jsonl|json|inline] [--max-tokens N] [-o out]
+  jdf embed    <file.{jdf,jdfx}> [--provider ollama|openai] [--model NAME] [--strategy \u2026] [--incremental] [-o out]
   jdf --help
 
 Commands:
   validate   Validate a .jdf / .jdfx file against the JDF schema
-  import     Convert a PDF, JSON, or Markdown file into JDF
+  convert    Convert a PDF, JSON, or Markdown file into JDF (alias: import)
+  chunk      Split a JDF document into retrieval-ready chunks (offline, deterministic)
+  embed      Compute embeddings for the chunks (local via Ollama by default)
 
 Flags:
-  -o, --output <path>   Explicit output path (extension picks .jdf vs .jdfx)
-      --json            Force pure JSON .jdf output (documents with embedded
-                        images stay as a single base64-inlined .jdf instead
-                        of a .jdfx bundle). Useful for RAG / CI consumers
-                        that prefer one text file over a zip.
+  -o, --output <path>   Explicit output path
+      --json            convert: force pure JSON .jdf output (inline base64
+                        instead of a .jdfx zip bundle)
+      --strategy <s>    chunk/embed: section (default) | element | fixed
+      --format <f>      chunk: jsonl (default) | json | inline
+      --max-tokens <n>  chunk/embed: soft cap per chunk (default 512)
+      --provider <p>    embed: ollama (default, local) | openai (remote API)
+      --model <name>    embed: model id (default: nomic-embed-text / text-embedding-3-small)
+      --incremental     embed: skip chunks whose content hash is unchanged
+      --no-auto-start   embed(ollama): don't auto-launch Ollama via Docker
+
+Environment (embed):
+  ollama:  OLLAMA_HOST (default http://localhost:11434)
+  openai:  OPENAI_API_KEY (required), OPENAI_BASE_URL (default api.openai.com)
 
 Examples:
   jdf validate spec/examples/hello-world.jdf
-  jdf import paper.pdf                        # PDF \u2192 JDF (or .jdfx for images)
-  jdf import contract.pdf --json | jq .       # PDF \u2192 pure JSON, pipe-friendly
-  jdf import response.json -o response.jdf    # LLM JSON output \u2192 validated JDF
-  jdf import README.md
+  jdf convert paper.pdf                        # PDF \u2192 JDF (or .jdfx for images)
+  jdf convert response.json -o response.jdf    # LLM JSON output \u2192 validated JDF
+  jdf chunk report.jdf                         # \u2192 report.chunks.jsonl (RAG-ready)
+  jdf chunk report.jdf --format inline         # embed the chunk index into the .jdf
+  jdf embed report.jdf                          # local embeddings via Ollama (auto-setup)
+  jdf embed report.jdf --provider openai --incremental
 `;
-var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["help", "h", "json", "verbose", "skip-validate"]);
+var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["help", "h", "json", "verbose", "skip-validate", "incremental", "no-auto-start"]);
 function parseArgs(argv) {
   const positional = [];
   const flags = {};
@@ -1540,10 +2148,13 @@ async function main() {
         const ok = await validate(positional[0]);
         process.exit(ok ? 0 : 1);
       }
+      // `convert` is the headline verb; `import` stays as a back-compat alias
+      // so existing scripts and docs keep working.
+      case "convert":
       case "import": {
         const input = positional[0];
         if (!input) {
-          console.error("Usage: jdf import <file.{pdf,json,md}> [-o output.jdf] [--json]");
+          console.error("Usage: jdf convert <file.{pdf,json,md}> [-o output.jdf] [--json]");
           process.exit(1);
         }
         const output = typeof flags.output === "string" ? flags.output : void 0;
@@ -1563,6 +2174,37 @@ async function main() {
           process.exit(1);
         }
       }
+      case "chunk": {
+        const input = positional[0];
+        if (!input) {
+          console.error("Usage: jdf chunk <file.{jdf,jdfx}> [--strategy section|element|fixed] [--format jsonl|json|inline] [--max-tokens N] [-o out]");
+          process.exit(1);
+        }
+        await chunkFile(input, {
+          strategy: typeof flags.strategy === "string" ? flags.strategy : void 0,
+          format: typeof flags.format === "string" ? flags.format : void 0,
+          maxTokens: typeof flags["max-tokens"] === "string" ? parseInt(flags["max-tokens"], 10) : void 0,
+          output: typeof flags.output === "string" ? flags.output : void 0
+        });
+        process.exit(0);
+      }
+      case "embed": {
+        const input = positional[0];
+        if (!input) {
+          console.error("Usage: jdf embed <file.{jdf,jdfx}> [--provider ollama|openai] [--model NAME] [--strategy \u2026] [--incremental] [--no-auto-start] [-o out]");
+          process.exit(1);
+        }
+        await embedFile(input, {
+          provider: typeof flags.provider === "string" ? flags.provider : void 0,
+          model: typeof flags.model === "string" ? flags.model : void 0,
+          strategy: typeof flags.strategy === "string" ? flags.strategy : void 0,
+          maxTokens: typeof flags["max-tokens"] === "string" ? parseInt(flags["max-tokens"], 10) : void 0,
+          incremental: flags.incremental === true,
+          autoStart: flags["no-auto-start"] !== true,
+          output: typeof flags.output === "string" ? flags.output : void 0
+        });
+        process.exit(0);
+      }
       default:
         console.error(`Unknown command: ${command}`);
         console.log(HELP);

package/dist/jdf-schema.json

@@ -26,9 +26,34 @@
       "type": "array",
       "minItems": 1,
       "items": { "$ref": "#/definitions/Page" }
-    }
+    },
+    "index": { "$ref": "#/definitions/DocumentIndex" }
   },
   "definitions": {
+    "DocumentIndex": {
+      "type": "object",
+      "description": "Optional precomputed RAG chunk index (jdf chunk --format inline). Data-only; renderers ignore it.",
+      "required": ["chunker", "chunks"],
+      "properties": {
+        "chunker": { "type": "string" },
+        "chunks": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "text", "hash"],
+            "properties": {
+              "id": { "type": "string" },
+              "text": { "type": "string" },
+              "path": { "type": "array", "items": { "type": "string" } },
+              "page": { "type": "integer" },
+              "types": { "type": "array", "items": { "type": "string" } },
+              "tokens": { "type": "integer" },
+              "hash": { "type": "string" }
+            }
+          }
+        }
+      }
+    },
     "Meta": {
       "type": "object",
       "required": ["title"],
@@ -42,7 +67,8 @@
         "pageSize": { "$ref": "#/definitions/PageSize" },
         "pageOrientation": { "$ref": "#/definitions/PageOrientation" },
         "margins": { "$ref": "#/definitions/Margins" },
-        "unit": { "$ref": "#/definitions/Unit" }
+        "unit": { "$ref": "#/definitions/Unit" },
+        "flow": { "type": "boolean", "description": "Document-level default: lay elements out top-to-bottom and auto-paginate overflow in PDF export. Per-page `flow` overrides." }
       }
     },
     "PageSize": {
@@ -335,7 +361,7 @@
         "headers": { "type": "array", "items": { "type": "string" } },
         "rows": {
           "type": "array",
-          "items": { "type": "array", "items": { "oneOf": [{ "type": "string" }, { "type": "object", "required": ["content"], "properties": { "content": { "type": "string" }, "style": { "$ref": "#/definitions/StyleRef" }, "colspan": { "type": "integer" }, "rowspan": { "type": "integer" } } }] } }
+          "items": { "type": "array", "items": { "oneOf": [{ "type": "string" }, { "type": "object", "required": ["content"], "properties": { "content": { "type": "string" }, "style": { "$ref": "#/definitions/StyleRef" }, "align": { "type": "string", "enum": ["left","center","right","justify"] }, "colspan": { "type": "integer" }, "rowspan": { "type": "integer" } } }] } }
         },
         "position": { "$ref": "#/definitions/Position" }, "width": { "type": "number" },
         "headerStyle": { "$ref": "#/definitions/StyleRef" },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uurtech/jdf-cli",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "description": "Command-line tool for the JDF (JSON Document Format) — validate and convert documents.",
   "license": "MIT",
   "author": "Ugur Kazdal",