@remigius42/morg 0.5.0 → 0.7.0

package/dist/core/uniorgToMdast/index.js

@@ -1,13 +1,27 @@
 import { stringify as stringifyYaml } from "yaml";
+import { warn } from "./shared.js";
+import { isModeLineComment, takeFileHeader, takeRenderedFileHeader } from "../frontmatterBlock.js";
 import { collectFootnoteLabels } from "./footnotes.js";
 import { transformNodes } from "./elements.js";
 /**
  * Transforms a uniorg AST to a mdast (Markdown AST).
  * @param uniorgAst The uniorg AST to transform.
- * @param options Controls org-ism serialization (`key:: value` lines).
+ * @param options Controls org-ism serialization (`key:: value` lines);
+ * `org`, the text a tree was parsed from, finds its frontmatter block.
  * @returns The transformed mdast.
  */
 export function transformUniorgAstToMdast(uniorgAst, options = {}) {
+    // called on its own, not by convertOrgToMarkdown, which takes the
+    // header off the org text: the header of the org text the tree was
+    // parsed from, or else the one transformMdastToUniorgAst built, from a
+    // copy, so the caller's tree stays as it is
+    if (options.takesMorgEntries === undefined) {
+        uniorgAst = { ...uniorgAst, children: [...uniorgAst.children] };
+        const header = options.org === undefined
+            ? takeRenderedFileHeader(uniorgAst)
+            : takeFileHeader(uniorgAst, options.org, options.onWarning);
+        options = { ...options, ...header };
+    }
     const ctx = {
         options,
         inlineFootnotes: [],
@@ -15,36 +29,15 @@ export function transformUniorgAstToMdast(uniorgAst, options = {}) {
     };
     collectFootnoteLabels(ctx, uniorgAst);
     const nodes = uniorgAst.children || [];
-    // leading #+KEY: value keywords map to md frontmatter, a native
-    // construct; JSON-encoded values restore their structure (ADR 0002)
-    const collected = new Map();
-    let first = 0;
-    while (nodes[first]?.type === "keyword") {
-        const keyword = nodes[first];
-        const key = keyword.key.toLowerCase();
-        // a keyword may legally repeat; collect the values instead of
-        // letting the last one win
-        collected.set(key, [
-            ...(collected.get(key) ?? []),
-            parseKeywordValue(keyword.value)
-        ]);
-        first++;
-    }
-    const frontmatter = Object.fromEntries([...collected].map(([key, values]) => [
-        key,
-        values.length === 1 ? values[0] : values
-    ]));
-    const children = transformNodes(ctx, nodes.slice(first));
+    const { yaml, rest } = takeFrontmatter(ctx, nodes);
+    const children = transformNodes(ctx, rest);
     // adjacent single-item task lists (one per converted TODO section)
     // merge into one list, or the output would not be a fixed point
     if (options.taskCheckboxes) {
         mergeAdjacentTaskLists(children);
     }
-    if (first > 0) {
-        children.unshift({
-            type: "yaml",
-            value: stringifyYaml(frontmatter).trimEnd()
-        });
+    if (yaml !== undefined) {
+        children.unshift({ type: "yaml", value: yaml });
     }
     for (const footnote of ctx.inlineFootnotes) {
         children.push({
@@ -56,12 +49,88 @@ export function transformUniorgAstToMdast(uniorgAst, options = {}) {
     }
     return { type: "root", children: children };
 }
-function parseKeywordValue(value) {
-    try {
-        return JSON.parse(value);
+// the frontmatter: the block's YAML, and the leading #+KEY: value
+// keywords, which can act in Emacs, in order, repeats included, as
+// morg's own entry (ADR 0005); where they cannot join it, they stay
+// keyword lines. Returns the YAML and the nodes left
+function takeFrontmatter(ctx, nodes) {
+    const block = ctx.options.frontmatter;
+    // set by the caller or transformUniorgAstToMdast, never undefined
+    const joins = ctx.options.takesMorgEntries === true;
+    const start = leadingKeywordsStart(nodes);
+    warnAboutFrontmatter(ctx, block, joins ? [] : nodes.slice(start));
+    const keywords = inertModeLineGuard(ctx, nodes, joins ? leadingKeywords(nodes.slice(start)) : [], start);
+    const entries = morgEntries(ctx.options.fileProperties ?? [], keywords);
+    const rest = [
+        ...nodes.slice(0, start),
+        ...nodes.slice(start + keywords.length)
+    ];
+    // an empty block adds nothing next to the entries
+    // on a line of their own: a keep-chomped (`|+`) scalar at the end of
+    // the block holds a newline already there
+    const yaml = entries && block
+        ? `${block.replace(/(?<!\n)$/, "\n")}${entries}`
+        : (entries ?? block);
+    // leading the Markdown body, with or without frontmatter above it
+    if (isInertModeLine(ctx, rest[0])) {
+        warn(ctx, "a -*- comment below the first line becomes the mode line in org");
+    }
+    return { yaml, rest };
+}
+// leading keywords may sit below a -*- comment, which leads the
+// Markdown body and so becomes the mode line (warned if it was none);
+// below another comment, they would come back above it
+function leadingKeywordsStart(nodes) {
+    return nodes[0] && isModeLineComment(nodes[0]) ? 1 : 0;
+}
+// a -*- comment below the file's first line, which Emacs ignores
+function isInertModeLine(ctx, node) {
+    return !ctx.options.startsWithModeLine && !!node && isModeLineComment(node);
+}
+// md→org makes a -*- comment that leads the Markdown body the file's
+// mode line: leading keywords directly above one stay lines, so it
+// does not lead it
+function inertModeLineGuard(ctx, nodes, keywords, start) {
+    if (!keywords.length ||
+        start !== 0 ||
+        !isInertModeLine(ctx, nodes[keywords.length])) {
+        return keywords;
+    }
+    warn(ctx, "leading keywords stay lines: a -*- comment below them is no mode line");
+    return [];
+}
+function leadingKeywords(nodes) {
+    const keywords = [];
+    while (nodes[keywords.length]?.type === "keyword") {
+        const keyword = nodes[keywords.length];
+        keywords.push([keyword.key, keyword.value]);
+    }
+    return keywords;
+}
+// morg's own entries as YAML text, or undefined without any
+function morgEntries(fileProperties, keywords) {
+    // an empty value as `KEY:`, as it reads back, not as `KEY: ""`
+    const items = (pairs) => pairs.map(([key, value]) => ({ [key]: value || null }));
+    const entries = {
+        ...(fileProperties.length && { morg_properties: items(fileProperties) }),
+        ...(keywords.length && { morg_keywords: items(keywords) })
+    };
+    // unfolded: a value folded over lines fits no keyword line again
+    return Object.keys(entries).length
+        ? stringifyYaml(entries, { lineWidth: 0, nullStr: "" }).trimEnd()
+        : undefined;
+}
+// `left`: the leading nodes the frontmatter could not take
+function warnAboutFrontmatter(ctx, block, left) {
+    if (block && /^---[ \t]*$/m.test(block)) {
+        warn(ctx, "frontmatter holds a --- line, which ends it early in Markdown");
+    }
+    const drawer = left[0]?.type === "property-drawer";
+    if (drawer) {
+        warn(ctx, "file-level drawer stays text: the frontmatter cannot take it");
     }
-    catch {
-        return value;
+    if (left[drawer ? 1 : 0]?.type === "keyword") {
+        warn(ctx, "leading keywords stay lines: the frontmatter cannot take them");
     }
 }
 function isTaskList(node) {

package/dist/core/uniorgToMdast/lists.js

@@ -61,17 +61,28 @@ function transformUniorgList(ctx, node) {
         return {
             type: "list",
             ordered,
-            start: ordered ? parseInt(firstBullet, 10) || 1 : null,
+            start: ordered ? parseInt(firstBullet, 10) : null,
             spread: false,
             children: run.map(item => transformUniorgListItem(ctx, item))
         };
     });
 }
+// uniorg keeps a list item's indentation in its code blocks' values;
+// md indents them itself. Only that much is dropped, as uniorg-stringify
+// does, so the code keeps its own
+function outdent(value, level) {
+    return value.replace(new RegExp(`^ {0,${level}}`, "gm"), "");
+}
 function transformUniorgListItem(ctx, item) {
     // md has no descriptive lists, so keep the ` :: ` syntax literally in the
     // item text; the return trip re-parses it as a descriptive list
     const tag = listItemTag(item);
     const children = transformNodes(ctx, (item.children || []).filter(child => child !== tag));
+    for (const child of children) {
+        if (child.type === "code") {
+            child.value = outdent(child.value, item.indent + item.bullet.length);
+        }
+    }
     if (tag) {
         const term = {
             type: "text",

package/dist/core/uniorgToMdast/objects.js

@@ -1,9 +1,16 @@
 import { toString as orgastToString } from "orgast-util-to-string";
 import { htmlEnabled, orgNodeToText, warn } from "./shared.js";
 import { transformFootnoteReference } from "./footnotes.js";
+import { unescapeOrgPath } from "../orgPath.js";
 const IMAGE_EXTENSION_RE = /\.(png|jpe?g|gif|svg|webp|avif|bmp|ico)$/i;
 export function transformUniorgObjects(ctx, children) {
     return (children || [])
+        .map((child, index) => 
+    // uniorg keeps the continuation line's indentation after a line
+    // break; md would render it as leading whitespace
+    child.type === "text" && children?.[index - 1]?.type === "line-break"
+        ? { ...child, value: child.value.replace(/^[ \t]+/, "") }
+        : child)
         .flatMap(child => transformUniorgObjectToMdastPhrasingContent(ctx, child))
         .filter(Boolean);
 }
@@ -74,12 +81,43 @@ function transformUniorgObjectToMdastPhrasingContent(ctx, node) {
             return null;
     }
 }
+// the inverse of md→org's decoding: % and # would be read as an escape
+// or the anchor, and a markdown url cannot hold a bare space. Org path
+// escapes (`[`, `]`, `:`) are undone first rather than escaped twice
+function encodeUrlPart(part) {
+    return unescapeOrgPath(part)
+        .replaceAll("%", "%25")
+        .replaceAll("#", "%23")
+        .replaceAll(" ", "%20");
+}
+// a path starting like `a:` would read as a url scheme in markdown
+function encodeUrlPath(path) {
+    const encoded = encodeUrlPart(path);
+    return /^[a-z][a-z0-9+.-]*:/i.test(encoded)
+        ? encoded.replaceAll(":", "%3A")
+        : encoded;
+}
+// an org file: link (or a ./ path) is a relative markdown link, the
+// search option becoming the #anchor
+function markdownUrl(node) {
+    if (node.linkType !== "file") {
+        return node.rawLink;
+    }
+    const target = node.rawLink.replace(/^file:/, "");
+    const search = target.indexOf("::");
+    return search === -1
+        ? encodeUrlPath(target)
+        : `${encodeUrlPath(target.slice(0, search))}#${encodeUrlPart(target.slice(search + 2))}`;
+}
 function transformUniorgLink(ctx, node) {
     const descriptionText = orgastToString(node);
+    const url = markdownUrl(node);
     // org has no dedicated image syntax; the common convention is a
-    // link to an image file, so map those to markdown images
-    if (IMAGE_EXTENSION_RE.test(node.rawLink)) {
-        return { type: "image", url: node.rawLink, alt: descriptionText };
+    // link to an image file, so map those to markdown images; a file
+    // link's ::search option is no part of the file name
+    const path = node.linkType === "file" ? node.rawLink.replace(/::.*$/s, "") : node.rawLink;
+    if (IMAGE_EXTENSION_RE.test(path)) {
+        return { type: "image", url, alt: descriptionText };
     }
     // a description equal to the url (a common Logseq pattern) is no
     // description: text === url makes remark-stringify emit an
@@ -100,8 +138,8 @@ function transformUniorgLink(ctx, node) {
         withoutMarkers(serializedDescription) === withoutMarkers(node.rawLink)) {
         return {
             type: "link",
-            url: node.rawLink,
-            children: [{ type: "text", value: node.rawLink }]
+            url,
+            children: [{ type: "text", value: url }]
         };
     }
     const children = transformUniorgObjects(ctx, node.children);
@@ -110,7 +148,7 @@ function transformUniorgLink(ctx, node) {
     const flattened = children.some(child => child.type === "link" || child.type === "image")
         ? [{ type: "text", value: descriptionText }]
         : children;
-    return { type: "link", url: node.rawLink, children: flattened };
+    return { type: "link", url, children: flattened };
 }
 function transformScriptMarkup(ctx, node) {
     // markdown has no equivalents; with useHtml render as raw html

package/dist/core/uniorgToMdast/shared.d.ts

@@ -6,6 +6,11 @@ export interface UniorgToMdastOptions {
     taskCheckboxes?: boolean;
     orgismKeys?: Record<string, string>;
     onWarning?: (message: string) => void;
+    frontmatter?: string;
+    fileProperties?: [string, string][];
+    takesMorgEntries?: boolean;
+    startsWithModeLine?: boolean;
+    org?: string;
 }
 export interface TransformContext {
     options: UniorgToMdastOptions;

package/dist/core/uniorgToMdast/shared.js

@@ -1,4 +1,4 @@
-import { toString as orgastToString } from "orgast-util-to-string";
+import { affiliatedEntries } from "../affiliated.js";
 import { unified } from "unified";
 import { uniorgStringify } from "uniorg-stringify";
 import { toggleEnabled } from "../../options.js";
@@ -41,15 +41,6 @@ export function trimTrailingNewline(value) {
 // affiliated keywords (#+CAPTION:, #+NAME:, #+ATTR_*) precede their
 // element as verbatim lines so the return trip re-attaches them natively
 export function affiliatedLines(node) {
-    const affiliated = node
-        .affiliated;
-    return Object.entries(affiliated ?? {}).flatMap(([key, value]) => {
-        const entries = Array.isArray(value) ? value : [value];
-        return entries.map(entry => {
-            const text = Array.isArray(entry)
-                ? entry.map(child => orgastToString(child)).join("")
-                : String(entry);
-            return `#+${key}: ${text}`;
-        });
-    });
+    const affiliated = node.affiliated;
+    return affiliatedEntries(affiliated).map(([key, value]) => `#+${key}: ${value}`);
 }

package/dist/core/uniorgToMdast/tables.js

@@ -16,12 +16,18 @@ function isAlignmentCookieRow(row) {
         cells.some(cell => cellText(cell) !== ""));
 }
 // org cell content keeps the aligning whitespace padding; markdown
-// cells are re-padded by the stringifier
-function trimCellPadding(node) {
-    if (node.type === "text") {
-        node.value = node.value.trim();
+// cells are re-padded by the stringifier. Only the cell's edges are
+// padding: a space between text and markup is content
+function trimCellPadding(children) {
+    const first = children[0];
+    if (first?.type === "text") {
+        first.value = first.value.trimStart();
     }
-    return node;
+    const last = children.at(-1);
+    if (last?.type === "text") {
+        last.value = last.value.trimEnd();
+    }
+    return children;
 }
 export function transformTable(ctx, node) {
     if (node.tableType === "table.el") {
@@ -59,7 +65,7 @@ export function transformTable(ctx, node) {
             type: "tableRow",
             children: (row.children || []).map(cell => ({
                 type: "tableCell",
-                children: transformUniorgObjects(ctx, cell.children).map(trimCellPadding)
+                children: trimCellPadding(transformUniorgObjects(ctx, cell.children))
             }))
         }))
     };

package/dist/markdownToOrg.js

@@ -5,8 +5,16 @@ import remarkFrontmatter from "remark-frontmatter";
 import remarkMath from "remark-math";
 import { uniorgStringify } from "uniorg-stringify";
 import { visit } from "unist-util-visit";
-import { transformMdastToUniorgAst } from "./core/mdastToUniorg/index.js";
+import { transformMdastToUniorgDraft } from "./core/mdastToUniorg/index.js";
 import { detectMarkdownStyle, STYLE_KEYWORD } from "./core/markdownStyle.js";
+import { escapeOrgMarkup } from "./core/markupBoundary.js";
+import { renderFileHeader } from "./core/frontmatterBlock.js";
+import { escapeLineSyntax } from "./core/lineSyntax.js";
+import { escapeFootnoteReferences } from "./core/footnoteReferences.js";
+import { escapeBackslashCommands } from "./core/backslashCommands.js";
+import { escapeTablePipes } from "./core/tablePipes.js";
+import { requireBracedScripts } from "./core/bracedScripts.js";
+import { keyValueEntries } from "./core/keyValueLines.js";
 /**
  * Converts a Markdown string to an Org-mode string.
  * @param markdown The Markdown string to convert.
@@ -15,14 +23,9 @@ import { detectMarkdownStyle, STYLE_KEYWORD } from "./core/markdownStyle.js";
  */
 export function convertMarkdownToOrg(markdown, options = {}) {
     // Phase 1: Parse Markdown to mdast
-    const mdast = unified()
-        .use(remarkParse)
-        .use(remarkGfm)
-        .use(remarkFrontmatter)
-        .use(remarkMath)
-        .parse(markdown);
+    const mdast = parseMarkdown(markdown, options.preset);
     // Phase 2: Generic mdast to uniorg-ast transformation
-    let uniorgAst = transformMdastToUniorgAst(mdast, {
+    let uniorgAst = transformMdastToUniorgDraft(mdast, {
         ...(options.preserveMdisms !== undefined && {
             preserveMdisms: options.preserveMdisms
         }),
@@ -52,12 +55,43 @@ export function convertMarkdownToOrg(markdown, options = {}) {
     if (options.preset?.applyToUniorg) {
         uniorgAst = options.preset.applyToUniorg(uniorgAst);
     }
+    // Phase 3b: literal footnote references, paragraph lines org would
+    // read as line syntax, literal markers org would read as markup, and
+    // markup touching a word character, need a zero-width space escape;
+    // a pipe in a table cell an entity; a literal backslash before a
+    // letter first, the entity being none
+    escapeBackslashCommands(uniorgAst);
+    escapeTablePipes(uniorgAst, options.onWarning);
+    escapeFootnoteReferences(uniorgAst);
+    escapeLineSyntax(uniorgAst);
+    escapeOrgMarkup(uniorgAst);
+    // Phase 3c: md text has no sub/superscripts; keep org from reading
+    // bare underscores and carets as such. Checked on the text as
+    // rendered: after the preset, whose text rewrites (wikilink aliases)
+    // org parses too, and after the escapes, next to which org reads them
+    requireBracedScripts(uniorgAst);
+    // Phase 3d: the file's header (ADR 0005): mode line and file-level
+    // drawer first, keywords apart from what follows, the frontmatter as
+    // a marked comment block; raw text, past every pass that rewrites text
+    renderFileHeader(uniorgAst);
     // Phase 4: Render uniorg-ast to Org-mode string
     const processor = unified().use(uniorgStringify);
     const orgContent = processor.stringify(uniorgAst);
     return orgContent;
 }
-// leads the document so it survives the frontmatter keywords following it
+// a preset's dialect conventions that need the Markdown source apply
+// to the parse, before the generic transform
+function parseMarkdown(markdown, preset) {
+    const mdast = unified()
+        .use(remarkParse)
+        .use(remarkGfm)
+        .use(remarkFrontmatter)
+        .use(remarkMath)
+        .parse(markdown);
+    preset?.applyToMdast?.(mdast, markdown);
+    return mdast;
+}
+// leads the document, ahead of restored keywords and the frontmatter block
 function recordStyleKeyword(uniorgAst, style) {
     if (!Object.keys(style).length) {
         return;
@@ -68,7 +102,6 @@ function recordStyleKeyword(uniorgAst, style) {
         value: JSON.stringify(style)
     });
 }
-const KEY_VALUE_LINE_RE = /^([\w-]+):: (.*)$/;
 function parseKeyValueParagraph(node) {
     if (node?.type !== "paragraph" || !node.children?.length) {
         return null;
@@ -76,19 +109,7 @@ function parseKeyValueParagraph(node) {
     if (!node.children.every(child => child.type === "text")) {
         return null;
     }
-    const lines = node.children
-        .map(child => child.value ?? "")
-        .join("")
-        .split("\n");
-    const entries = [];
-    for (const line of lines) {
-        const match = KEY_VALUE_LINE_RE.exec(line);
-        if (!match) {
-            return null;
-        }
-        entries.push([match[1], match[2]]);
-    }
-    return entries;
+    return keyValueEntries(node.children.map(child => child.value ?? "").join(""));
 }
 function makeTimestamp(rawValue) {
     return { type: "timestamp", rawValue };

package/dist/orgToMarkdown.js

@@ -1,11 +1,27 @@
 import { unified } from "unified";
-import uniorgParse from "uniorg-parse";
 import remarkStringify from "remark-stringify";
 import remarkGfm from "remark-gfm";
 import remarkFrontmatter from "remark-frontmatter";
 import remarkMath from "remark-math";
 import { transformUniorgAstToMdast } from "./core/uniorgToMdast/index.js";
 import { takeRecordedStyle } from "./core/markdownStyle.js";
+import { takeFileHeader } from "./core/frontmatterBlock.js";
+import { unescapeOrgMarkup } from "./core/markupBoundary.js";
+import { unescapeLineSyntax } from "./core/lineSyntax.js";
+import { unescapeFootnoteReferences } from "./core/footnoteReferences.js";
+import { unescapeBackslashCommands } from "./core/backslashCommands.js";
+import { unescapeTablePipes } from "./core/tablePipes.js";
+import { parseOrg } from "./core/bracedScripts.js";
+import { dropUnderscoreBulletGuards, guardUnderscoreBullets } from "./core/underscoreBullets.js";
+// a list item's paragraph after its nested list needs a blank line, or
+// md reads it as a lazy continuation of the nested list's last item
+function separateTextAfterNestedList(left, right, parent) {
+    return parent.type === "listItem" &&
+        left.type === "list" &&
+        right.type === "paragraph"
+        ? 1
+        : undefined;
+}
 /**
  * Converts an Org-mode string to a Markdown string.
  * @param org The Org-mode string to convert.
@@ -14,10 +30,23 @@ import { takeRecordedStyle } from "./core/markdownStyle.js";
  */
 export function convertOrgToMarkdown(org, options = {}) {
     // Phase 1: Parse Org-mode to uniorg-ast
-    let uniorgAst = unified().use(uniorgParse).parse(org);
+    // md text has no scripts, so ^:{} is implied there and consumed here
+    // (see markdownToOrg); uniorg misreads `_.` lines (see underscoreBullets)
+    const guarded = guardUnderscoreBullets(org);
+    let uniorgAst = parseOrg(guarded);
+    dropUnderscoreBulletGuards(uniorgAst);
     // Phase 1b: a recorded style is morg's own (ADR 0004), so consume it so
     // it does not travel on as frontmatter; explicit options still win
     const recordedStyle = takeRecordedStyle(uniorgAst);
+    // frontmatter travels in a block marked as morg's (ADR 0005)
+    // and so does a file-level drawer (org-roam's :ID:)
+    const fileHeader = takeFileHeader(uniorgAst, guarded, options.onWarning);
+    // Phase 1c: markdown needs no zero-width space escapes (inverse of md→org)
+    unescapeOrgMarkup(uniorgAst);
+    unescapeLineSyntax(uniorgAst);
+    unescapeFootnoteReferences(uniorgAst);
+    unescapeBackslashCommands(uniorgAst);
+    unescapeTablePipes(uniorgAst);
     // Phase 2: Extract dialect preset conventions, if any
     if (options.preset?.extractFromUniorg) {
         uniorgAst = options.preset.extractFromUniorg(uniorgAst);
@@ -34,7 +63,8 @@ export function convertOrgToMarkdown(org, options = {}) {
         ...(options.orgismKeys !== undefined && {
             orgismKeys: options.orgismKeys
         }),
-        ...(options.onWarning !== undefined && { onWarning: options.onWarning })
+        ...(options.onWarning !== undefined && { onWarning: options.onWarning }),
+        ...fileHeader
     });
     // Phase 4: Render mdast to Markdown string
     // bullet and rule "-" (not remark's default "*") are morg's canonical
@@ -46,11 +76,15 @@ export function convertOrgToMarkdown(org, options = {}) {
         rule: "-",
         ...recordedStyle,
         ...options.markdownStyle,
+        join: [separateTextAfterNestedList],
         handlers: {
             // key:: value blocks and preset inline passthroughs (e.g.
-            // wikilinks) are emitted verbatim, unescaped
+            // wikilinks) are emitted verbatim, unescaped; only a pipe inside
+            // a table cell is escaped, or it would split the cell
             keyValue: (node) => node.value,
-            verbatimInline: (node) => node.value
+            verbatimInline: (node, _parent, state) => state.stack.includes("tableCell")
+                ? node.value.replaceAll("|", "\\|")
+                : node.value
         }
     })
         .use(remarkGfm)