@remigius42/morg 0.5.0 → 0.7.0

package/dist/core/mdastToUniorg/phrasing.js

@@ -1,4 +1,7 @@
 import { mdismEnabled, warn } from "./context.js";
+import { visit } from "unist-util-visit";
+import { escapeOrgPath } from "../orgPath.js";
+import { orgParser, renderInline } from "../render.js";
 // html tags morg itself emits under useHtml; with interpretHtml a bare
 // open/close pair becomes the corresponding native org object
 const INLINE_HTML_ORG_TYPES = {
@@ -30,20 +33,87 @@ export function transformPhrasingChildren(ctx, children) {
             ? matchInlineHtmlPair(children, i)
             : null;
         if (pair) {
-            result.push({
+            result.push(...hoistEdgeWhitespace(joinLines({
                 type: pair.orgType,
                 children: transformPhrasingChildren(ctx, children.slice(i + 1, pair.end))
-            });
+            })));
             i = pair.end;
             continue;
         }
+        if (node.type === "inlineCode") {
+            result.push(...transformMdastInlineCode(ctx, node.value));
+            continue;
+        }
         const transformed = transformMdastPhrasingContentToUniorgObject(ctx, node);
         if (transformed) {
-            result.push(transformed);
+            result.push(...hoistEdgeWhitespace(joinLines(transformed)));
         }
     }
     return result;
 }
+const CONTAINER_MARKUP = new Set([
+    "bold",
+    "italic",
+    "strike-through",
+    "underline"
+]);
+// org markup spans at most two lines; beyond that its line endings
+// become spaces, as CommonMark renders them anyway (see
+// transformMdastInlineCode)
+function joinLines(node) {
+    if (!CONTAINER_MARKUP.has(node.type)) {
+        return node;
+    }
+    const texts = [];
+    visit(node, "text", (text) => {
+        texts.push(text);
+    });
+    const lineEndings = texts
+        .map(text => text.value)
+        .join("")
+        .split("\n").length - 1;
+    if (lineEndings > 1) {
+        for (const text of texts) {
+            text.value = text.value.replaceAll("\n", " ");
+        }
+    }
+    return node;
+}
+// strips the whitespace at the edges of `children`, returning it
+function takeEdgeWhitespace(children) {
+    const first = children[0];
+    const last = children.at(-1);
+    let before = "";
+    let after = "";
+    if (first?.type === "text") {
+        before = /^\s*/.exec(first.value)?.[0] ?? "";
+        first.value = first.value.slice(before.length);
+    }
+    if (last?.type === "text") {
+        after = /\s*$/.exec(last.value)?.[0] ?? "";
+        last.value = last.value.slice(0, last.value.length - after.length);
+    }
+    return { before, after };
+}
+// org markup may not start or end with whitespace, which a code span's
+// moved-out edge whitespace (see transformMdastInlineCode) can leave
+// inside it; it moves out further, to the markup's siblings
+function hoistEdgeWhitespace(node) {
+    if (!CONTAINER_MARKUP.has(node.type) || !("children" in node)) {
+        return [node];
+    }
+    const { before, after } = takeEdgeWhitespace(node.children);
+    if (!before && !after) {
+        return [node];
+    }
+    const content = node.children.filter(child => child.type !== "text" || child.value !== "");
+    const text = (value) => value ? [{ type: "text", value }] : [];
+    return [
+        ...text(before),
+        ...(content.length ? [{ ...node, children: content }] : []),
+        ...text(after)
+    ];
+}
 // an org bracket-link path cannot contain [ or ]; percent-encoding keeps
 // the url equivalent and, unlike org's backslash escaping, survives being
 // parsed back (uniorg does not decode `\[`). A silent normalization, not
@@ -51,6 +121,36 @@ export function transformPhrasingChildren(ctx, children) {
 function orgSafeUrl(url) {
     return url.replaceAll("[", "%5B").replaceAll("]", "%5D");
 }
+// a url without a scheme is a relative path in markdown, but a bare org
+// path is a fuzzy link (a heading search); org's file: type keeps it a
+// file. A #anchor becomes org's search option, which finds a
+// <<target>> or a headline of that name. `[[page]]` / `((uuid))` urls
+// are dialect references (logseq), not paths
+const SCHEME_RE = /^[a-z][a-z0-9+.-]*:/i;
+function isRelativePath(url) {
+    return url !== "" && !SCHEME_RE.test(url) && !/^(#|\/\/|\[|\()/.test(url);
+}
+// md urls are percent-encoded, org paths are not; a malformed escape
+// stays as written
+function decodeUrlPart(part) {
+    try {
+        return decodeURIComponent(part);
+    }
+    catch {
+        return part;
+    }
+}
+function orgLinkTarget(url) {
+    if (!isRelativePath(url)) {
+        return { rawLink: orgSafeUrl(url), linkType: "url" };
+    }
+    const hash = url.indexOf("#");
+    const path = escapeOrgPath(decodeUrlPart(hash === -1 ? url : url.slice(0, hash)));
+    const search = hash === -1
+        ? ""
+        : `::${escapeOrgPath(decodeUrlPart(url.slice(hash + 1)), true)}`;
+    return { rawLink: `file:${path}${search}`, linkType: "file" };
+}
 function transformMdastLink(ctx, linkNode) {
     const [only] = linkNode.children;
     // text equal to the url (autolinks) is no description; a plain
@@ -61,13 +161,13 @@ function transformMdastLink(ctx, linkNode) {
         ? []
         : transformPhrasingChildren(ctx, linkNode.children);
     // rawLink should just be the URL, uniorg-stringify adds the brackets
-    const url = orgSafeUrl(linkNode.url);
+    const { rawLink, linkType } = orgLinkTarget(linkNode.url);
     return {
         type: "link",
         format: "bracket", // Assuming bracket format for Markdown links
-        linkType: "url",
-        rawLink: url,
-        path: url,
+        linkType,
+        rawLink,
+        path: rawLink,
         children: linkChildren
     };
 }
@@ -101,16 +201,51 @@ function transformMdastImage(ctx, node) {
     if (node.title) {
         warn(ctx, `dropped image title "${node.title}" (${node.url})`);
     }
-    const url = orgSafeUrl(node.url);
+    const { rawLink } = orgLinkTarget(node.url);
     return {
         type: "link",
         format: "bracket",
         linkType: "file",
-        rawLink: url,
-        path: url,
+        rawLink,
+        path: rawLink,
         children: node.alt ? [{ type: "text", value: node.alt }] : []
     };
 }
+// org markup spans at most two lines; CommonMark renders a line ending
+// inside a code span as a space, so nothing is lost. Org markup may not
+// start or end with whitespace either: edge whitespace moves outside
+// the markers, a one-space shift in the rendered output
+function transformMdastInlineCode(ctx, value) {
+    const [, before = "", code = "", after = ""] = /^(\s*)(.*?)(\s*)$/s.exec(value.replace(/\r\n?|\n/g, " ")) ?? [];
+    if (!code) {
+        // org has no empty code markup
+        warn(ctx, "whitespace-only inline code kept as text");
+        return [{ type: "text", value: before }];
+    }
+    const type = codeType(code);
+    if (!type) {
+        warn(ctx, "inline code holding both ~ and = kept as text");
+        return [{ type: "text", value: before + code + after }];
+    }
+    return [
+        ...(before ? [{ type: "text", value: before }] : []),
+        { type, value: code },
+        ...(after ? [{ type: "text", value: after }] : [])
+    ];
+}
+// org ends code at the first `~` it may close on (`~a~ b~`); verbatim
+// keeps such code whole, and comes back as md code too, unless a `=`
+// ends it early in turn
+function codeType(code) {
+    return ["code", "verbatim"].find(type => !code.includes(type === "code" ? "~" : "=") ||
+        readsWhole({ type, value: code }));
+}
+// whether org reads `node`, rendered, back as the same node
+function readsWhole(node) {
+    const [paragraph] = orgParser.parse(renderInline(node)).children;
+    const [first, ...rest] = paragraph && "children" in paragraph ? paragraph.children : [];
+    return !rest.length && first?.type === node.type && first.value === node.value;
+}
 function transformMdastInlineMath(node) {
     const value = node.value;
     return {
@@ -144,8 +279,6 @@ function transformMdastPhrasingContentToUniorgObject(ctx, node) {
             return transformMdastLinkReference(ctx, node);
         case "imageReference":
             return transformMdastImageReference(ctx, node);
-        case "inlineCode":
-            return { type: "code", value: node.value };
         case "inlineMath":
             return transformMdastInlineMath(node);
         case "break":

package/dist/core/orgPath.d.ts

@@ -0,0 +1,9 @@
+/**
+ * md→org: a decoded file link path, or its search option, as org link
+ * text.
+ */
+export declare function escapeOrgPath(part: string, isSearch?: boolean): string;
+/**
+ * org→md: the inverse of `escapeOrgPath`.
+ */
+export declare function unescapeOrgPath(part: string): string;

package/dist/core/orgPath.js

@@ -0,0 +1,24 @@
+// an org bracket-link path cannot contain [ or ], and `::` starts its
+// search option. morg percent-encodes those (uniorg does not decode org's
+// own backslash escaping); a literal % in front of such an escape gets
+// one more `25`, so the encoding stays reversible (`%5B` ↔ `%255B`)
+const ESCAPE_RE = /%((?:25)*)(5B|5D|3A)/g;
+const ESCAPED = { "5B": "[", "5D": "]", "3A": ":" };
+/**
+ * md→org: a decoded file link path, or its search option, as org link
+ * text.
+ */
+export function escapeOrgPath(part, isSearch = false) {
+    const escaped = part
+        .replace(ESCAPE_RE, "%25$1$2")
+        .replaceAll("[", "%5B")
+        .replaceAll("]", "%5D");
+    // a colon next to another, or before the `::` separator
+    return isSearch ? escaped : escaped.replace(/:(?=:|$)/g, "%3A");
+}
+/**
+ * org→md: the inverse of `escapeOrgPath`.
+ */
+export function unescapeOrgPath(part) {
+    return part.replace(ESCAPE_RE, (_, more, code) => more ? `%${more.slice(2)}${code}` : (ESCAPED[code] ?? code));
+}

package/dist/core/render.d.ts

@@ -0,0 +1,33 @@
+import type { Parent } from "unist";
+export type Node = Parent["children"][number] & {
+    value?: string;
+};
+export declare const orgParser: import("unified").Processor<import("uniorg").OrgData, undefined, undefined, undefined, undefined>;
+export declare const positionParser: import("unified").Processor<import("uniorg").OrgData, undefined, undefined, undefined, undefined>;
+/**
+ * Parses `text` as an org document, or returns undefined where uniorg
+ * throws: it takes a line starting `_.` or `_)` for a bullet, then
+ * fails to read it (org has no such bullet; see underscoreBullets).
+ */
+export declare function tryParse(text: string, parser?: {
+    parse(text: string): unknown;
+}): Parent | undefined;
+export declare function isInline(node: Node | Parent): boolean;
+/**
+ * How org renders an inline node within its line.
+ */
+export declare function renderInline(node: Node): string;
+/**
+ * An inline node's delimiters around its children, as org renders them
+ * (`*` and `*`, `[[url][` and `]]`).
+ */
+export declare function delimiters(node: Node): [string, string];
+/**
+ * The org rendering of each child; a block element only ends a line.
+ */
+export declare function renderChildren(children: Node[]): string[];
+/**
+ * The index of the rendered child `offset` (into the joined rendering)
+ * lies in, and the offset within that child.
+ */
+export declare function locate(rendered: string[], offset: number): [number, number];

package/dist/core/render.js

@@ -0,0 +1,101 @@
+import { unified } from "unified";
+import uniorgParse from "uniorg-parse";
+import { uniorgStringify } from "uniorg-stringify";
+// built once: constructing a processor per parse dominates the cost
+export const orgParser = unified().use(uniorgParse).freeze();
+export const positionParser = unified()
+    .use(uniorgParse, { trackPosition: true })
+    .freeze();
+const stringifier = unified().use(uniorgStringify).freeze();
+/**
+ * Parses `text` as an org document, or returns undefined where uniorg
+ * throws: it takes a line starting `_.` or `_)` for a bullet, then
+ * fails to read it (org has no such bullet; see underscoreBullets).
+ */
+export function tryParse(text, parser = orgParser) {
+    try {
+        return parser.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+// uniorg's inline node types; anything else is a block element (in a
+// list item's flattened content: a nested list or code block)
+const INLINE_TYPES = new Set([
+    "text",
+    "bold",
+    "italic",
+    "underline",
+    "strike-through",
+    "code",
+    "verbatim",
+    "link",
+    "footnote-reference",
+    "latex-fragment",
+    "entity",
+    "timestamp",
+    "subscript",
+    "superscript",
+    "export-snippet",
+    "statistics-cookie",
+    "citation",
+    "line-break"
+]);
+export function isInline(node) {
+    return INLINE_TYPES.has(node.type);
+}
+// ends the rendering, so the paragraph's own trailing newline and
+// whitespace trimming stay out of it
+const SENTINEL = "\u0000";
+/**
+ * How org renders an inline node within its line.
+ */
+export function renderInline(node) {
+    if (node.type === "text") {
+        return node.value ?? "";
+    }
+    const rendered = String(stringifier.stringify({
+        type: "org-data",
+        children: [
+            {
+                type: "paragraph",
+                children: [node, { type: "text", value: SENTINEL }]
+            }
+        ]
+    }));
+    return rendered.slice(0, rendered.lastIndexOf(SENTINEL));
+}
+// stands in for an inline node's content
+const CONTENT = "\u0001";
+/**
+ * An inline node's delimiters around its children, as org renders them
+ * (`*` and `*`, `[[url][` and `]]`).
+ */
+export function delimiters(node) {
+    const [open = "", close = ""] = renderInline({
+        ...node,
+        children: [{ type: "text", value: CONTENT }]
+    }).split(CONTENT);
+    return [open, close];
+}
+/**
+ * The org rendering of each child; a block element only ends a line.
+ */
+export function renderChildren(children) {
+    return children.map(child => (isInline(child) ? renderInline(child) : "\n"));
+}
+/**
+ * The index of the rendered child `offset` (into the joined rendering)
+ * lies in, and the offset within that child.
+ */
+export function locate(rendered, offset) {
+    let start = 0;
+    for (const [index, part] of rendered.entries()) {
+        if (offset < start + part.length) {
+            return [index, offset - start];
+        }
+        start += part.length;
+    }
+    return [-1, 0];
+}

package/dist/core/tablePipes.d.ts

@@ -0,0 +1,17 @@
+import type { Parent } from "unist";
+/**
+ * md→org: a `|` in a table cell's text (md `\|`) would end the cell; org
+ * has no escaped `|`, but renders the `\vert` entity as one, `{}` ending
+ * its name before any letter. In code, where the entity would be
+ * literal, the lookalike stands in, which is reported. Runs after the
+ * preset, whose wikilink aliases (`[[Page|alias]]`) are no text.
+ * @param tree The document.
+ * @param onWarning Reports a lookalike written.
+ */
+export declare function escapeTablePipes(tree: Parent, onWarning?: (message: string) => void): void;
+/**
+ * org→md: the lookalike in code in a table cell back to the `|` it
+ * stands in for.
+ * @param tree The document.
+ */
+export declare function unescapeTablePipes(tree: Parent): void;

package/dist/core/tablePipes.js

@@ -0,0 +1,47 @@
+import { visit } from "unist-util-visit";
+// org has no escape inside ~code~ or =verbatim=: a `|` there would end
+// the cell, so a lookalike (U+2223 DIVIDES) stands in for it
+const LOOKALIKE = "∣";
+/**
+ * md→org: a `|` in a table cell's text (md `\|`) would end the cell; org
+ * has no escaped `|`, but renders the `\vert` entity as one, `{}` ending
+ * its name before any letter. In code, where the entity would be
+ * literal, the lookalike stands in, which is reported. Runs after the
+ * preset, whose wikilink aliases (`[[Page|alias]]`) are no text.
+ * @param tree The document.
+ * @param onWarning Reports a lookalike written.
+ */
+export function escapeTablePipes(tree, onWarning) {
+    let replaced = false;
+    visit(tree, "table-cell", (cell) => {
+        visit(cell, (node) => {
+            if (node.type === "text") {
+                node.value = node.value?.replaceAll("|", "\\vert{}");
+            }
+            else if (isCode(node) && node.value?.includes("|")) {
+                node.value = node.value.replaceAll("|", LOOKALIKE);
+                replaced = true;
+            }
+        });
+    });
+    if (replaced) {
+        onWarning?.("a | in code in a table cell becomes ∣ (U+2223) in org");
+    }
+}
+/**
+ * org→md: the lookalike in code in a table cell back to the `|` it
+ * stands in for.
+ * @param tree The document.
+ */
+export function unescapeTablePipes(tree) {
+    visit(tree, "table-cell", (cell) => {
+        visit(cell, (node) => {
+            if (isCode(node)) {
+                node.value = node.value?.replaceAll(LOOKALIKE, "|");
+            }
+        });
+    });
+}
+function isCode(node) {
+    return node.type === "code" || node.type === "verbatim";
+}

package/dist/core/underscoreBullets.d.ts

@@ -0,0 +1,10 @@
+import type { Parent } from "unist";
+/**
+ * org→md: guards the lines uniorg misreads before parsing.
+ */
+export declare function guardUnderscoreBullets(org: string): string;
+/**
+ * org→md: drops the guards again, also where uniorg reads no text (a
+ * src block's code). A zero-width space the author put there goes too.
+ */
+export declare function dropUnderscoreBulletGuards(tree: Parent): void;

package/dist/core/underscoreBullets.js

@@ -0,0 +1,32 @@
+import { visit } from "unist-util-visit";
+import { ZERO_WIDTH_SPACE } from "./markupBoundary.js";
+// ====================================================================
+// WORKAROUND for a bug in uniorg-parse 3.2.2 (upstream issue: not filed
+// yet, see TODO.md). Drop this module once a fixed version is in; the
+// canary in tests/uniorgWorkarounds.spec.ts fails then.
+// ====================================================================
+// Its `listItemRe` takes a line starting `_.` or `_)` for a list item
+// (`\w` includes `_`), but its `fullListItemRe`, like org itself, has
+// no such bullet. `parseListStructure` then throws (`match error`), or,
+// when a real bullet line follows, reads that one instead and silently
+// drops the line. Org reads the line as text, and so does uniorg with a
+// zero-width space in front (morg's line-start escape, see lineSyntax)
+const UNDERSCORE_BULLET_RE = /^([ \t]*)(?=_[.)](?:[ \t]|$))/gm;
+/**
+ * org→md: guards the lines uniorg misreads before parsing.
+ */
+export function guardUnderscoreBullets(org) {
+    return org.replace(UNDERSCORE_BULLET_RE, `$1${ZERO_WIDTH_SPACE}`);
+}
+const GUARD_RE = new RegExp(`(^|\\n)([ \\t]*)${ZERO_WIDTH_SPACE}(?=_[.)](?:[ \\t\\n]|$))`, "g");
+/**
+ * org→md: drops the guards again, also where uniorg reads no text (a
+ * src block's code). A zero-width space the author put there goes too.
+ */
+export function dropUnderscoreBulletGuards(tree) {
+    visit(tree, (node) => {
+        if (typeof node.value === "string") {
+            node.value = node.value.replace(GUARD_RE, "$1$2");
+        }
+    });
+}

package/dist/core/uniorgToMdast/elements.js

@@ -180,7 +180,7 @@ function transformPropertyDrawer(ctx, node) {
     }
     const isms = (node.children || [])
         .filter(child => child.type === "node-property")
-        .map(property => `${property.key}:: ${property.value}`);
+        .map(({ key, value }) => (value ? `${key}:: ${value}` : `${key}::`));
     return isms.length ? keyValueParagraph(isms) : null;
 }
 function transformParagraph(ctx, node) {

package/dist/core/uniorgToMdast/index.d.ts

@@ -1,11 +1,12 @@
 import type { Root as MdastRoot } from "mdast";
 import type { OrgData } from "uniorg";
-import type { UniorgToMdastOptions } from "./shared.js";
+import { type UniorgToMdastOptions } from "./shared.js";
 export type { UniorgToMdastOptions } from "./shared.js";
 /**
  * Transforms a uniorg AST to a mdast (Markdown AST).
  * @param uniorgAst The uniorg AST to transform.
- * @param options Controls org-ism serialization (`key:: value` lines).
+ * @param options Controls org-ism serialization (`key:: value` lines);
+ * `org`, the text a tree was parsed from, finds its frontmatter block.
  * @returns The transformed mdast.
  */
 export declare function transformUniorgAstToMdast(uniorgAst: OrgData, options?: UniorgToMdastOptions): MdastRoot;