@remigius42/morg 0.6.0 → 0.7.0

package/dist/markdownToOrg.js

@@ -5,13 +5,16 @@ import remarkFrontmatter from "remark-frontmatter";
 import remarkMath from "remark-math";
 import { uniorgStringify } from "uniorg-stringify";
 import { visit } from "unist-util-visit";
-import { transformMdastToUniorgAst } from "./core/mdastToUniorg/index.js";
+import { transformMdastToUniorgDraft } from "./core/mdastToUniorg/index.js";
 import { detectMarkdownStyle, STYLE_KEYWORD } from "./core/markdownStyle.js";
 import { escapeOrgMarkup } from "./core/markupBoundary.js";
+import { renderFileHeader } from "./core/frontmatterBlock.js";
 import { escapeLineSyntax } from "./core/lineSyntax.js";
 import { escapeFootnoteReferences } from "./core/footnoteReferences.js";
+import { escapeBackslashCommands } from "./core/backslashCommands.js";
 import { escapeTablePipes } from "./core/tablePipes.js";
 import { requireBracedScripts } from "./core/bracedScripts.js";
+import { keyValueEntries } from "./core/keyValueLines.js";
 /**
  * Converts a Markdown string to an Org-mode string.
  * @param markdown The Markdown string to convert.
@@ -20,14 +23,9 @@ import { requireBracedScripts } from "./core/bracedScripts.js";
  */
 export function convertMarkdownToOrg(markdown, options = {}) {
     // Phase 1: Parse Markdown to mdast
-    const mdast = unified()
-        .use(remarkParse)
-        .use(remarkGfm)
-        .use(remarkFrontmatter)
-        .use(remarkMath)
-        .parse(markdown);
+    const mdast = parseMarkdown(markdown, options.preset);
     // Phase 2: Generic mdast to uniorg-ast transformation
-    let uniorgAst = transformMdastToUniorgAst(mdast, {
+    let uniorgAst = transformMdastToUniorgDraft(mdast, {
         ...(options.preserveMdisms !== undefined && {
             preserveMdisms: options.preserveMdisms
         }),
@@ -60,8 +58,10 @@ export function convertMarkdownToOrg(markdown, options = {}) {
     // Phase 3b: literal footnote references, paragraph lines org would
     // read as line syntax, literal markers org would read as markup, and
     // markup touching a word character, need a zero-width space escape;
-    // a pipe in a table cell an entity
-    escapeTablePipes(uniorgAst);
+    // a pipe in a table cell an entity; a literal backslash before a
+    // letter first, the entity being none
+    escapeBackslashCommands(uniorgAst);
+    escapeTablePipes(uniorgAst, options.onWarning);
     escapeFootnoteReferences(uniorgAst);
     escapeLineSyntax(uniorgAst);
     escapeOrgMarkup(uniorgAst);
@@ -70,12 +70,28 @@ export function convertMarkdownToOrg(markdown, options = {}) {
     // rendered: after the preset, whose text rewrites (wikilink aliases)
     // org parses too, and after the escapes, next to which org reads them
     requireBracedScripts(uniorgAst);
+    // Phase 3d: the file's header (ADR 0005): mode line and file-level
+    // drawer first, keywords apart from what follows, the frontmatter as
+    // a marked comment block; raw text, past every pass that rewrites text
+    renderFileHeader(uniorgAst);
     // Phase 4: Render uniorg-ast to Org-mode string
     const processor = unified().use(uniorgStringify);
     const orgContent = processor.stringify(uniorgAst);
     return orgContent;
 }
-// leads the document so it survives the frontmatter keywords following it
+// a preset's dialect conventions that need the Markdown source apply
+// to the parse, before the generic transform
+function parseMarkdown(markdown, preset) {
+    const mdast = unified()
+        .use(remarkParse)
+        .use(remarkGfm)
+        .use(remarkFrontmatter)
+        .use(remarkMath)
+        .parse(markdown);
+    preset?.applyToMdast?.(mdast, markdown);
+    return mdast;
+}
+// leads the document, ahead of restored keywords and the frontmatter block
 function recordStyleKeyword(uniorgAst, style) {
     if (!Object.keys(style).length) {
         return;
@@ -86,7 +102,6 @@ function recordStyleKeyword(uniorgAst, style) {
         value: JSON.stringify(style)
     });
 }
-const KEY_VALUE_LINE_RE = /^([\w-]+):: (.*)$/;
 function parseKeyValueParagraph(node) {
     if (node?.type !== "paragraph" || !node.children?.length) {
         return null;
@@ -94,19 +109,7 @@ function parseKeyValueParagraph(node) {
     if (!node.children.every(child => child.type === "text")) {
         return null;
     }
-    const lines = node.children
-        .map(child => child.value ?? "")
-        .join("")
-        .split("\n");
-    const entries = [];
-    for (const line of lines) {
-        const match = KEY_VALUE_LINE_RE.exec(line);
-        if (!match) {
-            return null;
-        }
-        entries.push([match[1], match[2]]);
-    }
-    return entries;
+    return keyValueEntries(node.children.map(child => child.value ?? "").join(""));
 }
 function makeTimestamp(rawValue) {
     return { type: "timestamp", rawValue };

package/dist/orgToMarkdown.js

@@ -5,9 +5,12 @@ import remarkFrontmatter from "remark-frontmatter";
 import remarkMath from "remark-math";
 import { transformUniorgAstToMdast } from "./core/uniorgToMdast/index.js";
 import { takeRecordedStyle } from "./core/markdownStyle.js";
+import { takeFileHeader } from "./core/frontmatterBlock.js";
 import { unescapeOrgMarkup } from "./core/markupBoundary.js";
 import { unescapeLineSyntax } from "./core/lineSyntax.js";
 import { unescapeFootnoteReferences } from "./core/footnoteReferences.js";
+import { unescapeBackslashCommands } from "./core/backslashCommands.js";
+import { unescapeTablePipes } from "./core/tablePipes.js";
 import { parseOrg } from "./core/bracedScripts.js";
 import { dropUnderscoreBulletGuards, guardUnderscoreBullets } from "./core/underscoreBullets.js";
 // a list item's paragraph after its nested list needs a blank line, or
@@ -29,15 +32,21 @@ export function convertOrgToMarkdown(org, options = {}) {
     // Phase 1: Parse Org-mode to uniorg-ast
     // md text has no scripts, so ^:{} is implied there and consumed here
     // (see markdownToOrg); uniorg misreads `_.` lines (see underscoreBullets)
-    let uniorgAst = parseOrg(guardUnderscoreBullets(org));
+    const guarded = guardUnderscoreBullets(org);
+    let uniorgAst = parseOrg(guarded);
     dropUnderscoreBulletGuards(uniorgAst);
     // Phase 1b: a recorded style is morg's own (ADR 0004), so consume it so
     // it does not travel on as frontmatter; explicit options still win
     const recordedStyle = takeRecordedStyle(uniorgAst);
+    // frontmatter travels in a block marked as morg's (ADR 0005)
+    // and so does a file-level drawer (org-roam's :ID:)
+    const fileHeader = takeFileHeader(uniorgAst, guarded, options.onWarning);
     // Phase 1c: markdown needs no zero-width space escapes (inverse of md→org)
     unescapeOrgMarkup(uniorgAst);
     unescapeLineSyntax(uniorgAst);
     unescapeFootnoteReferences(uniorgAst);
+    unescapeBackslashCommands(uniorgAst);
+    unescapeTablePipes(uniorgAst);
     // Phase 2: Extract dialect preset conventions, if any
     if (options.preset?.extractFromUniorg) {
         uniorgAst = options.preset.extractFromUniorg(uniorgAst);
@@ -54,7 +63,8 @@ export function convertOrgToMarkdown(org, options = {}) {
         ...(options.orgismKeys !== undefined && {
             orgismKeys: options.orgismKeys
         }),
-        ...(options.onWarning !== undefined && { onWarning: options.onWarning })
+        ...(options.onWarning !== undefined && { onWarning: options.onWarning }),
+        ...fileHeader
     });
     // Phase 4: Render mdast to Markdown string
     // bullet and rule "-" (not remark's default "*") are morg's canonical

package/dist/presets/logseq.js

@@ -1,5 +1,8 @@
 import { visit } from "unist-util-visit";
 import { toString } from "orgast-util-to-string";
+import { isScalar, isSeq } from "yaml";
+import { fitsKeywordLine, isFrontmatterNode, isModeLine, isModeLineComment, takeFrontmatterEntries } from "../core/frontmatterBlock.js";
+import { keyValueEntries } from "../core/keyValueLines.js";
 /**
  * Logseq dialect preset: `heading::` properties and outline nesting.
  */
@@ -7,6 +10,7 @@ export function logseq(options = {}) {
     const nestUnderHeadings = options.nestUnderHeadings ?? true;
     return {
         name: "logseq",
+        applyToMdast: keepPagePropertySource,
         applyToUniorg: uniorgAst => applyLogseqSpecificsToUniorgAst(uniorgAst, nestUnderHeadings),
         extractFromUniorg: extractLogseqSpecificsFromUniorgAst
     };
@@ -87,6 +91,7 @@ function toBlockHeadline(node, level) {
  */
 export function applyLogseqSpecificsToUniorgAst(uniorgAst, nestUnderHeadings = true) {
     rewriteLabeledPageRefs(uniorgAst);
+    takePageProperties(uniorgAst);
     const children = uniorgAst.children;
     const result = [];
     let currentLevel = 0;
@@ -135,6 +140,130 @@ export function applyLogseqSpecificsToUniorgAst(uniorgAst, nestUnderHeadings = t
     uniorgAst.children = result;
     return uniorgAst;
 }
+// Logseq reads a page's first block of `key:: value` lines (md) and its
+// leading `#+key: value` lines (org) as page properties (ADR 0005)
+function pagePropertyEntries(text) {
+    const entries = keyValueEntries(text);
+    // a key org reads as no keyword name (begin_src) would not come back
+    return entries?.every(entry => fitsKeywordLine(...entry)) ? entries : null;
+}
+// md→org: the page property block's values are Logseq's, not Markdown;
+// its source text replaces the parse, so urls and markup stay verbatim
+function keepPagePropertySource(mdast, markdown) {
+    // below the frontmatter and an Emacs mode line
+    const node = mdast.children.find(child => child.type !== "yaml" &&
+        !(child.type === "html" &&
+            child.value.startsWith("<!--") &&
+            isModeLine(child.value)));
+    const start = node?.position?.start.offset;
+    const end = node?.position?.end.offset;
+    if (node?.type !== "paragraph" || start === undefined || end === undefined) {
+        return;
+    }
+    const source = markdown.slice(start, end);
+    if (pagePropertyEntries(source)) {
+        node.children = [{ type: "text", value: source }];
+    }
+}
+function takePageProperties(uniorgAst) {
+    const children = uniorgAst.children;
+    const keywords = takeFrontmatterKeywords(children);
+    // below the keywords, the file-level drawer and a mode line
+    const index = children.findIndex(node => !["keyword", "property-drawer"].includes(node.type) &&
+        !isModeLineComment(node) &&
+        !isFrontmatterNode(node));
+    const lines = pagePropertyLines(children[index]);
+    if (lines) {
+        children.splice(index, 1);
+        keywords.push(...lines);
+    }
+    // ahead of the block, so all of them lead the page
+    const at = children.findIndex(node => node.type !== "keyword");
+    children.splice(at === -1 ? children.length : at, 0, ...keywords.map(([key, value]) => ({ type: "keyword", key, value })));
+}
+function takeFrontmatterKeywords(children) {
+    const frontmatter = children.find(isFrontmatterNode);
+    if (!frontmatter) {
+        return [];
+    }
+    const keywords = takeFlatEntries(frontmatter);
+    // an emptied block goes
+    if (!frontmatter.yaml && keywords.length) {
+        children.splice(children.indexOf(frontmatter), 1);
+    }
+    return keywords;
+}
+function pagePropertyLines(node) {
+    // plain text only: markup in a key means the source was no key:: line
+    const children = node?.children;
+    if (node?.type !== "paragraph" ||
+        !children?.every(child => child.type === "text")) {
+        return null;
+    }
+    return pagePropertyEntries(toString(node));
+}
+// keywords that act in Emacs or in export (TODO states, startup and
+// export options, file inclusion, babel calls, a dynamic block's begin
+// and end lines, citations, a table of
+// contents, index entries, raw export lines and exporter options, of
+// org's own exporters, org-info.js and KOMA letter class files among
+// them, and of ox-hugo and org-re-reveal): as frontmatter they were
+// passive data, so they stay in the block. A denylist: other
+// third-party exporters' keys pass. Export metadata (`title`, `author`)
+// passes too, and `tags`, Logseq's page tags property
+const ACTING_KEYWORD_RE = /^(?:(?:SEQ_|TYP_)?TODO|STARTUP|OPTIONS|INCLUDE|SETUPFILE|BIND|MACRO|CALL|PROPERTY|LINK|CONSTANTS|PRIORITIES|BIBLIOGRAPHY|CITE_EXPORT|PRINT_BIBLIOGRAPHY|ARCHIVE|CATEGORY|COLUMNS|FILETAGS|EXCLUDE_TAGS|SELECT_TAGS|BEGIN|END|TOC|(?:C|F|K|P|T|V)?INDEX|INFOJS_OPT|LCO|EXPORT_\w+|(?:HTML|LATEX|BEAMER|ODT|TEXINFO|MAN|ASCII|MD|MARKDOWN|ICALENDAR|HUGO|REVEAL)(?:_\w+)?)$/i;
+// flat frontmatter entries are page properties too, a sequence written
+// as Logseq writes it (`a, b`); what a keyword line cannot hold stays
+function takeFlatEntries(frontmatter) {
+    const { keywords, yaml } = takeFrontmatterEntries(frontmatter.yaml, (key, value, text) => {
+        const flat = flatValue(value, text);
+        // a key that comes back as a key:: line as written: uniorg
+        // upper-cases it, Logseq reads it lower-cased
+        return flat !== null &&
+            /^[a-z0-9_-]+$/.test(text(key)) &&
+            fitsKeywordLine(text(key), flat) &&
+            !ACTING_KEYWORD_RE.test(text(key))
+            ? [[text(key), flat]]
+            : null;
+    });
+    frontmatter.yaml = yaml;
+    return keywords;
+}
+function flatValue(value, text) {
+    const single = (item) => isScalar(item) && item.value !== null ? text(item) : null;
+    if (!isSeq(value)) {
+        return single(value);
+    }
+    const items = value.items.map(single);
+    return items.length &&
+        items.every(item => item !== null && !item.includes(","))
+        ? items.join(", ")
+        : null;
+}
+// the reverse: leading keywords become the first block, verbatim, keys
+// lower-cased (uniorg upper-cases them, Logseq reads only lower case)
+function pageProperties(uniorgAst) {
+    const children = uniorgAst.children;
+    // below a mode line, as on the way in
+    const first = children.findIndex(node => !isModeLineComment(node));
+    let count = 0;
+    while (first !== -1 && children[first + count]?.type === "keyword") {
+        count++;
+    }
+    const leading = children.slice(first, first + count);
+    // a key no `key::` line holds (`CAPTION[short]`) stays a keyword
+    const properties = leading.filter(keyword => /^[\w-]+$/.test(keyword.key));
+    if (!properties.length) {
+        return;
+    }
+    const lines = properties.map(keyword => `${keyword.key.toLowerCase()}::${keyword.value ? ` ${keyword.value}` : ""}`);
+    children.splice(first, count, ...leading.filter(keyword => !properties.includes(keyword)), {
+        type: "paragraph",
+        children: [{ type: "verbatim-inline", value: lines.join("\n") }],
+        contentsBegin: 0,
+        contentsEnd: 0
+    });
+}
 // Logseq md keeps TODO/DONE as leading text markers and priorities as
 // [#A] text; in org they are the headline's TODO keyword and priority
 // (other Logseq markers like DOING are not org keywords and simply
@@ -162,6 +291,7 @@ function takeTaskMarker(headline) {
  * @returns The generic uniorg AST.
  */
 function extractLogseqSpecificsFromUniorgAst(uniorgAst) {
+    pageProperties(uniorgAst);
     extractInParent(uniorgAst);
     repairHighlights(uniorgAst);
     fuzzyLinksToPageRefs(uniorgAst);

package/dist/presets/types.d.ts

@@ -1,12 +1,15 @@
+import type { Root as MdastRoot } from "mdast";
 import type { OrgData } from "uniorg";
 /**
  * A dialect preset: transforms applied on top of the dialect-agnostic core.
- * `applyToUniorg` runs in the md→org pipeline after the generic transform;
+ * `applyToMdast` runs in the md→org pipeline before the generic
+ * transform, with the Markdown source at hand; `applyToUniorg` runs in the md→org pipeline after the generic transform;
  * `extractFromUniorg` runs in the org→md pipeline before the generic
  * transform, normalizing dialect conventions back to generic uniorg.
  */
 export interface Preset {
     name: string;
+    applyToMdast?: (mdast: MdastRoot, markdown: string) => void;
     applyToUniorg?: (uniorgAst: OrgData) => OrgData;
     extractFromUniorg?: (uniorgAst: OrgData) => OrgData;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remigius42/morg",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Bidirectional Markdown ↔ Org-mode converter with round-trip convergence",
   "keywords": [
     "org-mode",