@aicayzer/inkkit 0.0.1 → 0.0.2

package/dist/preserve.js

@@ -1,5 +1,172 @@
 import { parserCtx, remarkCtx } from '@milkdown/kit/core';
 import { serialize } from './dialect.js';
+function signature(node) {
+    return JSON.stringify(node, (key, value) => ['position', 'data', 'url', 'title', 'marker'].includes(key)
+        ? undefined
+        : value);
+}
+function ownShape(node) {
+    return JSON.stringify(node, (key, value) => ['position', 'data', 'children', 'value', 'marker'].includes(key)
+        ? undefined
+        : value);
+}
+function retainText(raw, before, after) {
+    before = before.replace(/\r\n?/g, '\n');
+    after = after.replace(/\r\n?/g, '\n');
+    let prefix = 0;
+    while (prefix < before.length &&
+        prefix < after.length &&
+        before[prefix] === after[prefix])
+        prefix++;
+    let suffix = 0;
+    while (suffix < before.length - prefix &&
+        suffix < after.length - prefix &&
+        before[before.length - 1 - suffix] === after[after.length - 1 - suffix])
+        suffix++;
+    const offsets = [0];
+    let decoded = '';
+    for (let index = 0; index < raw.length;) {
+        const escaped = /^\\([!"#$%&'()*+,\-./:;<=>?@[\]\\^_`{|}~])/.exec(raw.slice(index));
+        const entity = /^&(?:#[xX][\da-fA-F]+|#\d+|[A-Za-z][A-Za-z\d]+);/.exec(raw.slice(index));
+        let value;
+        let length;
+        if (escaped) {
+            value = escaped[1];
+            length = escaped[0].length;
+        }
+        else if (entity) {
+            const element = document.createElement('textarea');
+            element.innerHTML = entity[0];
+            value = element.value;
+            length = entity[0].length;
+        }
+        else if (raw.slice(index, index + 2) === '\r\n') {
+            value = '\n';
+            length = 2;
+        }
+        else {
+            value = raw[index];
+            length = 1;
+        }
+        decoded += value;
+        index += length;
+        for (let part = 0; part < value.length; part++)
+            offsets.push(index);
+    }
+    if (decoded !== before)
+        return after;
+    const inserted = after
+        .slice(prefix, after.length - suffix)
+        .replace(/[\\`*_[\]<>]/g, '\\$&');
+    return (raw.slice(0, offsets[prefix]) +
+        inserted +
+        raw.slice(offsets[before.length - suffix]));
+}
+function retainBoundaryWhitespace(raw, canonical) {
+    // A middle-of-line space becomes indentation or trimmed text at a block edge.
+    const token = '(?:[ \\t]|&#(?:[xX]0*(?:20|9)|0*(?:32|9));|&Tab;)';
+    const decoded = (value) => (value.match(new RegExp(token, 'g')) ?? [])
+        .map((value) => value === '\t' || /(?:9;|Tab;)$/.test(value) ? '\t' : ' ')
+        .join('');
+    const encode = (value) => value.replace(/[ \t]/g, (value) => (value === '\t' ? '&#x9;' : '&#x20;'));
+    const start = /^[ \t]+/.exec(raw)?.[0];
+    const rawStart = new RegExp(`^${token}+`).exec(raw)?.[0];
+    const encodedStart = new RegExp(`^${token}+`).exec(canonical)?.[0];
+    if (start &&
+        rawStart &&
+        encodedStart?.includes('&') &&
+        decoded(rawStart) === decoded(encodedStart))
+        raw = encode(start) + raw.slice(start.length);
+    const end = /[ \t]+$/.exec(raw)?.[0];
+    const rawEnd = new RegExp(`${token}+$`).exec(raw)?.[0];
+    const encodedEnd = new RegExp(`${token}+$`).exec(canonical)?.[0];
+    if (end &&
+        rawEnd &&
+        encodedEnd?.includes('&') &&
+        decoded(rawEnd) === decoded(encodedEnd))
+        raw = raw.slice(0, -end.length) + encode(end);
+    return raw;
+}
+// Patch changed leaves so an adjacent edit retains authored delimiters and labels.
+function retainSource(before, after, source, canonical, lineEnding) {
+    const start = before.position?.start.offset;
+    const end = before.position?.end.offset;
+    const newStart = after.position?.start.offset;
+    const newEnd = after.position?.end.offset;
+    if (start == null || end == null || newStart == null || newEnd == null)
+        throw new PreservationError('A source token has no location');
+    const raw = source.slice(start, end);
+    if (before.type === 'definition' &&
+        after.type === 'definition' &&
+        before.label === after.label &&
+        before.title === after.title) {
+        const destination = /^(\[[\s\S]*?\]:[ \t]*)(<[^>\n]*>|[^\s]+)([\s\S]*)$/.exec(raw);
+        if (destination)
+            return (destination[1] +
+                (destination[2].startsWith('<')
+                    ? `<${after.url}>`
+                    : String(after.url)) +
+                destination[3]);
+    }
+    if (before.type === 'text' && after.type === 'text')
+        return retainBoundaryWhitespace(retainText(raw, before.value, after.value), canonical.slice(newStart, newEnd));
+    if (signature(before) === signature(after) &&
+        ownShape(before) === ownShape(after))
+        return raw;
+    if (before.type === after.type &&
+        ownShape(before) === ownShape(after) &&
+        before.children &&
+        after.children &&
+        before.children.length === after.children.length) {
+        let output = '', previous = start;
+        before.children.forEach((child, index) => {
+            const childStart = child.position?.start.offset, childEnd = child.position?.end.offset;
+            if (childStart == null || childEnd == null)
+                throw new PreservationError('A source token has no location');
+            output +=
+                source.slice(previous, childStart) +
+                    retainSource(child, after.children[index], source, canonical, lineEnding);
+            previous = childEnd;
+        });
+        return output + source.slice(previous, end);
+    }
+    if (before.type === after.type &&
+        ownShape(before) === ownShape(after) &&
+        before.children &&
+        after.children) {
+        let prefix = 0, suffix = 0;
+        while (prefix < before.children.length &&
+            prefix < after.children.length &&
+            signature(before.children[prefix]) === signature(after.children[prefix]))
+            prefix++;
+        while (suffix < before.children.length - prefix &&
+            suffix < after.children.length - prefix &&
+            signature(before.children[before.children.length - 1 - suffix]) ===
+                signature(after.children[after.children.length - 1 - suffix]))
+            suffix++;
+        const from = prefix
+            ? before.children[prefix - 1].position?.end.offset
+            : start;
+        const to = suffix
+            ? before.children[before.children.length - suffix].position?.start.offset
+            : end;
+        const newFrom = prefix
+            ? after.children[prefix - 1].position?.end.offset
+            : newStart;
+        const newTo = suffix
+            ? after.children[after.children.length - suffix].position?.start.offset
+            : newEnd;
+        if (from != null && to != null && newFrom != null && newTo != null)
+            return retainBoundaryWhitespace(source.slice(start, from) +
+                canonical.slice(newFrom, newTo).replaceAll('\n', lineEnding) +
+                source.slice(to, end), canonical.slice(newStart, newEnd));
+    }
+    return canonical.slice(newStart, newEnd).replaceAll('\n', lineEnding);
+}
+function positionalNode(tree, transformed) {
+    return (tree.children.find((node) => node.position?.start.offset === transformed.position?.start.offset &&
+        node.position?.end.offset === transformed.position?.end.offset) ?? transformed);
+}
 export class PreservationError extends Error {
     constructor(message, options) {
         super(message, options);
@@ -16,6 +183,50 @@ function blockForm(ctx, node) {
         return '<br />';
     return form(serialize(ctx, node.type.schema.topNodeType.create(null, [node])));
 }
+function semanticSignature(doc) {
+    const clean = (value) => {
+        const content = value.content;
+        if (!content)
+            return value;
+        const tokens = [];
+        for (const child of content) {
+            if (child.type !== 'text') {
+                tokens.push(clean(child));
+                continue;
+            }
+            const parts = /^(\s*)([\s\S]*?)(\s*)$/.exec(String(child.text));
+            for (const [index, text] of parts.slice(1).entries()) {
+                if (!text)
+                    continue;
+                const token = {
+                    ...child,
+                    text,
+                    marks: index === 1 ? child.marks : undefined,
+                };
+                const previous = tokens.at(-1);
+                if (previous?.type === 'text' &&
+                    JSON.stringify(previous.marks) === JSON.stringify(token.marks))
+                    previous.text = String(previous.text) + text;
+                else
+                    tokens.push(token);
+            }
+        }
+        return { ...value, content: tokens };
+    };
+    const json = JSON.parse(JSON.stringify(doc.toJSON(), (key, value) => 
+    // Fresh list items default to loose even when their Markdown reopens tight.
+    [
+        'referenceContent',
+        'referenceType',
+        'marker',
+        'id',
+        'label',
+        'spread',
+    ].includes(key)
+        ? undefined
+        : value));
+    return JSON.stringify(clean(json));
+}
 // Edit distance distinguishes replacements from insertions, so replacing a block
 // retains its surrounding whitespace without assigning it a neighbour's source.
 function correspondence(before, after) {
@@ -86,10 +297,11 @@ export class Preservation {
             const processor = ctx.get(remarkCtx);
             const offset = source.startsWith('\uFEFF') ? 1 : 0;
             const body = source.slice(offset);
-            const tree = processor.runSync(processor.parse(source), {
+            const rawTree = processor.parse(source);
+            const tree = processor.runSync(structuredClone(rawTree), {
                 value: body,
             });
-            for (const node of tree.children) {
+            for (const [index, node] of tree.children.entries()) {
                 let start = node.position?.start.offset;
                 const end = node.position?.end.offset;
                 if (start == null || end == null)
@@ -100,13 +312,16 @@ export class Preservation {
                 if (/^[ \t]*$/.test(source.slice(firstColumn, start)))
                     start = firstColumn;
                 const text = source.slice(start, sourceEnd);
-                const parsed = parse(text);
+                const parsed = this.baseline.childCount === tree.children.length
+                    ? this.baseline.type.create(null, [this.baseline.child(index)])
+                    : parse(text);
                 this.blocks.push({
                     start,
                     end: sourceEnd,
                     form: parsed.childCount === 1
                         ? blockForm(ctx, parsed.firstChild)
                         : form(serialize(ctx, parsed)),
+                    ast: positionalNode(rawTree, node),
                 });
             }
         }
@@ -135,6 +350,12 @@ export class Preservation {
                 current.push(fingerprint);
             });
             const positions = correspondence(this.blocks.map((block) => block.form), current);
+            const processor = this.ctx.get(remarkCtx);
+            const currentRawTree = processor.parse(canonical);
+            const currentTree = processor.runSync(structuredClone(currentRawTree), {
+                value: canonical,
+            });
+            const body = this.source.replace(/^\uFEFF/, '');
             let output = '';
             let previous;
             current.forEach((text, index) => {
@@ -151,10 +372,13 @@ export class Preservation {
                 }
                 else
                     output += this.lineEnding.repeat(2);
-                output +=
-                    block && block.form === text
-                        ? this.source.slice(block.start, block.end)
-                        : text.replaceAll('\n', this.lineEnding);
+                if (block && block.form === text)
+                    output += this.source.slice(block.start, block.end);
+                else if (block && currentTree.children.length === current.length) {
+                    output += retainSource(block.ast, positionalNode(currentRawTree, currentTree.children[index]), body, canonical, this.lineEnding);
+                }
+                else
+                    output += text.replaceAll('\n', this.lineEnding);
                 previous = position;
             });
             if (previous === this.blocks.length - 1 && this.blocks.length)
@@ -162,7 +386,9 @@ export class Preservation {
             else if (output)
                 output += this.lineEnding;
             const parse = this.ctx.get(parserCtx);
-            if (serialize(this.ctx, parse(output)) === canonical)
+            const reopened = parse(output);
+            if (serialize(this.ctx, reopened) === canonical &&
+                semanticSignature(reopened) === semanticSignature(doc))
                 return output;
             throw new PreservationError('The edited Markdown cannot be reopened without changing its content');
         }

package/dist/reference-clipboard.d.ts

@@ -0,0 +1,8 @@
+import type { Ctx } from '@milkdown/kit/ctx';
+import { Fragment, type Node } from '@milkdown/kit/prose/model';
+export declare function selectionContent(doc: Node, content: Fragment): Fragment;
+export declare function selectionMarkdown(ctx: Ctx, doc: Node, content: Fragment): string;
+export declare function referenceMetadata(ctx: Ctx, doc: Node, content: Fragment): string | undefined;
+export declare function avoidReferenceCollisions(incoming: Node, destination: Node): Node;
+export declare function markdownFromHTML(html: string): string | undefined;
+export declare function withReferenceMetadata(html: string, markdown: string): string;

package/dist/reference-clipboard.js

@@ -0,0 +1,197 @@
+import { Fragment } from '@milkdown/kit/prose/model';
+import { serialize } from './dialect.js';
+import { normaliseLabel } from './references.js';
+function key(node) {
+    if (node.type.name === 'reference_definition')
+        return `link:${node.attrs.identifier}`;
+    if (node.type.name === 'footnote_definition')
+        return `footnote:${node.attrs.identifier}`;
+    return undefined;
+}
+function used(content) {
+    const result = new Set();
+    content.descendants((node) => {
+        if (node.type.name === 'footnote_reference')
+            result.add(`footnote:${node.attrs.identifier}`);
+        for (const mark of node.marks)
+            if (mark.type.name === 'link' && mark.attrs.identifier)
+                result.add(`link:${mark.attrs.identifier}`);
+    });
+    return result;
+}
+export function selectionContent(doc, content) {
+    const definitions = new Map();
+    doc.descendants((node) => {
+        const id = key(node);
+        if (id && !definitions.has(id))
+            definitions.set(id, node);
+    });
+    const required = used(content);
+    for (const id of required) {
+        const definition = definitions.get(id);
+        if (definition)
+            for (const dependency of used(definition.content))
+                required.add(dependency);
+    }
+    const complete = (node) => {
+        const id = key(node);
+        if (id && required.has(id) && definitions.has(id))
+            return definitions.get(id);
+        if (node.isText)
+            return node;
+        const children = [];
+        node.forEach((child) => children.push(complete(child)));
+        return node.copy(Fragment.fromArray(children));
+    };
+    const selected = [];
+    content.forEach((node) => selected.push(complete(node)));
+    content = Fragment.fromArray(selected);
+    const included = new Set();
+    content.descendants((node) => {
+        const id = key(node);
+        if (id)
+            included.add(id);
+    });
+    const extra = [];
+    for (const id of required) {
+        if (included.has(id))
+            continue;
+        const definition = definitions.get(id);
+        if (!definition)
+            continue;
+        included.add(id);
+        extra.push(definition);
+        for (const dependency of used(definition.content))
+            required.add(dependency);
+    }
+    return content.append(Fragment.fromArray(extra));
+}
+export function selectionMarkdown(ctx, doc, content) {
+    return serialize(ctx, doc.type.create(null, selectionContent(doc, content)));
+}
+export function referenceMetadata(ctx, doc, content) {
+    let references = false, index = 0;
+    const rewrite = (node) => {
+        if (key(node) ||
+            node.type.name === 'footnote_reference' ||
+            node.marks.some((mark) => mark.type.name === 'link' && mark.attrs.identifier))
+            references = true;
+        if (node.type.name === 'image')
+            return node.type.create({ ...node.attrs, src: `inkkit-clipboard-image:${index++}` }, null, node.marks);
+        if (node.isText)
+            return node;
+        const children = [];
+        node.forEach((child) => children.push(rewrite(child)));
+        return node.copy(Fragment.fromArray(children));
+    };
+    const valid = content.firstChild?.isInline
+        ? Fragment.from(doc.type.schema.nodes.paragraph.create(null, content))
+        : content;
+    const copied = rewrite(doc.type.create(null, selectionContent(doc, valid)));
+    return references ? serialize(ctx, copied) : undefined;
+}
+function allLabels(doc) {
+    const labels = new Map([
+        ['link', new Set()],
+        ['footnote', new Set()],
+    ]);
+    doc.descendants((node) => {
+        if (isCode(node))
+            return false;
+        const id = key(node);
+        if (id) {
+            const separator = id.indexOf(':');
+            labels.get(id.slice(0, separator)).add(id.slice(separator + 1));
+        }
+        if (node.type.name === 'footnote_reference')
+            labels.get('footnote').add(node.attrs.identifier);
+        for (const mark of node.marks)
+            if (mark.type.name === 'link' && mark.attrs.identifier)
+                labels.get('link').add(mark.attrs.identifier);
+        if (node.isText) {
+            for (const match of node.text.matchAll(rawReferences))
+                labels
+                    .get(match[1].startsWith('^') ? 'footnote' : 'link')
+                    .add(normaliseLabel(match[1].startsWith('^')
+                    ? match[1].slice(1)
+                    : match[2] || match[1]));
+        }
+    });
+    return labels;
+}
+const rawReferences = /(?<![\\!])\[([^\]]+)\](?:\[([^\]]*)\])?(?!\()/g;
+function isCode(node) {
+    return (node.type.name === 'code_block' ||
+        node.marks.some((mark) => mark.type.name === 'inlineCode'));
+}
+// Rename only the incoming document. Definitions belong to the insertion's undo transaction.
+export function avoidReferenceCollisions(incoming, destination) {
+    const occupied = allLabels(destination);
+    const incomingLabels = allLabels(incoming);
+    const replacements = new Map();
+    for (const [kind, labels] of incomingLabels) {
+        const reserved = new Set([...occupied.get(kind), ...labels]);
+        for (const identifier of labels) {
+            if (!occupied.get(kind).has(identifier))
+                continue;
+            let suffix = 2;
+            while (reserved.has(normaliseLabel(`${identifier}-${suffix}`)))
+                suffix++;
+            const label = `${identifier}-${suffix}`;
+            reserved.add(normaliseLabel(label));
+            replacements.set(`${kind}:${identifier}`, label);
+        }
+    }
+    if (!replacements.size)
+        return incoming;
+    const rewrite = (node) => {
+        if (isCode(node))
+            return node;
+        const kind = node.type.name.startsWith('footnote_') ? 'footnote' : 'link';
+        const label = replacements.get(`${kind}:${node.attrs.identifier}`);
+        const attrs = label
+            ? { ...node.attrs, label, identifier: normaliseLabel(label) }
+            : node.attrs;
+        const marks = node.marks.map((mark) => {
+            const replacement = replacements.get(`link:${mark.attrs.identifier}`);
+            return replacement && mark.type.name === 'link'
+                ? mark.type.create({
+                    ...mark.attrs,
+                    label: replacement,
+                    identifier: normaliseLabel(replacement),
+                    referenceType: 'full',
+                })
+                : mark;
+        });
+        if (node.isText) {
+            const text = node.text.replace(rawReferences, (raw, display, reference) => {
+                const footnote = display.startsWith('^')
+                    ? display.slice(1)
+                    : undefined;
+                const replacement = replacements.get(`${footnote ? 'footnote' : 'link'}:${normaliseLabel(footnote ?? (reference || display))}`);
+                return replacement
+                    ? footnote
+                        ? `[^${replacement}]`
+                        : `[${display}][${replacement}]`
+                    : raw;
+            });
+            return node.type.schema.text(text, marks);
+        }
+        const children = [];
+        node.forEach((child) => children.push(rewrite(child)));
+        return node.type.create(attrs, children, marks);
+    };
+    return rewrite(incoming);
+}
+export function markdownFromHTML(html) {
+    const template = document.createElement('template');
+    template.innerHTML = html;
+    const element = template.content.querySelector('[data-inkkit-markdown]');
+    return element?.getAttribute('data-inkkit-markdown') ?? undefined;
+}
+export function withReferenceMetadata(html, markdown) {
+    const container = document.createElement('div');
+    container.setAttribute('data-inkkit-markdown', markdown);
+    container.innerHTML = html;
+    return container.outerHTML;
+}

package/dist/references.d.ts

@@ -0,0 +1,14 @@
+import type { Node as ProseNode } from '@milkdown/kit/prose/model';
+export declare function normaliseLabel(label: string): string;
+export interface LocatedDefinition {
+    node: ProseNode;
+    pos: number;
+}
+export declare const referenceDefinitions: (doc: ProseNode) => Map<string, LocatedDefinition>;
+export declare const footnoteDefinitions: (doc: ProseNode) => Map<string, LocatedDefinition>;
+export declare const remarkReferencesPlugin: import("@milkdown/kit/utils").$Remark<"inkkitReferences", unknown>;
+export declare const referenceLinks: import("@milkdown/kit/utils").$MarkSchema<"link">;
+export declare const referenceDefinition: import("@milkdown/kit/utils").$Node;
+export declare const footnoteReference: import("@milkdown/kit/utils").$Node;
+export declare const footnoteDefinition: import("@milkdown/kit/utils").$Node;
+export declare const references: (import("@milkdown/kit/utils").$Ctx<import("@milkdown/kit/utils").GetMarkSchema, "link"> | import("@milkdown/kit/utils").$Mark | import("@milkdown/kit/utils").$Node | import("@milkdown/kit/utils").$Prose)[];