@inshapardaz/likhari-converters 0.1.45-dev.334 → 0.1.45-dev.337

package/dist/merge/continuation.d.ts

@@ -0,0 +1,25 @@
+import { type SNode } from '../shared/serialized.js';
+/**
+ * Whether a block's text stops mid-sentence: it does not end in a sentence mark (once any
+ * closing quote or bracket is set aside). A comma, colon, semicolon, dash or a plain word all
+ * count as unfinished. A block with no text at all counts as finished.
+ */
+export declare function endsMidSentence(block: SNode): boolean;
+/**
+ * Whether two blocks could be one paragraph: both are plain paragraphs, the second is not
+ * indented (an indent starts a new paragraph) and has text, and they agree on text direction
+ * and alignment. A direction left unset on either side is compatible with anything.
+ */
+export declare function canJoinParagraphs(previous: SNode | undefined, next: SNode | undefined): boolean;
+/** A paragraph with nothing in it: only whitespace and line breaks. */
+export declare function isEmptyParagraph(block: SNode | undefined): boolean;
+/**
+ * Whether a page's text ends with an empty line, which marks the end of a paragraph. Markdown
+ * cannot hold an empty paragraph (the parser drops it), so it is read from the raw text: the
+ * text must end with a blank line (`\n\n`), not just a final newline. For every other format it
+ * is a trailing empty paragraph in the parsed page.
+ */
+export declare function endsWithBlankLine(rawFormat: string, rawText: string, blocks: SNode[]): boolean;
+/** Appends `next`'s content to `previous` (which is changed), separated by one space unless one is already there. */
+export declare function joinParagraphs(previous: SNode, next: SNode): void;
+//# sourceMappingURL=continuation.d.ts.map

package/dist/merge/continuation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"continuation.d.ts","sourceRoot":"","sources":["../../src/merge/continuation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAc5D;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAGrD;AAKD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,KAAK,GAAG,SAAS,GAAG,OAAO,CAM/F;AAED,uEAAuE;AACvE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,GAAG,OAAO,CAElE;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAG9F;AAKD,qHAAqH;AACrH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,IAAI,CAOjE"}

package/dist/merge/continuation.js

@@ -0,0 +1,67 @@
+import { textNode } from '../shared/serialized.js';
+/** Marks that end a sentence, in Urdu and Latin script: ۔ ؟ ! ? . … and the CJK full stop. */
+const SENTENCE_END = /[۔؟!?.…。]/;
+/** Closing quotes and brackets that may follow the mark and still end the sentence: ” ’ " ' ) ] » */
+const CLOSERS = /[”’"')\]»]+$/;
+/** The text of a block, from its text nodes only. */
+function plainText(node) {
+    if (node.type === 'text')
+        return String(node.text ?? '');
+    if (node.type === 'linebreak')
+        return '\n';
+    return (node.children ?? []).map(plainText).join('');
+}
+/**
+ * Whether a block's text stops mid-sentence: it does not end in a sentence mark (once any
+ * closing quote or bracket is set aside). A comma, colon, semicolon, dash or a plain word all
+ * count as unfinished. A block with no text at all counts as finished.
+ */
+export function endsMidSentence(block) {
+    const text = plainText(block).trimEnd().replace(CLOSERS, '').trimEnd();
+    return text !== '' && !SENTENCE_END.test(text.charAt(text.length - 1));
+}
+/** The alignment a paragraph effectively has: '' and 'start' are the same thing. */
+const alignment = (block) => (block.format === '' || block.format == null ? 'start' : String(block.format));
+/**
+ * Whether two blocks could be one paragraph: both are plain paragraphs, the second is not
+ * indented (an indent starts a new paragraph) and has text, and they agree on text direction
+ * and alignment. A direction left unset on either side is compatible with anything.
+ */
+export function canJoinParagraphs(previous, next) {
+    if (!previous || !next || previous.type !== 'paragraph' || next.type !== 'paragraph')
+        return false;
+    if (plainText(next).trim() === '' || plainText(previous).trim() === '')
+        return false;
+    if (Number(next.indent ?? 0) > 0)
+        return false;
+    if (previous.direction && next.direction && previous.direction !== next.direction)
+        return false;
+    return alignment(previous) === alignment(next);
+}
+/** A paragraph with nothing in it: only whitespace and line breaks. */
+export function isEmptyParagraph(block) {
+    return !!block && block.type === 'paragraph' && plainText(block).trim() === '';
+}
+/**
+ * Whether a page's text ends with an empty line, which marks the end of a paragraph. Markdown
+ * cannot hold an empty paragraph (the parser drops it), so it is read from the raw text: the
+ * text must end with a blank line (`\n\n`), not just a final newline. For every other format it
+ * is a trailing empty paragraph in the parsed page.
+ */
+export function endsWithBlankLine(rawFormat, rawText, blocks) {
+    if (rawFormat === 'markdown')
+        return /(\r?\n[ \t]*){2,}$/.test(rawText);
+    return isEmptyParagraph(blocks[blocks.length - 1]);
+}
+const startsWithSpace = (children) => /^\s/.test(plainText({ type: 'x', children: children.slice(0, 1) }));
+const endsWithSpace = (children) => /\s$/.test(plainText({ type: 'x', children: children.slice(-1) }));
+/** Appends `next`'s content to `previous` (which is changed), separated by one space unless one is already there. */
+export function joinParagraphs(previous, next) {
+    const head = previous.children ?? [];
+    const tail = next.children ?? [];
+    const needsSpace = head.length > 0 && tail.length > 0 && !endsWithSpace(head) && !startsWithSpace(tail);
+    previous.children = [...head, ...(needsSpace ? [textNode(' ')] : []), ...tail];
+    // The merged paragraph keeps the first one's direction unless it had none.
+    if (!previous.direction && next.direction)
+        previous.direction = next.direction;
+}

package/dist/merge/index.d.ts

@@ -11,6 +11,24 @@ export interface MergePagesOptions {
     inputFormat?: FormatId;
     /** The format of the combined document. */
     outputFormat: FormatId;
+    /** Leave out pages with no content: nothing, whitespace, or only empty paragraphs. Off by default. */
+    skipBlankPages?: boolean;
+    /**
+     * Join pages that carry on the previous page's last paragraph into one paragraph, instead
+     * of breaking the page. Off by default. Both modes only join when the next page opens with an
+     * unindented paragraph of the same direction and alignment after a plain paragraph; headings,
+     * lists, quotes, tables and blank pages never join.
+     *
+     * - `'blank-line'`: a page that ends with an empty line or empty paragraph ends its paragraph,
+     *   so the next page starts a new one; a page that does not, continues. The marker is consumed,
+     *   so it does not appear in the output. Markdown is read from the raw text (it must end with a
+     *   blank line, `\n\n`), because Markdown has no empty paragraph; see the README.
+     * - `'punctuation'`: a guess from the text. The page continues when its last paragraph does
+     *   not end with `۔ ؟ ! ? . …` (closing quotes and brackets aside).
+     *
+     * A page break cannot sit inside a paragraph, so none is inserted where pages are joined.
+     */
+    joinParagraphs?: 'blank-line' | 'punctuation';
     /** Passed to every parse and to the final serialize (image URL resolver and the like). */
     context?: ConverterContext;
     /** Defaults to the registry with all four built-in formats. */
@@ -23,6 +41,8 @@ export declare function mergePagesToState(pages: PageInput[], options?: Omit<Mer
  * with a page break between each pair of pages, in the requested output format. Needs no
  * editor or DOM, so it can run in a server, a worker or a build script.
  *
+ * With `skipBlankPages`, blank pages are left out and no break is added for them.
+ *
  * Footnote ids are only unique within a page, so a page whose footnote ids clash with an
  * earlier page's gets them renamed (`1` becomes `1-2`), in references and notes alike; the
  * notes of all pages end up in one list at the end of the document.

package/dist/merge/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/merge/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAyB,KAAK,cAAc,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAG3D,kGAAkG;AAClG,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,QAAQ,CAAA;CAAE,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAChC,+EAA+E;IAC/E,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,2CAA2C;IAC3C,YAAY,EAAE,QAAQ,CAAC;IACvB,0FAA0F;IAC1F,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,mGAAmG;AACnG,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,OAAO,GAAE,IAAI,CAAC,iBAAiB,EAAE,cAAc,CAAM,GAAG,qBAAqB,CA4BlI;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAGjF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/merge/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAyB,KAAK,cAAc,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAI3D,kGAAkG;AAClG,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,QAAQ,CAAA;CAAE,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAChC,+EAA+E;IAC/E,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,2CAA2C;IAC3C,YAAY,EAAE,QAAQ,CAAC;IACvB,sGAAsG;IACtG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,EAAE,YAAY,GAAG,aAAa,CAAC;IAC9C,0FAA0F;IAC1F,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,mGAAmG;AACnG,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,OAAO,GAAE,IAAI,CAAC,iBAAiB,EAAE,cAAc,CAAM,GAAG,qBAAqB,CA8ClI;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAGjF"}

package/dist/merge/index.js

@@ -1,11 +1,14 @@
 import { defaultFormatRegistry } from '../registry.js';
 import { elementBase, makeState, rootChildren } from '../shared/serialized.js';
+import { canJoinParagraphs, endsMidSentence, endsWithBlankLine, isEmptyParagraph, joinParagraphs } from './continuation.js';
 /** Parses each page and returns the combined document as Lexical state, without serializing it. */
 export function mergePagesToState(pages, options = {}) {
-    const { inputFormat = 'markdown', context, registry = defaultFormatRegistry } = options;
+    const { inputFormat = 'markdown', skipBlankPages = false, joinParagraphs: joinMode, context, registry = defaultFormatRegistry } = options;
     const body = [];
     const footnotes = [];
     const takenFootnoteIds = new Set();
+    let kept = 0;
+    let previousEndedParagraph = false;
     pages.forEach((page, index) => {
         const { content, format = inputFormat } = typeof page === 'string' ? { content: page } : page;
         let blocks;
@@ -16,14 +19,33 @@ export function mergePagesToState(pages, options = {}) {
             const reason = error instanceof Error ? error.message : String(error);
             throw new Error(`mergePages: page ${index + 1} is not valid ${format}: ${reason}`);
         }
+        if (skipBlankPages && blocks.every(isBlank))
+            return;
         // Each page brings its own footnote list; they are combined into one at the end.
         const pageLists = blocks.filter((block) => block.type === 'footnote-list');
         const pageBody = blocks.filter((block) => block.type !== 'footnote-list');
         uniquifyFootnoteIds([...pageBody, ...pageLists], takenFootnoteIds);
         for (const list of pageLists)
             footnotes.push(...(list.children ?? []));
-        if (index > 0)
+        const previous = body[body.length - 1];
+        let continues = false;
+        if (joinMode === 'punctuation')
+            continues = !!previous && endsMidSentence(previous);
+        if (joinMode === 'blank-line') {
+            continues = !previousEndedParagraph;
+            previousEndedParagraph = endsWithBlankLine(format, content, [...pageBody]);
+            // The empty line only marks the end of the paragraph; it is not content to keep.
+            if (pageBody.some((block) => !isEmptyParagraph(block))) {
+                while (isEmptyParagraph(pageBody[pageBody.length - 1]))
+                    pageBody.pop();
+            }
+        }
+        const joins = kept > 0 && continues && canJoinParagraphs(previous, pageBody[0]);
+        if (joins)
+            joinParagraphs(previous, pageBody.shift());
+        else if (kept > 0)
             body.push({ type: 'page-break', version: 1 });
+        kept += 1;
         body.push(...pageBody);
     });
     return makeState(footnotes.length > 0 ? [...body, elementBase('footnote-list', footnotes)] : body);
@@ -33,6 +55,8 @@ export function mergePagesToState(pages, options = {}) {
  * with a page break between each pair of pages, in the requested output format. Needs no
  * editor or DOM, so it can run in a server, a worker or a build script.
  *
+ * With `skipBlankPages`, blank pages are left out and no break is added for them.
+ *
  * Footnote ids are only unique within a page, so a page whose footnote ids clash with an
  * earlier page's gets them renamed (`1` becomes `1-2`), in references and notes alike; the
  * notes of all pages end up in one list at the end of the document.
@@ -41,6 +65,12 @@ export function mergePages(pages, options) {
     const registry = options.registry ?? defaultFormatRegistry;
     return registry.serialize(options.outputFormat, mergePagesToState(pages, options), options.context);
 }
+/** A block with nothing in it: a paragraph holding only whitespace and line breaks. Anything else (an image, a rule, a table, a break) is content. */
+function isBlank(block) {
+    if (block.type !== 'paragraph')
+        return false;
+    return (block.children ?? []).every((child) => (child.type === 'text' ? String(child.text ?? '').trim() === '' : child.type === 'linebreak'));
+}
 /** Renames this page's footnote ids that an earlier page already uses, then claims all its ids. */
 function uniquifyFootnoteIds(blocks, taken) {
     const nodes = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@inshapardaz/likhari-converters",
-  "version": "0.1.45-dev.334",
+  "version": "0.1.45-dev.337",
   "description": "Format transformers (Markdown/HTML/plain-text/Lexical JSON) against the canonical Lexical EditorState, isolated from UI.",
   "license": "MIT",
   "repository": {