@fiduswriter/document 0.1.0-alpha.1

package/src/exporter/latex/escape_latex.js

@@ -0,0 +1,21 @@
+export const escapeLatexText = text =>
+    text
+        // Remove line breaks
+        .replace(/\r|\n/g, "")
+        // Escape characters that are protected in some way.
+        .replace(/\{/g, "\\{")
+        .replace(/\}/g, "\\}")
+        .replace(/\^/g, "\\textasciicircum{}")
+        .replace(/\$/g, "\\$")
+        .replace(/_/g, "\\_")
+        .replace(/~/g, "\\textasciitilde{}")
+        .replace(/#/g, "\\#")
+        .replace(/%/g, "\\%")
+        .replace(/&/g, "\\&")
+        .replace(/\\\\/g, "\\textbackslash")
+
+        // Remove control characters that somehow have ended up in the document
+        .replace(/\u000B/g, "")
+        .replace(/\u000C/g, "")
+        .replace(/\u000E/g, "")
+        .replace(/\u000F/g, "")

package/src/exporter/latex/index.js

@@ -0,0 +1,74 @@
+import {BibLatexExporter} from "biblatex-csl-converter"
+import download from "downloadjs"
+
+import {shortFileTitle} from "../../common/index.js"
+import {fixTables, removeHidden} from "../tools/doc_content.js"
+import {createSlug} from "../tools/file.js"
+import {ZipFileCreator} from "../tools/zip.js"
+import {LatexExporterConvert} from "./convert.js"
+import {readMe} from "./readme.js"
+/*
+ Exporter to LaTeX
+*/
+
+export class LatexExporter {
+    constructor(doc, bibDB, imageDB, updated) {
+        this.doc = doc
+        this.docTitle = shortFileTitle(this.doc.title, this.doc.path)
+        this.bibDB = bibDB
+        this.imageDB = imageDB
+        this.updated = updated
+
+        this.docContent = false
+        this.zipFileName = false
+        this.textFiles = []
+        this.httpFiles = []
+    }
+
+    init() {
+        this.zipFileName = `${createSlug(this.docTitle)}.latex.zip`
+        this.docContent = fixTables(removeHidden(this.doc.content))
+        this.converter = new LatexExporterConvert(
+            this,
+            this.imageDB,
+            this.bibDB,
+            this.doc.settings
+        )
+        this.conversion = this.converter.init(this.docContent)
+        if (Object.keys(this.conversion.usedBibDB).length > 0) {
+            const bibExport = new BibLatexExporter(this.conversion.usedBibDB)
+            this.textFiles.push({
+                filename: "bibliography.bib",
+                contents: bibExport.parse()
+            })
+        }
+        this.textFiles.push({
+            filename: "document.tex",
+            contents: this.conversion.latex
+        })
+        this.textFiles.push({filename: "README.txt", contents: readMe})
+        this.conversion.imageIds.forEach(id => {
+            this.httpFiles.push({
+                filename: this.imageDB.db[id].image.split("/").pop(),
+                url: this.imageDB.db[id].image
+            })
+        })
+        return this.createZip()
+    }
+
+    createZip() {
+        const zipper = new ZipFileCreator(
+            this.textFiles,
+            this.httpFiles,
+            undefined,
+            undefined,
+            this.updated
+        )
+
+        return zipper.init().then(blob => this.download(blob))
+    }
+
+    download(blob) {
+        return download(blob, this.zipFileName, "application/zip")
+    }
+}

package/src/exporter/latex/readme.js

@@ -0,0 +1,22 @@
+export const readMe = `We strongly recommend using LuaTeX instead of PDFTeX or XeLaTeX for document
+compilation.
+
+In order to compile the LaTeX file, you need to use at least TeXLive 2016. If
+there are citations, you additionally need Biber 2.7/BibLaTeX 3.7.
+
+On Ubuntu 18.04+ install these packages:
+
+> sudo apt install texlive-latex-base texlive-bibtex-extra biber texlive-latex-extra
+
+Extract all the files included in this ZIP into a folder.
+Run then these commands to create a PDF from within this folder:
+
+> lualatex document
+
+If there are citations, continue with these commands:
+
+> biber document
+> lualatex document
+
+Look at the output messages to determine whether you need to run laluatex again.
+`

package/src/exporter/native/shrink.js

@@ -0,0 +1,132 @@
+import {addAlert} from "../../common/index.js"
+import {docSchema} from "../../schema/document/index.js"
+import {toMiniJSON} from "../../schema/mini_json.js"
+// Generate a copy of the fidus doc, imageDB and bibDB with all clutter removed.
+export class ShrinkFidus {
+    /**
+     * @param {Object}  doc      - Full document object.
+     * @param {Object}  imageDB  - Image database, e.g. {db: {...}}.
+     * @param {Object}  bibDB    - Bibliography database, e.g. {db: {...}}.
+     * @param {boolean} [silent=false] - When true, suppresses the
+     *   "File export has been initiated" info alert.  Pass true when
+     *   shrinking multiple documents in a loop (e.g. one per book chapter)
+     *   and the caller already shows its own progress notification.
+     */
+    constructor(doc, imageDB, bibDB, silent = false) {
+        this.doc = doc
+        this.imageDB = imageDB
+        this.bibDB = bibDB
+        this.silent = silent
+        this.imageList = []
+        this.citeList = []
+    }
+
+    init() {
+        const shrunkImageDB = {},
+            httpIncludes = []
+
+        if (!this.silent) {
+            addAlert("info", gettext("File export has been initiated."))
+        }
+
+        this.walkTree(this.doc.content)
+
+        this.imageList = [...new Set(this.imageList)] // unique values
+
+        this.imageList.forEach(itemId => {
+            shrunkImageDB[itemId] = Object.assign({}, this.imageDB.db[itemId])
+            // Remove parts that are connected to a particular user/server
+            delete shrunkImageDB[itemId].cats
+            delete shrunkImageDB[itemId].thumbnail
+            delete shrunkImageDB[itemId].pk
+            delete shrunkImageDB[itemId].added
+            const imageUrl = shrunkImageDB[itemId].image
+            let filename
+            if (imageUrl.startsWith("blob:")) {
+                // Blob URL produced by decrypting an E2EE image client-side.
+                // The URL itself carries no useful file extension, so derive
+                // one from the image entry's MIME type instead.  Without this
+                // the server rejects the upload because get_encrypted_file_path
+                // requires a recognised extension.
+                const mime = shrunkImageDB[itemId].file_type || "image/png"
+                const mimeExtMap = {
+                    "image/png": "png",
+                    "image/jpeg": "jpg",
+                    "image/jpg": "jpg",
+                    "image/webp": "webp",
+                    "image/svg+xml": "svg",
+                    "image/gif": "gif",
+                    "image/avif": "avif"
+                }
+                const ext = mimeExtMap[mime] || "png"
+                filename = `images/${itemId}.${ext}`
+            } else {
+                filename = `images/${imageUrl.split("/").pop()}`
+            }
+            shrunkImageDB[itemId].image = filename
+            httpIncludes.push({
+                url: imageUrl,
+                filename
+            })
+        })
+
+        this.citeList = [...new Set(this.citeList)] // unique values
+
+        const shrunkBibDB = {}
+        this.citeList.forEach(itemId => {
+            shrunkBibDB[itemId] = Object.assign({}, this.bibDB.db[itemId])
+            // Remove the cats, as it is only a list of IDs for one
+            // particular user/server.
+            delete shrunkBibDB[itemId].cats
+        })
+
+        const docCopy = Object.assign({}, this.doc)
+
+        // Remove items that aren't needed.
+        delete docCopy.rights
+        delete docCopy.version
+        delete docCopy.comment_version
+        delete docCopy.owner
+        delete docCopy.id
+        delete docCopy.is_owner
+        delete docCopy.added
+        delete docCopy.updated
+        delete docCopy.revisions
+
+        docCopy.content = toMiniJSON(docSchema.nodeFromJSON(docCopy.content))
+
+        return new Promise(resolve =>
+            resolve({
+                doc: docCopy,
+                shrunkImageDB,
+                shrunkBibDB,
+                httpIncludes
+            })
+        )
+    }
+
+    walkTree(node) {
+        switch (node.type) {
+            case "citation":
+                this.citeList = this.citeList.concat(
+                    node.attrs.references.map(ref => ref.id)
+                )
+                break
+            case "image":
+                if (node.attrs.image !== false) {
+                    this.imageList.push(node.attrs.image)
+                }
+                break
+            case "footnote":
+                if (node.attrs?.footnote) {
+                    node.attrs.footnote.forEach(childNode =>
+                        this.walkTree(childNode)
+                    )
+                }
+                break
+        }
+        if (node.content) {
+            node.content.forEach(childNode => this.walkTree(childNode))
+        }
+    }
+}

package/src/exporter/odt/citations.js

@@ -0,0 +1,101 @@
+import {DOMParser, DOMSerializer} from "prosemirror-model"
+
+import {cslBibSchema} from "../../bibliography/csl_bib.js"
+import {FormatCitations} from "../../citations/format.js"
+import {fnSchema} from "../../schema/footnotes.js"
+import {descendantNodes} from "../tools/doc_content.js"
+
+export class ODTExporterCitations {
+    constructor(docContent, settings, styles, bibDB, csl, origCitInfos = []) {
+        this.docContent = docContent
+        this.settings = settings
+        this.styles = styles
+        this.bibDB = bibDB
+        this.csl = csl
+        // If citInfos were found in a previous run, they are stored here
+        // (for example: first citations in main document, then in footnotes)
+        this.origCitInfos = origCitInfos
+        this.citInfos = []
+        this.citationTexts = []
+        this.pmCits = []
+        this.citFm = false
+        this.pmBib = false
+    }
+
+    init() {
+        return this.formatCitations()
+    }
+
+    // Citations are highly interdependent -- so we need to format them all
+    // together before laying out the document.
+    formatCitations() {
+        if (this.origCitInfos.length) {
+            // Initial citInfos are taken from a previous run to include in
+            // bibliography, and they are removed before spitting out the
+            // citation entries for the given document.
+            // That way the bibliography should contain information from both.
+            this.citInfos = this.citInfos.concat(this.origCitInfos)
+        }
+
+        descendantNodes(this.docContent).forEach(node => {
+            if (node.type === "citation") {
+                this.citInfos.push(JSON.parse(JSON.stringify(node.attrs)))
+            }
+        })
+        this.citFm = new FormatCitations(
+            this.csl,
+            this.citInfos,
+            this.settings.citationstyle,
+            "",
+            this.bibDB,
+            false,
+            this.settings.language
+        )
+        return this.citFm.init().then(() => {
+            this.citationTexts = this.citFm.citationTexts
+            if (this.origCitInfos.length) {
+                // Remove all citation texts originating from original starting citInfos
+                this.citationTexts.splice(0, this.origCitInfos.length)
+            }
+            this.convertCitations()
+            return Promise.resolve()
+        })
+    }
+
+    convertCitations() {
+        // There could be some formatting in the citations, so we parse them through the PM schema for final formatting.
+        // We need to put the citations each in a paragraph so that it works with
+        // the fiduswriter schema and so that the converter doesn't mash them together.
+        if (this.citationTexts.length) {
+            let citationsHTML = ""
+            this.citationTexts.forEach(ct => {
+                citationsHTML += `<p>${ct}</p>`
+            })
+
+            // We create a standard footnote container DOM node,
+            // add the citations into it, and parse it back.
+            const fnNode = fnSchema.nodeFromJSON({type: "footnotecontainer"})
+            const serializer = DOMSerializer.fromSchema(fnSchema)
+            const dom = serializer.serializeNode(fnNode)
+            dom.innerHTML = citationsHTML
+            this.pmCits = DOMParser.fromSchema(fnSchema)
+                .parse(dom, {topNode: fnNode})
+                .toJSON().content
+        } else {
+            this.pmCits = []
+        }
+
+        // Now we do the same for the bibliography.
+        const cslBib = this.citFm.bibliography
+        if (cslBib && cslBib[1].length > 0) {
+            this.styles.addReferenceStyle(cslBib[0])
+            const bibNode = cslBibSchema.nodeFromJSON({type: "cslbib"})
+            const serializer = DOMSerializer.fromSchema(cslBibSchema)
+            const dom = serializer.serializeNode(bibNode)
+            dom.innerHTML = cslBib[1].join("")
+            this.pmBib = DOMParser.fromSchema(cslBibSchema)
+                .parse(dom, {topNode: bibNode})
+                .toJSON()
+        }
+    }
+}

package/src/exporter/odt/footnotes.js

@@ -0,0 +1,147 @@
+import {descendantNodes} from "../tools/doc_content.js"
+import {ODTExporterCitations} from "./citations.js"
+import {ODTExporterImages} from "./images.js"
+
+const DEFAULT_STYLE_FOOTNOTE = `
+    <style:style style:name="Footnote" style:family="paragraph" style:parent-style-name="Standard" style:class="extra">
+        <style:paragraph-properties fo:margin-left="0.2354in" fo:margin-right="0in" fo:text-indent="-0.2354in" style:auto-text-indent="false" text:number-lines="false" text:line-number="0" />
+        <style:text-properties fo:font-size="10pt" style:font-size-asian="10pt" style:font-size-complex="10pt" />
+    </style:style>
+    `
+
+const DEFAULT_STYLE_FOOTNOTE_ANCHOR = `
+    <style:style style:name="Footnote_20_anchor" style:display-name="Footnote anchor" style:family="text">
+        <style:text-properties style:text-position="super 58%" />
+    </style:style>
+    `
+const DEFAULT_STYLE_FOOTNOTE_SYMBOL = `
+    <style:style style:name="Footnote_20_Symbol" style:display-name="Footnote Symbol" style:family="text" />
+    `
+
+const DEFAULT_STYLE_FOOTNOTE_CONFIGURATION = `
+    <text:notes-configuration text:note-class="footnote" text:citation-style-name="Footnote_20_Symbol" text:citation-body-style-name="Footnote_20_anchor" style:num-format="1" text:start-value="0" text:footnotes-position="page" text:start-numbering-at="document" />
+    `
+
+export class ODTExporterFootnotes {
+    constructor(
+        docContent,
+        settings,
+        xml,
+        bodyCitations,
+        styles,
+        bibDB,
+        imageDB,
+        csl
+    ) {
+        this.docContent = docContent
+        this.settings = settings
+        this.xml = xml
+        this.bodyCitations = bodyCitations
+        this.styles = styles
+        this.bibDB = bibDB
+        this.imageDB = imageDB
+        this.csl = csl
+
+        this.pmBib = false
+        this.fnPmJSON = false
+        this.images = false
+        this.citations = false
+        this.footnotes = []
+        this.styleFilePath = "styles.xml"
+    }
+
+    init() {
+        this.findFootnotes()
+        if (
+            this.footnotes.length ||
+            (this.bodyCitations.citFm.citationType === "note" &&
+                this.bodyCitations.citInfos.length)
+        ) {
+            this.convertFootnotes()
+            // Include the citinfos from the main document so that they will be
+            // used for calculating the bibliography as well
+            this.citations = new ODTExporterCitations(
+                this.fnPmJSON,
+                this.settings,
+                this.styles,
+                this.bibDB,
+                this.csl,
+                this.bodyCitations.citInfos
+            )
+            this.images = new ODTExporterImages(
+                this.fnPmJSON,
+                this.xml,
+                this.imageDB
+            )
+
+            return this.citations
+                .init()
+                .then(() => {
+                    // Replace the main bibliography with the new one that includes
+                    // both citations in main document and in the footnotes.
+                    this.pmBib = this.citations.pmBib
+                    return this.images.init()
+                })
+                .then(() => {
+                    return this.addStyles()
+                })
+        } else {
+            // No footnotes were found.
+            return Promise.resolve()
+        }
+    }
+
+    addStyles() {
+        return this.xml.getXml(this.styleFilePath).then(styleXml => {
+            this.styleXml = styleXml
+            this.addStyle("Footnote", DEFAULT_STYLE_FOOTNOTE)
+            this.addStyle("Footnote_20_anchor", DEFAULT_STYLE_FOOTNOTE_ANCHOR)
+            this.addStyle("Footnote_20_Symbol", DEFAULT_STYLE_FOOTNOTE_SYMBOL)
+            this.setStyleConfig()
+            return Promise.resolve()
+        })
+    }
+
+    addStyle(styleName, xml) {
+        if (!this.styleXml.query("style:style", {"style:name": styleName})) {
+            const stylesEl = this.styleXml.query("office:styles")
+            stylesEl.appendXML(xml)
+        }
+    }
+
+    setStyleConfig() {
+        const oldFnStyleConfigEl = this.styleXml.query(
+            "text:notes-configuration",
+            {
+                "text:note-class": "footnote"
+            }
+        )
+        if (oldFnStyleConfigEl) {
+            oldFnStyleConfigEl.parentElement.removeChild(oldFnStyleConfigEl)
+        }
+        const stylesEl = this.styleXml.query("office:styles")
+        stylesEl.appendXML(DEFAULT_STYLE_FOOTNOTE_CONFIGURATION)
+    }
+
+    findFootnotes() {
+        descendantNodes(this.docContent).forEach(node => {
+            if (node.type === "footnote") {
+                this.footnotes.push(node.attrs.footnote)
+            }
+        })
+    }
+
+    convertFootnotes() {
+        const fnContent = []
+        this.footnotes.forEach(footnote => {
+            fnContent.push({
+                type: "footnotecontainer",
+                content: footnote
+            })
+        })
+        this.fnPmJSON = {
+            type: "doc",
+            content: fnContent
+        }
+    }
+}

package/src/exporter/odt/images.js

@@ -0,0 +1,115 @@
+import {get} from "../../common/index.js"
+import {descendantNodes} from "../tools/doc_content.js"
+import {svg2png} from "../tools/svg.js"
+
+export class ODTExporterImages {
+    constructor(docContent, xml, imageDB) {
+        this.docContent = docContent
+        this.xml = xml
+        this.imageDB = imageDB
+        this.images = {}
+        this.manifestXml = false
+    }
+
+    init() {
+        return this.xml.getXml("META-INF/manifest.xml").then(manifestXml => {
+            this.manifestXml = manifestXml
+            return this.exportImages()
+        })
+    }
+
+    // add an image to the list of files
+    addImage(imgFileName, image) {
+        imgFileName = this.addFileToManifest(imgFileName)
+        this.xml.addExtraFile(`Pictures/${imgFileName}`, image)
+        return imgFileName
+    }
+
+    // add a an image file to the manifest
+    addFileToManifest(imgFileName) {
+        const fileNameParts = imgFileName.split(".")
+        const fileNameEnding = fileNameParts.pop()
+        const fileNameStart = fileNameParts.join(".")
+        const manifestEl = this.manifestXml.query("manifest:manifest")
+        let imgManifest = manifestEl.query("manifest:file-entry", {
+            "manifest:full-path": `Pictures/${imgFileName}`
+        })
+        let counter = 0
+        while (imgManifest) {
+            // Name exists already, we change the name until we get a file name not yet included in manifest.
+            imgFileName = `${fileNameStart}_${counter++}.${fileNameEnding}`
+            imgManifest = manifestEl.query("manifest:file-entry", {
+                "manifest:full-path": `Pictures/${imgFileName}`
+            })
+        }
+        const string = `  <manifest:file-entry manifest:full-path="Pictures/${imgFileName}" manifest:media-type="image/${fileNameEnding}"/>`
+        manifestEl.appendXML(string)
+        return imgFileName
+    }
+
+    // Find all images used in file and add these to the export zip.
+    // TODO: This will likely fail on image types odt doesn't support such as
+    // SVG. Try out and fix.
+    exportImages() {
+        const usedImgs = []
+
+        descendantNodes(this.docContent).forEach(node => {
+            if (node.type === "image" && node.attrs.image !== false) {
+                if (!usedImgs.includes(node.attrs.image)) {
+                    usedImgs.push(node.attrs.image)
+                }
+            }
+        })
+
+        return new Promise(resolveExportImages => {
+            const p = []
+
+            usedImgs.forEach(image => {
+                const imgDBEntry = this.imageDB.db[image]
+                p.push(
+                    get(imgDBEntry.image)
+                        .then(response => response.blob())
+                        .then(blob => {
+                            const wImgId = this.addImage(
+                                imgDBEntry.image.split("/").pop(),
+                                blob
+                            )
+                            if (blob.type === "image/svg+xml") {
+                                // Add PNG version in addition to SVG
+                                return svg2png(blob).then(
+                                    ({blob: pngBlob, width, height}) => {
+                                        const pngWImgId = this.addImage(
+                                            imgDBEntry.image
+                                                .split("/")
+                                                .pop()
+                                                .replace(/.svg$/g, ".png"),
+                                            pngBlob
+                                        )
+                                        this.images[image] = {
+                                            id: pngWImgId,
+                                            width,
+                                            height,
+                                            title: imgDBEntry.title,
+                                            type: blob.type,
+                                            svg: wImgId
+                                        }
+                                    }
+                                )
+                            } else {
+                                this.images[image] = {
+                                    id: wImgId,
+                                    width: imgDBEntry.width,
+                                    height: imgDBEntry.height,
+                                    title: imgDBEntry.title,
+                                    type: blob.type,
+                                    svg: false
+                                }
+                            }
+                        })
+                )
+            })
+
+            Promise.all(p).then(() => resolveExportImages())
+        })
+    }
+}