mikser-io 6.0.0

package/src/plugins/files.js

@@ -0,0 +1,153 @@
+import path from 'node:path'
+import { mkdir, symlink, unlink, lstat, realpath } from 'fs/promises'
+import { globby } from 'globby'
+
+export default ({
+    runtime,
+    onLoaded,
+    useLogger,
+    onImport,
+    createEntity,
+    updateEntity,
+    deleteEntity,
+    watch,
+    onSync,
+    findEntity,
+    checksum,
+    trackProgress,
+    updateProgress,
+    constants: { ACTION },
+}) => {
+    const collection = 'files'
+    const type = 'file'
+
+    async function ensureLink(relativePath) {
+        const source = path.join(runtime.options.filesFolder, relativePath)
+        let uri = path.join(runtime.options.outputFolder, relativePath)
+        if (runtime.config.files?.outputFolder) uri = path.join(runtime.options.outputFolder, runtime.config.files.outputFolder, relativePath)
+        try {
+            await mkdir(path.dirname(uri), { recursive: true })
+            await symlink(path.resolve(source), uri, 'file')
+        } catch (err) {
+            if (err.code != 'EEXIST')
+                throw err
+        }
+        return { uri, source }
+    }
+
+    async function removeLink(relativePath) {
+        let uri = path.join(runtime.options.outputFolder, relativePath)
+        if (runtime.config.files?.outputFolder) uri = path.join(runtime.options.outputFolder, runtime.config.files.outputFolder, relativePath)
+        await unlink(path.resolve(uri))
+    }
+
+    async function link(source) {
+        const stat = await lstat(source)
+        if (stat.isSymbolicLink()) {
+            return await realpath(source)
+        }
+    }
+
+    onSync(collection, async ({ action, context }) => {
+        if (!context.relativePath) return false
+        const { relativePath } = context
+
+        const source = path.join(runtime.options.filesFolder, relativePath)
+        const format = path.extname(relativePath).substring(1).toLowerCase()
+        const id = path.join(`/${collection}`, relativePath)
+        let uri = path.join(runtime.options.outputFolder, relativePath)
+        let name = relativePath
+        if (runtime.config.files?.outputFolder) {
+            uri = path.join(runtime.options.outputFolder, runtime.config.files.outputFolder, relativePath)
+            name = path.join(runtime.config.files.outputFolder, relativePath)
+        }
+
+        let synced = true
+        switch (action) {
+            case ACTION.CREATE:
+                await ensureLink(relativePath)
+                await createEntity({
+                    id,
+                    uri,
+                    name,
+                    collection,
+                    type,
+                    format,
+                    source,
+                    checksum: await checksum(source),
+                    link: await link(source)
+                })
+                break
+            case ACTION.UPDATE:
+                const current = await findEntity({ id })
+                if (current?.checksum != checksum) {
+                    await updateEntity({
+                        id,
+                        uri,
+                        name: relativePath,
+                        collection,
+                        type,
+                        format,
+                        source,
+                        checksum: await checksum(source),
+                        link: await link(source)
+                    })
+                } else {
+                    synced = false
+                }
+                break
+            case ACTION.DELETE:
+                await removeLink(relativePath)
+                await deleteEntity({
+                    id,
+                    collection,
+                    type,
+                })
+                break
+        }
+        return synced
+    })
+
+    onLoaded(async () => {
+        const logger = useLogger()
+        runtime.options.files = runtime.config.files?.filesFolder || collection
+        runtime.options.filesFolder = path.join(runtime.options.workingFolder, runtime.options.files)
+
+        logger.info('Files folder: %s', runtime.options.filesFolder)
+        await mkdir(runtime.options.filesFolder, { recursive: true })
+
+        watch(collection, runtime.options.filesFolder)
+    })
+
+    onImport(async () => {
+        await mkdir(runtime.options.outputFolder, { recursive: true })
+        if (runtime.config.files?.outputFolder) await mkdir(path.join(runtime.options.outputFolder, runtime.config.files.outputFolder), { recursive: true })
+
+        const paths = await globby('**/*', { cwd: runtime.options.filesFolder })
+        trackProgress('Files import', paths.length)
+        return Promise.all(paths.map(async relativePath => {
+            const { uri, source } = await ensureLink(relativePath)
+            let name = relativePath
+            if (runtime.config.files?.outputFolder) {
+                name = path.join(runtime.config.files.outputFolder, relativePath)
+            }
+            await createEntity({
+                id: path.join(`/${collection}`, relativePath),
+                uri,
+                collection,
+                type,
+                format: path.extname(relativePath).substring(1).toLowerCase(),
+                name,
+                source,
+                checksum: await checksum(source),
+                link: await link(source)
+            })
+            updateProgress()
+        }))
+    })
+
+    return {
+        collection,
+        type
+    }
+}

package/src/plugins/front-matter.js

@@ -0,0 +1,24 @@
+import fm from 'front-matter'
+
+export default ({
+    onProcess,
+    useLogger,
+    useJournal,
+    updateEntry,
+    constants: { OPERATION }
+}) => {
+    onProcess(async () => {
+        const logger = useLogger()
+        for await (let { id, entity } of useJournal('Fron matter', [OPERATION.CREATE, OPERATION.UPDATE])) {
+            if (entity.content && fm.test(entity.content)) {
+                const info = fm(entity.content)
+                if (info.attributes) {
+                    entity.meta = Object.assign(entity.meta || {}, info.attributes)
+                    entity.content = info.body
+                    await updateEntry({ id, entity })
+                    logger.trace('Front matter %s: %s', entity.collection, entity.id)
+                }
+            }
+        }
+    })
+}

package/src/plugins/json.js

@@ -0,0 +1,20 @@
+export default ({ 
+    onProcess, 
+    useLogger, 
+    useJournal, 
+    updateEntry,
+    constants: { OPERATION }
+}) => {
+    onProcess(async () => {
+        const logger = useLogger()
+    
+        for await (let { id, entity } of useJournal('Json', [OPERATION.CREATE, OPERATION.UPDATE])) {
+            if (entity.content && entity.format == 'json') {
+                entity.meta = Object.assign(entity.meta || {}, JSON.parse(entity.content))
+                delete entity.content
+                await updateEntry({ id, entity })
+                logger.trace('Json %s: %s', entity.collection, entity.id)
+            }
+        }
+    })
+}

package/src/plugins/layouts.js

@@ -0,0 +1,368 @@
+import path from 'node:path'
+import { mkdir, writeFile, unlink } from 'node:fs/promises'
+import { globby } from 'globby'
+import _ from 'lodash'
+
+export default ({
+    runtime,
+    onLoaded,
+    useLogger,
+    onImport,
+    createEntity,
+    updateEntity,
+    deleteEntity,
+    watch,
+    onProcessed,
+    onBeforeRender,
+    useJournal,
+    renderEntities,
+    onComplete,
+    onSync,
+    matchEntity,
+    changeExtension,
+    constants: { ACTION, OPERATION, TASKS },
+}) => {
+    const collection = 'layouts'
+    const type = 'layout'
+
+    function getFormatInfo(relativePath) {
+        const template = path.extname(relativePath).substring(1).toLowerCase()
+        const withoutTemplate = relativePath.replace(path.extname(relativePath), '')
+        const formatExt = path.extname(withoutTemplate).substring(1).toLowerCase()
+        const [format, postprocessor] = formatExt.split('-')
+        const name = formatExt ? withoutTemplate.replace(path.extname(withoutTemplate), '') : withoutTemplate
+        return { name, format: format || 'html', template, postprocessor }
+    }
+
+    function addToSitemap(entity) {
+        const logger = useLogger()
+        const { sitemap } = runtime.state.layouts
+        const { href = '/' + entity.name, lang } = entity.meta || {}
+        if (lang) {
+            sitemap[href] = sitemap[href] || {};
+            let previous = sitemap[href][lang];
+            if (previous && (previous.id != entity.id)) {
+                logger.warn('Entity with equal href: [%s] %s and %s', previous.collection, previous.id, entity.id);
+            }
+            sitemap[href][lang] = entity
+        }
+        else {
+            let previous = sitemap[href];
+            if (previous && (previous.id != entity.id)) {
+                logger.warn('Entity with equal href: [%s] %s and %s', previous.collection, previous.id, entity.id);
+            }
+            sitemap[href] = entity
+        }
+    }
+
+    function removeFromSitemap(entity) {
+        const { sitemap } = runtime.state.layouts
+        for (let href in sitemap) {
+            let entry = sitemap[href]
+            if (entry.id) {
+                if (entry.id == entity.id) {
+                    delete sitemap[href]
+                    return
+                }
+            } else {
+                for (let lang in entry) {
+                    if (entry[lang].id == entity.id) {
+                        delete entry[lang]
+                        return
+                    }
+                }
+            }
+        }
+    }
+
+    function removePagesFromSitemap(entity) {
+        const entities = Array.from(getSitemapEntities())
+        for (let current of entities) {
+            if (entity.uri == current.uri) {
+                removeFromSitemap(current)
+            }
+        }
+    }
+
+    function* getSitemapEntities() {
+        const { sitemap } = runtime.state.layouts
+        for (let href in sitemap) {
+            let entry = sitemap[href]
+            if (entry.id) {
+                if (!entry.page || entry.page <= 1) {
+                    yield entry
+                }
+            } else {
+                for (let lang in entry) {
+                    if (!entry[lang].page || entry[lang].page <= 1) {
+                        yield entry[lang]
+                    }
+                }
+            }
+        }
+    }
+
+    onSync(collection, async ({ action, context }) => {
+        if (!context.relativePath) return false
+        const { relativePath } = context
+        let id = path.join(`/${collection}`, relativePath)
+        if (_.endsWith(id, '.js')) id = id.replace(new RegExp('.js$'), '')
+
+        const uri = path.join(runtime.options.layoutsFolder, relativePath)
+        const { layouts } = runtime.state.layouts
+        switch (action) {
+            case ACTION.CREATE:
+                var layout = {
+                    id,
+                    uri,
+                    collection,
+                    type,
+                    name: relativePath.replace(path.extname(relativePath), ''),
+                    ...getFormatInfo(relativePath)
+                }
+                layouts[layout.name] = layout
+                await createEntity(layout)
+                break
+            case ACTION.UPDATE:
+                var layout = {
+                    id,
+                    uri,
+                    collection,
+                    type,
+                    name: relativePath.replace(path.extname(relativePath), ''),
+                    ...getFormatInfo(relativePath)
+                }
+                layouts[layout.name] = layout
+                await updateEntity(layout)
+                break
+            case ACTION.DELETE:
+                var layout = {
+                    id,
+                    collection,
+                    type,
+                    format: path.extname(relativePath).substring(1).toLowerCase(),
+                }
+                for (let name in layouts) {
+                    if (layouts[name].id == layout.id) {
+                        delete layouts[name]
+                    }
+                }
+                await deleteEntity(layout)
+                break
+        }
+    })
+
+    onLoaded(async () => {
+        const logger = useLogger()
+
+        runtime.state.layouts = {
+            layouts: {},
+            sitemap: {}
+        }
+
+        runtime.options.layouts = runtime.config.layouts?.layoutsFolder || collection
+        runtime.options.layoutsFolder = path.join(runtime.options.workingFolder, runtime.options.layouts)
+        runtime.options.layoutsStateFolder = path.join(runtime.options.outputFolder, 'state')
+
+        logger.info('Layouts folder: %s', runtime.options.layoutsFolder)
+        await mkdir(runtime.options.layoutsFolder, { recursive: true })
+
+        watch(collection, runtime.options.layoutsFolder)
+    })
+
+    onImport(async () => {
+        const { layouts } = runtime.state.layouts
+        const paths = await globby('**/*', { cwd: runtime.options.layoutsFolder, ignore: ['**/*.js'] })
+        for (let relativePath of paths) {
+            const uri = path.join(runtime.options.layoutsFolder, relativePath)
+            const layout = {
+                id: path.join('/layouts', relativePath),
+                uri,
+                name: relativePath.replace(path.extname(relativePath), ''),
+                collection,
+                type,
+            }
+            Object.assign(layout, await getFormatInfo(relativePath))
+            layouts[layout.name] = layout
+            await createEntity(layout)
+        }
+    })
+
+    onProcessed(async (signal) => {
+        const logger = useLogger()
+        const { layouts } = runtime.state.layouts
+
+        for await (let { entity, operation } of useJournal('Layouts processing', [OPERATION.CREATE, OPERATION.UPDATE, OPERATION.DELETE], signal)) {
+            if (entity.collection == collection) continue
+            switch (operation) {
+                case OPERATION.CREATE:
+                case OPERATION.UPDATE:
+                    removePagesFromSitemap(entity)
+                    if (!entity.meta?.layout) {
+                        for (let pattern in runtime.config.layouts?.match || []) {
+                            if (matchEntity(entity, pattern)) {
+                                const layoutName = runtime.config.layouts?.match[pattern]
+                                entity.layout = layouts[layoutName]
+                                break
+                            }
+                        }
+                        if (!entity.layout && runtime.config.layouts?.autoLayouts && entity.name) {
+                            const nameChunks = entity.name.split('.')
+                            if (nameChunks?.length) {
+                                for (let index = 0; index < nameChunks.length; index++) {
+                                    const autoLayout = [
+                                        path.basename(entity.name).split('.').slice(index).join('.'),
+                                        path.basename(entity.id)
+                                    ]
+                                        .find(layout => layouts[layout])
+                                    if (autoLayout) {
+                                        entity.layout = layouts[autoLayout]
+                                        break
+                                    }
+                                }
+                            }
+                        }
+                    } else {
+                        entity.layout = layouts[entity.meta.layout]
+                    }
+                    if (entity.meta?.layout && !entity.layout) {
+                        logger.warn('Layout not found for %s: %s', entity.collection, entity.id)
+                    }
+
+                    if (entity.layout && entity.meta?.postprocessor) {
+                        entity.layout.postprocessor = entity.meta.postprocessor
+                    }
+
+                    if (entity.layout) {
+                        logger.debug('Layout matched for %s: %s', entity.collection, entity.id)
+                        addToSitemap(entity)
+                    } else if (entity.meta?.href) {
+                        logger.trace('Layout missing for %s: %s', entity.collection, entity.id)
+                        addToSitemap(entity)
+                    }
+                    break
+                case OPERATION.DELETE:
+                    removePagesFromSitemap(entity)
+                    break
+            }
+
+        }
+    })
+
+    onBeforeRender(async (signal) => {
+        const tasks = []
+        const entities = Array.from(getSitemapEntities())
+            .filter(entity => entity.layout)
+            .sort((a, b) => b.time - a.time)
+
+        for (let original of entities) {
+            if (signal.aborted) return
+
+            delete original.page
+            delete original.pages
+            delete original.destination
+
+            const entity = _.cloneDeep(original)
+            entity.destination = '/' + entity.name
+            let data
+            try {
+                var { load, plugins = [] } = await import(`${path.join(runtime.options.layoutsFolder, entity.layout.name)}.js?stamp=${Date.now()}`)
+                if (load) {
+                    data = await load(entity, signal)
+                }
+            } catch (err) {
+                if (err.code != 'ERR_MODULE_NOT_FOUND') throw err
+            }
+
+            if (data?.pages) {
+                if (!_.endsWith(entity.name, entity.format)) {
+                    for (let page = 0; page < data.pages - 1; page++) {
+                        const pageEntity = _.cloneDeep(entity)
+                        pageEntity.pages = data.pages
+                        if (page) {
+                            pageEntity.page = page + 1
+                            pageEntity.id = changeExtension(entity.id, `${pageEntity.page}.${entity.layout.format}`)
+                            if (entity.meta) {
+                                if (entity.meta.href) {
+                                    pageEntity.meta.href = `${entity.meta.href}.${pageEntity.page}`
+                                } else {
+                                    pageEntity.meta.href = `/${entity.name}.${pageEntity.page}`
+                                }
+                            }
+
+                            if (runtime.config.layouts?.cleanUrls && entity.layout.format == 'html') {
+                                pageEntity.destination = path.join(entity.destination.replace('index', ''), pageEntity.page.toString(), `index.${entity.layout.format}`)
+                            } else {
+                                pageEntity.destination += page ? `.${pageEntity.page}.${entity.layout.format}` : `.${entity.layout.format}`
+                            }
+                        } else {
+                            removePagesFromSitemap(original)
+                            pageEntity.page = 1
+                            if (runtime.config.layouts?.cleanUrls && !_.endsWith(entity.name, 'index') && entity.layout.format == 'html') {
+                                pageEntity.destination = path.join(entity.destination, `index.${entity.layout.format}`)
+                            } else {
+                                pageEntity.destination += `.${entity.layout.format}`
+                            }
+                        }
+                        addToSitemap(pageEntity)
+                        tasks.push({
+                            entity: pageEntity,
+                            options: {
+                                renderer: entity.layout.template,
+                                postprocessor: entity.layout.postprocessor,
+                                tasks: entity.meta?.task || TASKS.POOL
+                            },
+                            context: { data, plugins }
+                        })
+                    }
+                }
+            } else {
+                removePagesFromSitemap(original)
+                if (!_.endsWith(entity.name, entity.format)) {
+                    if (runtime.config.layouts?.cleanUrls && !_.endsWith(entity.name, 'index') && entity.layout.format == 'html') {
+                        entity.destination = path.join(entity.destination, `index.${entity.layout.format}`)
+                    } else {
+                        entity.destination += `.${entity.layout.format}`
+                    }
+                }
+                addToSitemap(entity)
+                if (entity.destination) {
+                    tasks.push({
+                        entity,
+                        options: {
+                            renderer: entity.layout.template,
+                            postprocessor: entity.layout.postprocessor,
+                            tasks: entity.meta?.task || TASKS.POOL
+                        },
+                        context: { data, plugins }
+                    })
+                }
+            }
+        }
+        await renderEntities(tasks)
+    })
+
+    onComplete(async ({ entity, options, output }) => {
+        const logger = useLogger()
+        if (entity.layout && !options?.ignore && output.result != null) {
+            const destinationFile = path.join(runtime.options.outputFolder, entity.destination)
+            await mkdir(path.dirname(destinationFile), { recursive: true })
+            try {
+                await unlink(destinationFile)
+            } catch { }
+            await writeFile(destinationFile, output.result)
+            logger.debug('Layout render finished: %s', entity.destination.replace(runtime.options.workingFolder, ''))
+            if (entity.origin) {
+                const originFile = path.join(runtime.options.outputFolder, entity.origin)
+                try {
+                    await unlink(originFile)
+                } catch { }
+            }
+        }
+    })
+
+    return {
+        collection,
+        type
+    }
+}

package/src/plugins/mapper.js

@@ -0,0 +1,29 @@
+import _ from 'lodash'
+
+export default ({
+    onProcess,
+    useLogger,
+    useJournal,
+    updateEntry,
+    matchEntity,
+    runtime,
+    constants: { OPERATION },
+}) => {
+    onProcess(async (signal) => {
+        const logger = useLogger()
+
+        for (let { match, map, operations = [OPERATION.CREATE, OPERATION.UPDATE] } of runtime.config.mapper?.mappers || []) {
+            for await (let { id, entity } of useJournal('Mapper', operations, signal)) {
+                if (entity && matchEntity(entity, match)) {
+                    logger.trace('Mapper: %s', entity.id)
+                    try {
+                        await map(entity)
+                        await updateEntry({ id, entity })
+                    } catch (err) {
+                        logger.error('Mapper error: %s %s', entity.name || entity.id, err.message)
+                    }
+                }
+            }
+        }
+    })
+}

package/src/plugins/post/pdf.js

@@ -0,0 +1,60 @@
+import path from 'node:path'
+
+const TEARDOWN_DELAY = 60_000
+
+let browser
+let teardownTimer
+
+export async function setup({ config, logger }) {
+    if (teardownTimer) {
+        clearTimeout(teardownTimer)
+        teardownTimer = undefined
+        logger.debug('Puppeteer browser reused')
+        return
+    }
+
+    const { default: puppeteer } = await import('puppeteer').catch(() => {
+        throw new Error('puppeteer is required for the pdf postprocessor — run: npm install puppeteer')
+    })
+    browser = await puppeteer.launch({
+        headless: true,
+        ...config?.launch
+    })
+    logger.debug('Puppeteer browser launched')
+}
+
+export async function postprocess({ entity, options, config, logger }) {
+    const sourcePath = path.join(options.outputFolder, entity.origin)
+
+    const page = await browser.newPage()
+    try {
+        await page.goto(`file://${sourcePath}`, {
+            waitUntil: 'networkidle0',
+            ...config?.navigation
+        })
+        return await page.pdf({
+            format: 'A4',
+            printBackground: true,
+            ...config?.pdf
+        })
+    } finally {
+        await page.close()
+    }
+}
+
+export async function teardown({ options, config, logger }) {
+    if (!options?.watch) {
+        await browser?.close()
+        browser = undefined
+        logger.debug('Puppeteer browser closed')
+        return
+    }
+    const delay = config?.teardownDelay ?? TEARDOWN_DELAY
+    teardownTimer = setTimeout(async () => {
+        teardownTimer = undefined
+        await browser?.close()
+        browser = undefined
+        logger.debug('Puppeteer browser closed')
+    }, delay)
+    logger.debug('Puppeteer browser teardown scheduled in %dms', delay)
+}

package/src/plugins/render/asset.js

@@ -0,0 +1,11 @@
+import path from 'node:path'
+
+export function load({ runtime, entity, state, options }) {
+    runtime.asset = (preset, url, format) => {
+        if (url[0] != '/') url = `/${url}`
+        const relative = `${state.assets.assetsFolder}/${preset}${format ? url.split('.').slice(0, -1).concat(format).join('.') : url}`
+        const destination = '/' + relative
+        const from = path.dirname(entity.destination || '/')
+        return { url: path.relative(from, destination) }
+    }
+}

package/src/plugins/render/file.js

@@ -0,0 +1,16 @@
+import { readFileSync } from 'node:fs'
+import { globby } from 'globby'
+
+export function load({ runtime }) {
+    runtime.readFile = (file) => {
+        const relativePath = file.name || file
+        return readFileSync(relativePath, { encoding: 'utf8' })
+    }
+    runtime.jsonFile = (file) => {
+        const relativePath = file.name || file
+        return JSON.parse(readFileSync(relativePath, { encoding: 'utf8' }))
+    }
+    runtime.glob = (pattern, options = {}) => {
+        return globby.sync(pattern, options)
+    }
+}