mikser-io 6.0.0

package/src/plugins/render/hbs.js

@@ -0,0 +1,52 @@
+import handlebars from 'handlebars'
+import helpers from 'handlebars-helpers'
+import { readFile } from 'fs/promises'
+
+export function load({ config, runtime, context }) {
+    handlebars.registerHelper(helpers(config?.helpers || [
+        'array',
+        'collection',
+        'object',
+        'comparison',
+        'date',
+        'markdown',
+        'match',
+        'math',
+        'number',
+        'regex',
+        'string',
+        'url'
+    ]))
+    for (let partial in context.layouts) {
+        if (context.layouts[partial].template == 'hbs' && partial.indexOf('partials') == 0) {
+            const partialLayout = readFile(context.layouts[partial].uri, 'utf8')
+            handlebars.registerPartial(partial, partialLayout)
+        }
+    }
+    handlebars.registerHelper('url', function(obj, options) {
+        // Called as {{url}} with no args — obj is the Handlebars options object,
+        // so read url from the current context (this)
+        if (!options) return this?.url ?? ''
+        if (!obj) return ''
+        if (typeof obj === 'string') return obj
+        return obj.url ?? ''
+    })
+
+    runtime.hbs = (source, sandbox) => {
+        const template = handlebars.compile(source)
+        return template(sandbox)
+    }
+}
+
+export async function render({ entity, runtime }) {
+    const source = await readFile(entity.layout.uri, 'utf8')
+    const sandbox = {}
+    for (let helper in runtime) {
+        if (typeof (runtime[helper]) == 'function') {
+            handlebars.registerHelper(helper, runtime[helper])
+        } else {
+            sandbox[helper] = runtime[helper]
+        }
+    }
+    return runtime.hbs(source, sandbox)
+}

package/src/plugins/render/href.js

@@ -0,0 +1,45 @@
+import path from 'node:path'
+
+export function load({ entity, runtime, state, options }) {
+    const { clear } = options
+
+    runtime.hrefLang = (href) => {
+        const { sitemap } = state.layouts
+        return sitemap[href]
+    }
+
+    runtime.hrefLangPage = (href, page) => {
+        if (page > 1) href += `.${page}`
+        return sitemap[href]
+    }
+
+    runtime.href = (href, lang) => {
+        if (!href || typeof href != 'string') return
+        if (typeof lang != 'string') lang = undefined
+
+        if (href.indexOf('http') == 0) return href
+        lang ||= entity.meta?.lang
+
+        let found = runtime.hrefLang(href)
+        if (!found) {
+            const from = path.dirname(entity.destination || '/')
+            return { url: path.relative(from, href) }
+        } else {
+            if (!found.id) {
+                found = found[lang]
+            }
+            if (found?.destination) {
+                const destination = clear ? found.destination.replace('index.html', '') : found.destination
+                const from = path.dirname(entity.destination || '/')
+                found.url = path.relative(from, destination)
+            }
+            return found
+        }
+    }
+
+    runtime.hrefPage = (href, page, lang) => {
+    }
+
+    runtime.prev = entity.page > 1 ? entity.page - 1 : false
+    runtime.next = entity.page + 1 < entity.pages ? entity.page + 1 : false
+}

package/src/plugins/render/preset.js

@@ -0,0 +1,13 @@
+import { mkdir } from 'node:fs/promises'
+import path from 'node:path'
+
+export async function load({ entity, runtime }) {
+    const preset = await import(`${entity.preset.uri}?stamp=${Date.now()}`)
+    runtime.preset = preset.default
+}
+
+export async function render({ entity, options, config, context, plugins, runtime, state, logger }) {
+    await mkdir(path.dirname(entity.destination), { recursive: true })
+    await runtime.preset({ entity, options, config, context, plugins, runtime, state, logger })
+    return entity.destination
+}

package/src/plugins/render/resource.js

@@ -0,0 +1,17 @@
+import path from 'node:path'
+
+export function load({ runtime, entity, state, options }) {
+    runtime.resource = (url) => {
+        const { resourceLib } = state.resources
+        for (let library in resourceLib) {
+            if (url.match(library)) {
+                const { origin } = new URL(url)
+                const name = url.replace(origin, `${resourceLib[library]}`)
+                const relative = url.replace(origin, `${state.resources.resourcesFolder}/${resourceLib[library]}`)
+                const destination = '/' + relative
+                const from = path.dirname(entity.destination || '/')
+                return { url: path.relative(from, destination), name }
+            }
+        }
+    }
+}

package/src/plugins/resources.js

@@ -0,0 +1,216 @@
+import { mkdir, symlink, rename, unlink } from 'node:fs/promises'
+import { createWriteStream } from 'node:fs'
+import lodash from 'lodash'
+import deepdash from 'deepdash'
+import axios from 'axios'
+import path from 'node:path'
+import { globby } from 'globby'
+import escapeStringRegexp from 'escape-string-regexp'
+import * as stream from 'stream'
+import { promisify } from 'util'
+import isUrl from 'is-url'
+import map from 'p-map'
+
+export default ({
+    useLogger,
+    useJournal,
+    onLoaded,
+    runtime,
+    stopProgress,
+    createEntity,
+    onProcessed,
+    onFinalize,
+    checksum,
+    trackProgress,
+    updateProgress,
+    updateEntry,
+    constants: { OPERATION },
+}) => {
+    const collection = 'resources'
+    const type = 'resource'
+
+    const _ = deepdash(lodash)
+
+    const finishedDownload = promisify(stream.finished)
+
+    onLoaded(async () => {
+        const logger = useLogger()
+
+        const resourcesName = runtime.config.resources?.resourcesFolder || collection
+        runtime.state.resources = {
+            resourceLib: {},
+            resourceMap: {},
+            resourcesFolder: runtime.config.resources?.outputFolder
+                ? path.join(runtime.config.resources.outputFolder, resourcesName)
+                : resourcesName,
+        }
+
+        runtime.options.resources = runtime.config.resources?.resourcesFolder || collection
+        runtime.options.resourcesFolder = path.join(runtime.options.workingFolder, runtime.options.resources)
+        logger.info('Resources folder: %s', runtime.options.resourcesFolder)
+
+        for (let library in (runtime.config.resources?.libraries || [])) {
+            let resource = runtime.config.resources.libraries[library]
+            runtime.state.resources.resourceLib[resource.match || escapeStringRegexp(resource.url)] = library
+        }
+    })
+
+    onProcessed(async (signal) => {
+        const logger = useLogger()
+        const { resourceLib, resourceMap } = runtime.state.resources
+
+        for await (let { id, entity } of useJournal('Resources provision', [OPERATION.CREATE, OPERATION.UPDATE], signal)) {
+            if (entity.collection != collection && entity.meta) {
+                resourceMap[entity.id] = []
+                _.eachDeep(entity.meta, resource => {
+                    if (typeof resource == 'string') {
+                        for (let library in resourceLib) {
+                            const match = new RegExp(library)
+                            if (resource.match(match)) {
+                                resourceMap[entity.id].push({ library, resource, entity })
+                            }
+                        }
+                    }
+                })
+                entity.resources = resourceMap[entity.id].map(({ resource }) => resource)
+                await updateEntry({ id, entity })
+            }
+        }
+        const resources = [].concat(...Object.values(resourceMap))
+        resources.length && logger.info('Resources: %d', resources.length)
+
+        const resourceDownloads = {}
+        const localResources = new Set()
+        trackProgress('Resources processing', resources.length)
+        for (let { library, resource, entity } of resources) {
+            if (signal?.aborted) {
+                stopProgress()
+                break
+            }
+
+            library = resourceLib[library]
+            if (isUrl(resource)) {
+                if (!resourceDownloads[resource]) {
+                    resourceDownloads[resource] = { library, entity }
+                }
+            } else {
+                try {
+                    const id = resource.indexOf(`/${library}`) == 0 ? resource : path.join(`/${library}`, resource)
+                    if (!localResources.has(id)) {
+                        await createEntity({
+                            id,
+                            uri: path.join(runtime.options.workingFolder, resource),
+                            collection,
+                            type,
+                            format: path.extname(resource).substring(1).toLowerCase(),
+                            name: resource.indexOf('/') == 0 ? resource.substring(1) : resource,
+                            source: path.join(runtime.options.workingFolder, resource),
+                            checksum: await checksum(path.join(runtime.options.workingFolder, resource))
+                        })
+                        logger.debug('Resource: %s %s', id, resource)
+                        localResources.add(id)
+                    }
+                } catch (err) {
+                    logger.error('Resource error: %s %s %s', entity.id, resource, err.message)
+                }
+            }
+            updateProgress()
+        }
+
+        const resourceFiles = await globby('**/*', { cwd: runtime.options.resourcesFolder })
+        const resourceFilesMap = new Set()
+        for (let resourceFile of resourceFiles) {
+            resourceFilesMap.add(resourceFile)
+        }
+
+        const downloads = Object.keys(resourceDownloads)
+        if (downloads.length) {
+            trackProgress('Resources download', downloads.length)
+            await mkdir(runtime.options.resourcesFolder, { recursive: true })
+            let link = path.join(runtime.options.outputFolder, runtime.options.resources)
+            if (runtime.config.resources?.outputFolder) link = path.join(runtime.options.outputFolder, runtime.config.resources?.outputFolder, runtime.options.resources)
+            try {
+                await mkdir(path.dirname(link), { recursive: true })
+                await symlink(path.resolve(runtime.options.resourcesFolder), link, 'dir')
+            } catch (err) {
+                if (err.code != 'EEXIST')
+                    throw err
+            }
+            let count = 0
+            await map(downloads, async url => {
+                const { library, entity } = resourceDownloads[url]
+                let { pathname } = new URL(url)
+                pathname = decodeURI(pathname)
+                const resource = path.join(runtime.options.resourcesFolder, library, pathname)
+                const uri = path.join(runtime.options.outputFolder, library, pathname)
+
+                let success = true
+                if (!resourceFilesMap.has(path.join(library, pathname))) {
+                    const resourceTemp = path.join(runtime.options.resourcesFolder, library, pathname + '.temp')
+                    logger.debug('Downloading resource: %s %s', entity.id, url)
+                    const config = runtime.config.resources.libraries[library]
+                    const request = {
+                        method: 'get',
+                        ...config,
+                        url,
+                        responseType: 'stream',
+                        signal
+                    }
+
+                    try {
+                        count++
+                        var response = await axios(request)
+                    } catch (err) {
+                        success == false
+                        if (axios.isCancel(err)) {
+                            logger.trace('Downloading canceled')
+                        } else {
+                            logger.error('Resource error: %s %s %s', entity.id, url, err.message)
+                        }
+                        return
+                    }
+
+                    if (response && success) {
+                        await mkdir(path.dirname(resource), { recursive: true })
+                        const writer = createWriteStream(resourceTemp)
+                        response.data.pipe(writer)
+                        await finishedDownload(writer)
+
+                        logger.debug('Resource: %s %s', entity.id, url)
+                        await rename(resourceTemp, resource)
+                    }
+                }
+
+                if (success) {
+                    await createEntity({
+                        id: path.join('/resources', library, pathname),
+                        uri,
+                        collection,
+                        type,
+                        format: path.extname(resource).substring(1).toLowerCase(),
+                        name: path.join(library, pathname),
+                        source: resource,
+                        checksum: await checksum(resource)
+                    })
+                }
+                updateProgress()
+            }, { concurrency: 10, signal })
+            count && logger.info('Downloaded: %d', count)
+        }
+    })
+
+    onFinalize(async () => {
+        runtime.state.resources.resourceMap = {}
+
+        const paths = await globby('**/*.temp', { cwd: runtime.options.resourcesFolder })
+        for (let relativePath of paths) {
+            let resourceTemp = path.join(runtime.options.resourcesFolder, relativePath)
+            await unlink(resourceTemp)
+        }
+    })
+
+    return {
+        collection,
+        type
+    }
+}

package/src/plugins/rest.js

@@ -0,0 +1,143 @@
+import path from 'node:path'
+import { writeFile, unlink, mkdir, access } from 'node:fs/promises'
+import { updateEntity } from '../lifecycle.js'
+import { findEntities } from '../catalog.js'
+
+export default ({
+    runtime,
+    onLoaded,
+    useLogger,
+}) => {
+    onLoaded(async () => {
+        const logger = useLogger()
+
+        const { default: express } = await import('express').catch(() => {
+            throw new Error('express is required for the rest plugin — run: npm install express')
+        })
+
+        const ownApp = !runtime.options.app
+        const app = runtime.options.app ?? express()
+
+        const router = express.Router()
+        router.use(express.json())
+
+        const token = runtime.config.rest?.token
+        const auth = (req, res, next) => {
+            if (!token) return next()
+            if (req.headers.authorization === `Bearer ${token}`) return next()
+            res.status(401).json({ error: 'Unauthorized' })
+        }
+
+        function collectionFolder(collection) {
+            return runtime.options[`${collection}Folder`]
+        }
+
+        router.get('/entities', async (req, res) => {
+            try {
+                const { page: rawPage, limit: rawLimit, ...filter } = req.query
+                const page = Math.max(1, parseInt(rawPage) || 1)
+                const limit = Math.min(100, Math.max(1, parseInt(rawLimit) || runtime.config.rest?.pageSize ?? 10))
+                const query = Object.keys(filter).length ? filter : undefined
+
+                const all = await findEntities(query)
+                const total = all.length
+                const totalPages = Math.ceil(total / limit)
+                const items = all.slice((page - 1) * limit, page * limit)
+
+                res.json({
+                    items,
+                    page,
+                    limit,
+                    total,
+                    totalPages,
+                    hasNext: page < totalPages,
+                    hasPrev: page > 1,
+                })
+            } catch (err) {
+                logger.error('REST list error: %s', err.message)
+                res.status(500).json({ error: err.message })
+            }
+        })
+
+        router.put('/entities', auth, async (req, res) => {
+            try {
+                const { collection, relativePath, content = '' } = req.body
+                const folder = collectionFolder(collection)
+                if (!folder) return res.status(400).json({ error: `Unknown collection: ${collection}` })
+                const uri = path.join(folder, relativePath)
+                await mkdir(path.dirname(uri), { recursive: true })
+                await writeFile(uri, content, 'utf8')
+                res.status(202).json({ ok: true })
+            } catch (err) {
+                logger.error('REST update error: %s', err.message)
+                res.status(500).json({ error: err.message })
+            }
+        })
+
+        router.delete('/entities', auth, async (req, res) => {
+            try {
+                const { collection, relativePath } = req.body
+                const folder = collectionFolder(collection)
+                if (!folder) return res.status(400).json({ error: `Unknown collection: ${collection}` })
+                const uri = path.join(folder, relativePath)
+                await unlink(uri)
+                res.status(202).json({ ok: true })
+            } catch (err) {
+                logger.error('REST delete error: %s', err.message)
+                res.status(500).json({ error: err.message })
+            }
+        })
+
+        router.post('/render', async (req, res) => {
+            try {
+                const entity = { ...req.body, _correlationId: crypto.randomUUID() }
+                const timeout = runtime.config.rest?.renderTimeout ?? 30_000
+
+                const output = await new Promise((resolve, reject) => {
+                    const timer = setTimeout(() => {
+                        runtime.removeHook('completed', hook)
+                        reject(new Error(`Render timeout for ${entity.id}`))
+                    }, timeout)
+
+                    const hook = async (entry) => {
+                        if (entry.entity._correlationId !== entity._correlationId) return
+                        clearTimeout(timer)
+                        runtime.removeHook('completed', hook)
+                        resolve(entry.output)
+                    }
+                    runtime.addHook('completed', hook)
+
+                    updateEntity(entity)
+                        .then(() => runtime.process())
+                        .catch(reject)
+                })
+
+                if (output?.result != null) {
+                    const isFile = await access(output.result).then(() => true).catch(() => false)
+                    if (isFile) {
+                        res.sendFile(output.result)
+                    } else {
+                        res.send(output.result)
+                    }
+                } else {
+                    res.status(204).send()
+                }
+            } catch (err) {
+                logger.error('REST render error: %s', err.message)
+                res.status(500).json({ error: err.message })
+            }
+        })
+
+        const base = runtime.config.rest?.base ?? '/mikser'
+        app.use(base, router)
+
+        if (ownApp) {
+            const port = runtime.config.rest?.port ?? 3001
+            app.listen(port, () => {
+                logger.info('REST plugin listening on port %d', port)
+            })
+        } else {
+            logger.info('REST plugin mounted on %s', base)
+        }
+    })
+}

package/src/plugins/shares.js

@@ -0,0 +1,40 @@
+import { mkdir, symlink, stat } from 'node:fs/promises'
+import path from 'node:path'
+
+export default ({
+    useLogger,
+    onLoaded,
+    runtime
+}) => {
+    onLoaded(async () => {
+        const logger = useLogger()
+
+        for (let item of (runtime.config.shares?.locations || [])) {
+            let source, destination
+            if (typeof item == 'string') {
+                source = destination = item
+            } else {
+                source = item.source
+                destination = item.destination
+            }
+
+            let destinationLocation = path.join(runtime.options.outputFolder, destination)
+            let destinationFolder = path.dirname(destinationLocation)
+            try {
+                const sourceLocation = path.join(runtime.options.workingFolder, source)
+                const sourceStat = await stat(sourceLocation)
+
+                await mkdir(destinationFolder, { recursive: true })
+                logger.info('Sharing: %s → %s', sourceLocation, destinationLocation)
+                if (sourceStat.isDirectory()) {
+                    await symlink(path.resolve(sourceLocation), destinationLocation, 'dir')
+                } else {
+                    await symlink(path.resolve(sourceLocation), destinationLocation, 'file')
+                }
+            } catch (err) {
+                if (err.code != 'EEXIST')
+                    throw err
+            }
+        }
+    })
+}

package/src/plugins/validator.js

@@ -0,0 +1,19 @@
+import _ from 'lodash'
+
+export default ({
+    onLoad,
+    onValidate,
+    runtime,
+    matchEntity,
+    constants: { OPERATION },
+}) => {
+    onLoad(() => {
+        for (let { match, validate, operations = [OPERATION.CREATE, OPERATION.UPDATE] } of runtime.config.validator?.validators || []) {
+            onValidate(operations, async entry => {
+                if (entity.meta && matchEntity(entry.entity, match)) {
+                    return await validate(entry.entity)
+                }
+            })
+        }
+    })
+}

package/src/plugins/yaml.js

@@ -0,0 +1,26 @@
+import YAML from 'yaml'
+
+export default ({
+    onProcess,
+    useLogger,
+    useJournal,
+    updateEntry,
+    constants: { OPERATION },
+}) => {
+    onProcess(async (signal) => {
+        const logger = useLogger()
+
+        for await (let { id, entity } of useJournal('Yaml', [OPERATION.CREATE, OPERATION.UPDATE], signal)) {
+            if (entity.content && (entity.format == 'yml' || entity.format == 'yaml')) {
+                try {
+                    entity.meta = Object.assign(entity.meta || {}, YAML.parse(entity.content))
+                    delete entity.content
+                    logger.trace('Yaml %s: %s', entity.collection, entity.id)
+                    await updateEntry({ id, entity })
+                } catch (err) {
+                    logger.error('Yaml error %s: %s %s', entity.collection, entity.id, err.message)
+                }
+            }
+        }
+    })
+}

package/src/plugins.js

@@ -0,0 +1,54 @@
+import { useLogger } from './engine.js'
+import { onLoad } from './lifecycle.js'
+import runtime from './runtime.js'
+import path from 'node:path'
+import fs from 'fs'
+
+import * as core from '../index.js'
+
+export async function loadPlugin(pluginName) {
+    const logger = useLogger()
+
+    const resolveLocations = [
+        path.join(path.dirname(import.meta.url), 'plugins', `${pluginName}.js`),
+        path.join(runtime.options.workingFolder, 'plugins', `${pluginName}.js`),
+        path.join(runtime.options.workingFolder, 'node_modules', `mikser-io-${pluginName}`, 'index.js'),
+    ]
+    for (let index = 0; index < resolveLocations.length; index++) {
+        const resolveLocation = resolveLocations[index]
+        if (fs.existsSync(resolveLocation.replace('file:', ''))) {
+            try {
+                const plugin = await import(resolveLocation)
+                const pluginRuntime = plugin.default(core)
+                runtime.engine[pluginName] = pluginRuntime
+                if (pluginRuntime) {
+                    logger.trace('Loaded %s plugin: %s', pluginName, pluginRuntime)
+                } else {
+                    logger.trace('Loaded %s plugin', pluginName)
+                }
+                return
+            } catch (err) {
+                logger.error('Plugin load error: [%s] %s', pluginName, err.message)
+                return
+            }
+        }
+    }
+    logger.error('Plugin not loaded: %s', pluginName)
+}
+
+onLoad(async () => {
+    const logger = useLogger()
+
+    runtime.options.plugins = runtime.options.plugins.concat(runtime.config.plugins).filter(plugin => plugin)
+
+    const userPlugins = runtime.options.plugins.filter(plugin => plugin.indexOf('render-') != 0 && plugin.indexOf('post-') != 0)
+    if (!userPlugins.length) {
+        logger.info('No plugins loaded')
+    } else {
+        logger.info('Loading plugins: %s', userPlugins)
+
+        for (let plugin of userPlugins) {
+            await loadPlugin(plugin)
+        }
+    }
+})

package/src/postprocess.js

@@ -0,0 +1,47 @@
+import path from 'node:path'
+import _ from 'lodash'
+
+export async function loadPlugin(pluginName, workingFolder) {
+    const resolveLocations = [
+        path.join(workingFolder, 'node_modules', `mikser-core-${pluginName}/index.js`),
+        path.join(workingFolder, 'plugins', `${pluginName}.js`),
+        path.join(path.dirname(import.meta.url), 'plugins', 'post', `${pluginName.replace('post-', '')}.js`)
+    ]
+    for (let resolveLocation of resolveLocations) {
+        try {
+            return await import(resolveLocation)
+        } catch (err) {
+            if (err.code != 'ERR_MODULE_NOT_FOUND') throw err
+        }
+    }
+}
+
+export default async ({ entity, options, config, context, state, logger }) => {
+
+    const { postprocessor } = options
+    const plugins = {}
+    let pluginsToLoad = [...context.plugins || []]
+    pluginsToLoad.push(`post-${postprocessor}`)
+    if (entity.meta?.plugins) {
+        pluginsToLoad.push(...entity.meta.plugins)
+    }
+    pluginsToLoad.push(...options.plugins)
+    pluginsToLoad = _.uniq(pluginsToLoad.filter(pluginName => pluginName && pluginName.indexOf('post-') == 0))
+
+    const runtime = {
+        [entity.type]: entity,
+        entity,
+        plugins,
+        config: config[`post-${postprocessor}`],
+        data: context.data,
+    }
+
+    for (let pluginName of pluginsToLoad) {
+        const plugin = await loadPlugin(pluginName, options.workingFolder)
+        plugins[pluginName] = plugin
+        if (plugin?.load) await plugin.load({ entity, options, config: config[pluginName], context, runtime, state, logger })
+    }
+
+    const postprocessorPlugin = plugins[`post-${postprocessor}`]
+    return await postprocessorPlugin?.postprocess({ entity, options, config, context, plugins, runtime, state, logger })
+}