mikser-io 6.0.0

package/src/render.js

@@ -0,0 +1,71 @@
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+import _ from 'lodash'
+
+export default async ({ entity, options, config, context, state, logger, port }) => {
+    logger = logger || {
+        info(...args) {
+            port.postMessage(JSON.stringify({ command: 'logger', data: { log: 'info', args } }))
+        },
+        warn(...args) {
+            port.postMessage(JSON.stringify({ command: 'logger', data: { log: 'warn', args } }))
+        },
+        error(...args) {
+            port.postMessage(JSON.stringify({ command: 'logger', data: { log: 'error', args } }))
+        },
+        trace(...args) {
+            port.postMessage(JSON.stringify({ command: 'logger', data: { log: 'trace', args } }))
+        },
+        notice(...args) {
+            port.postMessage(JSON.stringify({ command: 'logger', data: { log: 'notice', args } }))
+        }
+    }
+
+    async function loadPlugin(pluginName) {
+        const resolveLocations = [
+            path.join(options.workingFolder, 'node_modules', `mikser-core-${pluginName}/index.js`),
+            path.join(options.workingFolder, 'plugins', `${pluginName}.js`),
+            path.join(path.dirname(import.meta.url), 'plugins', 'render', `${pluginName.replace('render-', '')}.js`)
+        ]
+        for (let resolveLocation of resolveLocations) {
+            try {
+                return await import(resolveLocation)
+            } catch (err) {
+                if (err.code != 'ERR_MODULE_NOT_FOUND') {
+                    logger.error('Redner plugin error:', resolveLocation, err)
+                    throw err
+                }
+            }
+        }
+    }
+
+    const { renderer } = options
+    const plugins = {}
+    let pluginsToLoad = [...context.plugins || []]
+    pluginsToLoad.push(`render-${renderer}`)
+    if (entity.meta?.plugins) {
+        pluginsToLoad.push(...entity.meta.plugins)
+    }
+    pluginsToLoad.push(...options.plugins)
+    pluginsToLoad = _.uniq(pluginsToLoad.filter(pluginName => pluginName && pluginName.indexOf('render-') == 0))
+
+    const runtime = {
+        [entity.type]: entity,
+        entity,
+        plugins,
+        config: config[`render-${renderer}`],
+        data: context.data,
+        content() {
+            return readFileSync(entity.source, { encoding: 'utf8' })
+        }
+    }
+
+    for (let pluginName of pluginsToLoad) {
+        const plugin = await loadPlugin(pluginName)
+        plugins[pluginName] = plugin
+        if (plugin?.load) await plugin.load({ entity, options, config: config[pluginName], context, runtime, state, logger })
+    }
+
+    const rendererPlugin = plugins[`render-${renderer}`]
+    return await rendererPlugin?.render({ entity, options, config, context, plugins, runtime, state, logger })
+}

package/src/runtime.js

@@ -0,0 +1,153 @@
+import { Mutex } from 'await-semaphore'
+import { AbortError } from './utils.js'
+
+const runtime = {
+    stamp: Date.now(),
+    processTime: undefined,
+    engine: {},
+    options: {
+        plugins: []
+    },
+    config: {},
+    journal: [],
+    validators: [],
+    started: false,
+    mutex: new Mutex(),
+    abortController: undefined,
+    hooks: {
+        initialize: [],
+        initialized: [],
+        load: [],
+        loaded: [],
+        import: [],
+        validate: [],
+        imported: [],
+        process: [],
+        processed: [],
+        persist: [],
+        persisted: [],
+        beforeRender: [],
+        render: [],
+        afterRender: [],
+        beforePostprocess: [],
+        postprocess: [],
+        afterPostprocess: [],
+        cancel: [],
+        cancelled: [],
+        finalize: [],
+        finalized: [],
+        sync: [],
+        completed: [],
+    },
+
+    async callHooks(hooks, signal) {
+        for (let hook of hooks) {
+            if (signal?.aborted) throw new AbortError()
+            await hook(signal)
+        }
+    },
+
+    addHook(name, hook) {
+        if (!this.hooks[name]) throw new Error(`Unknown hook: ${name}`)
+        this.hooks[name].push(hook)
+        return hook
+    },
+
+    removeHook(name, hook) {
+        if (!this.hooks[name]) throw new Error(`Unknown hook: ${name}`)
+        const idx = this.hooks[name].indexOf(hook)
+        if (idx > -1) this.hooks[name].splice(idx, 1)
+    },
+
+    async start() {
+        await this.callHooks(this.hooks.initialize)
+        await this.callHooks(this.hooks.initialized)
+        await this.callHooks(this.hooks.load)
+        await this.callHooks(this.hooks.loaded)
+
+        await this.callHooks(this.hooks.import)
+        await this.callHooks(this.hooks.imported)
+
+        this.started = true
+        await this.process()
+    },
+
+    async process() {
+        if (this.abortController?.signal.aborted) return
+        else if (this.abortController) {
+            await this.cancel()
+        }
+        this.mutex.use(async () => {
+            try {
+                this.abortController = new AbortController()
+                const { signal } = this.abortController
+
+                await this.callHooks(this.hooks.process, signal)
+                await this.callHooks(this.hooks.processed, signal)
+                await this.callHooks(this.hooks.persist, signal)
+                await this.callHooks(this.hooks.persisted, signal)
+
+                await this.render(signal)
+            } catch (e) {
+                if (e.name !== 'AbortError') throw e
+                for (let hook of this.hooks.cancelled) await hook()
+            }
+        })
+    },
+
+    async render(signal) {
+        await this.callHooks(this.hooks.beforeRender, signal)
+        await this.callHooks(this.hooks.render, signal)
+        await this.callHooks(this.hooks.afterRender, signal)
+
+        await this.postprocess(signal)
+    },
+
+    async postprocess(signal) {
+        await this.callHooks(this.hooks.beforePostprocess, signal)
+        await this.callHooks(this.hooks.postprocess, signal)
+        await this.callHooks(this.hooks.afterPostprocess, signal)
+
+        await this.finalize(signal)
+    },
+
+    async cancel() {
+        this.abortController?.abort()
+        await this.callHooks(this.hooks.cancel)
+    },
+
+    async finalize(signal) {
+        await this.callHooks(this.hooks.finalize, signal)
+        await this.callHooks(this.hooks.finalized, signal)
+    },
+
+    async sync(operation) {
+        let synced
+        for (let hook of this.hooks.sync) {
+            const result = await hook(operation)
+            if (result === true) {
+                synced = true
+            } else if (result === false && !synced) {
+                synced = false
+            }
+        }
+        return synced === undefined || synced
+    },
+
+    async validate(entry) {
+        for (let hook of this.validators) {
+            if (!await hook(entry)) return false
+        }
+        return true
+    },
+
+    async complete(entry) {
+        let success = true
+        for (let hook of this.hooks.completed) {
+            if (await hook(entry) === false) success = false
+        }
+        entry.success = success
+    }
+}
+
+export default runtime

package/src/tracking.js

@@ -0,0 +1,74 @@
+import cliProgress from 'cli-progress'
+import runtime from './runtime.js'
+import { useLogger } from './engine.js'
+import formatTime from 'cli-progress/lib/format-time.js'
+import { onInitialized } from './lifecycle.js'
+import util from 'util'
+
+let progress = {}
+
+function log(log) {
+    const { bar, total, value, name } = progress
+    const isActive = bar?.isActive
+    isActive && bar?.stop()
+    log()
+    isActive && bar?.start(total, value, { name, details: '' })
+}
+
+onInitialized(() => {
+    const logger = useLogger()
+    if (runtime.options.info) {
+        logger.info = (...args) => log(() => console.log(...args))
+        logger.warn = (...args) => log(() => console.log('🟡 ' + args[0], ...args.slice(1)))
+        logger.error = (...args) => log(() => console.log('🔴 ' + args[0], ...args.slice(1)))
+        logger.notice = (...args) => log(() => console.log('🟢 ' + args[0], ...args.slice(1)))
+        logger.trace = (...args) => updateProgressDetails(util.format(...args))
+    }
+})
+
+export function trackProgress(name, total) {
+    const logger = useLogger()
+    logger.debug('%s started: %d', name, total)
+    progress.bar?.stop()
+    progress = {
+        name,
+        total,
+        value: 0,
+        stamp: Date.now(),
+        bar: runtime.options.info ? new cliProgress.SingleBar({
+            noTTYOutput: true,
+            hideCursor: true,
+            clearOnComplete: true,
+            barsize: 30,
+            format: '{name}: {bar} {percentage}% | ETA: {eta_formatted} {details}',
+        }, cliProgress.Presets.shades_grey) : null
+    }
+    progress.bar?.start(total, 0, { name, details: '' })
+}
+
+export function stopProgress() {
+    const logger = useLogger()
+    const { value, total, bar, name } = progress
+    bar?.stop()
+    if (value < total) {
+        logger.warn('%s unfinished: %d', name, total - value)
+    } else {
+        const time = Math.round((Date.now() - progress.stamp) / 1000)
+        logger.info('%s: %s', name, formatTime(time, { autopaddingChar: '' }))
+    }
+    progress = {}
+}
+
+export function updateProgress() {
+    progress.name && progress.value++
+    progress.bar?.increment()
+    if (progress?.value == progress?.total) {
+        stopProgress()
+    }
+}
+
+export function updateProgressDetails(details) {
+    const logger = useLogger()
+    logger.debug(details)
+    progress.bar?.update({ details: `| ${details}` })
+}

package/src/utils.js

@@ -0,0 +1,65 @@
+import { hashFile, hashingStream } from 'hasha'
+import { stat } from 'node:fs/promises'
+import TruncateStream from 'truncate-stream'
+import { createReadStream } from 'node:fs'
+import _ from 'lodash'
+import { minimatch } from 'minimatch'
+import path from 'path'
+
+export class AbortError extends Error {
+    constructor(message) {
+        super();
+        this.name = 'AbortError';
+        this.message = message;
+    }
+}
+
+export async function checksum(uri) {
+    const maxBytes = 300 * 1024
+    const { size } = await stat(uri)
+    if (size < maxBytes) {
+        return await hashFile(uri, { algorithm: 'md5' })
+    } else {
+        const truncate = new TruncateStream({ maxBytes })
+        const fileStream = createReadStream(uri)
+        fileStream.pipe(truncate)
+        const checksum = size.toString() + ':' + await hashingStream(truncate, { algorithm: 'md5' })
+        return checksum
+    }
+}
+
+export function normalize(object) {
+    return _.pickBy(
+        object,
+        (value, key) => {
+            let pick = value !== undefined &&
+                value !== '' &&
+                value !== null &&
+                key !== 'undefined' &&
+                key !== '' &&
+                key !== 'null' &&
+                (typeof (value) != 'number' || !isNaN(value))
+            return pick
+        }
+    )
+}
+
+export function matchEntity(entity, match) {
+    if (!match) return false
+    if (typeof match == 'function') return match(entity)
+    else if (typeof match == 'string') {
+        if (match.substring(0, 1) == '@/') {
+            return minimatch(entity.name, match.substring(2))
+        } else {
+            return minimatch(entity.id, match)
+        }
+    }
+    else if (typeof match == 'object') return _.isMatch(entity, match)
+    throw new Error('Ivalid match type')
+}
+
+export function changeExtension(file, format) {
+    let extension = path.extname(file)
+    let result = file.substring(0, file.length - extension.length) + '.' + format
+    return result
+}