mikser-io 6.0.0

package/src/journal.js

@@ -0,0 +1,108 @@
+import runtime from './runtime.js'
+import { onLoaded, onCancelled, onFinalized } from './lifecycle.js'
+import { unlink } from 'fs/promises'
+import knex from 'knex'
+import path from 'path'
+import { stopProgress, trackProgress, updateProgress } from './tracking.js'
+import { AbortError } from './utils.js'
+
+let journal
+
+export async function addEntry({ entity, operation, context, options }) {
+    await journal('operations').insert([{ entity, operation, context, options }])
+}
+
+export async function addEntries(entries) {
+    await journal.batchInsert('operations', entries.map(({ entity, operation, context, options }) => ({
+        entity: JSON.stringify(entity),
+        operation,
+        context: JSON.stringify(context),
+        options: JSON.stringify(options)
+    })), 10)
+}
+
+export async function updateEntry({ id, entity, output }) {
+    const data = {}
+    if (entity) data.entity = JSON.stringify(entity)
+    if (output) data.output = JSON.stringify(output)
+    await journal('operations').where({ id }).update(data)
+}
+
+export async function* useJournal(name, operations, signal) {
+    let query = journal('operations')
+    if (operations?.length) {
+        query.whereIn('operation', operations)
+    }
+    let [total] = await query.clone().count()
+    total = total['count(*)']
+    if (!total) return
+
+    trackProgress(name, total)
+
+    let offset = 0
+    const limit = 1000
+    let count = 0
+    do {
+        count = 0
+        const entries = await query.clone().orderBy('id').select().offset(offset).limit(limit)
+        for (let { id, entity, operation, context, options, output } of entries) {
+            if (signal?.aborted) {
+                stopProgress()
+                throw new AbortError()
+            }
+            count++
+
+            updateProgress()
+            yield {
+                id,
+                entity: JSON.parse(entity),
+                operation,
+                context: JSON.parse(context),
+                options: JSON.parse(options),
+                output: JSON.parse(output)
+            }
+        }
+        offset += limit
+    } while (count == limit)
+}
+
+export async function clearJournal(aborted) {
+    await journal('operations').del()
+    if (!aborted) {
+        if (runtime.options.watch !== true) {
+            journal.destroy()
+        }
+    }
+}
+
+onLoaded(async () => {
+    const filename = path.join(runtime.options.runtimeFolder, `journal.db`)
+    try {
+        await unlink(filename)
+    } catch { }
+
+    journal = knex({
+        client: 'sqlite3',
+        connection: {
+            filename
+        },
+        useNullAsDefault: true
+    })
+
+    await journal.schema.createTable('operations', table => {
+        table.increments('id')
+        table.string('operation').index()
+        table.json('entity')
+        table.json('context')
+        table.json('options')
+        table.json('output')
+    })
+})
+
+onFinalized(async (signal) => {
+    await clearJournal(signal.aborted)
+})
+
+onCancelled(async () => {
+    await clearJournal(true)
+})

package/src/lifecycle.js

@@ -0,0 +1,299 @@
+import runtime from './runtime.js'
+import { OPERATION } from './constants.js'
+import { useLogger } from './engine.js'
+import { addEntry, addEntries } from './journal.js'
+
+export async function createEntity(entity) {
+    const logger = useLogger()
+    entity.stamp = runtime.stamp
+    entity.time = Date.now()
+    const entry = { operation: OPERATION.CREATE, entity }
+    if (await runtime.validate(entry)) {
+        logger.debug('Create %s entity: %s', entity.collection, entity.id)
+        await addEntry(entry)
+    }
+}
+
+export async function deleteEntity({ id, collection, type }) {
+    const logger = useLogger()
+    const entry = { operation: OPERATION.DELETE, entity: { id, type, collection } }
+    if (await runtime.validate(entry)) {
+        logger.debug('Delete %s entity: %s %s', collection, type, id)
+        await addEntry(entry)
+    }
+}
+
+export async function updateEntity(entity) {
+    const logger = useLogger()
+    entity.stamp = runtime.stamp
+    entity.time = Date.now()
+    const entry = { operation: OPERATION.UPDATE, entity }
+    if (await runtime.validate(entry)) {
+        logger.debug('Update %s entity: %s', entity.collection, entity.id)
+        await addEntry(entry)
+    }
+}
+
+export async function postprocessEntity(entity, options = {}, context = {}) {
+    const logger = useLogger()
+    const entry = { operation: OPERATION.POSTPROCESS, entity, options, context }
+    logger.debug('Postprocess %s entity: [%s] %s → %s', entity.collection, options.postprocessor, entity.id, entity.destination)
+    await addEntry(entry)
+}
+
+export async function postprocessEntities(tasks) {
+    const logger = useLogger()
+    if (!tasks.length) return
+    const entries = []
+    for (let { entity, options = {}, context = {} } of tasks) {
+        const entry = { operation: OPERATION.POSTPROCESS, entity, options, context }
+        logger.debug('Postprocess %s entity: [%s] %s → %s', entity.collection, options.postprocessor, entity.id, entity.destination)
+        entries.push(entry)
+    }
+    await addEntries(entries)
+}
+
+export async function renderEntities(tasks) {
+    const logger = useLogger()
+    if (!tasks.length) return
+    const entries = []
+    for (let { entity, options = {}, context = {} } of tasks) {
+        const entry = { operation: OPERATION.RENDER, entity, options, context, }
+        if (options.ignore) {
+            logger.trace('Render %s entity: [%s] %s → %s %s', entity.collection, options.renderer, entity.id, entity.destination, !options.ignore)
+        } else {
+            logger.debug('Render %s entity: [%s] %s → %s %s', entity.collection, options.renderer, entity.id, entity.destination, !options.ignore)
+        }
+        entries.push(entry)
+    }
+    await addEntries(entries)
+}
+
+export async function renderEntity(entity, options = {}, context = {}) {
+    const logger = useLogger()
+    const entry = { operation: OPERATION.RENDER, entity, options, context, }
+    if (options.ignore) {
+        logger.trace('Render %s entity: [%s] %s → %s %s', entity.collection, options.renderer, entity.id, entity.destination, !options.ignore)
+    } else {
+        logger.debug('Render %s entity: [%s] %s → %s %s', entity.collection, options.renderer, entity.id, entity.destination, !options.ignore)
+    }
+    await addEntry(entry)
+}
+
+export async function onInitialize(callback) {
+    runtime.hooks.initialize.push(callback)
+}
+
+export async function onInitialized(callback) {
+    runtime.hooks.initialized.push(callback)
+}
+
+export async function onLoad(callback) {
+    runtime.hooks.load.push(callback)
+}
+
+export async function onLoaded(callback) {
+    runtime.hooks.loaded.push(callback)
+}
+
+export async function onImport(callback) {
+    runtime.hooks.import.push(callback)
+}
+
+export async function onImported(callback) {
+    runtime.hooks.imported.push(callback)
+}
+
+export async function onProcess(callback, once) {
+    if (!once) runtime.hooks.process.push(callback)
+    else {
+        let called = false
+        runtime.hooks.process.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onProcessed(callback, once) {
+    if (!once) runtime.hooks.processed.push(callback)
+    else {
+        let called = false
+        runtime.hooks.processed.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onPersist(callback, once) {
+    if (!once) runtime.hooks.persist.push(callback)
+    else {
+        let called = false
+        runtime.hooks.persist.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onPersisted(callback, once) {
+    if (!once) runtime.hooks.persisted.push(callback)
+    else {
+        let called = false
+        runtime.hooks.persisted.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onCancel(callback) {
+    runtime.hooks.cancel.push(callback)
+}
+
+export async function onCancelled(callback) {
+    runtime.hooks.cancelled.push(callback)
+}
+
+export async function onBeforeRender(callback, once) {
+    if (!once) runtime.hooks.beforeRender.push(callback)
+    else {
+        let called = false
+        runtime.hooks.beforeRender.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onRender(callback, once) {
+    if (!once) runtime.hooks.render.push(callback)
+    else {
+        let called = false
+        runtime.hooks.render.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onAfterRender(callback, once) {
+    if (!once) runtime.hooks.afterRender.push(callback)
+    else {
+        let called = false
+        runtime.hooks.afterRender.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onBeforePostprocess(callback, once) {
+    if (!once) runtime.hooks.beforePostprocess.push(callback)
+    else {
+        let called = false
+        runtime.hooks.beforePostprocess.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onPostprocess(callback, once) {
+    if (!once) runtime.hooks.postprocess.push(callback)
+    else {
+        let called = false
+        runtime.hooks.postprocess.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onAfterPostprocess(callback, once) {
+    if (!once) runtime.hooks.afterPostprocess.push(callback)
+    else {
+        let called = false
+        runtime.hooks.afterPostprocess.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onFinalize(callback, once) {
+    if (!once) runtime.hooks.finalize.push(callback)
+    else {
+        let called = false
+        runtime.hooks.finalize.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export async function onFinalized(callback, once) {
+    if (!once) runtime.hooks.finalized.push(callback)
+    else {
+        let called = false
+        runtime.hooks.finalized.push((signal) => {
+            if (!called) {
+                called = true
+                callback(signal)
+            }
+        })
+    }
+}
+
+export function onSync(name, callback) {
+    runtime.hooks.sync.push(async (operation) => {
+        if (operation.name == name) {
+            return await callback(operation)
+        }
+    })
+}
+
+export function onValidate(operations, callback) {
+    const logger = useLogger()
+    runtime.validators.push(async (entry) => {
+        if (operations.indexOf(entry.operation) != -1) {
+            try {
+                const message = await callback(entry)
+                if (message) {
+                    logger.warn('Validation problem: [%s] %s %s', entry.operation, entry.entity.name, message)
+                }
+                return true
+            } catch (err) {
+                logger.error('Validation error: [%s] %s %s', entry.operation, entry.entity.name, err.message)
+                return false
+            }
+        }
+    })
+}
+
+export function onComplete(callback) {
+    runtime.hooks.completed.push(callback)
+}

package/src/manager.js

@@ -0,0 +1,119 @@
+import runtime from './runtime.js'
+import chokidar from 'chokidar'
+import cron from 'node-cron'
+import { onProcess, onFinalized } from './lifecycle.js'
+import { useLogger } from './engine.js'
+import { ACTION } from './constants.js'
+
+const tasks = []
+
+export async function createdHook(name, context) {
+    if (!runtime.started) return
+
+    const synced = await runtime.sync({
+        action: ACTION.CREATE,
+        name,
+        context
+    })
+
+    if (synced) {
+        clearTimeout(runtime.runtime.processTimeout)
+        runtime.runtime.processTimeout = setTimeout(() => runtime.process(), 1000)
+    }
+}
+
+export async function updatedHook(name, context) {
+    if (!runtime.started) return
+
+    const synced = await runtime.sync({
+        action: ACTION.UPDATE,
+        name,
+        context
+    })
+
+    if (synced) {
+        clearTimeout(runtime.runtime.processTimeout)
+        runtime.runtime.processTimeout = setTimeout(() => runtime.process(), 1000)
+    }
+}
+
+export async function triggeredHook(name, context) {
+    if (!runtime.started) return
+
+    const synced = await runtime.sync({
+        action: ACTION.TRIGGER,
+        name,
+        context
+    })
+
+    if (synced) {
+        clearTimeout(runtime.runtime.processTimeout)
+        runtime.runtime.processTimeout = setTimeout(() => runtime.process(), 1000)
+    }
+}
+
+export async function deletedHook(name, context) {
+    if (!runtime.started) return
+
+    const synced = await runtime.sync({
+        action: ACTION.DELETE,
+        name,
+        context
+    })
+
+    if (synced) {
+        clearTimeout(runtime.runtime.processTimeout)
+        runtime.runtime.processTimeout = setTimeout(() => runtime.process(), 1000)
+    }
+}
+
+export function watch(name, folder, options = { interval: 1000, binaryInterval: 3000, ignored: /[\/\\]\./, ignoreInitial: true }) {
+    if (runtime.options.watch !== true) return
+
+    chokidar.watch(folder, options)
+        .on('all', () => {
+            clearTimeout(runtime.runtime.processTimeout)
+        })
+        .on('add', async fullPath => {
+            const relativePath = fullPath.replace(`${folder}/`, '')
+            createdHook(name, { relativePath })
+        })
+        .on('change', async fullPath => {
+            const relativePath = fullPath.replace(`${folder}/`, '')
+            updatedHook(name, { relativePath })
+        })
+        .on('unlink', async fullPath => {
+            const relativePath = fullPath.replace(`${folder}/`, '')
+            deletedHook(name, { relativePath })
+        })
+}
+
+export function schedule(name, expression, context) {
+    if (runtime.options.watch !== true) return
+    const logger = useLogger()
+    const taks = cron.schedule(expression, async () => {
+        logger.info('Scheduled task executed: %s %s', name, expression)
+        triggeredHook(name, context)
+    }, {
+        scheduled: false
+    })
+    tasks.push(taks)
+}
+
+onProcess(() => {
+    if (!tasks.length) return
+    const logger = useLogger()
+    logger.debug('Stopping scheduled tasks: %d', tasks.length)
+    for (let task of tasks) {
+        task.stop()
+    }
+})
+
+onFinalized(() => {
+    if (!tasks.length) return
+    const logger = useLogger()
+    logger.debug('Starting scheduled tasks: %d', tasks.length)
+    for (let task of tasks) {
+        task.start()
+    }
+})

package/src/plugins/api.js

@@ -0,0 +1,176 @@
+import path from 'path'
+import { hash } from 'hasha'
+import _ from 'lodash'
+
+export default ({
+    runtime,
+    useLogger,
+    onImport,
+    onLoaded,
+    onSync,
+    createEntity,
+    updateEntity,
+    deleteEntity,
+    findEntity,
+    findEntities,
+    schedule,
+    normalize,
+    trackProgress,
+    updateProgress,
+}) => {
+    const format = 'api'
+
+    async function syncEntities(apiName) {
+        const logger = useLogger()
+        const syncTime = Date.now()
+        const {
+            collection = apiName,
+            type = 'document',
+            readMany,
+            uri = ''
+        } = runtime.config.api[apiName]
+
+        let synced = 0
+        let removed = 0
+        try {
+            const recent = new Set()
+            const entities = await readMany(runtime)
+            trackProgress(`Api sync ${apiName}`, entities.length)
+            for (let meta of entities) {
+                if (collection && type && meta.id) {
+                    const name = path.join(collection, meta.name || meta.id.toString())
+                    const id = path.join('/api', collection, meta.id.toString())
+                    if (recent.has(id)) {
+                        logger.error(meta, 'Duplicate entity found: %s', id)
+                        continue
+                    }
+                    recent.add(id)
+                    const entity = normalize({
+                        id,
+                        uri: `${uri}/${meta.id}`,
+                        name,
+                        collection,
+                        type,
+                        format,
+                        meta
+                    })
+
+                    entity.checksum = await hash(JSON.stringify(entity.meta), { algorithm: 'md5' })
+                    const current = await findEntity({ id })
+                    if (current) {
+                        if (entity.checksum != current.checksum) {
+                            await updateEntity(entity)
+                            synced++
+                        }
+                    } else {
+                        await createEntity(entity)
+                        synced++
+                    }
+                }
+                updateProgress()
+            }
+
+            const entitiesToRemove = await findEntities(entity =>
+                entity.type == type &&
+                entity.format == format &&
+                entity.collection == collection &&
+                entity.time < syncTime &&
+                !recent.has(entity.id)
+            )
+            if (entitiesToRemove.length) trackProgress(`Api remove ${apiName}`, entitiesToRemove.length)
+            for (let entity of entitiesToRemove) {
+                deleteEntity(entity)
+                removed++
+                updateProgress()
+            }
+            if (synced || removed) {
+                logger.debug('Syncing api [%s] synced: %d, removed: %d', collection, synced, removed)
+            }
+        } catch (err) {
+            logger.error('Api sync [%s] error: %s', collection, err.message)
+        }
+        return synced > 0 || removed > 0
+    }
+
+    async function syncEntity(apiName, apiId) {
+        const logger = useLogger()
+        const {
+            collection = apiName,
+            type = 'document',
+            readOne,
+            uri = ''
+        } = runtime.config.api[apiName]
+
+        try {
+            const id = path.join('/api', collection, apiId.toString())
+            const current = await findEntity({ id })
+            const meta = await readOne(apiId, runtime)
+            if (meta?.id) {
+                const name = path.join(collection, meta.name || meta.id.toString())
+                const entity = normalize({
+                    id,
+                    uri: `${uri}/${meta.id}`,
+                    name,
+                    collection,
+                    type,
+                    format,
+                    meta
+                })
+                entity.checksum = await hash(JSON.stringify(entity.meta), { algorithm: 'md5' })
+                if (current) {
+                    if (entity.checksum != current.checksum) {
+                        logger.info('Api update: %s', id)
+                        await updateEntity(entity)
+                    }
+                } else {
+                    logger.info('Api create: %s', id)
+                    await createEntity(entity)
+                }
+            } else {
+                if (current) {
+                    logger.info('Api delete: %s', id)
+                    await deleteEntity(entity)
+                }
+            }
+        } catch (err) {
+            logger.error('Api sync entity [%s] error: %s', collection, err.message)
+        }
+    }
+
+    onLoaded(async () => {
+        const logger = useLogger()
+        for (let apiName in runtime.config.api || {}) {
+            const { cron } = runtime.config.api[apiName]
+            if (cron) {
+                logger.info('Schedule api: [%s] %s', apiName, cron)
+                schedule(apiName, cron)
+            }
+
+            onSync(apiName, async ({ context }) => {
+                if (context?.id) {
+                    return syncEntity(apiName, context.id)
+                } else {
+                    return syncEntities(apiName)
+                }
+            })
+
+            const { origin } = new URL(runtime.config.api[apiName].uri)
+            onSync(origin, async ({ context }) => {
+                if (context.uri) {
+                    logger.info('Syncing api: [%s] %s', apiName, context.uri)
+                    return syncEntities(apiName)
+                }
+            })
+        }
+    })
+
+    onImport(async () => {
+        for (let apiName in runtime.config.api || {}) {
+            await syncEntities(apiName)
+        }
+    })
+
+    return {
+        format
+    }
+}