mikser-io 6.4.1 → 6.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +2 -1
- package/package.json +1 -1
- package/src/api.js +165 -0
- package/src/plugins/api.js +154 -155
- package/src/plugins/observer.js +176 -0
- package/src/plugins/rest.js +0 -143
package/index.js
CHANGED
package/package.json
CHANGED
package/src/api.js
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
// Public, transport-agnostic primitives that the API plugin's HTTP
|
|
2
|
+
// endpoints are thin wrappers over. Library users embedding mikser
|
|
3
|
+
// programmatically can import these directly.
|
|
4
|
+
|
|
5
|
+
import { randomUUID } from 'node:crypto'
|
|
6
|
+
import { writeFile, unlink, mkdir } from 'node:fs/promises'
|
|
7
|
+
import path from 'node:path'
|
|
8
|
+
import { updateEntity as defaultUpdateEntity } from './lifecycle.js'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Bind to the runtime and return an on-demand renderer that pipelines
|
|
12
|
+
* concurrent calls into the minimum number of `runtime.process()` cycles.
|
|
13
|
+
* The returned binding is stateful — each call to useRenderer() owns its
|
|
14
|
+
* own pending queue and `completed`-hook lifecycle. Mount once per
|
|
15
|
+
* consumer (the API plugin mounts one; a library service mounts its own).
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* const { render } = useRenderer(runtime)
|
|
19
|
+
* const { output, entity } = await render(entityShape)
|
|
20
|
+
*
|
|
21
|
+
* @param {object} runtime - the mikser runtime singleton
|
|
22
|
+
* @param {object} [opts]
|
|
23
|
+
* @param {Function} [opts.updateEntity] - override lifecycle.updateEntity (mostly for testing)
|
|
24
|
+
* @param {number} [opts.defaultTimeout] - per-render timeout in ms (default 30_000)
|
|
25
|
+
* @returns {{ render: (entity, { timeout? }?) => Promise<{ output, entity }> }}
|
|
26
|
+
*/
|
|
27
|
+
export function useRenderer(runtime, {
|
|
28
|
+
updateEntity = defaultUpdateEntity,
|
|
29
|
+
defaultTimeout = 30_000,
|
|
30
|
+
} = {}) {
|
|
31
|
+
let pending = []
|
|
32
|
+
let cycleRunning = false
|
|
33
|
+
|
|
34
|
+
async function runBatch() {
|
|
35
|
+
if (cycleRunning || pending.length === 0) return
|
|
36
|
+
cycleRunning = true
|
|
37
|
+
|
|
38
|
+
const batch = pending
|
|
39
|
+
pending = []
|
|
40
|
+
|
|
41
|
+
const remaining = new Map(batch.map(b => [b.correlationId, b]))
|
|
42
|
+
const completedHooks = runtime.hooks.completed
|
|
43
|
+
const hook = async (entry) => {
|
|
44
|
+
const cid = entry.entity?._correlationId
|
|
45
|
+
if (!cid) return
|
|
46
|
+
const item = remaining.get(cid)
|
|
47
|
+
if (!item) return
|
|
48
|
+
remaining.delete(cid)
|
|
49
|
+
clearTimeout(item.timer)
|
|
50
|
+
item.resolve({ output: entry.output, entity: entry.entity })
|
|
51
|
+
}
|
|
52
|
+
completedHooks.push(hook)
|
|
53
|
+
|
|
54
|
+
for (const item of batch) {
|
|
55
|
+
item.timer = setTimeout(() => {
|
|
56
|
+
if (remaining.delete(item.correlationId)) {
|
|
57
|
+
item.reject(new Error(`Render timeout for ${item.entity.id}`))
|
|
58
|
+
}
|
|
59
|
+
}, item.timeout)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
for (const item of batch) {
|
|
64
|
+
await updateEntity(item.entity).catch(item.reject)
|
|
65
|
+
}
|
|
66
|
+
await runtime.process()
|
|
67
|
+
} catch (err) {
|
|
68
|
+
for (const item of remaining.values()) {
|
|
69
|
+
clearTimeout(item.timer)
|
|
70
|
+
item.reject(err)
|
|
71
|
+
}
|
|
72
|
+
remaining.clear()
|
|
73
|
+
} finally {
|
|
74
|
+
for (const item of remaining.values()) {
|
|
75
|
+
clearTimeout(item.timer)
|
|
76
|
+
item.reject(new Error(`Render did not complete for ${item.entity.id}`))
|
|
77
|
+
}
|
|
78
|
+
const idx = completedHooks.indexOf(hook)
|
|
79
|
+
if (idx >= 0) completedHooks.splice(idx, 1)
|
|
80
|
+
cycleRunning = false
|
|
81
|
+
if (pending.length) setImmediate(runBatch)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Submit an entity for rendering. Resolves with `{ output, entity }`
|
|
87
|
+
* where `output.result` is whatever the renderer/postprocessor returned
|
|
88
|
+
* (a string for HTML/text outputs, a Buffer for PDFs, etc.).
|
|
89
|
+
*
|
|
90
|
+
* Requests arriving concurrently are coalesced into the next available
|
|
91
|
+
* `runtime.process()` cycle — within that cycle, mikser's worker pool
|
|
92
|
+
* renders the batch in parallel.
|
|
93
|
+
*
|
|
94
|
+
* @param {object} entity - any entity-shaped object
|
|
95
|
+
* @param {object} [opts]
|
|
96
|
+
* @param {number} [opts.timeout] - override the default timeout
|
|
97
|
+
* @returns {Promise<{output, entity}>}
|
|
98
|
+
*/
|
|
99
|
+
function render(entity, { timeout = defaultTimeout } = {}) {
|
|
100
|
+
return new Promise((resolve, reject) => {
|
|
101
|
+
const correlationId = randomUUID()
|
|
102
|
+
pending.push({
|
|
103
|
+
entity: { ...entity, _correlationId: correlationId },
|
|
104
|
+
correlationId,
|
|
105
|
+
timeout,
|
|
106
|
+
resolve,
|
|
107
|
+
reject,
|
|
108
|
+
timer: null,
|
|
109
|
+
})
|
|
110
|
+
if (!cycleRunning) setImmediate(runBatch)
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return { render }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Bind to a single collection's source folder and return file-level
|
|
119
|
+
* `write` / `remove` operations against it. Each collection plugin sets
|
|
120
|
+
* `runtime.options.<name>Folder` during its onLoaded hook; this looks
|
|
121
|
+
* that up lazily, so it's safe to call useCollection() anywhere after
|
|
122
|
+
* `runtime.start()`.
|
|
123
|
+
*
|
|
124
|
+
* Distinct from `lifecycle.updateEntity` / `lifecycle.deleteEntity` —
|
|
125
|
+
* those write journal entries. These write actual files; in watch mode
|
|
126
|
+
* the resulting fs change is what kicks the next sync→process cycle.
|
|
127
|
+
*
|
|
128
|
+
* @example
|
|
129
|
+
* const documents = useCollection(runtime, 'documents')
|
|
130
|
+
* await documents.write('en/draft.md', '# Hi')
|
|
131
|
+
* await documents.remove('en/old.md')
|
|
132
|
+
*
|
|
133
|
+
* @param {object} runtime - the mikser runtime singleton
|
|
134
|
+
* @param {string} name - collection name (e.g. 'documents')
|
|
135
|
+
* @returns {{
|
|
136
|
+
* name: string,
|
|
137
|
+
* folder: string,
|
|
138
|
+
* write(relativePath: string, content?: string): Promise<string>,
|
|
139
|
+
* remove(relativePath: string): Promise<void>,
|
|
140
|
+
* }}
|
|
141
|
+
*/
|
|
142
|
+
export function useCollection(runtime, name) {
|
|
143
|
+
function resolveFolder() {
|
|
144
|
+
const folder = runtime?.options?.[`${name}Folder`]
|
|
145
|
+
if (!folder) throw new Error(`Unknown collection: ${name}`)
|
|
146
|
+
return folder
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
name,
|
|
151
|
+
get folder() { return resolveFolder() },
|
|
152
|
+
|
|
153
|
+
async write(relativePath, content = '') {
|
|
154
|
+
const uri = path.join(resolveFolder(), relativePath)
|
|
155
|
+
await mkdir(path.dirname(uri), { recursive: true })
|
|
156
|
+
await writeFile(uri, content, 'utf8')
|
|
157
|
+
return uri
|
|
158
|
+
},
|
|
159
|
+
|
|
160
|
+
async remove(relativePath) {
|
|
161
|
+
const uri = path.join(resolveFolder(), relativePath)
|
|
162
|
+
await unlink(uri)
|
|
163
|
+
},
|
|
164
|
+
}
|
|
165
|
+
}
|
package/src/plugins/api.js
CHANGED
|
@@ -1,176 +1,175 @@
|
|
|
1
|
-
import path from 'path'
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
1
|
+
import path from 'node:path'
|
|
2
|
+
import { access } from 'node:fs/promises'
|
|
3
|
+
import { useRenderer, useCollection } from '../api.js'
|
|
4
|
+
|
|
5
|
+
// MIME type lookup used when streaming a postprocessor's output back over
|
|
6
|
+
// HTTP. The renderer's output extension lives on entity.destination
|
|
7
|
+
// (assigned by the layouts plugin), so we use it as the source of truth.
|
|
8
|
+
const MIME_BY_EXT = {
|
|
9
|
+
pdf: 'application/pdf',
|
|
10
|
+
html: 'text/html; charset=utf-8',
|
|
11
|
+
xml: 'application/xml; charset=utf-8',
|
|
12
|
+
xhtml: 'application/xhtml+xml; charset=utf-8',
|
|
13
|
+
rss: 'application/rss+xml; charset=utf-8',
|
|
14
|
+
atom: 'application/atom+xml; charset=utf-8',
|
|
15
|
+
json: 'application/json; charset=utf-8',
|
|
16
|
+
css: 'text/css; charset=utf-8',
|
|
17
|
+
js: 'application/javascript; charset=utf-8',
|
|
18
|
+
svg: 'image/svg+xml',
|
|
19
|
+
png: 'image/png',
|
|
20
|
+
jpg: 'image/jpeg',
|
|
21
|
+
jpeg: 'image/jpeg',
|
|
22
|
+
webp: 'image/webp',
|
|
23
|
+
gif: 'image/gif',
|
|
24
|
+
mp4: 'video/mp4',
|
|
25
|
+
webm: 'video/webm',
|
|
26
|
+
txt: 'text/plain; charset=utf-8',
|
|
27
|
+
md: 'text/markdown; charset=utf-8',
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function mimeForEntity(entity) {
|
|
31
|
+
if (!entity?.destination) return null
|
|
32
|
+
const ext = path.extname(entity.destination).toLowerCase().replace(/^\./, '')
|
|
33
|
+
return MIME_BY_EXT[ext] ?? null
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Decide how to send the render output over HTTP. Exported (and pure-ish)
|
|
37
|
+
// so tests can exercise the branching without spinning up a real server.
|
|
38
|
+
export async function sendRenderOutput(res, output, entity) {
|
|
39
|
+
if (output == null || output.result == null) {
|
|
40
|
+
return res.status(204).send()
|
|
41
|
+
}
|
|
42
|
+
const result = output.result
|
|
43
|
+
const mime = mimeForEntity(entity)
|
|
44
|
+
|
|
45
|
+
if (Buffer.isBuffer(result)) {
|
|
46
|
+
if (mime) res.type(mime)
|
|
47
|
+
return res.send(result)
|
|
48
|
+
}
|
|
49
|
+
if (typeof result === 'string') {
|
|
50
|
+
// A few postprocessors might return an absolute path to a generated
|
|
51
|
+
// file rather than its contents. Only attempt this when the string
|
|
52
|
+
// looks plausibly path-shaped — short, starts with a slash, and the
|
|
53
|
+
// file actually exists. Otherwise treat it as content.
|
|
54
|
+
if (result.length < 4096 && (result.startsWith('/') || /^[A-Za-z]:[\\/]/.test(result))) {
|
|
55
|
+
try {
|
|
56
|
+
await access(result)
|
|
57
|
+
return res.sendFile(result)
|
|
58
|
+
} catch { /* not a path, fall through */ }
|
|
59
|
+
}
|
|
60
|
+
if (mime) res.type(mime)
|
|
61
|
+
return res.send(result)
|
|
62
|
+
}
|
|
63
|
+
// Anything else (plain object, etc.) is sent as JSON.
|
|
64
|
+
return res.json(result)
|
|
65
|
+
}
|
|
4
66
|
|
|
5
67
|
export default ({
|
|
6
68
|
runtime,
|
|
7
|
-
useLogger,
|
|
8
|
-
onImport,
|
|
9
69
|
onLoaded,
|
|
10
|
-
|
|
11
|
-
createEntity,
|
|
70
|
+
useLogger,
|
|
12
71
|
updateEntity,
|
|
13
|
-
deleteEntity,
|
|
14
|
-
findEntity,
|
|
15
72
|
findEntities,
|
|
16
|
-
schedule,
|
|
17
|
-
normalize,
|
|
18
|
-
trackProgress,
|
|
19
|
-
updateProgress,
|
|
20
73
|
}) => {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
async function syncEntities(apiName) {
|
|
74
|
+
onLoaded(async () => {
|
|
24
75
|
const logger = useLogger()
|
|
25
|
-
const syncTime = Date.now()
|
|
26
|
-
const {
|
|
27
|
-
collection = apiName,
|
|
28
|
-
type = 'document',
|
|
29
|
-
readMany,
|
|
30
|
-
uri = ''
|
|
31
|
-
} = runtime.config.api[apiName]
|
|
32
|
-
|
|
33
|
-
let synced = 0
|
|
34
|
-
let removed = 0
|
|
35
|
-
try {
|
|
36
|
-
const recent = new Set()
|
|
37
|
-
const entities = await readMany(runtime)
|
|
38
|
-
trackProgress(`Api sync ${apiName}`, entities.length)
|
|
39
|
-
for (let meta of entities) {
|
|
40
|
-
if (collection && type && meta.id) {
|
|
41
|
-
const name = path.join(collection, meta.name || meta.id.toString())
|
|
42
|
-
const id = path.join('/api', collection, meta.id.toString())
|
|
43
|
-
if (recent.has(id)) {
|
|
44
|
-
logger.error(meta, 'Duplicate entity found: %s', id)
|
|
45
|
-
continue
|
|
46
|
-
}
|
|
47
|
-
recent.add(id)
|
|
48
|
-
const entity = normalize({
|
|
49
|
-
id,
|
|
50
|
-
uri: `${uri}/${meta.id}`,
|
|
51
|
-
name,
|
|
52
|
-
collection,
|
|
53
|
-
type,
|
|
54
|
-
format,
|
|
55
|
-
meta
|
|
56
|
-
})
|
|
57
|
-
|
|
58
|
-
entity.checksum = await hash(JSON.stringify(entity.meta), { algorithm: 'md5' })
|
|
59
|
-
const current = await findEntity({ id })
|
|
60
|
-
if (current) {
|
|
61
|
-
if (entity.checksum != current.checksum) {
|
|
62
|
-
await updateEntity(entity)
|
|
63
|
-
synced++
|
|
64
|
-
}
|
|
65
|
-
} else {
|
|
66
|
-
await createEntity(entity)
|
|
67
|
-
synced++
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
updateProgress()
|
|
71
|
-
}
|
|
72
76
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
if (
|
|
87
|
-
|
|
88
|
-
}
|
|
89
|
-
} catch (err) {
|
|
90
|
-
logger.error('Api sync [%s] error: %s', collection, err.message)
|
|
77
|
+
const { default: express } = await import('express').catch(() => {
|
|
78
|
+
throw new Error('express is required for the api plugin — run: npm install express')
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
const ownApp = !runtime.options.app
|
|
82
|
+
const app = runtime.options.app ?? express()
|
|
83
|
+
|
|
84
|
+
const router = express.Router()
|
|
85
|
+
router.use(express.json())
|
|
86
|
+
|
|
87
|
+
const token = runtime.config.api?.token
|
|
88
|
+
const auth = (req, res, next) => {
|
|
89
|
+
if (!token) return next()
|
|
90
|
+
if (req.headers.authorization === `Bearer ${token}`) return next()
|
|
91
|
+
res.status(401).json({ error: 'Unauthorized' })
|
|
91
92
|
}
|
|
92
|
-
return synced > 0 || removed > 0
|
|
93
|
-
}
|
|
94
93
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
94
|
+
// Reuse the transport-agnostic primitives from src/api.js so the
|
|
95
|
+
// library entry point and the API endpoints share the exact same
|
|
96
|
+
// batching/timeouts/error semantics.
|
|
97
|
+
const { render } = useRenderer(runtime, {
|
|
98
|
+
updateEntity,
|
|
99
|
+
defaultTimeout: runtime.config.api?.renderTimeout ?? 30_000,
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
router.get('/entities', async (req, res) => {
|
|
103
|
+
try {
|
|
104
|
+
const { page: rawPage, limit: rawLimit, ...filter } = req.query
|
|
105
|
+
const page = Math.max(1, parseInt(rawPage) || 1)
|
|
106
|
+
const limit = Math.min(100, Math.max(1, parseInt(rawLimit) || (runtime.config.api?.pageSize ?? 10)))
|
|
107
|
+
const query = Object.keys(filter).length ? filter : undefined
|
|
108
|
+
|
|
109
|
+
const all = await findEntities(query)
|
|
110
|
+
const total = all.length
|
|
111
|
+
const totalPages = Math.ceil(total / limit)
|
|
112
|
+
const items = all.slice((page - 1) * limit, page * limit)
|
|
113
|
+
|
|
114
|
+
res.json({
|
|
115
|
+
items,
|
|
116
|
+
page,
|
|
117
|
+
limit,
|
|
118
|
+
total,
|
|
119
|
+
totalPages,
|
|
120
|
+
hasNext: page < totalPages,
|
|
121
|
+
hasPrev: page > 1,
|
|
118
122
|
})
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
logger.info('Api update: %s', id)
|
|
123
|
-
await updateEntity(entity)
|
|
124
|
-
}
|
|
125
|
-
} else {
|
|
126
|
-
logger.info('Api create: %s', id)
|
|
127
|
-
await createEntity(entity)
|
|
128
|
-
}
|
|
129
|
-
} else {
|
|
130
|
-
if (current) {
|
|
131
|
-
logger.info('Api delete: %s', id)
|
|
132
|
-
await deleteEntity(entity)
|
|
133
|
-
}
|
|
123
|
+
} catch (err) {
|
|
124
|
+
logger.error('API list error: %s', err.message)
|
|
125
|
+
res.status(500).json({ error: err.message })
|
|
134
126
|
}
|
|
135
|
-
}
|
|
136
|
-
logger.error('Api sync entity [%s] error: %s', collection, err.message)
|
|
137
|
-
}
|
|
138
|
-
}
|
|
127
|
+
})
|
|
139
128
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
129
|
+
router.put('/entities', auth, async (req, res) => {
|
|
130
|
+
try {
|
|
131
|
+
const { collection, relativePath, content = '' } = req.body
|
|
132
|
+
await useCollection(runtime, collection).write(relativePath, content)
|
|
133
|
+
res.status(202).json({ ok: true })
|
|
134
|
+
} catch (err) {
|
|
135
|
+
logger.error('API update error: %s', err.message)
|
|
136
|
+
res.status(/Unknown collection/.test(err.message) ? 400 : 500).json({ error: err.message })
|
|
147
137
|
}
|
|
138
|
+
})
|
|
148
139
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
140
|
+
router.delete('/entities', auth, async (req, res) => {
|
|
141
|
+
try {
|
|
142
|
+
const { collection, relativePath } = req.body
|
|
143
|
+
await useCollection(runtime, collection).remove(relativePath)
|
|
144
|
+
res.status(202).json({ ok: true })
|
|
145
|
+
} catch (err) {
|
|
146
|
+
logger.error('API delete error: %s', err.message)
|
|
147
|
+
res.status(/Unknown collection/.test(err.message) ? 400 : 500).json({ error: err.message })
|
|
148
|
+
}
|
|
149
|
+
})
|
|
156
150
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
151
|
+
router.post('/render', auth, async (req, res) => {
|
|
152
|
+
try {
|
|
153
|
+
const { output, entity } = await render(req.body)
|
|
154
|
+
await sendRenderOutput(res, output, entity)
|
|
155
|
+
} catch (err) {
|
|
156
|
+
logger.error('API render error: %s', err.message)
|
|
157
|
+
if (!res.headersSent) {
|
|
158
|
+
res.status(500).json({ error: err.message })
|
|
162
159
|
}
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
|
|
160
|
+
}
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
const base = runtime.config.api?.base ?? '/api'
|
|
164
|
+
app.use(base, router)
|
|
166
165
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
166
|
+
if (ownApp) {
|
|
167
|
+
const port = runtime.config.api?.port ?? 3001
|
|
168
|
+
app.listen(port, () => {
|
|
169
|
+
logger.info('API plugin listening on port %d', port)
|
|
170
|
+
})
|
|
171
|
+
} else {
|
|
172
|
+
logger.info('API plugin mounted on %s', base)
|
|
170
173
|
}
|
|
171
174
|
})
|
|
172
|
-
|
|
173
|
-
return {
|
|
174
|
-
format
|
|
175
|
-
}
|
|
176
|
-
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import path from 'path'
|
|
2
|
+
import { hash } from 'hasha'
|
|
3
|
+
import _ from 'lodash'
|
|
4
|
+
|
|
5
|
+
export default ({
|
|
6
|
+
runtime,
|
|
7
|
+
useLogger,
|
|
8
|
+
onImport,
|
|
9
|
+
onLoaded,
|
|
10
|
+
onSync,
|
|
11
|
+
createEntity,
|
|
12
|
+
updateEntity,
|
|
13
|
+
deleteEntity,
|
|
14
|
+
findEntity,
|
|
15
|
+
findEntities,
|
|
16
|
+
schedule,
|
|
17
|
+
normalize,
|
|
18
|
+
trackProgress,
|
|
19
|
+
updateProgress,
|
|
20
|
+
}) => {
|
|
21
|
+
const format = 'observer'
|
|
22
|
+
|
|
23
|
+
async function syncEntities(observerName) {
|
|
24
|
+
const logger = useLogger()
|
|
25
|
+
const syncTime = Date.now()
|
|
26
|
+
const {
|
|
27
|
+
collection = observerName,
|
|
28
|
+
type = 'document',
|
|
29
|
+
readMany,
|
|
30
|
+
uri = ''
|
|
31
|
+
} = runtime.config.observer[observerName]
|
|
32
|
+
|
|
33
|
+
let synced = 0
|
|
34
|
+
let removed = 0
|
|
35
|
+
try {
|
|
36
|
+
const recent = new Set()
|
|
37
|
+
const entities = await readMany(runtime)
|
|
38
|
+
trackProgress(`Observer sync ${observerName}`, entities.length)
|
|
39
|
+
for (let meta of entities) {
|
|
40
|
+
if (collection && type && meta.id) {
|
|
41
|
+
const name = path.join(collection, meta.name || meta.id.toString())
|
|
42
|
+
const id = path.join('/observer', collection, meta.id.toString())
|
|
43
|
+
if (recent.has(id)) {
|
|
44
|
+
logger.error(meta, 'Duplicate entity found: %s', id)
|
|
45
|
+
continue
|
|
46
|
+
}
|
|
47
|
+
recent.add(id)
|
|
48
|
+
const entity = normalize({
|
|
49
|
+
id,
|
|
50
|
+
uri: `${uri}/${meta.id}`,
|
|
51
|
+
name,
|
|
52
|
+
collection,
|
|
53
|
+
type,
|
|
54
|
+
format,
|
|
55
|
+
meta
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
entity.checksum = await hash(JSON.stringify(entity.meta), { algorithm: 'md5' })
|
|
59
|
+
const current = await findEntity({ id })
|
|
60
|
+
if (current) {
|
|
61
|
+
if (entity.checksum != current.checksum) {
|
|
62
|
+
await updateEntity(entity)
|
|
63
|
+
synced++
|
|
64
|
+
}
|
|
65
|
+
} else {
|
|
66
|
+
await createEntity(entity)
|
|
67
|
+
synced++
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
updateProgress()
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const entitiesToRemove = await findEntities(entity =>
|
|
74
|
+
entity.type == type &&
|
|
75
|
+
entity.format == format &&
|
|
76
|
+
entity.collection == collection &&
|
|
77
|
+
entity.time < syncTime &&
|
|
78
|
+
!recent.has(entity.id)
|
|
79
|
+
)
|
|
80
|
+
if (entitiesToRemove.length) trackProgress(`Observer remove ${observerName}`, entitiesToRemove.length)
|
|
81
|
+
for (let entity of entitiesToRemove) {
|
|
82
|
+
deleteEntity(entity)
|
|
83
|
+
removed++
|
|
84
|
+
updateProgress()
|
|
85
|
+
}
|
|
86
|
+
if (synced || removed) {
|
|
87
|
+
logger.debug('Syncing api [%s] synced: %d, removed: %d', collection, synced, removed)
|
|
88
|
+
}
|
|
89
|
+
} catch (err) {
|
|
90
|
+
logger.error('Observer sync [%s] error: %s', collection, err.message)
|
|
91
|
+
}
|
|
92
|
+
return synced > 0 || removed > 0
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function syncEntity(observerName, apiId) {
|
|
96
|
+
const logger = useLogger()
|
|
97
|
+
const {
|
|
98
|
+
collection = observerName,
|
|
99
|
+
type = 'document',
|
|
100
|
+
readOne,
|
|
101
|
+
uri = ''
|
|
102
|
+
} = runtime.config.observer[observerName]
|
|
103
|
+
|
|
104
|
+
try {
|
|
105
|
+
const id = path.join('/observer', collection, apiId.toString())
|
|
106
|
+
const current = await findEntity({ id })
|
|
107
|
+
const meta = await readOne(apiId, runtime)
|
|
108
|
+
if (meta?.id) {
|
|
109
|
+
const name = path.join(collection, meta.name || meta.id.toString())
|
|
110
|
+
const entity = normalize({
|
|
111
|
+
id,
|
|
112
|
+
uri: `${uri}/${meta.id}`,
|
|
113
|
+
name,
|
|
114
|
+
collection,
|
|
115
|
+
type,
|
|
116
|
+
format,
|
|
117
|
+
meta
|
|
118
|
+
})
|
|
119
|
+
entity.checksum = await hash(JSON.stringify(entity.meta), { algorithm: 'md5' })
|
|
120
|
+
if (current) {
|
|
121
|
+
if (entity.checksum != current.checksum) {
|
|
122
|
+
logger.info('Observer update: %s', id)
|
|
123
|
+
await updateEntity(entity)
|
|
124
|
+
}
|
|
125
|
+
} else {
|
|
126
|
+
logger.info('Observer create: %s', id)
|
|
127
|
+
await createEntity(entity)
|
|
128
|
+
}
|
|
129
|
+
} else {
|
|
130
|
+
if (current) {
|
|
131
|
+
logger.info('Observer delete: %s', id)
|
|
132
|
+
await deleteEntity(entity)
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
} catch (err) {
|
|
136
|
+
logger.error('Observer sync entity [%s] error: %s', collection, err.message)
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
onLoaded(async () => {
|
|
141
|
+
const logger = useLogger()
|
|
142
|
+
for (let observerName in runtime.config.observer || {}) {
|
|
143
|
+
const { cron } = runtime.config.observer[observerName]
|
|
144
|
+
if (cron) {
|
|
145
|
+
logger.info('Schedule observer: [%s] %s', observerName, cron)
|
|
146
|
+
schedule(observerName, cron)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
onSync(observerName, async ({ context }) => {
|
|
150
|
+
if (context?.id) {
|
|
151
|
+
return syncEntity(observerName, context.id)
|
|
152
|
+
} else {
|
|
153
|
+
return syncEntities(observerName)
|
|
154
|
+
}
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
const { origin } = new URL(runtime.config.observer[observerName].uri)
|
|
158
|
+
onSync(origin, async ({ context }) => {
|
|
159
|
+
if (context.uri) {
|
|
160
|
+
logger.info('Syncing observer: [%s] %s', observerName, context.uri)
|
|
161
|
+
return syncEntities(observerName)
|
|
162
|
+
}
|
|
163
|
+
})
|
|
164
|
+
}
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
onImport(async () => {
|
|
168
|
+
for (let observerName in runtime.config.observer || {}) {
|
|
169
|
+
await syncEntities(observerName)
|
|
170
|
+
}
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
return {
|
|
174
|
+
format
|
|
175
|
+
}
|
|
176
|
+
}
|
package/src/plugins/rest.js
DELETED
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
import path from 'node:path'
|
|
2
|
-
import { writeFile, unlink, mkdir, access } from 'node:fs/promises'
|
|
3
|
-
import { updateEntity } from '../lifecycle.js'
|
|
4
|
-
import { findEntities } from '../catalog.js'
|
|
5
|
-
|
|
6
|
-
export default ({
|
|
7
|
-
runtime,
|
|
8
|
-
onLoaded,
|
|
9
|
-
useLogger,
|
|
10
|
-
}) => {
|
|
11
|
-
onLoaded(async () => {
|
|
12
|
-
const logger = useLogger()
|
|
13
|
-
|
|
14
|
-
const { default: express } = await import('express').catch(() => {
|
|
15
|
-
throw new Error('express is required for the rest plugin — run: npm install express')
|
|
16
|
-
})
|
|
17
|
-
|
|
18
|
-
const ownApp = !runtime.options.app
|
|
19
|
-
const app = runtime.options.app ?? express()
|
|
20
|
-
|
|
21
|
-
const router = express.Router()
|
|
22
|
-
router.use(express.json())
|
|
23
|
-
|
|
24
|
-
const token = runtime.config.rest?.token
|
|
25
|
-
const auth = (req, res, next) => {
|
|
26
|
-
if (!token) return next()
|
|
27
|
-
if (req.headers.authorization === `Bearer ${token}`) return next()
|
|
28
|
-
res.status(401).json({ error: 'Unauthorized' })
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
function collectionFolder(collection) {
|
|
32
|
-
return runtime.options[`${collection}Folder`]
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
router.get('/entities', async (req, res) => {
|
|
36
|
-
try {
|
|
37
|
-
const { page: rawPage, limit: rawLimit, ...filter } = req.query
|
|
38
|
-
const page = Math.max(1, parseInt(rawPage) || 1)
|
|
39
|
-
const limit = Math.min(100, Math.max(1, parseInt(rawLimit) || (runtime.config.rest?.pageSize ?? 10)))
|
|
40
|
-
const query = Object.keys(filter).length ? filter : undefined
|
|
41
|
-
|
|
42
|
-
const all = await findEntities(query)
|
|
43
|
-
const total = all.length
|
|
44
|
-
const totalPages = Math.ceil(total / limit)
|
|
45
|
-
const items = all.slice((page - 1) * limit, page * limit)
|
|
46
|
-
|
|
47
|
-
res.json({
|
|
48
|
-
items,
|
|
49
|
-
page,
|
|
50
|
-
limit,
|
|
51
|
-
total,
|
|
52
|
-
totalPages,
|
|
53
|
-
hasNext: page < totalPages,
|
|
54
|
-
hasPrev: page > 1,
|
|
55
|
-
})
|
|
56
|
-
} catch (err) {
|
|
57
|
-
logger.error('REST list error: %s', err.message)
|
|
58
|
-
res.status(500).json({ error: err.message })
|
|
59
|
-
}
|
|
60
|
-
})
|
|
61
|
-
|
|
62
|
-
router.put('/entities', auth, async (req, res) => {
|
|
63
|
-
try {
|
|
64
|
-
const { collection, relativePath, content = '' } = req.body
|
|
65
|
-
const folder = collectionFolder(collection)
|
|
66
|
-
if (!folder) return res.status(400).json({ error: `Unknown collection: ${collection}` })
|
|
67
|
-
const uri = path.join(folder, relativePath)
|
|
68
|
-
await mkdir(path.dirname(uri), { recursive: true })
|
|
69
|
-
await writeFile(uri, content, 'utf8')
|
|
70
|
-
res.status(202).json({ ok: true })
|
|
71
|
-
} catch (err) {
|
|
72
|
-
logger.error('REST update error: %s', err.message)
|
|
73
|
-
res.status(500).json({ error: err.message })
|
|
74
|
-
}
|
|
75
|
-
})
|
|
76
|
-
|
|
77
|
-
router.delete('/entities', auth, async (req, res) => {
|
|
78
|
-
try {
|
|
79
|
-
const { collection, relativePath } = req.body
|
|
80
|
-
const folder = collectionFolder(collection)
|
|
81
|
-
if (!folder) return res.status(400).json({ error: `Unknown collection: ${collection}` })
|
|
82
|
-
const uri = path.join(folder, relativePath)
|
|
83
|
-
await unlink(uri)
|
|
84
|
-
res.status(202).json({ ok: true })
|
|
85
|
-
} catch (err) {
|
|
86
|
-
logger.error('REST delete error: %s', err.message)
|
|
87
|
-
res.status(500).json({ error: err.message })
|
|
88
|
-
}
|
|
89
|
-
})
|
|
90
|
-
|
|
91
|
-
router.post('/render', async (req, res) => {
|
|
92
|
-
try {
|
|
93
|
-
const entity = { ...req.body, _correlationId: crypto.randomUUID() }
|
|
94
|
-
const timeout = runtime.config.rest?.renderTimeout ?? 30_000
|
|
95
|
-
|
|
96
|
-
const output = await new Promise((resolve, reject) => {
|
|
97
|
-
const timer = setTimeout(() => {
|
|
98
|
-
runtime.removeHook('completed', hook)
|
|
99
|
-
reject(new Error(`Render timeout for ${entity.id}`))
|
|
100
|
-
}, timeout)
|
|
101
|
-
|
|
102
|
-
const hook = async (entry) => {
|
|
103
|
-
if (entry.entity._correlationId !== entity._correlationId) return
|
|
104
|
-
clearTimeout(timer)
|
|
105
|
-
runtime.removeHook('completed', hook)
|
|
106
|
-
resolve(entry.output)
|
|
107
|
-
}
|
|
108
|
-
runtime.addHook('completed', hook)
|
|
109
|
-
|
|
110
|
-
updateEntity(entity)
|
|
111
|
-
.then(() => runtime.process())
|
|
112
|
-
.catch(reject)
|
|
113
|
-
})
|
|
114
|
-
|
|
115
|
-
if (output?.result != null) {
|
|
116
|
-
const isFile = await access(output.result).then(() => true).catch(() => false)
|
|
117
|
-
if (isFile) {
|
|
118
|
-
res.sendFile(output.result)
|
|
119
|
-
} else {
|
|
120
|
-
res.send(output.result)
|
|
121
|
-
}
|
|
122
|
-
} else {
|
|
123
|
-
res.status(204).send()
|
|
124
|
-
}
|
|
125
|
-
} catch (err) {
|
|
126
|
-
logger.error('REST render error: %s', err.message)
|
|
127
|
-
res.status(500).json({ error: err.message })
|
|
128
|
-
}
|
|
129
|
-
})
|
|
130
|
-
|
|
131
|
-
const base = runtime.config.rest?.base ?? '/mikser'
|
|
132
|
-
app.use(base, router)
|
|
133
|
-
|
|
134
|
-
if (ownApp) {
|
|
135
|
-
const port = runtime.config.rest?.port ?? 3001
|
|
136
|
-
app.listen(port, () => {
|
|
137
|
-
logger.info('REST plugin listening on port %d', port)
|
|
138
|
-
})
|
|
139
|
-
} else {
|
|
140
|
-
logger.info('REST plugin mounted on %s', base)
|
|
141
|
-
}
|
|
142
|
-
})
|
|
143
|
-
}
|