mikser-io 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +49 -0
- package/app.js +7 -0
- package/index.js +11 -0
- package/mikser-lockup-stacked.svg +14 -0
- package/package.json +64 -0
- package/src/catalog.js +56 -0
- package/src/config.js +37 -0
- package/src/constants.js +20 -0
- package/src/engine.js +317 -0
- package/src/journal.js +108 -0
- package/src/lifecycle.js +299 -0
- package/src/manager.js +119 -0
- package/src/plugins/api.js +176 -0
- package/src/plugins/assets.js +346 -0
- package/src/plugins/commands.js +75 -0
- package/src/plugins/data.js +138 -0
- package/src/plugins/documents.js +96 -0
- package/src/plugins/files.js +153 -0
- package/src/plugins/front-matter.js +24 -0
- package/src/plugins/json.js +20 -0
- package/src/plugins/layouts.js +368 -0
- package/src/plugins/mapper.js +29 -0
- package/src/plugins/post/pdf.js +60 -0
- package/src/plugins/render/asset.js +11 -0
- package/src/plugins/render/file.js +16 -0
- package/src/plugins/render/hbs.js +52 -0
- package/src/plugins/render/href.js +45 -0
- package/src/plugins/render/preset.js +13 -0
- package/src/plugins/render/resource.js +17 -0
- package/src/plugins/resources.js +216 -0
- package/src/plugins/rest.js +143 -0
- package/src/plugins/shares.js +40 -0
- package/src/plugins/validator.js +19 -0
- package/src/plugins/yaml.js +26 -0
- package/src/plugins.js +54 -0
- package/src/postprocess.js +47 -0
- package/src/render.js +71 -0
- package/src/runtime.js +153 -0
- package/src/tracking.js +74 -0
- package/src/utils.js +65 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Almero Digital Marketing
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="mikser-lockup-stacked.svg" alt="mikser" width="198" />
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
# Mikser Documentation
|
|
6
|
+
|
|
7
|
+
Mikser is a precision content engine for Node.js — built around a strict lifecycle, a composable plugin system, and zero compromise on output control. Every document, asset, and template flows through the same deterministic pipeline: import → process → render → finalize. Plugins hook in at any phase; nothing runs outside the cycle. The result is a system that scales from a single markdown blog to a multi-language, multi-format publishing platform — without ever losing sight of what it produced and why.
|
|
8
|
+
|
|
9
|
+
## Documentation Index
|
|
10
|
+
|
|
11
|
+
| Document | Audience | Description |
|
|
12
|
+
|----------|----------|-------------|
|
|
13
|
+
| [Getting Started](./documentation/getting-started.md) | Users | Installation, first project, basic usage |
|
|
14
|
+
| [Configuration](./documentation/configuration.md) | Users | All CLI options and config file reference |
|
|
15
|
+
| [Lifecycle](./documentation/lifecycle.md) | Users & Developers | Complete lifecycle phases and hook system |
|
|
16
|
+
| [Plugins](./documentation/plugins.md) | Users & Developers | Built-in plugins, writing custom plugins |
|
|
17
|
+
| [Entities](./documentation/entities.md) | Users & Developers | Entity model, operations, journal, catalog |
|
|
18
|
+
| [Rendering](./documentation/rendering.md) | Users & Developers | Render pipeline, render plugins, render modes |
|
|
19
|
+
| [Watch Mode](./documentation/watch-mode.md) | Users | File watching, scheduled tasks, incremental builds |
|
|
20
|
+
| [Architecture](./documentation/architecture.md) | Developers | System design, module structure, extension points |
|
|
21
|
+
| [API Reference](./documentation/api-reference.md) | Developers | Complete public API reference |
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm install mikser-io
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```js
|
|
30
|
+
// mikser.config.js
|
|
31
|
+
export default {
|
|
32
|
+
plugins: ['documents', 'layouts'],
|
|
33
|
+
layouts: {
|
|
34
|
+
cleanUrls: true
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
npx mikser
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Core Concepts
|
|
44
|
+
|
|
45
|
+
- **Lifecycle** — Processing runs through fixed phases: initialize → load → import → process → persist → render → finalize. Plugins hook into any phase.
|
|
46
|
+
- **Entities** — Everything is an entity (document, file, layout, asset). Entities flow through the journal and are tracked in the catalog.
|
|
47
|
+
- **Plugins** — Functionality is delivered via plugins. Built-in plugins handle common sources (documents, files, layouts, assets). Custom plugins can be added to any project.
|
|
48
|
+
- **Runtime Singleton** — A plain module-level object holds all global state and coordinates the lifecycle. The ES module cache guarantees every importer gets the same instance.
|
|
49
|
+
- **Watch Mode** — In watch mode, file changes trigger incremental re-processing without restarting.
|
package/app.js
ADDED
package/index.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { default as runtime } from './src/runtime.js'
|
|
2
|
+
export * as constants from './src/constants.js'
|
|
3
|
+
export * from './src/utils.js'
|
|
4
|
+
export * from './src/journal.js'
|
|
5
|
+
export * from './src/lifecycle.js'
|
|
6
|
+
export * from './src/catalog.js'
|
|
7
|
+
export * from './src/config.js'
|
|
8
|
+
export * from './src/plugins.js'
|
|
9
|
+
export * from './src/manager.js'
|
|
10
|
+
export * from './src/tracking.js'
|
|
11
|
+
export * from './src/engine.js'
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="396" height="330" viewBox="0 0 396 330" role="img" aria-label="mikser">
|
|
2
|
+
<title>mikser</title>
|
|
3
|
+
<g transform="translate(98 20)" style="isolation:isolate">
|
|
4
|
+
<g style="mix-blend-mode:multiply">
|
|
5
|
+
<circle cx="100" cy="72" r="46" fill="#0A0907"></circle>
|
|
6
|
+
<circle cx="124.249" cy="114" r="46" fill="#0A0907" opacity="0.85"></circle>
|
|
7
|
+
<circle cx="75.751" cy="114" r="46" fill="#FF3F00"></circle>
|
|
8
|
+
</g>
|
|
9
|
+
</g>
|
|
10
|
+
<g transform="translate(13 310.32)">
|
|
11
|
+
<path d="M75.46-55.77L75.46-55.77Q82.83-55.77 86.68-51.92Q90.53-48.07 90.53-40.70L90.53-40.70L90.53 0L74.47 0L74.47-35.64Q74.36-39.05 73.04-40.54Q71.72-42.02 68.42-42.02L68.42-42.02Q66.55-42.02 64.73-41.52Q62.92-41.03 60.61-39.82L60.61-39.82Q58.85-38.83 56.43-37.40L56.43-37.40L56.43 0L40.81 0L40.81-35.64Q40.81-39.16 39.33-40.59Q37.84-42.02 34.76-42.02L34.76-42.02Q32.89-42.02 31.07-41.52Q29.26-41.03 26.95-39.82L26.95-39.82Q25.19-38.94 22.88-37.62L22.88-37.62L22.88 0L6.71 0L6.71-54.34L19.58-54.34L20.79-46.09Q25.85-50.82 30.58-53.24L30.58-53.24Q35.64-55.77 41.58-55.77L41.58-55.77Q48.84-55.77 52.69-52.03L52.69-52.03Q54.67-49.94 55.66-46.86L55.66-46.86Q60.17-50.93 64.57-53.24L64.57-53.24Q69.63-55.77 75.46-55.77ZM101.86-54.34L117.92-54.34L117.92 0L101.86 0L101.86-54.34ZM106.37-78.32L113.41-78.32Q118.25-78.32 118.25-73.48L118.25-73.48L118.25-67.54Q118.25-62.70 113.41-62.70L113.41-62.70L106.37-62.70Q101.53-62.70 101.53-67.54L101.53-67.54L101.53-73.48Q101.53-78.32 106.37-78.32L106.37-78.32ZM163.90-54.34L182.05-54.34L167.86-35.42Q166.87-33.88 165.33-32.45Q163.79-31.02 162.80-30.36L162.80-30.36L162.80-30.03Q163.79-29.37 165.33-27.66Q166.87-25.96 167.86-24.20L167.86-24.20L183.92 0L165.77 0L150.37-25.30L145.09-25.30Q145.20-24.53 145.31-23.65L145.31-23.65Q145.75-20.02 145.75-17.05L145.75-17.05L145.75 0L129.58 0L129.58-77L145.75-77L145.86-45.21Q145.86-41.36 145.42-37.73L145.42-37.73Q145.31-36.41 145.09-35.20L145.09-35.20L150.48-35.20L163.90-54.34ZM209.44-55.77L209.44-55.77Q213.40-55.77 218.02-55.49Q222.64-55.22 227.04-54.78Q231.44-54.34 234.85-53.68L234.85-53.68L233.75-43.56Q228.47-43.67 223.14-43.78Q217.80-43.89 212.74-43.89L212.74-43.89Q208.45-43.89 206.14-43.73Q203.83-43.56 202.90-42.73Q201.96-41.91 201.96-39.82L201.96-39.82Q201.96-37.29 203.61-36.47Q205.26-35.64 209.11-34.65L209.11-34.65L223.30-31.24Q230.12-29.48 233.42-26.02Q236.72-22.55 236.72-15.62L236.72-15.62Q236.72-8.80 234.19-5.17Q231.66-1.54 226.16-0.17Q220.66 1.21 211.97 1.21L211.97 1.21Q208.56 1.21 201.85 0.88Q195.14 0.55 187.22-0.77L187.22-0.77L188.21-10.89Q190.30-10.78 193.33-10.72Q196.35-10.67 199.98-10.67L199.98-10.67L207.24-10.67Q212.96-10.67 215.93-10.95Q218.90-11.22 220.00-12.21Q221.10-13.20 221.10-15.18L221.10-15.18Q221.10-17.71 219.23-18.54Q217.36-19.36 213.29-20.46L213.29-20.46L199.54-23.87Q194.26-25.30 191.40-27.50Q188.54-29.70 187.39-32.84Q186.23-35.97 186.23-40.37L186.23-40.37Q186.23-45.98 188.54-49.39Q190.85-52.80 195.91-54.29Q200.97-55.77 209.44-55.77ZM270.71-55.77L270.71-55.77Q284.46-55.77 290.46-51.09Q296.45-46.42 296.45-37.18L296.45-37.18Q296.56-29.81 292.71-26.07Q288.86-22.33 279.40-22.33L279.40-22.33L260.04-22.33Q260.37-19.80 260.92-18.15L260.92-18.15Q262.13-14.41 265.05-12.98Q267.96-11.55 273.13-11.55L273.13-11.55Q276.87-11.55 282.43-11.82Q287.98-12.10 293.59-12.76L293.59-12.76L295.13-2.75Q291.94-1.21 287.87-0.33Q283.80 0.55 279.51 0.94Q275.22 1.32 271.26 1.32L271.26 1.32Q260.92 1.32 254.65-1.71Q248.38-4.73 245.58-11Q242.77-17.27 242.77-27.06L242.77-27.06Q242.77-37.73 245.58-44Q248.38-50.27 254.54-53.02Q260.70-55.77 270.71-55.77ZM259.71-31.46L259.71-31.46L275.00-31.46Q278.63-31.46 279.68-33.05Q280.72-34.65 280.72-37.51L280.72-37.51Q280.72-41.36 278.58-42.84Q276.43-44.33 271.26-44.33L271.26-44.33Q266.75-44.44 264.22-43.17Q261.69-41.91 260.70-38.50L260.70-38.50Q259.93-35.86 259.71-31.46ZM304.92-54.34L317.57-54.34L319.11-46.09Q324.39-50.71 329.89-53.13L329.89-53.13Q335.72-55.77 341.22-55.77L341.22-55.77L344.08-55.77L342.54-40.81L338.03-40.81Q334.18-40.81 330.00-39.71L330.00-39.71Q326.37-38.72 321.09-36.85L321.09-36.85L321.09 0L304.92 0L304.92-54.34Z" fill="#0A0907"></path>
|
|
12
|
+
<path d="M354.31-16.72L358.49-16.72Q361.68-16.72 362.94-15.46Q364.21-14.19 364.21-11L364.21-11L364.21-5.72Q364.21-2.53 362.94-1.26Q361.68 0 358.49 0L358.49 0L354.31 0Q351.12 0 349.86-1.26Q348.59-2.53 348.59-5.72L348.59-5.72L348.59-11Q348.59-14.19 349.86-15.46Q351.12-16.72 354.31-16.72L354.31-16.72Z" fill="#FF3F00"></path>
|
|
13
|
+
</g>
|
|
14
|
+
</svg>
|
package/package.json
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "mikser-io",
|
|
3
|
+
"version": "6.0.0",
|
|
4
|
+
"description": "<p align=\"center\"> <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"scripts": {
|
|
7
|
+
"test": "node --no-warnings app.js --working-folder test"
|
|
8
|
+
},
|
|
9
|
+
"bin": {
|
|
10
|
+
"mikser": "app.js"
|
|
11
|
+
},
|
|
12
|
+
"repository": {
|
|
13
|
+
"type": "git",
|
|
14
|
+
"url": "git+https://github.com/almero-digital-marketing/mikser-io.git"
|
|
15
|
+
},
|
|
16
|
+
"type": "module",
|
|
17
|
+
"author": "",
|
|
18
|
+
"license": "ISC",
|
|
19
|
+
"dependencies": {
|
|
20
|
+
"await-semaphore": "^0.1.3",
|
|
21
|
+
"axios": "^1.16.0",
|
|
22
|
+
"chokidar": "^5.0.0",
|
|
23
|
+
"cli-progress": "^3.12.0",
|
|
24
|
+
"commander": "^14.0.3",
|
|
25
|
+
"deepdash": "^5.3.9",
|
|
26
|
+
"escape-string-regexp": "^5.0.0",
|
|
27
|
+
"execa": "^9.6.1",
|
|
28
|
+
"front-matter": "^4.0.2",
|
|
29
|
+
"globby": "^16.2.0",
|
|
30
|
+
"handlebars": "^4.7.9",
|
|
31
|
+
"handlebars-helpers": "^0.10.0",
|
|
32
|
+
"hasha": "^7.0.0",
|
|
33
|
+
"is-url": "^1.2.4",
|
|
34
|
+
"knex": "^3.2.10",
|
|
35
|
+
"line-reader": "^0.4.0",
|
|
36
|
+
"lodash": "^4.18.1",
|
|
37
|
+
"lowdb": "^7.0.1",
|
|
38
|
+
"minimatch": "^10.2.5",
|
|
39
|
+
"node-cron": "^4.2.1",
|
|
40
|
+
"p-map": "^7.0.4",
|
|
41
|
+
"p-queue": "^9.2.0",
|
|
42
|
+
"pino": "^10.3.1",
|
|
43
|
+
"pino-pretty": "^13.1.3",
|
|
44
|
+
"piscina": "^5.1.4",
|
|
45
|
+
"sqlite3": "^6.0.1",
|
|
46
|
+
"truncate-stream": "^1.0.2",
|
|
47
|
+
"yaml": "^2.8.4"
|
|
48
|
+
},
|
|
49
|
+
"optionalDependencies": {
|
|
50
|
+
"puppeteer": "^24.43.0",
|
|
51
|
+
"express": "^4.21.0"
|
|
52
|
+
},
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"fluent-ffmpeg": "^2.1.3",
|
|
55
|
+
"sharp": "^0.34.5"
|
|
56
|
+
},
|
|
57
|
+
"directories": {
|
|
58
|
+
"test": "test"
|
|
59
|
+
},
|
|
60
|
+
"bugs": {
|
|
61
|
+
"url": "https://github.com/almero-digital-marketing/mikser-io/issues"
|
|
62
|
+
},
|
|
63
|
+
"homepage": "https://github.com/almero-digital-marketing/mikser-io#readme"
|
|
64
|
+
}
|
package/src/catalog.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import runtime from './runtime.js'
|
|
2
|
+
import { useLogger } from './engine.js'
|
|
3
|
+
import { onLoaded, onPersist, onFinalized } from './lifecycle.js'
|
|
4
|
+
import { useJournal } from './journal.js'
|
|
5
|
+
import { OPERATION } from './constants.js'
|
|
6
|
+
import { Low } from 'lowdb'
|
|
7
|
+
import path from 'node:path'
|
|
8
|
+
import { JSONFile } from 'lowdb/node'
|
|
9
|
+
import _ from 'lodash'
|
|
10
|
+
|
|
11
|
+
let catalog
|
|
12
|
+
|
|
13
|
+
onLoaded(async () => {
|
|
14
|
+
const adapter = new JSONFile(path.join(runtime.options.runtimeFolder, `catalog.json`))
|
|
15
|
+
catalog = new Low(adapter, {
|
|
16
|
+
entities: [],
|
|
17
|
+
})
|
|
18
|
+
catalog.chain = _.chain(catalog).get('data')
|
|
19
|
+
runtime.catalog = catalog
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
onPersist(async () => {
|
|
23
|
+
const logger = useLogger()
|
|
24
|
+
for await (let { operation, entity } of useJournal('Catalog')) {
|
|
25
|
+
switch (operation) {
|
|
26
|
+
case OPERATION.CREATE:
|
|
27
|
+
logger.trace('Database %s %s: %s', entity.collection, operation, entity.id)
|
|
28
|
+
catalog.data.entities.push(entity)
|
|
29
|
+
break
|
|
30
|
+
case OPERATION.UPDATE:
|
|
31
|
+
logger.trace('Database %s %s: %s', entity.collection, operation, entity.id)
|
|
32
|
+
catalog.chain.get('entities').find({ id: entity.id }).assign(entity).value()
|
|
33
|
+
break
|
|
34
|
+
case OPERATION.DELETE:
|
|
35
|
+
logger.trace('Database %s %s: %s', entity.collection, operation, entity.id)
|
|
36
|
+
catalog.chain.get('entities').remove({ id: entity.id }).value()
|
|
37
|
+
break
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
onFinalized(async () => {
|
|
43
|
+
await catalog.write()
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
export async function findEntity(query) {
|
|
47
|
+
if (!query) return
|
|
48
|
+
return catalog.chain.get('entities').find(query).value()
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function findEntities(query) {
|
|
52
|
+
if (!query) {
|
|
53
|
+
return catalog.chain.get('entities').value()
|
|
54
|
+
}
|
|
55
|
+
return catalog.chain.get('entities').filter(query).value()
|
|
56
|
+
}
|
package/src/config.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import runtime from './runtime.js'
|
|
2
|
+
import { useLogger } from './engine.js'
|
|
3
|
+
import { onLoad } from './lifecycle.js'
|
|
4
|
+
import path from 'node:path'
|
|
5
|
+
|
|
6
|
+
onLoad(async () => {
|
|
7
|
+
const logger = useLogger()
|
|
8
|
+
const configFile = path.resolve(runtime.options.config)
|
|
9
|
+
logger.info('Config: %s', configFile)
|
|
10
|
+
try {
|
|
11
|
+
const config = await import(configFile)
|
|
12
|
+
if (typeof config.default == 'function') {
|
|
13
|
+
runtime.config = await config.default(runtime)
|
|
14
|
+
} else if (typeof config.default == 'object') {
|
|
15
|
+
runtime.config = config.default
|
|
16
|
+
}
|
|
17
|
+
} catch (err) {
|
|
18
|
+
if (err.code != 'ERR_MODULE_NOT_FOUND') throw err
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const plugins = runtime.options.plugins.concat(runtime.config.plugins).filter((plugin) => plugin)
|
|
22
|
+
for (const plugin of plugins) {
|
|
23
|
+
if (!runtime.config[plugin]) {
|
|
24
|
+
try {
|
|
25
|
+
const pluginConfig = path.join(runtime.options.workingFolder, 'config', `${plugin}.config.js`)
|
|
26
|
+
const config = await import(pluginConfig)
|
|
27
|
+
if (typeof config.default == 'function') {
|
|
28
|
+
runtime.config[plugin] = await config.default(runtime)
|
|
29
|
+
} else if (typeof config.default == 'object') {
|
|
30
|
+
runtime.config[plugin] = config.default
|
|
31
|
+
}
|
|
32
|
+
} catch (err) {
|
|
33
|
+
if (err.code != 'ERR_MODULE_NOT_FOUND') throw err
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
})
|
package/src/constants.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export const OPERATION = {
|
|
2
|
+
CREATE: 'create',
|
|
3
|
+
UPDATE: 'update',
|
|
4
|
+
DELETE: 'delete',
|
|
5
|
+
RENDER: 'render',
|
|
6
|
+
POSTPROCESS: 'postprocess',
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export const ACTION = {
|
|
10
|
+
CREATE: 'create',
|
|
11
|
+
UPDATE: 'update',
|
|
12
|
+
DELETE: 'delete',
|
|
13
|
+
TRIGGER: 'trigger',
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export const TASKS = {
|
|
17
|
+
QUEUE: 'queue',
|
|
18
|
+
WORKER: 'worker',
|
|
19
|
+
POOL: 'pool',
|
|
20
|
+
}
|
package/src/engine.js
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import pino from 'pino'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import { Command } from 'commander'
|
|
4
|
+
import { rm, lstat, realpath, mkdir, writeFile } from 'fs/promises'
|
|
5
|
+
import { existsSync } from 'fs'
|
|
6
|
+
import _ from 'lodash'
|
|
7
|
+
import Piscina from 'piscina'
|
|
8
|
+
import runtime from './runtime.js'
|
|
9
|
+
import { onInitialize, onInitialized, onRender, onCancel, onCancelled, onFinalized, onLoaded, onAfterRender, onBeforePostprocess, onPostprocess, postprocessEntities } from './lifecycle.js'
|
|
10
|
+
import { useJournal, updateEntry } from './journal.js'
|
|
11
|
+
import { globby } from 'globby'
|
|
12
|
+
import { OPERATION, TASKS } from './constants.js'
|
|
13
|
+
import { changeExtension } from './utils.js'
|
|
14
|
+
import render from './render.js'
|
|
15
|
+
import postprocess, { loadPlugin as loadPostPlugin } from './postprocess.js'
|
|
16
|
+
import map from 'p-map'
|
|
17
|
+
import Queue from 'p-queue'
|
|
18
|
+
import packageInfo from '../package.json' with { type: 'json' }
|
|
19
|
+
|
|
20
|
+
export async function setup(options) {
|
|
21
|
+
runtime.options.threads = options?.threads !== undefined ? options.threads : 4
|
|
22
|
+
runtime.engine = {
|
|
23
|
+
logger: options?.logger || pino({
|
|
24
|
+
transport: {
|
|
25
|
+
target: 'pino-pretty'
|
|
26
|
+
},
|
|
27
|
+
}),
|
|
28
|
+
commander: new Command(),
|
|
29
|
+
renderWorkers: new Piscina({
|
|
30
|
+
filename: new URL('./render.js', import.meta.url).href,
|
|
31
|
+
maxThreads: runtime.options.threads
|
|
32
|
+
}),
|
|
33
|
+
queue: new Queue({ concurrency: 1 })
|
|
34
|
+
}
|
|
35
|
+
runtime.state = {}
|
|
36
|
+
|
|
37
|
+
onInitialize(async () => {
|
|
38
|
+
runtime.engine.commander?.version(packageInfo.version)
|
|
39
|
+
.option('-i --working-folder <folder>', 'set mikser working folder', './')
|
|
40
|
+
.option('-p --plugins [plugins...]', 'list of mikser plugins to load', [])
|
|
41
|
+
.option('-c --config <file>', 'set mikser mikser.config.js location', './mikser.config.js')
|
|
42
|
+
.option('-m --mode <mode>', 'set mikser runtime mode', 'development')
|
|
43
|
+
.option('-r --clear', 'clear current state before execution', false)
|
|
44
|
+
.option('-o --output-folder <folder>', 'set mikser output folder relative to working folder', 'out')
|
|
45
|
+
.option('-w --watch', 'watch entities for changes', false)
|
|
46
|
+
.option('-d --debug', 'display debug statements')
|
|
47
|
+
.option('-t --trace', 'display trace statements')
|
|
48
|
+
.option('-e --runtime-folder <folder>', 'set mikser runtime folder relative to working folder', 'runtime')
|
|
49
|
+
|
|
50
|
+
Object.assign(runtime.options, options || runtime.engine.commander.parse(process.argv).opts())
|
|
51
|
+
runtime.options.info = true
|
|
52
|
+
if (runtime.options.debug) {
|
|
53
|
+
runtime.engine.logger.level = 'debug'
|
|
54
|
+
runtime.options.info = false
|
|
55
|
+
}
|
|
56
|
+
if (runtime.options.trace) {
|
|
57
|
+
runtime.engine.logger.level = 'trace'
|
|
58
|
+
runtime.options.debug = false
|
|
59
|
+
runtime.options.info = false
|
|
60
|
+
}
|
|
61
|
+
runtime.engine.logger.notice = runtime.engine.logger.info
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
onInitialized(async () => {
|
|
65
|
+
const logger = useLogger()
|
|
66
|
+
|
|
67
|
+
runtime.options.workingFolder = path.resolve(runtime.options.workingFolder)
|
|
68
|
+
process.chdir(runtime.options.workingFolder)
|
|
69
|
+
|
|
70
|
+
runtime.options.runtimeFolder = path.join(runtime.options.workingFolder, runtime.options.runtimeFolder || 'runtime')
|
|
71
|
+
runtime.options.outputFolder = path.join(runtime.options.workingFolder, runtime.options.outputFolder || 'out')
|
|
72
|
+
|
|
73
|
+
logger.info('Working folder: %s', runtime.options.workingFolder)
|
|
74
|
+
logger.info('Output folder: %s', runtime.options.outputFolder)
|
|
75
|
+
|
|
76
|
+
if (runtime.options.clear) {
|
|
77
|
+
try {
|
|
78
|
+
logger.info('Clearing folders')
|
|
79
|
+
await rm(runtime.options.outputFolder, { recursive: true })
|
|
80
|
+
await rm(runtime.options.runtimeFolder, { recursive: true })
|
|
81
|
+
} catch (err) {
|
|
82
|
+
if (err.code != 'ENOENT')
|
|
83
|
+
throw err
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
await mkdir(runtime.options.runtimeFolder, { recursive: true })
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
onLoaded(async () => {
|
|
90
|
+
const logger = useLogger()
|
|
91
|
+
logger.debug(runtime.options, 'Mikser options')
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
onRender(async (signal) => {
|
|
95
|
+
const logger = useLogger()
|
|
96
|
+
const renderJobs = new Set()
|
|
97
|
+
await map(useJournal('Rendering', [OPERATION.RENDER], signal), async entry => {
|
|
98
|
+
const { id, entity, options, context } = entry
|
|
99
|
+
const jobId = entity.id + ':' + entity.destination
|
|
100
|
+
if (!renderJobs.has(jobId) && !options.ignore) {
|
|
101
|
+
renderJobs.add(jobId)
|
|
102
|
+
const renderOptions = {
|
|
103
|
+
entity,
|
|
104
|
+
options: {
|
|
105
|
+
tasks: TASKS.POOL,
|
|
106
|
+
...runtime.options,
|
|
107
|
+
...options,
|
|
108
|
+
},
|
|
109
|
+
config: _.pickBy(runtime.config, (value, key) => _.startsWith(key, 'render-')),
|
|
110
|
+
context,
|
|
111
|
+
state: runtime.state
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
let result
|
|
115
|
+
switch (renderOptions.options.tasks) {
|
|
116
|
+
case TASKS.POOL:
|
|
117
|
+
renderOptions.logger = logger
|
|
118
|
+
renderOptions.signal = signal
|
|
119
|
+
if (!signal.aborted) {
|
|
120
|
+
result = await render(renderOptions)
|
|
121
|
+
}
|
|
122
|
+
break
|
|
123
|
+
case TASKS.QUEUE:
|
|
124
|
+
renderOptions.logger = logger
|
|
125
|
+
renderOptions.signal = signal
|
|
126
|
+
if (!signal.aborted) {
|
|
127
|
+
result = await runtime.engine.queue.add(() => render(renderOptions), { signal })
|
|
128
|
+
}
|
|
129
|
+
break
|
|
130
|
+
case TASKS.WORKER:
|
|
131
|
+
const mc = new MessageChannel();
|
|
132
|
+
mc.port2.onmessage = event => {
|
|
133
|
+
const message = JSON.parse(event.data)
|
|
134
|
+
if (message.command == 'logger') {
|
|
135
|
+
runtime.engine.logger[message.data.log](...message.data.args)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
mc.port2.unref()
|
|
139
|
+
renderOptions.port = mc.port1
|
|
140
|
+
result = await runtime.engine.renderWorkers.run(
|
|
141
|
+
renderOptions,
|
|
142
|
+
{ signal, transferList: [mc.port1] }
|
|
143
|
+
)
|
|
144
|
+
break
|
|
145
|
+
}
|
|
146
|
+
if (!signal.aborted) {
|
|
147
|
+
entry.output = {
|
|
148
|
+
success: true,
|
|
149
|
+
result,
|
|
150
|
+
}
|
|
151
|
+
await runtime.complete(entry)
|
|
152
|
+
await updateEntry({ id, output: entry.output })
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
logger.debug('Rendered: [%s] %s → %s', options.renderer, entity.name || entity.id, entity.destination)
|
|
156
|
+
} catch (err) {
|
|
157
|
+
if (!signal.aborted) {
|
|
158
|
+
await updateEntry({ id, output: { success: false } })
|
|
159
|
+
logger.error('Render error: %s %s', entity.id, err.message)
|
|
160
|
+
}
|
|
161
|
+
logger.debug('Render canceled')
|
|
162
|
+
}
|
|
163
|
+
} else {
|
|
164
|
+
await updateEntry({ id, output: { success: true } })
|
|
165
|
+
}
|
|
166
|
+
}, {
|
|
167
|
+
concurrency: runtime.options.threads,
|
|
168
|
+
signal
|
|
169
|
+
})
|
|
170
|
+
renderJobs.size && logger.info('Rendered: %d', renderJobs.size)
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
onAfterRender(async () => {
|
|
174
|
+
const results = new Map()
|
|
175
|
+
for await (let { output, entity } of useJournal('Output', [OPERATION.RENDER])) {
|
|
176
|
+
if (output?.success) {
|
|
177
|
+
const jobId = entity.id + ':' + entity.destination
|
|
178
|
+
results.set(jobId, entity)
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
const renderOutput = path.join(runtime.options.runtimeFolder, `render-details.json`)
|
|
182
|
+
await writeFile(renderOutput, JSON.stringify(Array.from(results.values())), 'utf8')
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
onBeforePostprocess(async (signal) => {
|
|
186
|
+
const tasks = []
|
|
187
|
+
for await (const { entity, options, context, output } of useJournal('Queuing postprocess', [OPERATION.RENDER], signal)) {
|
|
188
|
+
if (output?.success && options.postprocessor) {
|
|
189
|
+
tasks.push({
|
|
190
|
+
entity: {
|
|
191
|
+
...entity,
|
|
192
|
+
origin: entity.destination,
|
|
193
|
+
destination: changeExtension(entity.destination, options.postprocessor)
|
|
194
|
+
},
|
|
195
|
+
options: { postprocessor: options.postprocessor, tasks: options.tasks },
|
|
196
|
+
context
|
|
197
|
+
})
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
if (tasks.length) await postprocessEntities(tasks)
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
onPostprocess(async (signal) => {
|
|
204
|
+
const logger = useLogger()
|
|
205
|
+
const config = _.pickBy(runtime.config, (value, key) => _.startsWith(key, 'post-'))
|
|
206
|
+
|
|
207
|
+
const postPlugins = {}
|
|
208
|
+
for (const pluginName of runtime.options.plugins.filter(p => p.startsWith('post-'))) {
|
|
209
|
+
const plugin = await loadPostPlugin(pluginName, runtime.options.workingFolder)
|
|
210
|
+
if (plugin) {
|
|
211
|
+
postPlugins[pluginName] = plugin
|
|
212
|
+
if (plugin.setup) await plugin.setup({ options: runtime.options, config: config[pluginName], state: runtime.state, logger })
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const postprocessJobs = new Set()
|
|
217
|
+
try {
|
|
218
|
+
await map(useJournal('Postprocessing', [OPERATION.POSTPROCESS], signal), async entry => {
|
|
219
|
+
const { id, entity, options, context } = entry
|
|
220
|
+
const jobId = entity.id + ':' + entity.destination
|
|
221
|
+
if (!postprocessJobs.has(jobId) && !options.ignore) {
|
|
222
|
+
postprocessJobs.add(jobId)
|
|
223
|
+
const postprocessOptions = {
|
|
224
|
+
entity,
|
|
225
|
+
options: {
|
|
226
|
+
tasks: TASKS.POOL,
|
|
227
|
+
...runtime.options,
|
|
228
|
+
...options,
|
|
229
|
+
},
|
|
230
|
+
config,
|
|
231
|
+
context,
|
|
232
|
+
state: runtime.state
|
|
233
|
+
}
|
|
234
|
+
try {
|
|
235
|
+
let result
|
|
236
|
+
switch (postprocessOptions.options.tasks) {
|
|
237
|
+
case TASKS.POOL:
|
|
238
|
+
postprocessOptions.logger = logger
|
|
239
|
+
postprocessOptions.signal = signal
|
|
240
|
+
if (!signal.aborted) {
|
|
241
|
+
result = await postprocess(postprocessOptions)
|
|
242
|
+
}
|
|
243
|
+
break
|
|
244
|
+
case TASKS.QUEUE:
|
|
245
|
+
postprocessOptions.logger = logger
|
|
246
|
+
postprocessOptions.signal = signal
|
|
247
|
+
if (!signal.aborted) {
|
|
248
|
+
result = await runtime.engine.queue.add(() => postprocess(postprocessOptions), { signal })
|
|
249
|
+
}
|
|
250
|
+
break
|
|
251
|
+
}
|
|
252
|
+
if (!signal.aborted) {
|
|
253
|
+
entry.output = { success: true }
|
|
254
|
+
if (result) entry.output.result = result
|
|
255
|
+
await runtime.complete(entry)
|
|
256
|
+
await updateEntry({ id, output: entry.output })
|
|
257
|
+
}
|
|
258
|
+
logger.debug('Postprocessed: [%s] %s → %s', options.postprocessor, entity.name || entity.id, entity.destination)
|
|
259
|
+
} catch (err) {
|
|
260
|
+
if (!signal.aborted) {
|
|
261
|
+
await updateEntry({ id, output: { success: false } })
|
|
262
|
+
logger.error('Postprocess error: %s %s', entity.id, err.message)
|
|
263
|
+
}
|
|
264
|
+
logger.debug('Postprocess canceled')
|
|
265
|
+
}
|
|
266
|
+
} else {
|
|
267
|
+
await updateEntry({ id, output: { success: true } })
|
|
268
|
+
}
|
|
269
|
+
}, {
|
|
270
|
+
concurrency: runtime.options.threads,
|
|
271
|
+
signal
|
|
272
|
+
})
|
|
273
|
+
postprocessJobs.size && logger.info('Postprocessed: %d', postprocessJobs.size)
|
|
274
|
+
} finally {
|
|
275
|
+
for (const [pluginName, plugin] of Object.entries(postPlugins)) {
|
|
276
|
+
if (plugin.teardown) await plugin.teardown({ options: runtime.options, config: config[pluginName], state: runtime.state, logger })
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
onCancel(async () => {
|
|
282
|
+
if (runtime.engine.renderWorkers.queueSize) {
|
|
283
|
+
await new Promise(resolve => {
|
|
284
|
+
runtime.engine.renderWorkers.once('drain', resolve)
|
|
285
|
+
})
|
|
286
|
+
}
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
onFinalized(async () => {
|
|
290
|
+
const logger = useLogger()
|
|
291
|
+
|
|
292
|
+
const paths = await globby('**/*', { cwd: runtime.options.outputFolder, followSymbolicLinks: false })
|
|
293
|
+
for (let relativePath of paths) {
|
|
294
|
+
let source = path.join(runtime.options.outputFolder, relativePath)
|
|
295
|
+
const linkStat = await lstat(source)
|
|
296
|
+
if (linkStat.isSymbolicLink()) {
|
|
297
|
+
const destination = await realpath(source)
|
|
298
|
+
if (!existsSync(destination)) {
|
|
299
|
+
await unlink(source)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
logger.notice('Mikser completed')
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
onCancelled(async () => {
|
|
307
|
+
const logger = useLogger()
|
|
308
|
+
logger.notice('Mikser restarted')
|
|
309
|
+
})
|
|
310
|
+
|
|
311
|
+
console.info('Mikser: %s', packageInfo.version)
|
|
312
|
+
return runtime
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export function useLogger() {
|
|
316
|
+
return runtime.engine.logger
|
|
317
|
+
}
|