lakutata 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -30
- package/README.zh-CN.md +130 -0
- package/dist/cjs/exports/QueueComponent.d.ts +1 -0
- package/dist/cjs/exports/QueueComponent.js +20 -0
- package/dist/esm/exports/QueueComponent.js +1 -0
- package/dist/types/exports/QueueComponent.d.ts +1 -0
- package/doc/en/Application.md +107 -0
- package/doc/en/CLI.md +274 -0
- package/doc/en/Cache.md +379 -0
- package/doc/en/Command.md +110 -0
- package/doc/en/ContainerdIntegration.md +17 -16
- package/doc/en/Controllers.md +256 -0
- package/doc/en/DBus.md +291 -0
- package/doc/en/DTOValidation.md +72 -43
- package/doc/en/Database.md +188 -0
- package/doc/en/DatabaseDrivers.md +287 -0
- package/doc/en/Decorator.md +52 -40
- package/doc/en/DockerIntegration.md +7 -6
- package/doc/en/Entities.md +240 -0
- package/doc/en/Entrypoint.md +216 -0
- package/doc/en/Exceptions.md +174 -0
- package/doc/en/GRPC.md +251 -0
- package/doc/en/GettingStarted.md +165 -0
- package/doc/en/HTTP.md +507 -0
- package/doc/en/HelperFunctions.md +11 -11
- package/doc/en/I18n.md +119 -115
- package/doc/en/Injection.md +144 -0
- package/doc/en/Installers.md +280 -0
- package/doc/en/Logger.md +206 -0
- package/doc/en/MCP.md +293 -0
- package/doc/en/Middlewares.md +213 -0
- package/doc/en/Migrations.md +202 -0
- package/doc/en/MongoEmbedded.md +149 -0
- package/doc/en/Monitor.md +164 -0
- package/doc/en/Mutex.md +27 -25
- package/doc/en/NATS.md +246 -0
- package/doc/en/NativeAddons.md +212 -0
- package/doc/en/Objects.md +143 -0
- package/doc/en/Overview.md +62 -85
- package/doc/en/Packaging.md +193 -0
- package/doc/en/ProjectStructure.md +124 -0
- package/doc/en/Queue.md +269 -0
- package/doc/en/Schedules.md +217 -0
- package/doc/en/Security.md +10 -9
- package/doc/en/ServiceCalls.md +504 -0
- package/doc/en/Time.md +19 -20
- package/doc/en/UpgradingFrom1.md +435 -0
- package/doc/en/UpgradingFrom2.md +315 -0
- package/doc/en/Workspaces.md +189 -0
- package/doc/en/examples/BackgroundJobs.md +677 -0
- package/doc/en/examples/CommandLineTool.md +664 -0
- package/doc/en/examples/DesktopApp.md +809 -0
- package/doc/en/examples/DeviceService.md +607 -0
- package/doc/en/examples/MCPServer.md +653 -0
- package/doc/en/examples/README.md +16 -0
- package/doc/en/examples/RestAPI.md +490 -0
- package/doc/en/examples/ServiceCalls.md +829 -0
- package/doc/en/examples/WebApp.md +568 -0
- package/doc/zh/Application.md +75 -0
- package/doc/zh/CLI.md +250 -0
- package/doc/zh/Cache.md +328 -0
- package/doc/zh/Command.md +78 -0
- package/doc/zh/ContainerdIntegration.md +9 -9
- package/doc/zh/Controllers.md +228 -0
- package/doc/zh/DBus.md +257 -0
- package/doc/zh/DTOValidation.md +49 -19
- package/doc/zh/Database.md +175 -0
- package/doc/zh/DatabaseDrivers.md +261 -0
- package/doc/zh/Decorator.md +25 -18
- package/doc/zh/DockerIntegration.md +6 -6
- package/doc/zh/Entities.md +224 -0
- package/doc/zh/Entrypoint.md +193 -0
- package/doc/zh/Exceptions.md +158 -0
- package/doc/zh/GRPC.md +226 -0
- package/doc/zh/GettingStarted.md +148 -0
- package/doc/zh/HTTP.md +445 -0
- package/doc/zh/HelperFunctions.md +2 -2
- package/doc/zh/I18n.md +26 -26
- package/doc/zh/Injection.md +129 -0
- package/doc/zh/Installers.md +232 -0
- package/doc/zh/Logger.md +188 -0
- package/doc/zh/MCP.md +265 -0
- package/doc/zh/Middlewares.md +198 -0
- package/doc/zh/Migrations.md +178 -0
- package/doc/zh/MongoEmbedded.md +118 -0
- package/doc/zh/Monitor.md +145 -0
- package/doc/zh/Mutex.md +10 -10
- package/doc/zh/NATS.md +207 -0
- package/doc/zh/NativeAddons.md +180 -0
- package/doc/zh/Objects.md +119 -0
- package/doc/zh/Overview.md +46 -71
- package/doc/zh/Packaging.md +154 -0
- package/doc/zh/ProjectStructure.md +102 -0
- package/doc/zh/Queue.md +214 -0
- package/doc/zh/Schedules.md +193 -0
- package/doc/zh/Security.md +6 -6
- package/doc/zh/ServiceCalls.md +459 -0
- package/doc/zh/Time.md +2 -2
- package/doc/zh/UpgradingFrom1.md +347 -0
- package/doc/zh/UpgradingFrom2.md +230 -0
- package/doc/zh/Workspaces.md +150 -0
- package/doc/zh/examples/BackgroundJobs.md +618 -0
- package/doc/zh/examples/CommandLineTool.md +618 -0
- package/doc/zh/examples/DesktopApp.md +748 -0
- package/doc/zh/examples/DeviceService.md +546 -0
- package/doc/zh/examples/MCPServer.md +604 -0
- package/doc/zh/examples/README.md +14 -0
- package/doc/zh/examples/RestAPI.md +458 -0
- package/doc/zh/examples/ServiceCalls.md +765 -0
- package/doc/zh/examples/WebApp.md +520 -0
- package/package.json +46 -17
- package/doc/en/ActionPattern.md +0 -408
- package/doc/en/Core.md +0 -312
- package/doc/en/ORMIntegration.md +0 -341
- package/doc/zh/ActionPattern.md +0 -334
- package/doc/zh/Core.md +0 -256
- package/doc/zh/ORMIntegration.md +0 -319
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# Lakutata Migrations
|
|
2
|
+
|
|
3
|
+
`synchronize: true` changes the tables to match the entities each time the database connects: handy in development, but
|
|
4
|
+
unsafe in production, where a renamed column would be dropped with its data. In production the schema changes through
|
|
5
|
+
**migrations**: classes that each make one change, run in order, and recorded in the database so that each runs once.
|
|
6
|
+
|
|
7
|
+
## 1. A migration
|
|
8
|
+
|
|
9
|
+
A migration implements `MigrationInterface` from `lakutata/orm`. `up()` makes the change, `down()` reverts it:
|
|
10
|
+
|
|
11
|
+
```typescript
|
|
12
|
+
import type {MigrationInterface, QueryRunner} from 'lakutata/orm'
|
|
13
|
+
|
|
14
|
+
export class AddBookAuthor1792161460658 implements MigrationInterface {
|
|
15
|
+
public name: string = 'AddBookAuthor1792161460658'
|
|
16
|
+
|
|
17
|
+
public async up(queryRunner: QueryRunner): Promise<void> {
|
|
18
|
+
await queryRunner.query('ALTER TABLE "books" ADD "author" character varying')
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
public async down(queryRunner: QueryRunner): Promise<void> {
|
|
22
|
+
await queryRunner.query('ALTER TABLE "books" DROP COLUMN "author"')
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
- The name of a migration ends with a timestamp in milliseconds (13 digits): the migrations run in the order of their
|
|
28
|
+
timestamps. Two migrations with the same name are an error.
|
|
29
|
+
- The `migrations` option of the database lists them: the classes, or globs of their compiled files, relative to the
|
|
30
|
+
working directory.
|
|
31
|
+
|
|
32
|
+
```typescript
|
|
33
|
+
import {buildDatabaseOptions} from 'lakutata/com/database'
|
|
34
|
+
|
|
35
|
+
export const db = buildDatabaseOptions({
|
|
36
|
+
type: 'postgres',
|
|
37
|
+
url: process.env.DATABASE_URL,
|
|
38
|
+
entities: [],
|
|
39
|
+
migrations: ['build/migrations/*.js']
|
|
40
|
+
})
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
- The migrations run are recorded in a table named `migrations` (`migrationsTableName` changes its name). A
|
|
44
|
+
migration is run once, even if its timestamp is older than the last one run.
|
|
45
|
+
|
|
46
|
+
## 2. The `lakutata` command
|
|
47
|
+
|
|
48
|
+
The projects created by `lakutata` have scripts for the migrations:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
npm run migration:generate # a migration of the changes of the entities
|
|
52
|
+
npm run migration:create -- seed # an empty migration, src/migrations/<timestamp>-seed.ts
|
|
53
|
+
npm run migration:run # run the pending migrations
|
|
54
|
+
npm run migration:show # list the migrations, run and pending
|
|
55
|
+
npm run migration:revert # revert the last migration run
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
They call `lakutata migration <action>`:
|
|
59
|
+
|
|
60
|
+
| Option | Default | Meaning |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| `--dir` | `src/migrations` | The directory of the migrations written (`generate`, `create`) |
|
|
63
|
+
| `--script` | `dev` | The npm script that builds and starts the application |
|
|
64
|
+
| `--database` | Every database | The database of the command; required to revert with several databases |
|
|
65
|
+
| `--transaction` | The `migrationsTransactionMode` of the database | `all`, `each` or `none` (section 6) |
|
|
66
|
+
| `--fake` | `false` | Record the migrations as run (or reverted) without running them |
|
|
67
|
+
| `--package` | The current directory | The package of the workspace to run the command in |
|
|
68
|
+
|
|
69
|
+
`create` writes an empty migration without starting the application. The other actions run the npm script of the
|
|
70
|
+
application with the environment variables of section 3: the application itself does the command.
|
|
71
|
+
|
|
72
|
+
## 3. The commands of the application
|
|
73
|
+
|
|
74
|
+
An application started with one of these environment variables does the command instead of launching: it reads its
|
|
75
|
+
options, connects to its databases, does the command, and exits. Its `bootstrap` and its `onLaunched` handlers do not
|
|
76
|
+
run, and no server starts.
|
|
77
|
+
|
|
78
|
+
| Variable | Command |
|
|
79
|
+
|---|---|
|
|
80
|
+
| `MIGRATION_GENERATE=<directory>` | Write a migration of the differences between the entities and the tables (section 4) |
|
|
81
|
+
| `MIGRATION_RUN=1` | Run the pending migrations |
|
|
82
|
+
| `MIGRATION_REVERT=1` | Revert the last migration run |
|
|
83
|
+
| `MIGRATION_SHOW=1` | List the migrations of the options, run (`[X]`) and pending (`[ ]`) |
|
|
84
|
+
| `MIGRATION_DATABASE=<name>` | The database of the command: the name of its component or provider (every database by default) |
|
|
85
|
+
| `MIGRATION_TRANSACTION=all\|each\|none` | The transactions of `MIGRATION_RUN` and `MIGRATION_REVERT` (section 6) |
|
|
86
|
+
| `MIGRATION_FAKE=1` | Record the migrations as run (or reverted) without running them |
|
|
87
|
+
|
|
88
|
+
```text
|
|
89
|
+
$ MIGRATION_SHOW=1 node build/App.js
|
|
90
|
+
[db] [ ] Migrations1792161460658
|
|
91
|
+
[db] 1 pending migration
|
|
92
|
+
$ MIGRATION_RUN=1 node build/App.js
|
|
93
|
+
[db] Migration Migrations1792161460658 has been run
|
|
94
|
+
$ MIGRATION_REVERT=1 node build/App.js
|
|
95
|
+
[db] Migration Migrations1792161460658 has been reverted
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
- A flag is set by `1`, `true` or `yes`: `MIGRATION_RUN=0` starts the application as usual.
|
|
99
|
+
- The commands work on the `Database` components and providers declared at the top of the options (`components`,
|
|
100
|
+
`providers`), each named by its key.
|
|
101
|
+
- The exit code is 0 when the command succeeded, 1 when it failed. With several databases, `MIGRATION_REVERT` needs
|
|
102
|
+
`MIGRATION_DATABASE`.
|
|
103
|
+
|
|
104
|
+
## 4. Generating a migration
|
|
105
|
+
|
|
106
|
+
`MIGRATION_GENERATE` compares the entities with the tables of the database, and writes the SQL of the differences as a
|
|
107
|
+
migration:
|
|
108
|
+
|
|
109
|
+
```text
|
|
110
|
+
$ MIGRATION_GENERATE=src/migrations node build/App.js
|
|
111
|
+
Migration /home/app/src/migrations/1792161460658-db.ts has been generated successfully.
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
```typescript
|
|
115
|
+
import { MigrationInterface, QueryRunner } from "lakutata/orm";
|
|
116
|
+
|
|
117
|
+
export class Migrations1792161460658 implements MigrationInterface {
|
|
118
|
+
name = 'Migrations1792161460658'
|
|
119
|
+
|
|
120
|
+
public async up(queryRunner: QueryRunner): Promise<void> {
|
|
121
|
+
await queryRunner.query(`
|
|
122
|
+
CREATE TABLE "books" (
|
|
123
|
+
"id" SERIAL NOT NULL,
|
|
124
|
+
"title" character varying NOT NULL,
|
|
125
|
+
CONSTRAINT "PK_f3f2f25a099d24e12545b70b022" PRIMARY KEY ("id")
|
|
126
|
+
)
|
|
127
|
+
`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
public async down(queryRunner: QueryRunner): Promise<void> {
|
|
131
|
+
await queryRunner.query(`
|
|
132
|
+
DROP TABLE "books"
|
|
133
|
+
`);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
- The database must be reachable, and up to date with the migrations already written: the differences are computed
|
|
140
|
+
from its tables.
|
|
141
|
+
- The file is `<directory>/<timestamp>-<database name>.ts`, a TypeScript class named after the directory
|
|
142
|
+
(`src/migrations` gives `Migrations<timestamp>`). Build it with the application: the `migrations` option lists the
|
|
143
|
+
compiled files.
|
|
144
|
+
- Read the migration before running it: a renamed property becomes a column dropped and a column added, with the data
|
|
145
|
+
of the old one lost. Edit the SQL when needed.
|
|
146
|
+
- When nothing changed, the command writes nothing and exits with the code 1 (`No changes in database schema were
|
|
147
|
+
found`).
|
|
148
|
+
- With several databases, each database whose tables differ gets its own file. Give each database a `migrations` option
|
|
149
|
+
that lists only its own files (`'build/migrations/*-db.js'`), or the databases would run each other's migrations.
|
|
150
|
+
|
|
151
|
+
## 5. Running the migrations at startup
|
|
152
|
+
|
|
153
|
+
Instead of a command, the database can run its pending migrations each time it connects, with `migrationsRun: true`.
|
|
154
|
+
The order is then: `dropSchema` (if set), the migrations, then `synchronize` (if set).
|
|
155
|
+
|
|
156
|
+
- The `Database` component connects once: listed in `bootstrap`, it runs the migrations while the application launches.
|
|
157
|
+
- The `Database` provider connects each time it is resolved, so it checks the migrations each time.
|
|
158
|
+
|
|
159
|
+
The component and the provider also have the methods `runMigrations({transaction, fake})`,
|
|
160
|
+
`undoLastMigration({transaction, fake})` and `showMigrations()` (which resolves to whether migrations are pending).
|
|
161
|
+
|
|
162
|
+
## 6. Transactions
|
|
163
|
+
|
|
164
|
+
`migrationsTransactionMode`, an option of the database, or `MIGRATION_TRANSACTION` for one command:
|
|
165
|
+
|
|
166
|
+
| Mode | The pending migrations run… |
|
|
167
|
+
|---|---|
|
|
168
|
+
| `all` (default) | In one transaction: a failure reverts all of them |
|
|
169
|
+
| `each` | Each in its own transaction: a failure keeps the migrations run before it |
|
|
170
|
+
| `none` | Without a transaction |
|
|
171
|
+
|
|
172
|
+
- A migration can set its own `transaction` property (`true` or `false`) in the modes `each` and `none`; in the mode
|
|
173
|
+
`all`, this is an error.
|
|
174
|
+
- PostgreSQL and SQLite run the changes of the schema inside transactions. MySQL and MariaDB commit each change of the
|
|
175
|
+
schema at once: a failed migration can leave part of its changes applied.
|
|
176
|
+
|
|
177
|
+
## 7. MongoDB
|
|
178
|
+
|
|
179
|
+
The MongoDB databases (`mongodb`, `mongo-embedded`) have no schema to compare: `MIGRATION_GENERATE` writes nothing for
|
|
180
|
+
them, and their migrations are written by hand. `queryRunner.query()` is not supported; the entity manager of MongoDB
|
|
181
|
+
does the work:
|
|
182
|
+
|
|
183
|
+
```typescript
|
|
184
|
+
import type {MigrationInterface, MongoEntityManager, QueryRunner} from 'lakutata/orm'
|
|
185
|
+
|
|
186
|
+
export class RenameTitle1792161460700 implements MigrationInterface {
|
|
187
|
+
public name: string = 'RenameTitle1792161460700'
|
|
188
|
+
|
|
189
|
+
public async up(queryRunner: QueryRunner): Promise<void> {
|
|
190
|
+
const manager: MongoEntityManager = queryRunner.manager as MongoEntityManager
|
|
191
|
+
await manager.updateMany('notes', {}, {$rename: {title: 'name'}})
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
public async down(queryRunner: QueryRunner): Promise<void> {
|
|
195
|
+
const manager: MongoEntityManager = queryRunner.manager as MongoEntityManager
|
|
196
|
+
await manager.updateMany('notes', {}, {$rename: {name: 'title'}})
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
MongoDB runs them without transactions: a failed migration leaves the migrations run before it, and its own changes made
|
|
202
|
+
before the failure.
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# Lakutata Embedded MongoDB
|
|
2
|
+
|
|
3
|
+
The `mongo-embedded` driver gives an application a MongoDB of its own, inside its process: the entities, the
|
|
4
|
+
repositories and the query operators of MongoDB in TypeORM, with the documents stored in the files of a directory. There
|
|
5
|
+
is no server to install or to run, which suits desktop applications, devices, command line tools and tests.
|
|
6
|
+
|
|
7
|
+
It is a part of lakutata, not of TypeORM. It runs the queries with [mingo](https://github.com/kofrasa/mingo), a
|
|
8
|
+
JavaScript implementation of the query language of MongoDB.
|
|
9
|
+
|
|
10
|
+
## 1. A first database
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
npm install lakutata mingo bson
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
```typescript
|
|
17
|
+
import {Application} from 'lakutata'
|
|
18
|
+
import {buildDatabaseOptions, type Database} from 'lakutata/com/database'
|
|
19
|
+
import {Column, Entity, Index, ObjectIdColumn} from 'lakutata/decorator/orm'
|
|
20
|
+
import type {MongoRepository, ObjectId} from 'lakutata/orm'
|
|
21
|
+
|
|
22
|
+
@Entity('notes')
|
|
23
|
+
class Note {
|
|
24
|
+
@ObjectIdColumn()
|
|
25
|
+
public id: ObjectId
|
|
26
|
+
|
|
27
|
+
@Index()
|
|
28
|
+
@Column()
|
|
29
|
+
public folder: string
|
|
30
|
+
|
|
31
|
+
@Column()
|
|
32
|
+
public title: string
|
|
33
|
+
|
|
34
|
+
@Column()
|
|
35
|
+
public words: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
Application
|
|
39
|
+
.run(() => ({
|
|
40
|
+
id: 'notes.app',
|
|
41
|
+
name: 'Notes',
|
|
42
|
+
components: {
|
|
43
|
+
db: buildDatabaseOptions({
|
|
44
|
+
type: 'mongo-embedded',
|
|
45
|
+
//A directory, created when missing
|
|
46
|
+
database: 'data/notes',
|
|
47
|
+
entities: [Note],
|
|
48
|
+
//Creates the indices of the entities
|
|
49
|
+
synchronize: true
|
|
50
|
+
})
|
|
51
|
+
},
|
|
52
|
+
bootstrap: ['db']
|
|
53
|
+
}))
|
|
54
|
+
.onLaunched(async (app: Application): Promise<void> => {
|
|
55
|
+
const db: Database = await app.getObject<Database>('db')
|
|
56
|
+
const notes: MongoRepository<Note> = db.getMongoRepository(Note)
|
|
57
|
+
await notes.save([
|
|
58
|
+
{folder: 'work', title: 'Plan', words: 120},
|
|
59
|
+
{folder: 'work', title: 'Review', words: 40},
|
|
60
|
+
{folder: 'home', title: 'Shopping', words: 15}
|
|
61
|
+
])
|
|
62
|
+
//The query operators of MongoDB: ["Plan"]
|
|
63
|
+
console.log((await notes.find({where: {folder: 'work', words: {$gte: 50}}})).map((note: Note): string => note.title))
|
|
64
|
+
//An aggregation: [{_id: 'home', words: 15}, {_id: 'work', words: 160}]
|
|
65
|
+
console.log(await notes.aggregate([{$group: {_id: '$folder', words: {$sum: '$words'}}}, {$sort: {_id: 1}}]).toArray())
|
|
66
|
+
await notes.updateMany({folder: 'home'}, {$set: {folder: 'personal'}})
|
|
67
|
+
})
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The entities, the repositories and the queries are the ones of MongoDB in TypeORM (see
|
|
71
|
+
[MongoDB](DatabaseDrivers.md#mongodb)): an `@ObjectIdColumn()` typed with the `ObjectId` of `lakutata/orm`, a
|
|
72
|
+
`MongoRepository` with the methods of MongoDB. An application can move to a MongoDB server by changing its options only.
|
|
73
|
+
|
|
74
|
+
## 2. Options
|
|
75
|
+
|
|
76
|
+
| Option | Default | Meaning |
|
|
77
|
+
|---|---|---|
|
|
78
|
+
| `database` | `':memory:'` | The directory of the database, or `':memory:'` for a database that is not saved |
|
|
79
|
+
| `sync` | `'interval'` | When the writes are flushed to the disk: `'interval'`, `'always'` or `'never'` (section 3) |
|
|
80
|
+
| `syncInterval` | `100` | The interval of the flushes of `sync: 'interval'`, in milliseconds |
|
|
81
|
+
| `cacheSize` | 64 MiB | The memory of the documents kept in memory after they are read, in bytes; `0` keeps none |
|
|
82
|
+
| `compaction` | `{minimumSize: 16 MiB, ratio: 2}` | When the file of a collection is compacted; `false` never (section 3) |
|
|
83
|
+
| `snapshot` | `{minimumSize: 64 MiB, ratio: 0.25}` | When the indices of a collection are saved while it is open; `false` only when it closes (section 3) |
|
|
84
|
+
| `worker` | `true` | Run the database in a worker thread (section 4) |
|
|
85
|
+
| `workerResourceLimits` | None | The memory limits of the worker thread (section 4) |
|
|
86
|
+
|
|
87
|
+
The options common to every database (`entities`, `synchronize`, `logging`…) apply too.
|
|
88
|
+
|
|
89
|
+
## 3. How the documents are stored
|
|
90
|
+
|
|
91
|
+
- Each collection has its own file in the directory. A write is appended to the file in one write; the documents are
|
|
92
|
+
found by their `_id` through an index held in memory, outside the heap of JavaScript.
|
|
93
|
+
- **Durability.** With `sync: 'interval'`, the writes are flushed to the disk every 100 milliseconds: a write that has
|
|
94
|
+
resolved survives a crash of the application, but the last interval of writes can be lost if the computer crashes or
|
|
95
|
+
loses power. `'always'` flushes before each write resolves (the writes made together share one flush); `'never'`
|
|
96
|
+
leaves the flushes to the system.
|
|
97
|
+
- **Crashes.** A write cut by a crash is ignored when the database opens again: the database is as it was before that
|
|
98
|
+
write.
|
|
99
|
+
- **Compaction.** An update or a delete leaves the old version of the document in the file. Once the file is
|
|
100
|
+
`minimumSize` and `ratio` times larger than its live documents, it is compacted: its live documents are copied to a
|
|
101
|
+
new file, while the reads and the writes go on.
|
|
102
|
+
- **Snapshots.** The indices are saved to a snapshot when the database closes, and while it is open once `minimumSize`
|
|
103
|
+
bytes and a `ratio` of the file have been written since the last one. Opening the database reads the snapshot, then
|
|
104
|
+
only the records written after it. Without a snapshot (after a crash before the first one), the indices are built
|
|
105
|
+
again from the whole file: about 60 seconds for 10 million documents with two indices.
|
|
106
|
+
- **One process.** A directory is locked by the process that opens it. A second process fails to open it, with an error
|
|
107
|
+
naming the process that holds it.
|
|
108
|
+
|
|
109
|
+
## 4. The worker thread
|
|
110
|
+
|
|
111
|
+
By default the database runs in a worker thread: its queries, its writes and its files never block the event loop of
|
|
112
|
+
the application. Each call and its results cross between the threads encoded as BSON, which costs about 20 µs a call.
|
|
113
|
+
|
|
114
|
+
- `worker: false` runs the database in the thread of the application. Its calls are faster, and the queries can then
|
|
115
|
+
use JavaScript functions (`$where`, `$function`, `$accumulator`), which cannot be sent to a worker thread.
|
|
116
|
+
- The worker thread has the memory limits of the application unless `workerResourceLimits` sets its own
|
|
117
|
+
(`maxOldGenerationSizeMb`, `maxYoungGenerationSizeMb`, `codeRangeSizeMb`, `stackSizeMb`). A worker thread that
|
|
118
|
+
exceeds them is stopped, not the application: its calls fail and its directory is unlocked. A limit of 256 MB or less
|
|
119
|
+
may stop the whole process on Node.js 24 instead; Bun ignores these limits.
|
|
120
|
+
|
|
121
|
+
## 5. Large collections
|
|
122
|
+
|
|
123
|
+
- **Long writes.** `insertMany`, `updateMany`, `deleteMany`, and the creation of an index on existing documents run in
|
|
124
|
+
batches of 1000 documents, letting the application run between them. As in MongoDB without a transaction, a write
|
|
125
|
+
that fails on a document leaves the documents before it written.
|
|
126
|
+
- **`insertMany`.** As in MongoDB, it stops at its first duplicate key, or inserts the other documents with
|
|
127
|
+
`ordered: false`. It then fails with `writeErrors`, `insertedCount` and `insertedIds`.
|
|
128
|
+
- **Cursors.** A cursor without a sort (`createCursor`, `createEntityCursor`) read with `next()`, `hasNext()` or
|
|
129
|
+
`for await` reads 1000 documents at a time; `toArray()` reads all of them at once.
|
|
130
|
+
- **Sorting.** A sort without an index holds all the documents it sorts in memory: index the fields that large
|
|
131
|
+
collections are sorted on. A sort with a `limit` reads the order of an index.
|
|
132
|
+
- **Grouping.** A `$group` at the start of a pipeline, after stages that work on one document at a time (`$match`,
|
|
133
|
+
`$project`, `$unwind`…), reads the documents a batch at a time: its memory is the one of its groups.
|
|
134
|
+
- **Index keys** are limited to about 1000 bytes: a write giving a larger key fails before it is written, with the code
|
|
135
|
+
17280 of MongoDB.
|
|
136
|
+
- Integers beyond 2^53 are kept as `Long`. In a sort done in memory (without an index, or without a `limit`), they come
|
|
137
|
+
after the other numbers, in the order of their text.
|
|
138
|
+
|
|
139
|
+
## 6. What it does not do
|
|
140
|
+
|
|
141
|
+
As with a MongoDB server in TypeORM:
|
|
142
|
+
|
|
143
|
+
- no relations, no query builders;
|
|
144
|
+
- no transactions: **`transaction()` runs its function without a transaction**;
|
|
145
|
+
- `synchronize` creates the indices, and the migrations are written by hand (see [Migrations](Migrations.md)).
|
|
146
|
+
|
|
147
|
+
The `ObjectId` of the ids is the one exported by `lakutata/orm`. The queries accept the `ObjectId` of the package
|
|
148
|
+
`bson` too, but `instanceof` tells them apart: an id read from the database is not an instance of the `ObjectId` of
|
|
149
|
+
`import {ObjectId} from 'bson'`.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# Lakutata Monitors
|
|
2
|
+
|
|
3
|
+
The monitors of `lakutata/com/monitor` measure the process an application runs in: its CPU usage, its memory, the delays
|
|
4
|
+
of its event loop, and the time its HTTP requests take. Each one is a component that keeps the statistics of its
|
|
5
|
+
measures since it was created, or since its last `reset()`, in constant memory. They need nothing more than `lakutata`.
|
|
6
|
+
|
|
7
|
+
## 1. Declaring and reading the monitors
|
|
8
|
+
|
|
9
|
+
```typescript
|
|
10
|
+
import {Application, type ILogger} from 'lakutata'
|
|
11
|
+
import {buildEntrypoints, Controller} from 'lakutata/com/entrypoint'
|
|
12
|
+
import {GET} from 'lakutata/decorator/ctrl'
|
|
13
|
+
import {Inject} from 'lakutata/decorator/di'
|
|
14
|
+
import {expressHTTPEntrypoint} from 'lakutata/adapter/express'
|
|
15
|
+
import {CpuMonitor, EventLoopMonitor, HttpRequestMonitor, MemoryMonitor} from 'lakutata/com/monitor'
|
|
16
|
+
|
|
17
|
+
class MetricsController extends Controller {
|
|
18
|
+
@Inject('cpu')
|
|
19
|
+
protected readonly cpu: CpuMonitor
|
|
20
|
+
|
|
21
|
+
@Inject('memory')
|
|
22
|
+
protected readonly memory: MemoryMonitor
|
|
23
|
+
|
|
24
|
+
@Inject('eventLoop')
|
|
25
|
+
protected readonly eventLoop: EventLoopMonitor
|
|
26
|
+
|
|
27
|
+
@Inject('httpRequests')
|
|
28
|
+
protected readonly httpRequests: HttpRequestMonitor
|
|
29
|
+
|
|
30
|
+
@GET('/metrics')
|
|
31
|
+
public async metrics(): Promise<Record<string, unknown>> {
|
|
32
|
+
return {
|
|
33
|
+
cpu: this.cpu.statistics,
|
|
34
|
+
memory: this.memory.statistics,
|
|
35
|
+
eventLoop: this.eventLoop.statistics,
|
|
36
|
+
httpRequests: this.httpRequests.statistics
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
Application
|
|
42
|
+
.run(() => ({
|
|
43
|
+
id: 'api.app',
|
|
44
|
+
name: 'API',
|
|
45
|
+
components: {
|
|
46
|
+
cpu: {class: CpuMonitor},
|
|
47
|
+
memory: {class: MemoryMonitor},
|
|
48
|
+
eventLoop: {class: EventLoopMonitor},
|
|
49
|
+
httpRequests: {class: HttpRequestMonitor},
|
|
50
|
+
entrypoint: buildEntrypoints({
|
|
51
|
+
controllers: [MetricsController],
|
|
52
|
+
http: expressHTTPEntrypoint({port: 8080})
|
|
53
|
+
})
|
|
54
|
+
},
|
|
55
|
+
//Created when the application launches, so that they measure from the start
|
|
56
|
+
bootstrap: ['cpu', 'memory', 'eventLoop', 'httpRequests', 'entrypoint']
|
|
57
|
+
}))
|
|
58
|
+
.onLaunched(async (app: Application, log: ILogger): Promise<void> => {
|
|
59
|
+
const cpu: CpuMonitor = await app.getObject<CpuMonitor>('cpu')
|
|
60
|
+
const eventLoop: EventLoopMonitor = await app.getObject<EventLoopMonitor>('eventLoop')
|
|
61
|
+
//Every minute, the statistics of the minute, then a new period (/metrics starts over with it)
|
|
62
|
+
setInterval((): void => {
|
|
63
|
+
log.info({cpuAvg: cpu.statistics.usageAvg, eventLoopMax: eventLoop.statistics.max}, 'the last minute')
|
|
64
|
+
cpu.reset()
|
|
65
|
+
eventLoop.reset()
|
|
66
|
+
}, 60000)
|
|
67
|
+
})
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
- Declare each monitor you need with its `class`, under the name of your choice. A monitor is a singleton, so every
|
|
71
|
+
object that injects it shares its statistics.
|
|
72
|
+
- A monitor measures from its creation: list it in `bootstrap`. Otherwise it is created the first time it is used, and
|
|
73
|
+
measures from then on.
|
|
74
|
+
- The monitors have no options. An option given to one is ignored, with a warning when the application starts.
|
|
75
|
+
- `statistics` is computed at each read. `reset()` starts a new period: the statistics then cover the measures taken
|
|
76
|
+
since.
|
|
77
|
+
- Their timers do not keep the process alive, and they stop when the application stops.
|
|
78
|
+
|
|
79
|
+
## 2. What they measure
|
|
80
|
+
|
|
81
|
+
| Monitor | Measures | Sampled | Unit |
|
|
82
|
+
|---|---|---|---|
|
|
83
|
+
| `CpuMonitor` | The CPU time of the process | Every second | Percent of one core |
|
|
84
|
+
| `MemoryMonitor` | The memory of the process: resident, heap, external | At its creation, then every second | Bytes |
|
|
85
|
+
| `EventLoopMonitor` | The delays of the event loop, and the share of the time it is busy | Continuously | Milliseconds, percent |
|
|
86
|
+
| `HttpRequestMonitor` | The duration of the HTTP requests the process answers | Each request | Milliseconds |
|
|
87
|
+
|
|
88
|
+
The statistics of the CPU, the memory and the HTTP requests hold the latest value, the minimum, the maximum and the
|
|
89
|
+
mean, which are exact, and percentiles (P50, P90, P95, P99) with 3 significant digits. The statistics of the CPU and of
|
|
90
|
+
the HTTP requests are all 0 before their first measure.
|
|
91
|
+
|
|
92
|
+
## 3. `CpuMonitor`
|
|
93
|
+
|
|
94
|
+
| Field | Meaning |
|
|
95
|
+
|---|---|
|
|
96
|
+
| `usage` | The latest sample |
|
|
97
|
+
| `usageMin`, `usageMax`, `usageAvg` | The minimum, the maximum and the mean of the samples |
|
|
98
|
+
| `usageP50`, `usageP90`, `usageP95`, `usageP99` | The percentiles of the samples |
|
|
99
|
+
|
|
100
|
+
- The usage is the CPU time of the whole process (user and system, every thread) over the elapsed time, in percent of
|
|
101
|
+
one core: 100 is one core fully busy, and a process using several cores (worker threads, the thread pool of Node.js)
|
|
102
|
+
goes above 100. The values are rounded to 5 decimals.
|
|
103
|
+
- The first sample is taken one second after the monitor is created.
|
|
104
|
+
- When the event loop is blocked, the next sample comes late and covers the whole blocked time. It then counts once per
|
|
105
|
+
second it covers, so that the mean and the percentiles weigh the busy periods by their duration.
|
|
106
|
+
|
|
107
|
+
## 4. `MemoryMonitor`
|
|
108
|
+
|
|
109
|
+
| Fields | Meaning |
|
|
110
|
+
|---|---|
|
|
111
|
+
| `physicalTotal` | The memory of the machine (`os.totalmem()`), not the limit of a container |
|
|
112
|
+
| `physicalUsed`, `physicalUsage` | The resident memory of the process (`rss`), and its percentage of `physicalTotal` |
|
|
113
|
+
| `heapTotal`, `heapUsed`, `heapUsage` | The heap of V8 and its used part, and their ratio in percent |
|
|
114
|
+
| `externalUsed` | The memory of the C++ objects bound to JavaScript objects, the `Buffer`s among them |
|
|
115
|
+
| `physicalUsedMin`, `…Max`, `…Avg`, `…P50`, `…P90`, `…P95`, `…P99` | The statistics of `physicalUsed` |
|
|
116
|
+
| `heapUsedMin`, `…Max`, `…Avg`, `…P50`, `…P90`, `…P95`, `…P99` | The statistics of `heapUsed` |
|
|
117
|
+
| `externalUsedMin`, `…Max`, `…Avg`, `…P50`, `…P90`, `…P95`, `…P99` | The statistics of `externalUsed` |
|
|
118
|
+
|
|
119
|
+
- The values are in bytes; the means and the two percentages are rounded to 2 decimals.
|
|
120
|
+
- `heapUsage` compares the used heap with the heap allocated so far, not with its limit: a high value alone is not a
|
|
121
|
+
problem. A leak shows as a `heapUsedMin` that rises from one period to the next.
|
|
122
|
+
- `reset()` keeps the latest values (`physicalUsed`, `heapUsed`…); the other statistics start again.
|
|
123
|
+
|
|
124
|
+
## 5. `EventLoopMonitor`
|
|
125
|
+
|
|
126
|
+
| Field | Meaning |
|
|
127
|
+
|---|---|
|
|
128
|
+
| `min`, `max`, `avg`, `stdDev` | The minimum, the maximum, the mean and the standard deviation of the delays |
|
|
129
|
+
| `p50`, `p90`, `p95`, `p99` | The percentiles of the delays |
|
|
130
|
+
| `utilRate` | The share of the time the event loop was busy, in percent |
|
|
131
|
+
|
|
132
|
+
- The delay is the interval between the runs of a timer of 1 ms: about 1 ms when the event loop is idle, more when
|
|
133
|
+
synchronous work (a long computation, synchronous I/O, a large JSON) holds it. `max` is the longest block.
|
|
134
|
+
- `utilRate` covers the whole period since the creation or the last `reset()`: near 100, the event loop has no idle
|
|
135
|
+
time left. Right after a `reset()`, it reads 100, the event loop being busy running the code that called it.
|
|
136
|
+
- The values come from the histogram of Node.js (`monitorEventLoopDelay`): they are not rounded, and they are
|
|
137
|
+
approximate, the minimum, the maximum and the mean included.
|
|
138
|
+
- The timer of 1 ms costs about 1.5 % of one core while the application is idle: on a device that runs on a battery,
|
|
139
|
+
declare this monitor only when you need it.
|
|
140
|
+
|
|
141
|
+
## 6. `HttpRequestMonitor`
|
|
142
|
+
|
|
143
|
+
| Field | Meaning |
|
|
144
|
+
|---|---|
|
|
145
|
+
| `count` | The number of requests answered |
|
|
146
|
+
| `min`, `max`, `avg`, `stdDev` | The minimum, the maximum, the mean and the standard deviation of their durations |
|
|
147
|
+
| `p50`, `p90`, `p95`, `p99` | The percentiles of their durations |
|
|
148
|
+
|
|
149
|
+
- It counts the requests answered by every HTTP server of the process (the HTTP adapters, the HTTP service, JSON-RPC
|
|
150
|
+
over HTTP, the server of Socket.IO, a server of your own), errors included, HTTPS too. It does not count the requests
|
|
151
|
+
of HTTP/2, those that the process sends, nor those it never answers.
|
|
152
|
+
- It gives no breakdown by route or by status.
|
|
153
|
+
- It only counts the requests that start after its creation: a request in progress when the monitor is created is not
|
|
154
|
+
counted. Read from an action, the statistics do not include the request being answered.
|
|
155
|
+
- The durations are in milliseconds, rounded to 2 decimals.
|
|
156
|
+
|
|
157
|
+
## 7. Worker threads
|
|
158
|
+
|
|
159
|
+
A worker thread that declares monitors measures:
|
|
160
|
+
|
|
161
|
+
- the delays and the busy time of its own event loop (`EventLoopMonitor`);
|
|
162
|
+
- the requests of its own HTTP servers (`HttpRequestMonitor`);
|
|
163
|
+
- the CPU and the resident memory of the whole process (`CpuMonitor`, `physicalUsed` of `MemoryMonitor`), but its own
|
|
164
|
+
heap (`heapUsed`).
|
package/doc/en/Mutex.md
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# Lakutata Mutex
|
|
2
2
|
|
|
3
|
-
The **Mutex component** (`lakutata/com/mutex`, package `@lakutata/mutex`) gives
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
The **Mutex component** (`lakutata/com/mutex`, package `@lakutata/mutex`) gives an application named locks: the work
|
|
4
|
+
protected by a lock runs in one place at a time. Depending on the driver, a lock is shared by the components of a
|
|
5
|
+
process, by the processes of a machine, or by every instance of the application. The component also serves
|
|
6
|
+
`@Synchronized`, which runs a controller action under a lock, and the `exclusive` schedules. The README of
|
|
7
|
+
`@lakutata/mutex` gives the details.
|
|
7
8
|
|
|
8
9
|
## 1. Configuration
|
|
9
10
|
|
|
@@ -33,14 +34,14 @@ Application.run(() => ({
|
|
|
33
34
|
|
|
34
35
|
| Driver | Shared by | Notes |
|
|
35
36
|
|---|---|---|
|
|
36
|
-
| `{type: 'memory'}` (default) |
|
|
37
|
-
| `{type: 'file', directory}` |
|
|
38
|
-
| `{type: 'redis', url}` |
|
|
39
|
-
| `{type: 'postgres', host, database, username, password, mode}` |
|
|
40
|
-
| `{type: 'mysql', host, database, username, password, mode}` |
|
|
37
|
+
| `{type: 'memory'}` (default) | The components of a process | Development, tests, a single instance |
|
|
38
|
+
| `{type: 'file', directory?}` | The processes of a machine | Lock files, created exclusively (in `lakutata-mutex` under the temporary directory of the system unless `directory` is given); the lock of a process that died is taken over at once |
|
|
39
|
+
| `{type: 'redis', url}` | Every instance | Redis 4+ or Valkey, package `redis`; fencing tokens; use `noeviction` (an evicted key is a lost lock) |
|
|
40
|
+
| `{type: 'postgres', host, database, username, password, mode}` | Every instance | Package `pg`; `mode: 'advisory'` (default): advisory locks on a connection of its own, released when it closes, not usable through PgBouncer in transaction mode; `mode: 'table'`: rows of a table, which expire, with fencing tokens |
|
|
41
|
+
| `{type: 'mysql', host, database, username, password, mode}` | Every instance | Package `mysql2`; `mode: 'advisory'` (default): `GET_LOCK` on a connection of its own, which the nodes of a cluster do not share; `mode: 'table'`: rows of a table, which expire, with fencing tokens |
|
|
41
42
|
|
|
42
|
-
The clients (`redis`, `pg`, `mysql2`) are optional peer dependencies
|
|
43
|
-
|
|
43
|
+
The clients (`redis`, `pg`, `mysql2`) are optional peer dependencies. A driver connects at the first acquisition; when
|
|
44
|
+
its package is missing, that acquisition fails with `MutexDriverNotFoundException`.
|
|
44
45
|
|
|
45
46
|
## 2. Usage
|
|
46
47
|
|
|
@@ -88,20 +89,20 @@ class Orders extends Component {
|
|
|
88
89
|
- **Lost locks**: when a renewal fails until the lock expires, or the connection holding an advisory lock closes,
|
|
89
90
|
`lock.held` becomes `false` and `lock.signal` is aborted with a `MutexLostException`. The work is not interrupted: it
|
|
90
91
|
should watch the signal.
|
|
91
|
-
- **Fencing tokens**: `lock.fence` increases with each acquisition (memory, Redis, the `table` modes):
|
|
92
|
-
|
|
92
|
+
- **Fencing tokens**: `lock.fence` increases with each acquisition (memory, Redis, the `table` modes): a store written
|
|
93
|
+
under a lock can use it to refuse the writes of an older holder.
|
|
93
94
|
- **Not reentrant**: acquiring a lock the component already holds waits for its release, like any other holder.
|
|
94
95
|
- **Lifecycle**: the component releases its locks and closes its driver when it is destroyed (the application
|
|
95
96
|
stopping).
|
|
96
97
|
|
|
97
98
|
## 3. Synchronized actions
|
|
98
99
|
|
|
99
|
-
`@Synchronized(name, options?)` runs a controller action
|
|
100
|
-
access control passed, released
|
|
101
|
-
their transport
|
|
102
|
-
a function `(input, context) => string`. The options are `mutex` (the
|
|
103
|
-
`wait` and `ttl`. A request not
|
|
104
|
-
when
|
|
100
|
+
`@Synchronized(name, options?)` runs a controller action under a lock, acquired after its input is validated and its
|
|
101
|
+
access control has passed, and released when the action returns. The requests that need the same lock run one at a time,
|
|
102
|
+
across every instance and whatever their transport; the other requests run in parallel. The name is a template whose
|
|
103
|
+
`{field}` are fields of the validated input, or a function `(input, context) => string`. The options are `mutex` (the
|
|
104
|
+
name of the component, `'mutex'` by default), `timeout`, `wait` and `ttl`. A request that does not get the lock in time
|
|
105
|
+
fails with `MutexTimeoutException` (HTTP 409); it stops waiting when it is aborted.
|
|
105
106
|
|
|
106
107
|
```typescript
|
|
107
108
|
import {Controller} from 'lakutata/com/entrypoint'
|
|
@@ -132,11 +133,12 @@ class OrderController extends Controller {
|
|
|
132
133
|
|
|
133
134
|
## 4. Exclusive schedules
|
|
134
135
|
|
|
135
|
-
A scheduled action declared `exclusive` runs in one instance of the application at a time: each run tries to acquire
|
|
136
|
-
|
|
137
|
-
the instances not
|
|
138
|
-
|
|
139
|
-
|
|
136
|
+
A scheduled action declared `exclusive` runs in one instance of the application at a time: each run tries to acquire the
|
|
137
|
+
lock `schedule:<key>` (the key is the name of the action by default) from the component `mutex` (or the one
|
|
138
|
+
`exclusive: {mutex, key}` names), and the instances that do not get it skip the run. Its intervals are aligned on the
|
|
139
|
+
clock (every 5 minutes means at :00, :05, :10…), so that every instance plans the same times. After a run, the lock is
|
|
140
|
+
held until half the period has passed since the planned time (30 s at most), so that an instance whose clock is a little
|
|
141
|
+
behind does not run it again. Without its lock component, the action does not run, and the error is logged.
|
|
140
142
|
|
|
141
143
|
```typescript
|
|
142
144
|
import {Controller} from 'lakutata/com/entrypoint'
|
|
@@ -156,7 +158,7 @@ class ReportController extends Controller {
|
|
|
156
158
|
```
|
|
157
159
|
|
|
158
160
|
The entrypoint needs a schedule adapter: `buildEntrypoints({controllers: [ReportController], schedule:
|
|
159
|
-
scheduleEntrypoint()})` (see [
|
|
161
|
+
scheduleEntrypoint()})` (see [Schedules](Schedules.md)).
|
|
160
162
|
|
|
161
163
|
## 5. Exceptions
|
|
162
164
|
|