mikroscope 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +7 -0
- package/README.md +337 -0
- package/dist/cli.mjs +124 -0
- package/openapi/openapi.json +1433 -0
- package/openapi/openapi.yaml +949 -0
- package/package.json +70 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Copyright 2026-present Mikael Vesavuori
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
# MikroScope
|
|
2
|
+
|
|
3
|
+
**Log sidecar using NDJSON: ingest, query, retention, and alerts.**
|
|
4
|
+
|
|
5
|
+
MikroScope runs next to your service, writes/reads `.ndjson` logs, indexes them into SQLite, and exposes an HTTP API for search and aggregation.
|
|
6
|
+
|
|
7
|
+
## What You Get
|
|
8
|
+
|
|
9
|
+
- **Ingest API**: Accepts logs over HTTP and writes `.ndjson` files
|
|
10
|
+
- Lets backend and frontend send logs to one place
|
|
11
|
+
- **SQLite index**: Continuously indexes raw logs
|
|
12
|
+
- Fast queries without giving up raw source logs
|
|
13
|
+
- **Query + aggregate API**: Filter logs and group by level/event/field/correlation
|
|
14
|
+
- Quick troubleshooting and basic analytics
|
|
15
|
+
- **Health + docs endpoints**: Runtime health plus OpenAPI/interactive reference
|
|
16
|
+
- Easier operations and integration
|
|
17
|
+
- **Retention + maintenance**: Cleans old DB/log records and checks free disk
|
|
18
|
+
- Keeps long-running deployments stable
|
|
19
|
+
- **Webhook alerts**: Sends notifications for error spikes or outages
|
|
20
|
+
- Basic operational alerting without extra tooling
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
| Requirement | Notes |
|
|
25
|
+
|-------------------------------|----------------------------------------------|
|
|
26
|
+
| Node.js `>= 24` | Required to run MikroScope |
|
|
27
|
+
| npm | Required for npm install / npx flows |
|
|
28
|
+
| `curl` or `wget` | Used by installer to download release assets |
|
|
29
|
+
| `tar` (or `unzip` for `.zip`) | Needed to extract release archive |
|
|
30
|
+
|
|
31
|
+
| Method | Recommended for | Command |
|
|
32
|
+
|---------------------------|-------------------------------------------|------------------------------------|
|
|
33
|
+
| npm global install | Node/npm environments with persistent CLI | `npm install -g mikroscope` |
|
|
34
|
+
| npx | One-off execution without global install | `npx mikroscope --help` |
|
|
35
|
+
| One-line installer | VM/server installs without npm dependency | See command below |
|
|
36
|
+
| Non-interactive installer | CI/provisioning via release assets | See command below |
|
|
37
|
+
| Manual release archive | Pinned/manual installs | See "Manual Release Install" below |
|
|
38
|
+
|
|
39
|
+
Installer behavior:
|
|
40
|
+
|
|
41
|
+
| Step | Result |
|
|
42
|
+
|-------------------------|------------------------------------------------------------|
|
|
43
|
+
| Download latest release | Fetches latest tagged archive from GitHub Releases |
|
|
44
|
+
| Verify checksum | Uses `SHA256SUMS.txt` when available |
|
|
45
|
+
| Expand archive | Installs binaries/docs under your chosen install directory |
|
|
46
|
+
| Prompt for config | Writes host/port/path/auth settings to a local env file |
|
|
47
|
+
| Create wrapper | Adds a `mikroscope` command in your chosen bin directory |
|
|
48
|
+
|
|
49
|
+
npm install examples:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# Install once globally
|
|
53
|
+
npm install -g mikroscope
|
|
54
|
+
mikroscope --help
|
|
55
|
+
|
|
56
|
+
# Run without installing globally
|
|
57
|
+
npx mikroscope --help
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
One-line installer:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
curl -fsSL https://raw.githubusercontent.com/mikaelvesavuori/mikroscope/main/scripts/install.sh | sh
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Non-interactive installer:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
MIKROSCOPE_INSTALL_NONINTERACTIVE=1 \
|
|
70
|
+
MIKROSCOPE_INSTALL_DIR="$HOME/.local/share/mikroscope" \
|
|
71
|
+
MIKROSCOPE_BIN_DIR="$HOME/.local/bin" \
|
|
72
|
+
curl -fsSL https://raw.githubusercontent.com/mikaelvesavuori/mikroscope/main/scripts/install.sh | sh
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
If `mikroscope` is not found after install, add your chosen bin directory to `PATH` (the installer prints the exact export command).
|
|
76
|
+
|
|
77
|
+
### Manual Release Install
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
VERSION=0.0.1
|
|
81
|
+
curl -LO "https://github.com/mikaelvesavuori/mikroscope/releases/download/v${VERSION}/mikroscope-${VERSION}.tar.gz"
|
|
82
|
+
curl -LO "https://github.com/mikaelvesavuori/mikroscope/releases/download/v${VERSION}/SHA256SUMS.txt"
|
|
83
|
+
shasum -a 256 -c SHA256SUMS.txt
|
|
84
|
+
tar -xzf "mikroscope-${VERSION}.tar.gz"
|
|
85
|
+
cd "mikroscope-${VERSION}"
|
|
86
|
+
./mikroscope serve --host 127.0.0.1 --port 4310 --logs ./logs --db ./data/mikroscope.db
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Quick Start
|
|
90
|
+
|
|
91
|
+
1. Start MikroScope with API auth and ingest producer mappings:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
mikroscope serve \
|
|
95
|
+
--host 127.0.0.1 \
|
|
96
|
+
--port 4310 \
|
|
97
|
+
--logs ./logs \
|
|
98
|
+
--db ./data/mikroscope.db \
|
|
99
|
+
--api-token "api-token-123" \
|
|
100
|
+
--ingest-producers "backend-token=backend-api,frontend-token=frontend-web"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
1. Send logs from one producer:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
curl -sS "http://127.0.0.1:4310/api/ingest" \
|
|
107
|
+
-H "Authorization: Bearer backend-token" \
|
|
108
|
+
-H "Content-Type: application/json" \
|
|
109
|
+
--data '[{"timestamp":"2026-02-18T10:20:00.000Z","level":"INFO","event":"order.created","message":"Order created","orderId":"ORD-123"}]'
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
1. Query logs:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
curl -sS "http://127.0.0.1:4310/api/logs?field=producerId&value=backend-api&limit=10" \
|
|
116
|
+
-H "Authorization: Bearer api-token-123"
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
1. Open docs and health:
|
|
120
|
+
|
|
121
|
+
| URL | Purpose |
|
|
122
|
+
|--------------------------------------|------------------------------------|
|
|
123
|
+
| `http://127.0.0.1:4310/health` | Service health/status payload |
|
|
124
|
+
| `http://127.0.0.1:4310/docs` | Interactive API reference (Scalar) |
|
|
125
|
+
| `http://127.0.0.1:4310/openapi.json` | OpenAPI 3.1 JSON |
|
|
126
|
+
| `http://127.0.0.1:4310/openapi.yaml` | OpenAPI 3.1 YAML |
|
|
127
|
+
|
|
128
|
+
If `/docs` is blank (for example blocked CDN scripts), use `/openapi.json` directly.
|
|
129
|
+
|
|
130
|
+
## CLI Commands
|
|
131
|
+
|
|
132
|
+
| Command | Use case |
|
|
133
|
+
|------------------------------------------------------------------------------|-----------------------------------|
|
|
134
|
+
| `mikroscope serve --logs ./logs --db ./data/mikroscope.db` | Start HTTP/HTTPS API service |
|
|
135
|
+
| `mikroscope index --logs ./logs --db ./data/mikroscope.db` | One-time full index from raw logs |
|
|
136
|
+
| `mikroscope query --db ./data/mikroscope.db --level ERROR --limit 50` | Query logs from CLI |
|
|
137
|
+
| `mikroscope aggregate --db ./data/mikroscope.db --group-by level --limit 10` | Aggregate logs from CLI |
|
|
138
|
+
|
|
139
|
+
## Auth and `producerId` Model
|
|
140
|
+
|
|
141
|
+
| Route | Auth options | `producerId` behavior |
|
|
142
|
+
|---------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------|
|
|
143
|
+
| `POST /api/ingest` | `Bearer <ingest-token>` mapped by `--ingest-producers`, or Basic auth if configured | Always server-assigned. Incoming `producerId` in payload is ignored/overridden. |
|
|
144
|
+
| `GET /api/logs` | `Bearer <api-token>` and/or Basic auth (if enabled) | N/A |
|
|
145
|
+
| `GET /api/logs/aggregate` | `Bearer <api-token>` and/or Basic auth (if enabled) | N/A |
|
|
146
|
+
| `POST /api/reindex` | `Bearer <api-token>` and/or Basic auth (if enabled) | N/A |
|
|
147
|
+
|
|
148
|
+
Notes:
|
|
149
|
+
|
|
150
|
+
| Case | Outcome |
|
|
151
|
+
|----------------------------------------------------------|---------------------------------------------------|
|
|
152
|
+
| Basic auth is used on ingest | `producerId` becomes the authenticated username |
|
|
153
|
+
| `--ingest-producers` is empty and Basic auth is disabled | `/api/ingest` returns `404` (endpoint disabled) |
|
|
154
|
+
| Async ingest queue enabled (`--ingest-async-queue`) | Ingest responses return `202` with `queued: true` |
|
|
155
|
+
|
|
156
|
+
## Ingest Contract
|
|
157
|
+
|
|
158
|
+
| Item | Value |
|
|
159
|
+
|-------------------------|-------------------------------------------------------------------------|
|
|
160
|
+
| Endpoint | `POST /api/ingest` |
|
|
161
|
+
| Content type | `application/json` |
|
|
162
|
+
| Accepted payload shapes | `[{...}]` or `{ "logs": [{...}] }` |
|
|
163
|
+
| Max payload size | Controlled by `--ingest-max-body-bytes` (default `1048576`) |
|
|
164
|
+
| Required fields per log | No strict schema at ingest layer; invalid/non-object items are rejected |
|
|
165
|
+
| Server-added fields | `producerId`, `ingestedAt` |
|
|
166
|
+
| Storage path | `logs/ingest/<producerId>/YYYY-MM-DD.ndjson` |
|
|
167
|
+
|
|
168
|
+
## API Summary
|
|
169
|
+
|
|
170
|
+
| Method | Path | Auth | Purpose |
|
|
171
|
+
|--------|-----------------------|-------------------------------------------|------------------------------------------|
|
|
172
|
+
| `GET` | `/health` | None | Runtime health and policy/status details |
|
|
173
|
+
| `POST` | `/api/ingest` | Ingest bearer token mapping or Basic auth | Accept and persist inbound logs |
|
|
174
|
+
| `GET` | `/api/logs` | API bearer token and/or Basic auth | Paginated log query |
|
|
175
|
+
| `GET` | `/api/logs/aggregate` | API bearer token and/or Basic auth | Bucketed counts |
|
|
176
|
+
| `POST` | `/api/reindex` | API bearer token and/or Basic auth | Full DB reset + reindex from logs |
|
|
177
|
+
| `GET` | `/openapi.json` | None | OpenAPI 3.1 JSON document |
|
|
178
|
+
| `GET` | `/openapi.yaml` | None | OpenAPI 3.1 YAML document |
|
|
179
|
+
| `GET` | `/docs` | None | Scalar-rendered interactive API docs |
|
|
180
|
+
|
|
181
|
+
Query parameters for `/api/logs`:
|
|
182
|
+
|
|
183
|
+
| Parameter | Type | Description |
|
|
184
|
+
|-----------|---------------|---------------------------------------------|
|
|
185
|
+
| `from` | ISO timestamp | Lower bound |
|
|
186
|
+
| `to` | ISO timestamp | Upper bound |
|
|
187
|
+
| `level` | string | `DEBUG`, `INFO`, `WARN`, `ERROR`, or custom |
|
|
188
|
+
| `audit` | boolean | Audit-only filter |
|
|
189
|
+
| `field` | string | Top-level JSON field key |
|
|
190
|
+
| `value` | string | Top-level JSON field value |
|
|
191
|
+
| `limit` | number | Max rows (capped at `1000`) |
|
|
192
|
+
| `cursor` | string | Pagination cursor from previous result |
|
|
193
|
+
|
|
194
|
+
Query parameters for `/api/logs/aggregate`:
|
|
195
|
+
|
|
196
|
+
| Parameter | Type | Description |
|
|
197
|
+
|-----------------------------------------------------------|--------|------------------------------------------|
|
|
198
|
+
| `groupBy` | enum | `level`, `event`, `field`, `correlation` |
|
|
199
|
+
| `groupField` | string | Required when `groupBy=field` |
|
|
200
|
+
| `from`, `to`, `level`, `audit`, `field`, `value`, `limit` | mixed | Same filtering behavior as `/api/logs` |
|
|
201
|
+
|
|
202
|
+
## Configuration Reference
|
|
203
|
+
|
|
204
|
+
### Core and Security
|
|
205
|
+
|
|
206
|
+
| Flag | Default | Description |
|
|
207
|
+
|-----------------------|------------------------|------------------------------------------------|
|
|
208
|
+
| `--logs` | `./logs` | NDJSON root directory |
|
|
209
|
+
| `--db` | `./data/mikroscope.db` | SQLite database file |
|
|
210
|
+
| `--host` | `127.0.0.1` | Bind host |
|
|
211
|
+
| `--port` | `4310` | Bind port |
|
|
212
|
+
| `--https` | `false` | Enable HTTPS listener |
|
|
213
|
+
| `--tls-cert` | none | TLS certificate path (required with `--https`) |
|
|
214
|
+
| `--tls-key` | none | TLS key path (required with `--https`) |
|
|
215
|
+
| `--api-token` | none | Bearer token for `/api/*` routes |
|
|
216
|
+
| `--auth-username` | none | Basic auth username for `/api/*` |
|
|
217
|
+
| `--auth-password` | none | Basic auth password for `/api/*` |
|
|
218
|
+
| `--cors-allow-origin` | `*` | CORS allow list (comma-separated origins) |
|
|
219
|
+
|
|
220
|
+
### Ingest and Indexing
|
|
221
|
+
|
|
222
|
+
| Flag | Default | Description |
|
|
223
|
+
|---------------------------|-----------|---------------------------------------------|
|
|
224
|
+
| `--ingest-producers` | none | Ingest auth map as `token=producerId` pairs |
|
|
225
|
+
| `--ingest-max-body-bytes` | `1048576` | Max ingest request body size |
|
|
226
|
+
| `--ingest-interval-ms` | `2000` | Incremental ingest cadence |
|
|
227
|
+
| `--disable-auto-ingest` | `false` | Disable periodic incremental ingest |
|
|
228
|
+
| `--ingest-async-queue` | `false` | Enable async ingest write/index queue |
|
|
229
|
+
| `--ingest-queue-flush-ms` | `25` | Async queue flush cadence |
|
|
230
|
+
|
|
231
|
+
### Retention and Maintenance
|
|
232
|
+
|
|
233
|
+
| Flag | Default | Description |
|
|
234
|
+
|------------------------------|-------------|----------------------------------------------------|
|
|
235
|
+
| `--db-retention-days` | `30` | Retain non-audit indexed rows for N days |
|
|
236
|
+
| `--db-audit-retention-days` | `365` | Retain audit indexed rows for N days |
|
|
237
|
+
| `--log-retention-days` | `30` | Retain non-audit raw `.ndjson` files for N days |
|
|
238
|
+
| `--log-audit-retention-days` | `365` | Retain audit raw `.ndjson` files for N days |
|
|
239
|
+
| `--audit-backup-dir` | none | Optional backup target before deleting audit files |
|
|
240
|
+
| `--maintenance-interval-ms` | `21600000` | Maintenance cadence |
|
|
241
|
+
| `--min-free-bytes` | `268435456` | Minimum free bytes for DB/log paths |
|
|
242
|
+
|
|
243
|
+
### Alerting
|
|
244
|
+
|
|
245
|
+
| Flag | Default | Description |
|
|
246
|
+
|-------------------------------------|----------|-------------------------------------------------|
|
|
247
|
+
| `--alert-webhook-url` | none | Webhook target for alert payloads |
|
|
248
|
+
| `--alert-interval-ms` | `30000` | Alert evaluation cadence |
|
|
249
|
+
| `--alert-window-minutes` | `5` | Error threshold lookback window |
|
|
250
|
+
| `--alert-error-threshold` | `20` | Error count threshold in alert window |
|
|
251
|
+
| `--alert-no-logs-threshold-minutes` | `0` | Alert when no logs for N minutes (`0` disables) |
|
|
252
|
+
| `--alert-cooldown-ms` | `300000` | Minimum delay between same-rule notifications |
|
|
253
|
+
| `--alert-webhook-timeout-ms` | `5000` | Webhook timeout per request |
|
|
254
|
+
| `--alert-webhook-retry-attempts` | `3` | Webhook retry attempts per alert |
|
|
255
|
+
| `--alert-webhook-backoff-ms` | `250` | Base backoff between webhook retries |
|
|
256
|
+
|
|
257
|
+
## Example Integrations
|
|
258
|
+
|
|
259
|
+
| Producer | Example |
|
|
260
|
+
|-----------------|-----------------------------------------------------------------|
|
|
261
|
+
| Backend service | `Authorization: Bearer backend-token` mapped to `backend-api` |
|
|
262
|
+
| Frontend app | `Authorization: Bearer frontend-token` mapped to `frontend-web` |
|
|
263
|
+
|
|
264
|
+
Backend example:
|
|
265
|
+
|
|
266
|
+
```ts
|
|
267
|
+
await fetch("http://127.0.0.1:4310/api/ingest", {
|
|
268
|
+
method: "POST",
|
|
269
|
+
headers: {
|
|
270
|
+
authorization: "Bearer backend-token",
|
|
271
|
+
"content-type": "application/json",
|
|
272
|
+
},
|
|
273
|
+
body: JSON.stringify([
|
|
274
|
+
{
|
|
275
|
+
timestamp: new Date().toISOString(),
|
|
276
|
+
level: "INFO",
|
|
277
|
+
event: "order.created",
|
|
278
|
+
message: "Order created",
|
|
279
|
+
orderId: "ORD-123",
|
|
280
|
+
},
|
|
281
|
+
]),
|
|
282
|
+
});
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
Frontend example:
|
|
286
|
+
|
|
287
|
+
```ts
|
|
288
|
+
await fetch("http://127.0.0.1:4310/api/ingest", {
|
|
289
|
+
method: "POST",
|
|
290
|
+
headers: {
|
|
291
|
+
authorization: "Bearer frontend-token",
|
|
292
|
+
"content-type": "application/json",
|
|
293
|
+
},
|
|
294
|
+
body: JSON.stringify({
|
|
295
|
+
logs: [
|
|
296
|
+
{
|
|
297
|
+
timestamp: new Date().toISOString(),
|
|
298
|
+
level: "INFO",
|
|
299
|
+
event: "ui.click",
|
|
300
|
+
message: "Checkout clicked",
|
|
301
|
+
route: "/checkout",
|
|
302
|
+
},
|
|
303
|
+
],
|
|
304
|
+
}),
|
|
305
|
+
});
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
## VM Deployment (systemd)
|
|
309
|
+
|
|
310
|
+
Prepared units are in `/deploy/systemd`:
|
|
311
|
+
|
|
312
|
+
| File | Purpose |
|
|
313
|
+
|----------------------------------------------|---------------------------|
|
|
314
|
+
| `/deploy/systemd/mikroscope.service` | Long-running API sidecar |
|
|
315
|
+
| `/deploy/systemd/mikroscope-reindex.service` | One-shot full reindex job |
|
|
316
|
+
| `/deploy/systemd/mikroscope-reindex.timer` | Scheduled reindex trigger |
|
|
317
|
+
| `/deploy/systemd/mikroscope.env.example` | Environment template |
|
|
318
|
+
|
|
319
|
+
Quick start steps are documented in `/deploy/systemd/README.md`.
|
|
320
|
+
|
|
321
|
+
## Operations
|
|
322
|
+
|
|
323
|
+
| Document | Purpose |
|
|
324
|
+
|-------------------------|----------------------------------------------------------|
|
|
325
|
+
| `/OPS_RUNBOOK.md` | Health checks, backup policy, restore, incident playbook |
|
|
326
|
+
| `/openapi/openapi.yaml` | OpenAPI 3.1 source |
|
|
327
|
+
| `/openapi/openapi.json` | OpenAPI 3.1 JSON |
|
|
328
|
+
|
|
329
|
+
## From Source (Optional)
|
|
330
|
+
|
|
331
|
+
Use this only if you want to develop MikroScope itself.
|
|
332
|
+
|
|
333
|
+
```bash
|
|
334
|
+
npm install
|
|
335
|
+
npm run index
|
|
336
|
+
npm run start
|
|
337
|
+
```
|
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// MikroScope - See LICENSE file for copyright and license details.
|
|
3
|
+
function ee(e,n,t){return Math.max(1,Math.min(t,Math.trunc(e||n)))}var te=1e3;function Ge(e){return Buffer.from(JSON.stringify(e),"utf8").toString("base64url")}function We(e){if(e)try{let n=JSON.parse(Buffer.from(e,"base64url").toString("utf8"));return typeof n.timestamp!="string"||typeof n.id!="number"||!Number.isFinite(n.id)||n.id<=0?void 0:{id:Math.trunc(n.id),timestamp:n.timestamp}}catch{return}}var w=class{constructor(n){this.db=n}queryLogsPage(n){let t=We(n.cursor),r=ee(n.limit,100,te),s=this.db.queryPage({...n,limit:r},t),i=s.entries[s.entries.length-1];return{entries:s.entries,hasMore:s.hasMore,limit:s.limit,nextCursor:s.hasMore&&i?Ge({id:i.id,timestamp:i.timestamp}):void 0}}aggregateLogs(n,t,r){let s=ee(n.limit,25,te);return this.db.aggregate({...n,limit:s},t,r)}countLogs(n){return this.db.count(n)}};import{createReadStream as ne}from"node:fs";import{readdir as Ke,stat as $e}from"node:fs/promises";import{extname as je,join as ze,relative as re,resolve as M,sep as W}from"node:path";import se from"node:readline";function Je(e){let n=typeof e;return e===null||n==="string"||n==="number"||n==="boolean"}function qe(e){return typeof e=="string"?e:e==null?"":JSON.stringify(e)}function Ye(e){return typeof e=="string"&&e.length>0?e:new Date().toISOString()}function Xe(e){return typeof e=="string"&&e.length>0?e.toUpperCase():"INFO"}function xe(e){return typeof e.event=="string"&&e.event.length>0?e.event:typeof e.message=="string"&&e.message.length>0?e.message:"log.event"}function Ve(e,n){let t=e.audit;if(typeof t=="boolean")return t;if(typeof t=="number")return t!==0;if(typeof t=="string"){let s=t.trim().toLowerCase();if(s==="true"||s==="1"||s==="yes")return!0;if(s==="false"||s==="0"||s==="no")return!1}return n.toLowerCase().includes(`${W}audit${W}`)}async function ie(e){let n=[],t=[M(e)];for(;t.length>0;){let r=t.pop();if(!r)continue;let s=await Ke(r,{withFileTypes:!0}).catch(i=>{if(typeof i=="object"&&i!==null&&"code"in i&&i.code==="ENOENT")return[];throw i});for(let i of s){let o=ze(r,i.name);i.isDirectory()?t.push(o):i.isFile()&&je(i.name).toLowerCase()===".ndjson"&&n.push(o)}}return n.sort((r,s)=>r.localeCompare(s)),n}function oe(e){return{filesScanned:0,linesScanned:0,recordsInserted:0,recordsSkipped:0,parseErrors:0,startedAt:new Date().toISOString(),finishedAt:"",mode:e}}var v=class{constructor(n){this.db=n}incrementalState=new Map;resetIncrementalState(){this.incrementalState.clear()}async indexDirectory(n){let t=oe("full"),r=await ie(n);t.filesScanned=r.length;for(let s of r)await this.indexFileFull(n,s,t);return t.finishedAt=new Date().toISOString(),t}async indexDirectoryIncremental(n){let t=oe("incremental"),r=await ie(n),s=new Set;t.filesScanned=r.length;for(let i of r){let o=M(i);s.add(o),await this.indexFileIncremental(n,o,t)}for(let i of this.incrementalState.keys())s.has(i)||this.incrementalState.delete(i);return t.finishedAt=new Date().toISOString(),t}async indexFileFull(n,t,r){let s=ne(t,{encoding:"utf8"}),i=se.createInterface({input:s,crlfDelay:1/0}),o=0,a=re(M(n),M(t)).split(W).join("/");for await(let u of i)o++,r.linesScanned++,this.processLine({filePath:t,line:u,lineNumber:o,report:r,sourceFile:a})}async indexFileIncremental(n,t,r){let s=re(M(n),M(t)).split(W).join("/"),i;try{i=await $e(t)}catch(y){if(typeof y=="object"&&y!==null&&"code"in y&&y.code==="ENOENT"){this.incrementalState.delete(t);return}throw y}let o=this.incrementalState.get(t),a=o&&Number.isFinite(o.byteOffset)&&o.byteOffset>=0,u=!!(o&&(i.size<o.byteOffset||Number.isFinite(o.mtimeMs)&&i.mtimeMs!==o.mtimeMs&&i.size===o.byteOffset)),l=a&&!u&&o&&i.size>=o.byteOffset;o&&a&&!l&&this.db.deleteEntriesForSourceFile(s);let g=l?o.byteOffset:0,d=l?o.lineNumber:0,h=0,f=ne(t,{encoding:"utf8",start:g});f.on("data",y=>{typeof y=="string"?h+=Buffer.byteLength(y,"utf8"):h+=y.length});let I=se.createInterface({input:f,crlfDelay:1/0});for await(let y of I)d++,r.linesScanned++,this.processLine({filePath:t,line:y,lineNumber:d,report:r,sourceFile:s});this.incrementalState.set(t,{byteOffset:g+h,fileSize:i.size,lineNumber:d,mtimeMs:i.mtimeMs})}processLine(n){let t=n.line.trim();if(!t)return;let r;try{r=JSON.parse(t)}catch{n.report.parseErrors++;return}let s=this.db.upsertEntry({timestamp:Ye(r.timestamp),level:Xe(r.level),event:xe(r),message:qe(r.message),isAudit:Ve(r,n.filePath),dataJson:JSON.stringify(r),sourceFile:n.sourceFile,lineNumber:n.lineNumber});if(!s.inserted){n.report.recordsSkipped++;return}for(let[i,o]of Object.entries(r))Je(o)&&this.db.upsertField(s.entryId,i,String(o));n.report.recordsInserted++}};import{mkdirSync as Ze}from"node:fs";import{dirname as et}from"node:path";import{DatabaseSync as tt}from"node:sqlite";var A=class{constructor(n){this.dbPath=n;Ze(et(n),{recursive:!0}),this.db=new tt(n),this.db.exec("PRAGMA journal_mode = WAL;"),this.db.exec("PRAGMA synchronous = NORMAL;"),this.db.exec("PRAGMA temp_store = MEMORY;"),this.initialize()}db;close(){this.db.close()}getStats(){let n=this.db.prepare("SELECT COUNT(*) AS count FROM log_entries").get(),t=this.db.prepare("SELECT COUNT(*) AS count FROM log_fields").get(),r=this.db.prepare("PRAGMA page_count").get(),s=this.db.prepare("PRAGMA page_size").get(),i=Number(r.page_count||0),o=Number(s.page_size||0);return{entryCount:Number(n.count||0),fieldCount:Number(t.count||0),pageCount:i,pageSize:o,approximateSizeBytes:i*o}}pruneOlderThan(n){return this.pruneByRetention({normalCutoffIso:n,auditCutoffIso:n})}pruneByRetention(n){this.db.exec("BEGIN IMMEDIATE TRANSACTION");try{let t=this.db.prepare(`
|
|
4
|
+
DELETE FROM log_fields
|
|
5
|
+
WHERE entry_id IN (
|
|
6
|
+
SELECT id
|
|
7
|
+
FROM log_entries
|
|
8
|
+
WHERE (is_audit = 0 AND timestamp < ?)
|
|
9
|
+
OR (is_audit = 1 AND timestamp < ?)
|
|
10
|
+
)
|
|
11
|
+
`).run(n.normalCutoffIso,n.auditCutoffIso),r=this.db.prepare("DELETE FROM log_entries WHERE is_audit = 0 AND timestamp < ?").run(n.normalCutoffIso),s=this.db.prepare("DELETE FROM log_entries WHERE is_audit = 1 AND timestamp < ?").run(n.auditCutoffIso);this.db.exec("COMMIT");let i=Number(r.changes||0),o=Number(s.changes||0);return{normalCutoffIso:n.normalCutoffIso,auditCutoffIso:n.auditCutoffIso,normalEntriesDeleted:i,auditEntriesDeleted:o,entriesDeleted:i+o,fieldsDeleted:Number(t.changes||0)}}catch(t){throw this.db.exec("ROLLBACK"),t}}vacuum(){this.db.exec("VACUUM"),this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)")}upsertEntry(n){let r=this.db.prepare(`
|
|
12
|
+
INSERT OR IGNORE INTO log_entries
|
|
13
|
+
(timestamp, level, event, message, is_audit, data_json, source_file, line_number)
|
|
14
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
15
|
+
`).run(n.timestamp,n.level,n.event,n.message,n.isAudit?1:0,n.dataJson,n.sourceFile,n.lineNumber);if(r.changes>0)return{entryId:Number(r.lastInsertRowid),inserted:!0};let s=this.db.prepare("SELECT id FROM log_entries WHERE source_file = ? AND line_number = ?").get(n.sourceFile,n.lineNumber);if(!s)throw new Error("Could not read existing log entry after insert ignore.");return{entryId:s.id,inserted:!1}}upsertField(n,t,r){this.db.prepare("INSERT OR IGNORE INTO log_fields(entry_id, key, value_text) VALUES (?, ?, ?)").run(n,t,r)}reset(){this.db.exec("BEGIN IMMEDIATE TRANSACTION");try{let n=this.db.prepare("DELETE FROM log_fields").run(),t=this.db.prepare("DELETE FROM log_entries").run();return this.db.exec("COMMIT"),{entriesDeleted:Number(t.changes||0),fieldsDeleted:Number(n.changes||0)}}catch(n){throw this.db.exec("ROLLBACK"),n}}deleteEntriesForSourceFile(n){this.db.exec("BEGIN IMMEDIATE TRANSACTION");try{let t=this.db.prepare(`
|
|
16
|
+
DELETE FROM log_fields
|
|
17
|
+
WHERE entry_id IN (
|
|
18
|
+
SELECT id FROM log_entries WHERE source_file = ?
|
|
19
|
+
)
|
|
20
|
+
`).run(n),r=this.db.prepare("DELETE FROM log_entries WHERE source_file = ?").run(n);return this.db.exec("COMMIT"),{sourceFile:n,entriesDeleted:Number(r.changes||0),fieldsDeleted:Number(t.changes||0)}}catch(t){throw this.db.exec("ROLLBACK"),t}}queryPage(n,t){let r=this.buildFilterSql(n,"f"),s=[...r.args],i=[...r.whereClauses],o=this.resolveLimit(n.limit,1e3);t&&(i.push("(e.timestamp < ? OR (e.timestamp = ? AND e.id < ?))"),s.push(t.timestamp,t.timestamp,t.id));let a=i.length>0?`WHERE ${i.join(" AND ")}`:"",u=`
|
|
21
|
+
SELECT DISTINCT
|
|
22
|
+
e.id,
|
|
23
|
+
e.timestamp,
|
|
24
|
+
e.level,
|
|
25
|
+
e.event,
|
|
26
|
+
e.message,
|
|
27
|
+
e.data_json,
|
|
28
|
+
e.source_file,
|
|
29
|
+
e.line_number
|
|
30
|
+
FROM log_entries e
|
|
31
|
+
${r.joinSql}
|
|
32
|
+
${a}
|
|
33
|
+
ORDER BY e.timestamp DESC, e.id DESC
|
|
34
|
+
LIMIT ?
|
|
35
|
+
`;s.push(o+1);let l=this.db.prepare(u).all(...s),g=l.length>o,d=g?l.slice(0,o):l;return{entries:this.mapRows(d),hasMore:g,limit:o}}count(n){let t=this.buildFilterSql(n,"f"),r=t.whereClauses.length>0?`WHERE ${t.whereClauses.join(" AND ")}`:"",s=`
|
|
36
|
+
SELECT COUNT(DISTINCT e.id) AS count
|
|
37
|
+
FROM log_entries e
|
|
38
|
+
${t.joinSql}
|
|
39
|
+
${r}
|
|
40
|
+
`,i=this.db.prepare(s).get(...t.args);return Number(i?.count||0)}aggregate(n,t,r){let s=this.buildFilterSql(n,"ff"),i=this.resolveLimit(n.limit,1e3),o="",a="",u="",l=[];if(t==="level")o="e.level AS key, COUNT(DISTINCT e.id) AS count",a="GROUP BY e.level";else if(t==="event")o="e.event AS key, COUNT(DISTINCT e.id) AS count",a="GROUP BY e.event";else if(t==="correlation")o="COALESCE(corr.value_text, req.value_text, '(missing)') AS key, COUNT(DISTINCT e.id) AS count",a="GROUP BY COALESCE(corr.value_text, req.value_text, '(missing)')",u=`
|
|
41
|
+
LEFT JOIN log_fields corr ON corr.entry_id = e.id AND corr.key = 'correlationId'
|
|
42
|
+
LEFT JOIN log_fields req ON req.entry_id = e.id AND req.key = 'requestId'
|
|
43
|
+
`;else{let f=(r||"").trim();if(!f)throw new Error("groupField is required when groupBy=field");o="COALESCE(gf.value_text, '(missing)') AS key, COUNT(DISTINCT e.id) AS count",a="GROUP BY COALESCE(gf.value_text, '(missing)')",u="LEFT JOIN log_fields gf ON gf.entry_id = e.id AND gf.key = ?",l.push(f)}let g=s.whereClauses.length>0?`WHERE ${s.whereClauses.join(" AND ")}`:"",d=`
|
|
44
|
+
SELECT
|
|
45
|
+
${o}
|
|
46
|
+
FROM log_entries e
|
|
47
|
+
${u}
|
|
48
|
+
${s.joinSql}
|
|
49
|
+
${g}
|
|
50
|
+
${a}
|
|
51
|
+
ORDER BY count DESC, key ASC
|
|
52
|
+
LIMIT ?
|
|
53
|
+
`;return l.push(...s.args,i),this.db.prepare(d).all(...l).map(f=>({key:f.key??"(missing)",count:Number(f.count||0)}))}initialize(){this.db.exec(`
|
|
54
|
+
CREATE TABLE IF NOT EXISTS log_entries (
|
|
55
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
56
|
+
timestamp TEXT NOT NULL,
|
|
57
|
+
level TEXT NOT NULL,
|
|
58
|
+
event TEXT NOT NULL,
|
|
59
|
+
message TEXT NOT NULL,
|
|
60
|
+
is_audit INTEGER NOT NULL DEFAULT 0,
|
|
61
|
+
data_json TEXT NOT NULL,
|
|
62
|
+
source_file TEXT NOT NULL,
|
|
63
|
+
line_number INTEGER NOT NULL,
|
|
64
|
+
indexed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
65
|
+
UNIQUE(source_file, line_number)
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
CREATE TABLE IF NOT EXISTS log_fields (
|
|
69
|
+
entry_id INTEGER NOT NULL,
|
|
70
|
+
key TEXT NOT NULL,
|
|
71
|
+
value_text TEXT NOT NULL,
|
|
72
|
+
UNIQUE(entry_id, key, value_text),
|
|
73
|
+
FOREIGN KEY(entry_id) REFERENCES log_entries(id) ON DELETE CASCADE
|
|
74
|
+
);
|
|
75
|
+
`),this.ensureAuditColumn(),this.db.exec(`
|
|
76
|
+
CREATE INDEX IF NOT EXISTS idx_log_entries_timestamp ON log_entries(timestamp);
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_log_entries_level_timestamp ON log_entries(level, timestamp);
|
|
78
|
+
CREATE INDEX IF NOT EXISTS idx_log_entries_event_timestamp ON log_entries(event, timestamp);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS idx_log_entries_audit_timestamp ON log_entries(is_audit, timestamp);
|
|
80
|
+
CREATE INDEX IF NOT EXISTS idx_log_fields_key_value ON log_fields(key, value_text);
|
|
81
|
+
CREATE INDEX IF NOT EXISTS idx_log_fields_entry_key ON log_fields(entry_id, key);
|
|
82
|
+
`)}ensureAuditColumn(){this.db.prepare("PRAGMA table_info(log_entries)").all().some(r=>r.name==="is_audit")||this.db.exec("ALTER TABLE log_entries ADD COLUMN is_audit INTEGER NOT NULL DEFAULT 0")}buildFilterSql(n,t){let r=[],s=[];n.from&&(r.push("e.timestamp >= ?"),s.push(n.from)),n.to&&(r.push("e.timestamp <= ?"),s.push(n.to)),n.level&&(r.push("e.level = ?"),s.push(n.level.toUpperCase())),n.audit!==void 0&&(r.push("e.is_audit = ?"),s.push(n.audit?1:0));let i="";return n.field&&n.value!==void 0&&(i=`JOIN log_fields ${t} ON ${t}.entry_id = e.id`,r.push(`${t}.key = ?`),r.push(`${t}.value_text = ?`),s.push(n.field,n.value)),{args:s,joinSql:i,whereClauses:r}}mapRows(n){return n.map(t=>({id:t.id,timestamp:t.timestamp,level:t.level,event:t.event,message:t.message,data:JSON.parse(t.data_json),sourceFile:t.source_file,lineNumber:t.line_number}))}resolveLimit(n,t){return Math.max(1,Math.min(t,Math.trunc(n||100)))}};import{copyFileSync as gt,mkdirSync as Le,readdirSync as mt,readFileSync as x,rmSync as ft,statfsSync as pt,statSync as ht,unlinkSync as yt,writeFileSync as bt}from"node:fs";import{createServer as St}from"node:http";import{createServer as Et}from"node:https";import{basename as It,dirname as V,join as Q,relative as At,resolve as O,sep as we}from"node:path";import{fileURLToPath as Rt}from"node:url";function ae(e){if(!e||e.trim().length===0)return["*"];let n=e.split(",").map(t=>t.trim()).filter(t=>t.length>0);return n.length>0?n:["*"]}function L(e,n,t){let r=e.headers.origin,s=t.includes("*"),i=typeof r=="string"?t.find(a=>a===r):void 0,o=s?"*":i;o&&n.setHeader("access-control-allow-origin",o),!s&&o&&n.setHeader("vary","Origin"),n.setHeader("access-control-allow-methods","GET,POST,OPTIONS"),n.setHeader("access-control-allow-headers","authorization,content-type"),n.setHeader("access-control-max-age","600")}function p(e,n,t,r,s){L(e,n,s);let i=JSON.stringify(r);n.statusCode=t,n.setHeader("content-type","application/json; charset=utf-8"),n.end(i)}async function ue(e,n){return await new Promise((t,r)=>{let s=[],i=0,o=!1;e.on("data",a=>{if(o)return;let u=Buffer.from(a);if(i+=u.length,i>n){o=!0,r(new Error(`Payload too large. Max body size is ${n} bytes.`));return}s.push(u)}),e.on("end",()=>{if(o)return;let a=Buffer.concat(s).toString("utf8").trim();if(!a){t([]);return}try{t(JSON.parse(a))}catch{r(new Error("Invalid JSON payload."))}}),e.on("error",a=>{o||r(a)})})}function le(e){let n=(e.authUsername??process.env.MIKROSCOPE_AUTH_USERNAME??"").trim(),t=e.authPassword??process.env.MIKROSCOPE_AUTH_PASSWORD;if(n&&!t||!n&&t)throw new Error("Basic auth requires both authUsername and authPassword.");return!n||!t?{enabled:!1}:{enabled:!0,password:t,username:n}}function de(e){let n=e.headers.authorization;if(!n)return;let[t,r]=n.split(" ");if(!(t?.toLowerCase()!=="bearer"||!r))return r}function nt(e){let n=e.headers.authorization;if(!n)return;let[t,r]=n.split(" ");if(t?.toLowerCase()!=="basic"||!r)return;let s="";try{s=Buffer.from(r,"base64").toString("utf8")}catch{return}let i=s.indexOf(":");if(!(i<=0))return{username:s.slice(0,i),password:s.slice(i+1)}}function ce(e,n){if(!n.enabled||!n.username||n.password===void 0)return;let t=nt(e);if(t&&t.username===n.username&&t.password===n.password)return t.username}function ge(e,n,t){let r=typeof n=="string"&&n.length>0,s=t.enabled;return!r&&!s||r&&de(e)===n?!0:ce(e,t)!==void 0}function me(e,n,t){let r=ce(e,n);if(r)return r;let s=de(e);if(s)return t.get(s)}import{mkdirSync as rt}from"node:fs";import{appendFile as st}from"node:fs/promises";import{join as fe,resolve as it}from"node:path";var ot=2e3,at=1048576,ut=25,pe=/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;function q(e,n){let t=Number(e);return!Number.isFinite(t)||t<0?n:t}function he(e,n){if(typeof e=="boolean")return e;if(typeof e!="string")return n;let t=e.trim().toLowerCase();return t==="true"||t==="1"||t==="yes"?!0:t==="false"||t==="0"||t==="no"?!1:n}function ye(e){let n=he(e.disableAutoIngest??process.env.MIKROSCOPE_DISABLE_AUTO_INGEST,!1),t=Math.max(250,q(e.ingestIntervalMs??process.env.MIKROSCOPE_INGEST_INTERVAL_MS,ot));return{enabled:!n,intervalMs:t}}function lt(e){if(!e||e.trim().length===0)return new Map;let n=new Map,t=e.split(",").map(r=>r.trim()).filter(r=>r.length>0);for(let r of t){let s=r.indexOf("=");if(s<=0||s===r.length-1)throw new Error(`Invalid ingest producer mapping "${r}". Expected "token=producerId".`);let i=r.slice(0,s).trim(),o=r.slice(s+1).trim();if(!i||!o)throw new Error(`Invalid ingest producer mapping "${r}". Expected non-empty token and producerId.`);if(!pe.test(o))throw new Error(`Invalid producerId "${o}". Allowed pattern: ${pe.source}`);if(n.has(i))throw new Error("Duplicate ingest producer mapping token detected.");n.set(i,o)}return n}function be(e){let n=lt(e.ingestProducers??process.env.MIKROSCOPE_INGEST_PRODUCERS),t=Math.max(1024,q(e.ingestMaxBodyBytes??process.env.MIKROSCOPE_INGEST_MAX_BODY_BYTES,at));return{enabled:n.size>0,maxBodyBytes:t,producerByToken:n}}function Se(e){let n=he(e.ingestAsyncQueue??process.env.MIKROSCOPE_INGEST_ASYNC_QUEUE,!1),t=Math.max(0,q(e.ingestQueueFlushMs??process.env.MIKROSCOPE_INGEST_QUEUE_FLUSH_MS,ut));return{enabled:n,flushMs:t}}function Ee(){return{running:!1,runs:0,recordsInsertedLastRun:0,recordsInsertedTotal:0,recordsSkippedLastRun:0,recordsSkippedTotal:0,parseErrorsLastRun:0,parseErrorsTotal:0,linesScannedLastRun:0,linesScannedTotal:0,filesScannedLastRun:0,filesScannedTotal:0}}function Ie(){return{batchesFlushed:0,batchesQueued:0,draining:!1,pending:[],pendingRecords:0,recordsFlushed:0,recordsQueued:0}}async function K(e){if(e.ingest.running)return;e.ingest.running=!0,e.ingest.runs++,e.ingest.lastRunAt=new Date().toISOString();let n=performance.now();try{let t=await e.indexer.indexDirectoryIncremental(e.logsPath);e.ingest.recordsInsertedLastRun=t.recordsInserted,e.ingest.recordsInsertedTotal+=t.recordsInserted,e.ingest.recordsSkippedLastRun=t.recordsSkipped,e.ingest.recordsSkippedTotal+=t.recordsSkipped,e.ingest.parseErrorsLastRun=t.parseErrors,e.ingest.parseErrorsTotal+=t.parseErrors,e.ingest.linesScannedLastRun=t.linesScanned,e.ingest.linesScannedTotal+=t.linesScanned,e.ingest.filesScannedLastRun=t.filesScanned,e.ingest.filesScannedTotal+=t.filesScanned,e.ingest.lastMode=t.mode,e.ingest.lastSuccessAt=new Date().toISOString(),e.ingest.lastError=void 0}catch(t){e.ingest.lastError=t instanceof Error?t.message:String(t)}finally{e.ingest.lastDurationMs=Number((performance.now()-n).toFixed(2)),e.ingest.running=!1}}function Ae(e){if(Array.isArray(e))return e;if(typeof e!="object"||e===null||Array.isArray(e))return;let n=e.logs;return Array.isArray(n)?n:void 0}function Re(e,n,t){if(typeof e!="object"||e===null||Array.isArray(e))return;let r={...e};return r.producerId=n,r.ingestedAt=t,r}async function dt(e,n,t){if(t.length===0)return;let r=it(e),s=new Date().toISOString().slice(0,10),i=fe(r,"ingest",n);rt(i,{recursive:!0});let o=fe(i,`${s}.ndjson`),a=t.map(u=>JSON.stringify(u)).join(`
|
|
83
|
+
`);await st(o,`${a}
|
|
84
|
+
`,"utf8")}function ct(e){let n=new Map;for(let r of e){let s=n.get(r.producerId);if(s){s.push(...r.records);continue}n.set(r.producerId,[...r.records])}let t=[];for(let[r,s]of n.entries())t.push({producerId:r,records:s});return t}async function Y(e,n){if(n.length===0)return;let t=ct(n);for(let s of t)await dt(e.logsPath,s.producerId,s.records);let r=t.reduce((s,i)=>s+i.records.length,0);e.ingestQueueState.recordsFlushed+=r,e.ingestQueueState.batchesFlushed+=t.length,e.ingestQueueState.lastFlushAt=new Date().toISOString(),e.ingestQueueState.lastError=void 0,await K(e)}function Oe(e){if(e.ingestQueueState.draining||e.ingestQueueState.timer)return;let n=setTimeout(()=>{e.ingestQueueState.timer=void 0,X(e)},e.ingestQueuePolicy.flushMs);n.unref?.(),e.ingestQueueState.timer=n}function Te(e,n){n.records.length!==0&&(e.ingestQueueState.pending.push(n),e.ingestQueueState.pendingRecords+=n.records.length,e.ingestQueueState.batchesQueued++,e.ingestQueueState.recordsQueued+=n.records.length,Oe(e))}async function X(e){if(!e.ingestQueueState.draining){e.ingestQueueState.draining=!0;try{for(;e.ingestQueueState.pending.length>0;){let n=e.ingestQueueState.pending.splice(0,e.ingestQueueState.pending.length),t=n.reduce((r,s)=>r+s.records.length,0);e.ingestQueueState.pendingRecords-=t;try{await Y(e,n)}catch(r){e.ingestQueueState.lastError=r instanceof Error?r.message:String(r),e.ingestQueueState.pending.unshift(...n),e.ingestQueueState.pendingRecords+=t;break}}}finally{e.ingestQueueState.draining=!1,e.ingestQueueState.pending.length>0&&Oe(e)}}}var R=class extends Error{retryable;constructor(n,t){super(n),this.name="AlertWebhookError",this.retryable=t}};function Ot(e,n,t){if(!e)return n;let r=Number.parseInt(e,10);return!Number.isFinite(r)||r<=0?n:typeof t=="number"?Math.min(t,r):r}var $=1440*60*1e3,Tt=30,wt=365,_t=30,Dt=365,Pt=360*60*1e3,Mt=256*1024*1024,vt=3e4,Lt=5,Nt=20,Ct=0,kt=300*1e3,Ft=5e3,Bt=3,Qt=250,Ut=1e3,Ht=Q("openapi","openapi.json"),Gt=Q("openapi","openapi.yaml");function _e(e){return typeof e=="bigint"?Number(e):e}function S(e,n){let t=Number(e);return!Number.isFinite(t)||t<0?n:t}function De(e,n){Le(e,{recursive:!0});let t=Q(e,`.mikroscope-write-probe-${process.pid}-${Date.now()}`);try{bt(t,"ok","utf8"),yt(t)}catch(r){throw new Error(`Path preflight failed for ${n} (${e}): ${r instanceof Error?r.message:String(r)}`)}}function j(e,n,t){let r=pt(e),s=_e(r.bavail)*_e(r.bsize);if(s<n)throw new Error(`Path preflight failed for ${t} (${e}): insufficient free space (${s} < ${n})`);return s}function Wt(e){let n=O(e),t=[],r=[n];for(;r.length>0;){let s=r.pop();if(!s)continue;let i;try{i=mt(s,{withFileTypes:!0})}catch(o){if(typeof o=="object"&&o!==null&&"code"in o&&o.code==="ENOENT")continue;throw o}for(let o of i){let a=String(o.name),u=Q(s,a);o.isDirectory()?r.push(u):o.isFile()&&a.toLowerCase().endsWith(".ndjson")&&t.push(u)}}return t}function Kt(){return{running:!1,runs:0,filesDeletedLastRun:0,filesDeletedTotal:0,normalFilesDeletedLastRun:0,normalFilesDeletedTotal:0,auditFilesDeletedLastRun:0,auditFilesDeletedTotal:0,entriesDeletedLastRun:0,entriesDeletedTotal:0,normalEntriesDeletedLastRun:0,normalEntriesDeletedTotal:0,auditEntriesDeletedLastRun:0,auditEntriesDeletedTotal:0,fieldsDeletedLastRun:0,fieldsDeletedTotal:0,vacuumRuns:0}}function $t(e){let n=e.alertWebhookUrl??process.env.MIKROSCOPE_ALERT_WEBHOOK_URL,t=typeof n=="string"&&n.trim().length>0?n.trim():void 0;return{enabled:!!t,webhookUrl:t,intervalMs:Math.max(1e3,S(e.alertIntervalMs??process.env.MIKROSCOPE_ALERT_INTERVAL_MS,vt)),windowMinutes:Math.max(1,S(e.alertWindowMinutes??process.env.MIKROSCOPE_ALERT_WINDOW_MINUTES,Lt)),errorThreshold:Math.max(1,S(e.alertErrorThreshold??process.env.MIKROSCOPE_ALERT_ERROR_THRESHOLD,Nt)),noLogsThresholdMinutes:Math.max(0,S(e.alertNoLogsThresholdMinutes??process.env.MIKROSCOPE_ALERT_NO_LOGS_THRESHOLD_MINUTES,Ct)),cooldownMs:Math.max(1e3,S(e.alertCooldownMs??process.env.MIKROSCOPE_ALERT_COOLDOWN_MS,kt)),webhookTimeoutMs:Math.max(250,S(e.alertWebhookTimeoutMs??process.env.MIKROSCOPE_ALERT_WEBHOOK_TIMEOUT_MS,Ft)),webhookRetryAttempts:Math.max(1,Math.trunc(S(e.alertWebhookRetryAttempts??process.env.MIKROSCOPE_ALERT_WEBHOOK_RETRY_ATTEMPTS,Bt))),webhookBackoffMs:Math.max(25,S(e.alertWebhookBackoffMs??process.env.MIKROSCOPE_ALERT_WEBHOOK_BACKOFF_MS,Qt))}}function jt(){return{running:!1,runs:0,sent:0,suppressed:0,lastTriggerAtByRule:{}}}function zt(e){let n=e.auditBackupDirectory??process.env.MIKROSCOPE_AUDIT_BACKUP_DIR,t=typeof n=="string"&&n.trim().length>0?O(n):void 0;return{dbRetentionDays:S(e.dbRetentionDays??process.env.MIKROSCOPE_DB_RETENTION_DAYS,Tt),dbAuditRetentionDays:S(e.dbAuditRetentionDays??process.env.MIKROSCOPE_DB_AUDIT_RETENTION_DAYS,wt),logRetentionDays:S(e.logRetentionDays??process.env.MIKROSCOPE_LOG_RETENTION_DAYS,_t),logAuditRetentionDays:S(e.logAuditRetentionDays??process.env.MIKROSCOPE_LOG_AUDIT_RETENTION_DAYS,Dt),auditBackupDirectory:t,maintenanceIntervalMs:Math.max(1e3,S(e.maintenanceIntervalMs??process.env.MIKROSCOPE_MAINTENANCE_INTERVAL_MS,Pt))}}function Jt(e){let n=e.toLowerCase();return n.includes(`${we}audit${we}`)?!0:It(n).includes("audit")}function qt(e){let n=Math.max(1,S(e.minFreeBytes??process.env.MIKROSCOPE_MIN_FREE_BYTES,Mt)),t=V(O(e.dbPath)),r=O(e.logsPath);De(t,"dbDirectory"),De(r,"logsDirectory");let s=j(t,n,"dbDirectory"),i=j(r,n,"logsDirectory");return{dbDirectory:t,dbDirectoryFreeBytes:s,logsDirectory:r,logsDirectoryFreeBytes:i,minFreeBytes:n}}function Yt(e,n,t,r){if(n<=0&&t<=0)return{normalDeleted:0,auditDeleted:0};let s=Date.now()-n*$,i=Date.now()-t*$,o=0,a=0;for(let u of Wt(e)){let l=Jt(u);if(!(l?t>0:n>0))continue;let d=ht(u),h=l?i:s;if(d.mtimeMs<h){if(l&&r){let f=At(O(e),O(u)),I=Q(r,f);Le(V(I),{recursive:!0}),gt(u,I)}ft(u,{force:!0}),l?a++:o++}}return{normalDeleted:o,auditDeleted:a}}function Pe(e,n){if(e.maintenance.running)return;e.maintenance.running=!0,e.maintenance.runs++,e.maintenance.lastRunAt=new Date().toISOString();let t=performance.now();try{let r=Yt(e.logsPath,n.logRetentionDays,n.logAuditRetentionDays,n.auditBackupDirectory),s=new Date(Date.now()-n.dbRetentionDays*$).toISOString(),i=new Date(Date.now()-n.dbAuditRetentionDays*$).toISOString(),o=e.db.pruneByRetention({normalCutoffIso:s,auditCutoffIso:i}),a=r.normalDeleted+r.auditDeleted;e.maintenance.filesDeletedLastRun=a,e.maintenance.filesDeletedTotal+=a,e.maintenance.normalFilesDeletedLastRun=r.normalDeleted,e.maintenance.normalFilesDeletedTotal+=r.normalDeleted,e.maintenance.auditFilesDeletedLastRun=r.auditDeleted,e.maintenance.auditFilesDeletedTotal+=r.auditDeleted,e.maintenance.entriesDeletedLastRun=o.entriesDeleted,e.maintenance.entriesDeletedTotal+=o.entriesDeleted,e.maintenance.normalEntriesDeletedLastRun=o.normalEntriesDeleted,e.maintenance.normalEntriesDeletedTotal+=o.normalEntriesDeleted,e.maintenance.auditEntriesDeletedLastRun=o.auditEntriesDeleted,e.maintenance.auditEntriesDeletedTotal+=o.auditEntriesDeleted,e.maintenance.fieldsDeletedLastRun=o.fieldsDeleted,e.maintenance.fieldsDeletedTotal+=o.fieldsDeleted,(o.entriesDeleted>0||a>0)&&(e.db.vacuum(),e.maintenance.vacuumRuns++),e.maintenance.lastSuccessAt=new Date().toISOString(),e.maintenance.lastError=void 0}catch(r){e.maintenance.lastError=r instanceof Error?r.message:String(r)}finally{e.maintenance.lastDurationMs=Number((performance.now()-t).toFixed(2)),e.maintenance.running=!1}}function Xt(e){return e===408||e===429||e>=500}function xt(e){return new Promise(n=>{setTimeout(n,e)})}async function Vt(e,n,t){let r;for(let s=1;s<=t.webhookRetryAttempts;s++){let i=new AbortController,o=setTimeout(()=>{i.abort()},t.webhookTimeoutMs);try{let u=await fetch(e,{method:"POST",headers:{"content-type":"application/json; charset=utf-8"},body:JSON.stringify(n),signal:i.signal});if(u.ok)return;let g=(await u.text().catch(()=>"")).slice(0,240);throw new R(`Alert webhook failed (${u.status})${g?`: ${g}`:""}`,Xt(u.status))}catch(u){let l=u instanceof R?u:u instanceof Error&&u.name==="AbortError"?new R(`Alert webhook timeout after ${t.webhookTimeoutMs}ms`,!0):u instanceof Error?new R(u.message,!0):new R(String(u),!0);if(!l.retryable||s>=t.webhookRetryAttempts)throw l;r=l}finally{clearTimeout(o)}let a=Math.round(t.webhookBackoffMs*2**(s-1));await xt(a)}throw r||new R("Alert webhook failed after retries",!1)}async function Me(e){if(!e.alertPolicy.enabled||!e.alertPolicy.webhookUrl||e.alerting.running)return;e.alerting.running=!0,e.alerting.runs++,e.alerting.lastRunAt=new Date().toISOString();let n=Date.now(),t=performance.now();try{let r=new Date(n-e.alertPolicy.windowMinutes*6e4).toISOString(),s=e.queryService.countLogs({from:r,level:"ERROR"}),i=e.queryService.countLogs({from:r}),o=[];if(s>=e.alertPolicy.errorThreshold&&o.push({rule:"error_threshold",severity:"critical",details:{errorCount:s,threshold:e.alertPolicy.errorThreshold,totalWindowCount:i,windowMinutes:e.alertPolicy.windowMinutes}}),e.alertPolicy.noLogsThresholdMinutes>0){let a=new Date(n-e.alertPolicy.noLogsThresholdMinutes*6e4).toISOString();e.queryService.countLogs({from:a})===0&&o.push({rule:"no_logs",severity:"warning",details:{thresholdMinutes:e.alertPolicy.noLogsThresholdMinutes}})}for(let a of o){let u=e.alerting.lastTriggerAtByRule[a.rule],l=u?Date.parse(u):NaN;if(Number.isFinite(l)&&n-l<e.alertPolicy.cooldownMs){e.alerting.suppressed++;continue}await Vt(e.alertPolicy.webhookUrl,{source:"mikroscope",rule:a.rule,severity:a.severity,triggeredAt:new Date(n).toISOString(),serviceUrl:e.url,details:a.details},e.alertPolicy);let g=new Date().toISOString();e.alerting.lastTriggerAtByRule[a.rule]=g,e.alerting.sent++}e.alerting.lastSuccessAt=new Date().toISOString(),e.alerting.lastError=void 0}catch(r){e.alerting.lastError=r instanceof Error?r.message:String(r)}finally{e.alerting.lastDurationMs=Number((performance.now()-t).toFixed(2)),e.alerting.running=!1}}function ve(e){let n=e.searchParams.get("audit");return{audit:n===null?void 0:n.toLowerCase()==="true"||n==="1"?!0:n.toLowerCase()==="false"||n==="0"?!1:void 0,cursor:e.searchParams.get("cursor")||void 0,field:e.searchParams.get("field")||void 0,from:e.searchParams.get("from")||void 0,level:e.searchParams.get("level")||void 0,limit:Ot(e.searchParams.get("limit"),100,Ut),to:e.searchParams.get("to")||void 0,value:e.searchParams.get("value")||void 0}}function Zt(e){if(e==="level"||e==="event"||e==="field"||e==="correlation")return e}function Ne(e){let n=V(Rt(import.meta.url)),t=[O(process.cwd(),e),O(n,"..",e)];for(let r of t)try{let s=x(r,"utf8");if(s.length===0)continue;return{content:s,path:r}}catch{}}function en(){return Ne(Gt)}function tn(){return Ne(Ht)}function nn(e){return`<!doctype html>
|
|
85
|
+
<html lang="en">
|
|
86
|
+
<head>
|
|
87
|
+
<meta charset="utf-8" />
|
|
88
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
89
|
+
<title>MikroScope API Docs</title>
|
|
90
|
+
<style>
|
|
91
|
+
html, body {
|
|
92
|
+
margin: 0;
|
|
93
|
+
padding: 0;
|
|
94
|
+
width: 100%;
|
|
95
|
+
height: 100%;
|
|
96
|
+
}
|
|
97
|
+
#app {
|
|
98
|
+
width: 100%;
|
|
99
|
+
height: 100%;
|
|
100
|
+
}
|
|
101
|
+
</style>
|
|
102
|
+
</head>
|
|
103
|
+
<body>
|
|
104
|
+
<div id="app" style="font-family: system-ui, sans-serif; padding: 12px 16px;">
|
|
105
|
+
<h1 style="margin: 0 0 6px;">MikroScope API Docs</h1>
|
|
106
|
+
<p style="margin: 0;">
|
|
107
|
+
Loading interactive docs. If this page remains plain, open
|
|
108
|
+
<a href="${e}">${e}</a>.
|
|
109
|
+
</p>
|
|
110
|
+
</div>
|
|
111
|
+
<script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script>
|
|
112
|
+
<script>
|
|
113
|
+
if (typeof Scalar !== "undefined" && typeof Scalar.createApiReference === "function") {
|
|
114
|
+
Scalar.createApiReference("#app", { url: "${e}" });
|
|
115
|
+
}
|
|
116
|
+
</script>
|
|
117
|
+
</body>
|
|
118
|
+
</html>`}async function rn(e,n,t){let r=new URL(e.url||"/",`${t.protocol}://localhost`);if(L(e,n,t.corsAllowOrigins),e.method==="OPTIONS"&&(r.pathname==="/health"||r.pathname==="/openapi.json"||r.pathname==="/openapi.yaml"||r.pathname==="/docs"||r.pathname==="/docs/"||r.pathname.startsWith("/api/"))){n.statusCode=204,n.end();return}if(r.pathname==="/openapi.yaml"&&e.method==="GET"){if(!t.openApiSpec)return p(e,n,404,{error:"OpenAPI specification not found."},t.corsAllowOrigins);L(e,n,t.corsAllowOrigins),n.statusCode=200,n.setHeader("content-type","application/yaml; charset=utf-8"),n.end(t.openApiSpec.content);return}if(r.pathname==="/openapi.json"&&e.method==="GET"){if(!t.openApiJson)return p(e,n,404,{error:"OpenAPI JSON document not found."},t.corsAllowOrigins);L(e,n,t.corsAllowOrigins),n.statusCode=200,n.setHeader("content-type","application/json; charset=utf-8"),n.end(t.openApiJson.content);return}if((r.pathname==="/docs"||r.pathname==="/docs/")&&e.method==="GET"){if(!t.openApiSpec&&!t.openApiJson)return p(e,n,404,{error:"OpenAPI document not found."},t.corsAllowOrigins);let s=t.openApiJson?"/openapi.json":"/openapi.yaml";L(e,n,t.corsAllowOrigins),n.statusCode=200,n.setHeader("content-type","text/html; charset=utf-8"),n.end(nn(s));return}if(r.pathname==="/health"){let s=t.db.getStats(),i=j(t.preflight.dbDirectory,1,"dbDirectory"),o=j(t.preflight.logsDirectory,1,"logsDirectory");return p(e,n,200,{ok:!0,service:"mikroscope",uptimeSec:Number(((Date.now()-t.startedAtMs)/1e3).toFixed(2)),ingest:t.ingest,auth:{apiTokenEnabled:!!t.apiToken,basicEnabled:t.basicAuth.enabled},ingestPolicy:t.ingestPolicy,ingestEndpoint:{enabled:t.ingestAuthPolicy.enabled||t.basicAuth.enabled,maxBodyBytes:t.ingestAuthPolicy.maxBodyBytes,producerCount:t.ingestAuthPolicy.producerByToken.size,queue:{enabled:t.ingestQueuePolicy.enabled,flushMs:t.ingestQueuePolicy.flushMs,draining:t.ingestQueueState.draining,pendingBatches:t.ingestQueueState.pending.length,pendingRecords:t.ingestQueueState.pendingRecords,recordsFlushed:t.ingestQueueState.recordsFlushed,recordsQueued:t.ingestQueueState.recordsQueued,lastError:t.ingestQueueState.lastError,lastFlushAt:t.ingestQueueState.lastFlushAt}},alerting:t.alerting,alertPolicy:{...t.alertPolicy,webhookUrl:t.alertPolicy.webhookUrl?"[configured]":void 0},maintenance:t.maintenance,retentionDays:{db:t.maintenancePolicy.dbRetentionDays,dbAudit:t.maintenancePolicy.dbAuditRetentionDays,logs:t.maintenancePolicy.logRetentionDays,logsAudit:t.maintenancePolicy.logAuditRetentionDays},backup:{auditDirectory:t.maintenancePolicy.auditBackupDirectory},storage:{dbApproximateSizeBytes:s.approximateSizeBytes,dbDirectoryFreeBytes:i,logsDirectoryFreeBytes:o,minFreeBytes:t.preflight.minFreeBytes}},t.corsAllowOrigins)}if(r.pathname==="/api/ingest"&&e.method==="POST"){if(!t.ingestAuthPolicy.enabled&&!t.basicAuth.enabled)return p(e,n,404,{error:"Ingest endpoint is not enabled."},t.corsAllowOrigins);let s=me(e,t.basicAuth,t.ingestAuthPolicy.producerByToken);if(!s)return p(e,n,401,{error:"Unauthorized"},t.corsAllowOrigins);let i;try{i=await ue(e,t.ingestAuthPolicy.maxBodyBytes)}catch(d){let h=d instanceof Error?d.message:String(d),f=h.startsWith("Payload too large")?413:400;return p(e,n,f,{error:h},t.corsAllowOrigins)}let o=Ae(i);if(!o)return p(e,n,400,{error:"Invalid ingest payload. Expected an array or an object with a logs array."},t.corsAllowOrigins);let a=new Date().toISOString(),u=[],l=0;for(let d of o){let h=Re(d,s,a);if(!h){l++;continue}u.push(h)}let g=!1;return u.length>0&&(t.ingestQueuePolicy.enabled?(Te(t,{producerId:s,records:u}),g=!0):await Y(t,[{producerId:s,records:u}])),p(e,n,g?202:200,{accepted:u.length,queued:g,producerId:s,receivedAt:a,rejected:l},t.corsAllowOrigins)}if(r.pathname.startsWith("/api/")&&!ge(e,t.apiToken,t.basicAuth))return p(e,n,401,{error:"Unauthorized"},t.corsAllowOrigins);if(r.pathname==="/api/reindex"&&e.method==="POST"){t.indexer.resetIncrementalState();let s=t.db.reset(),i=await t.indexer.indexDirectory(t.logsPath);return p(e,n,200,{report:i,reset:s},t.corsAllowOrigins)}if(r.pathname==="/api/logs"&&e.method==="GET"){let s=t.queryService.queryLogsPage(ve(r));return p(e,n,200,s,t.corsAllowOrigins)}if(r.pathname==="/api/logs/aggregate"&&e.method==="GET"){let s=Zt(r.searchParams.get("groupBy"));if(!s)return p(e,n,400,{error:"Invalid groupBy. Expected level, event, field, or correlation."},t.corsAllowOrigins);let i=r.searchParams.get("groupField")||void 0;if(s==="field"&&(!i||i.trim().length===0))return p(e,n,400,{error:"Missing required groupField when groupBy=field."},t.corsAllowOrigins);let o=ve(r),a=t.queryService.aggregateLogs({audit:o.audit,field:o.field,from:o.from,level:o.level,limit:o.limit,to:o.to,value:o.value},s,i);return p(e,n,200,{buckets:a,groupBy:s,groupField:i},t.corsAllowOrigins)}p(e,n,404,{error:"Not found"},t.corsAllowOrigins)}function sn(e,n,t){if(e==="https"){if(!n||!t)throw new Error("HTTPS requires both tlsCertPath and tlsKeyPath");let r={cert:x(n,"utf8"),key:x(t,"utf8")};return Et(r)}return St()}async function on(e,n,t){await new Promise((i,o)=>{e.once("error",o),e.listen(t,n,()=>{e.off("error",o),i()})});let r=e.address();if(!r||typeof r=="string")throw new Error("Failed to resolve server address");return{host:n,port:r.port}}async function Ce(e){let n=e.host||"127.0.0.1",t=e.protocol||"http",r=e.attachSignalHandlers??!0,s=qt(e),i=ye(e),o=be(e),a=le(e),u=Se(e),l=Ee(),g=Ie(),d=$t(e),h=jt(),f=zt(e),I=Kt(),y=tn(),z=en(),D=new A(e.dbPath),U=new v(D),J=new w(D);await K({indexer:U,ingest:l,logsPath:e.logsPath});let N=sn(t,e.tlsCertPath,e.tlsKeyPath),b={apiToken:e.apiToken,basicAuth:a,alertPolicy:d,alerting:h,corsAllowOrigins:ae(e.corsAllowOrigin??process.env.MIKROSCOPE_CORS_ALLOW_ORIGIN),db:D,ingest:l,ingestAuthPolicy:o,ingestQueuePolicy:u,ingestQueueState:g,ingestPolicy:i,logsPath:e.logsPath,maintenancePolicy:f,maintenance:I,openApiJson:y,openApiSpec:z,preflight:s,protocol:t,indexer:U,queryService:J,startedAtMs:Date.now(),url:`${t}://${n}:${e.port}`};Pe(b,f);let H=setInterval(()=>{Pe(b,f)},f.maintenanceIntervalMs);H.unref();let C=i.enabled?setInterval(()=>{K(b)},i.intervalMs):void 0;C?.unref();let k=d.enabled?setInterval(()=>{Me(b)},d.intervalMs):void 0;k?.unref(),N.on("request",(E,T)=>{rn(E,T,b).catch(B=>{p(E,T,500,{error:B instanceof Error?B.message:String(B)},b.corsAllowOrigins)})});let P=await on(N,n,e.port),F=`${t}://${P.host}:${P.port}`;b.url=F,d.enabled&&Me(b);let G=async()=>{clearInterval(H),C&&clearInterval(C),k&&clearInterval(k),await new Promise(E=>{N.close(()=>E())}),b.ingestQueueState.timer&&(clearTimeout(b.ingestQueueState.timer),b.ingestQueueState.timer=void 0),await X(b),D.close()};if(r){let E=!1,T=async()=>{E||(E=!0,await G(),process.exit(0))};process.on("SIGINT",T),process.on("SIGTERM",T)}return process.stdout.write(`[mikroscope] listening on ${F} logsPath=${e.logsPath} dbPath=${e.dbPath}
|
|
119
|
+
`),{close:G,host:P.host,port:P.port,protocol:t,url:F}}function an(e){let n={_:[]};for(let t=0;t<e.length;t++){let r=e[t];if(!r.startsWith("--")){n._.push(r);continue}let s=r.slice(2),i=e[t+1];if(!i||i.startsWith("--")){n[s]="true";continue}n[s]=i,t++}return n}function _(e,n,t){let r=e[n];return typeof r!="string"||r.length===0?t:r}function m(e,n,t){let r=e[n];if(typeof r!="string")return t;let s=Number.parseInt(r,10);return!Number.isFinite(s)||s<0?t:s}function Z(e,n,t){let r=e[n];if(typeof r!="string")return t;let s=r.toLowerCase();return s==="true"||s==="1"?!0:s==="false"||s==="0"?!1:t}function c(e,n){let t=e[n];return typeof t=="string"&&t.length>0?t:void 0}function Fe(e,n){let t=e[n];if(typeof t!="string"||t.length===0)return;let r=t.toLowerCase();if(r==="true"||r==="1")return!0;if(r==="false"||r==="0")return!1}function ke(){process.stdout.write(["MikroScope CLI","","Commands:"," serve Start HTTP/HTTPS API service"," index Index NDJSON logs into SQLite"," query Query paginated logs from SQLite"," aggregate Aggregate indexed logs by level/event/field","","Flags:"," --db SQLite file path (default: ./data/mikroscope.db)"," --logs NDJSON log directory (default: ./logs)"," --host Bind host for `serve` (default: 127.0.0.1)"," --port Server port for `serve` (default: 4310)"," --https Enable HTTPS for `serve` (default: false)"," --tls-cert TLS certificate path (required with --https)"," --tls-key TLS private key path (required with --https)"," --api-token Bearer token required for /api/* routes (optional)"," --auth-username Basic auth username for /api/* routes (optional)"," --auth-password Basic auth password for /api/* routes (optional)"," --cors-allow-origin CORS allow-list (comma separated origins, default: *)"," --db-retention-days Retain indexed DB rows for N days (default: 30)"," --db-audit-retention-days Retain indexed audit rows for N days (default: 365)"," --log-retention-days Delete raw non-audit .ndjson files older than N days (default: 30)"," --log-audit-retention-days Delete raw audit .ndjson files older than N days (default: 365)"," --audit-backup-dir Copy audit files here before retention delete (optional)"," --maintenance-interval-ms Maintenance cadence in ms (default: 21600000)"," --min-free-bytes Minimum free bytes required for db/log paths (default: 268435456)"," --ingest-interval-ms Incremental ingest cadence in ms (default: 2000)"," --disable-auto-ingest Disable periodic incremental ingest (default: false)"," --ingest-producers Ingest auth map as token=producerId pairs (comma separated)"," --ingest-max-body-bytes Max bytes accepted by /api/ingest payloads (default: 1048576)"," --ingest-async-queue Enable async ingest write/index queue (default: false)"," --ingest-queue-flush-ms Queue flush cadence in ms (default: 25)"," --alert-webhook-url Send alert payloads to this webhook URL (optional)"," --alert-interval-ms Alert evaluation cadence in ms (default: 30000)"," --alert-window-minutes Error threshold lookback window in minutes (default: 5)"," --alert-error-threshold ERROR logs threshold in alert window (default: 20)"," --alert-no-logs-threshold-minutes Trigger alert when no logs in N minutes (default: 0=off)"," --alert-cooldown-ms Min milliseconds between same rule notifications (default: 300000)"," --alert-webhook-timeout-ms Webhook request timeout per attempt (default: 5000)"," --alert-webhook-retry-attempts Max webhook attempts per alert (default: 3)"," --alert-webhook-backoff-ms Base retry backoff in ms (default: 250)"," --from ISO timestamp lower bound (query)"," --to ISO timestamp upper bound (query)"," --level DEBUG|INFO|WARN|ERROR (query)"," --audit true|false (query)"," --field Top-level field key (query)"," --value Top-level field value (query)"," --limit Max rows (query default: 100, aggregate default: 25)"," --cursor Page cursor token from previous query response"," --group-by level|event|field|correlation (aggregate)"," --group-field Required when --group-by field (aggregate)",""].join(`
|
|
120
|
+
`))}async function un(e){let n=_(e,"db","./data/mikroscope.db"),t=_(e,"logs","./logs"),r=new A(n),s=new v(r);try{let i=await s.indexDirectory(t);process.stdout.write(`${JSON.stringify({report:i},null,2)}
|
|
121
|
+
`)}finally{r.close()}}async function ln(e){let n=_(e,"db","./data/mikroscope.db"),t={audit:Fe(e,"audit"),cursor:c(e,"cursor"),from:c(e,"from"),to:c(e,"to"),level:c(e,"level"),field:c(e,"field"),value:c(e,"value"),limit:m(e,"limit",100)},r=new A(n),s=new w(r);try{let i=s.queryLogsPage(t);process.stdout.write(`${JSON.stringify(i,null,2)}
|
|
122
|
+
`)}finally{r.close()}}function dn(e){if(e==="level"||e==="event"||e==="field"||e==="correlation")return e}async function cn(e){let n=_(e,"db","./data/mikroscope.db"),t=dn(c(e,"group-by"));if(!t)throw new Error("Missing or invalid --group-by. Use level, event, field, or correlation.");let r=new A(n),s=new w(r);try{let i=s.aggregateLogs({audit:Fe(e,"audit"),field:c(e,"field"),from:c(e,"from"),level:c(e,"level"),limit:m(e,"limit",25),to:c(e,"to"),value:c(e,"value")},t,c(e,"group-field"));process.stdout.write(`${JSON.stringify({buckets:i,groupBy:t},null,2)}
|
|
123
|
+
`)}finally{r.close()}}async function gn(e){let n=_(e,"db","./data/mikroscope.db"),t=_(e,"logs","./logs"),r=_(e,"host",process.env.MIKROSCOPE_HOST||"127.0.0.1"),s=m(e,"port",4310),o=Z(e,"https",process.env.MIKROSCOPE_HTTPS==="1")?"https":"http",a=c(e,"tls-cert")||process.env.MIKROSCOPE_TLS_CERT_PATH,u=c(e,"tls-key")||process.env.MIKROSCOPE_TLS_KEY_PATH,l=c(e,"api-token")||process.env.MIKROSCOPE_API_TOKEN,g=c(e,"auth-username")||process.env.MIKROSCOPE_AUTH_USERNAME,d=c(e,"auth-password")||process.env.MIKROSCOPE_AUTH_PASSWORD,h=c(e,"cors-allow-origin")||process.env.MIKROSCOPE_CORS_ALLOW_ORIGIN,f=m(e,"db-retention-days",30),I=m(e,"db-audit-retention-days",365),y=m(e,"log-retention-days",30),z=m(e,"log-audit-retention-days",365),D=c(e,"audit-backup-dir")||process.env.MIKROSCOPE_AUDIT_BACKUP_DIR,U=m(e,"maintenance-interval-ms",360*60*1e3),J=m(e,"min-free-bytes",256*1024*1024),N=m(e,"ingest-interval-ms",2e3),b=Z(e,"disable-auto-ingest",!1),H=c(e,"ingest-producers")||process.env.MIKROSCOPE_INGEST_PRODUCERS,C=m(e,"ingest-max-body-bytes",1048576),k=Z(e,"ingest-async-queue",process.env.MIKROSCOPE_INGEST_ASYNC_QUEUE==="1"||process.env.MIKROSCOPE_INGEST_ASYNC_QUEUE==="true"),P=m(e,"ingest-queue-flush-ms",25),F=c(e,"alert-webhook-url")||process.env.MIKROSCOPE_ALERT_WEBHOOK_URL,G=m(e,"alert-interval-ms",3e4),E=m(e,"alert-window-minutes",5),T=m(e,"alert-error-threshold",20),B=m(e,"alert-no-logs-threshold-minutes",0),Be=m(e,"alert-cooldown-ms",300*1e3),Qe=m(e,"alert-webhook-timeout-ms",5e3),Ue=m(e,"alert-webhook-retry-attempts",3),He=m(e,"alert-webhook-backoff-ms",250);await Ce({apiToken:l,authPassword:d,authUsername:g,alertCooldownMs:Be,alertErrorThreshold:T,alertIntervalMs:G,alertNoLogsThresholdMinutes:B,alertWebhookUrl:F,alertWebhookBackoffMs:He,alertWebhookRetryAttempts:Ue,alertWebhookTimeoutMs:Qe,alertWindowMinutes:E,corsAllowOrigin:h,auditBackupDirectory:D,attachSignalHandlers:!0,dbPath:n,dbAuditRetentionDays:I,dbRetentionDays:f,host:r,ingestIntervalMs:N,disableAutoIngest:b,ingestProducers:H,ingestMaxBodyBytes:C,ingestAsyncQueue:k,ingestQueueFlushMs:P,logAuditRetentionDays:z,logRetentionDays:y,logsPath:t,maintenanceIntervalMs:U,minFreeBytes:J,port:s,protocol:o,tlsCertPath:a,tlsKeyPath:u})}async function mn(){let e=an(process.argv.slice(2)),n=e._[0];if(!n||n==="help"||n==="--help"||n==="-h"){ke();return}if(n==="serve"){await gn(e);return}if(n==="index"){await un(e);return}if(n==="query"){await ln(e);return}if(n==="aggregate"){await cn(e);return}ke(),process.exitCode=1}mn().catch(e=>{process.stderr.write(`[mikroscope] fatal: ${e instanceof Error?e.stack||e.message:String(e)}
|
|
124
|
+
`),process.exit(1)});
|