@hyperwatch/hyperwatch 4.3.1 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/config/apache_syslog_example.js +1 -1
- package/config/default.js +1 -1
- package/config/example.js +4 -4
- package/config/express_websocket_example.js +1 -1
- package/config/websocket_client_example.js +52 -0
- package/docs/configuration.md +67 -6
- package/docs/express-embedding.md +137 -0
- package/docs/input.md +50 -24
- package/docs/tutorials/apache_input.md +16 -14
- package/docs/tutorials/express_input.md +10 -8
- package/package.json +16 -14
- package/scripts/fetch-anthropic-ips.js +60 -0
- package/src/app/api.js +129 -3
- package/src/app/index.js +8 -4
- package/src/app/mount.js +115 -0
- package/src/app/websocket.js +12 -9
- package/src/app/ws-server.js +123 -0
- package/src/constants.js +8 -5
- package/src/data/amazon-searchbot-ips.json +304 -0
- package/src/data/amazonbot-ips.json +775 -1
- package/src/data/chatgpt-user-ips.json +115 -112
- package/src/data/claude-bot-ips.json +28 -0
- package/src/data/cloudfront-ips.json +14 -0
- package/src/data/gptbot-ips.json +0 -3
- package/src/data/openai-searchbot-ips.json +4 -0
- package/src/index.js +6 -1
- package/src/input/http.js +4 -0
- package/src/input/syslog.js +5 -1
- package/src/input/websocket.js +57 -21
- package/src/lib/aggregator.js +112 -19
- package/src/lib/formatter.js +10 -1
- package/src/lib/log-buffer.js +45 -0
- package/src/lib/persistence.js +11 -1
- package/src/lib/pipeline.js +140 -14
- package/src/lib/recent-map.js +23 -0
- package/src/modules/address.js +59 -2
- package/src/modules/dnsbl.js +11 -1
- package/src/modules/history.js +4 -28
- package/src/modules/hostname.js +12 -4
- package/src/modules/identity.js +92 -9
- package/src/modules/index.js +14 -5
- package/src/modules/signature.js +49 -14
- package/src/modules/sparkline.js +7 -3
- package/src/modules/status.js +6 -1
package/README.md
CHANGED
|
@@ -12,9 +12,9 @@ Hyperwatch is built on a real-time stream processor handling logs from inputs of
|
|
|
12
12
|
|
|
13
13
|
## Install
|
|
14
14
|
|
|
15
|
-
Make sure you have Node.js version >=
|
|
15
|
+
Make sure you have Node.js version >= 24.
|
|
16
16
|
|
|
17
|
-
We recommend using [nvm](https://github.com/
|
|
17
|
+
We recommend using [nvm](https://github.com/nvm-sh/nvm): `nvm install && nvm use`.
|
|
18
18
|
|
|
19
19
|
```bash
|
|
20
20
|
git clone https://github.com/hyperwatch/hyperwatch.git
|
|
@@ -42,7 +42,7 @@ In order to do this, you need to create a new configuration file such as `config
|
|
|
42
42
|
|
|
43
43
|
See [Input Configuration](./docs/input.md) for the list of available input types and how to configure them.
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
Modules (live log streams, User-Agent parsing, GeoIP, aggregations, …) and a few constants are configured with a `.hyperwatchrc` file. Only the `status` module is active by default. To learn more, head to [Global Configuration](./docs/configuration.md).
|
|
46
46
|
|
|
47
47
|
### Start with custom configuration
|
|
48
48
|
|
|
@@ -50,7 +50,7 @@ There are also a couple of constants you might configure with a simple config fi
|
|
|
50
50
|
npm start config/custom
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
The Hyperwatch API and
|
|
53
|
+
The Hyperwatch API and WebSocket will be served from port `3000` by default. Open `http://localhost:3000/status` to check the status of your inputs.
|
|
54
54
|
|
|
55
55
|
You can change that using an environment variable:
|
|
56
56
|
|
|
@@ -58,6 +58,11 @@ You can change that using an environment variable:
|
|
|
58
58
|
PORT=80 npm start config/custom
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
+
## Tutorials
|
|
62
|
+
|
|
63
|
+
- [Monitor web traffic with syslog input from Apache](./docs/tutorials/apache_input.md)
|
|
64
|
+
- [Monitor web traffic with Node/Express middleware integration](./docs/tutorials/express_input.md)
|
|
65
|
+
|
|
61
66
|
## License
|
|
62
67
|
|
|
63
68
|
[Apache License, version 2](LICENSE)
|
|
@@ -14,7 +14,7 @@ module.exports = function (hyperwatch) {
|
|
|
14
14
|
const syslogApacheHyperwatchCombinedInput = input.syslog.create({
|
|
15
15
|
port: 1518,
|
|
16
16
|
parse: format.apache.parser({
|
|
17
|
-
format: format.apache.formats.
|
|
17
|
+
format: format.apache.formats.hyperwatch_combined,
|
|
18
18
|
}),
|
|
19
19
|
});
|
|
20
20
|
|
package/config/default.js
CHANGED
|
@@ -27,7 +27,7 @@ const defaultConfig = function (hyperwatch) {
|
|
|
27
27
|
name: 'Syslog (nginx hyperwatch_combined format)',
|
|
28
28
|
port: 1515,
|
|
29
29
|
parse: format.nginx.parser({
|
|
30
|
-
format: format.nginx.formats.
|
|
30
|
+
format: format.nginx.formats.hyperwatch_combined,
|
|
31
31
|
}),
|
|
32
32
|
});
|
|
33
33
|
|
package/config/example.js
CHANGED
|
@@ -27,7 +27,7 @@ const exampleConfig = function (hyperwatch) {
|
|
|
27
27
|
name: 'Syslog (nginx hyperwatch_combined format)',
|
|
28
28
|
port: 1515,
|
|
29
29
|
parse: format.nginx.parser({
|
|
30
|
-
format: format.nginx.formats.
|
|
30
|
+
format: format.nginx.formats.hyperwatch_combined,
|
|
31
31
|
}),
|
|
32
32
|
});
|
|
33
33
|
|
|
@@ -60,7 +60,7 @@ const exampleConfig = function (hyperwatch) {
|
|
|
60
60
|
name: 'Syslog (apache hyperwatch_combined format)',
|
|
61
61
|
port: 1518,
|
|
62
62
|
parse: format.apache.parser({
|
|
63
|
-
format: format.apache.formats.
|
|
63
|
+
format: format.apache.formats.hyperwatch_combined,
|
|
64
64
|
}),
|
|
65
65
|
});
|
|
66
66
|
|
|
@@ -121,7 +121,7 @@ const exampleConfig = function (hyperwatch) {
|
|
|
121
121
|
// name: 'File input (nginx hyperwatch_combined format)',
|
|
122
122
|
// path: '/var/log/nginx/hyperwatch.log',
|
|
123
123
|
// parse: format.nginx.parser({
|
|
124
|
-
// format: format.nginx.formats.
|
|
124
|
+
// format: format.nginx.formats.hyperwatch_combined,
|
|
125
125
|
// }),
|
|
126
126
|
// });
|
|
127
127
|
|
|
@@ -145,7 +145,7 @@ const exampleConfig = function (hyperwatch) {
|
|
|
145
145
|
// name: 'File input (nginx hyperwatch_combined format)',
|
|
146
146
|
// path: '/var/log/apache2/hyperwatch.log',
|
|
147
147
|
// parse: format.nginx.parser({
|
|
148
|
-
// format: format.apache.formats.
|
|
148
|
+
// format: format.apache.formats.hyperwatch_combined,
|
|
149
149
|
// }),
|
|
150
150
|
// });
|
|
151
151
|
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Subscribe to the raw logs of an app embedding Hyperwatch (see
|
|
3
|
+
* docs/express-embedding.md) with a WebSocket client input, enrich them, and
|
|
4
|
+
* print them.
|
|
5
|
+
*
|
|
6
|
+
* HYPERWATCH_SECRET=… npm start config/websocket_client_example
|
|
7
|
+
*
|
|
8
|
+
* Environment:
|
|
9
|
+
* - HYPERWATCH_URL: the app's raw log stream
|
|
10
|
+
* (default: ws://localhost:3000/_hyperwatch/logs/raw, an app running locally)
|
|
11
|
+
* - HYPERWATCH_USERNAME / HYPERWATCH_SECRET: the app's Basic Auth credentials
|
|
12
|
+
* (the username defaults to hyperwatch)
|
|
13
|
+
* - PORT: where this watcher serves its API (default: 4000)
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
module.exports = function (hyperwatch) {
|
|
17
|
+
const { pipeline, input, logger } = hyperwatch;
|
|
18
|
+
|
|
19
|
+
const {
|
|
20
|
+
HYPERWATCH_URL = 'ws://localhost:3000/_hyperwatch/logs/raw',
|
|
21
|
+
HYPERWATCH_USERNAME = 'hyperwatch',
|
|
22
|
+
HYPERWATCH_SECRET,
|
|
23
|
+
} = process.env;
|
|
24
|
+
|
|
25
|
+
hyperwatch.init({
|
|
26
|
+
// Not 3000, so the watcher doesn't clash with an app running locally
|
|
27
|
+
port: 4000,
|
|
28
|
+
modules: {
|
|
29
|
+
cloudflare: { active: true },
|
|
30
|
+
geoip: { active: true },
|
|
31
|
+
agent: { active: true },
|
|
32
|
+
hostname: { active: true },
|
|
33
|
+
identity: { active: true },
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
// Fetch the logs
|
|
38
|
+
pipeline.registerInput(
|
|
39
|
+
input.websocket.create({
|
|
40
|
+
type: 'client',
|
|
41
|
+
address: HYPERWATCH_URL,
|
|
42
|
+
username: HYPERWATCH_USERNAME,
|
|
43
|
+
password: HYPERWATCH_SECRET,
|
|
44
|
+
reconnectOnClose: true,
|
|
45
|
+
})
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
// Output them, enriched
|
|
49
|
+
pipeline
|
|
50
|
+
.getNode('main')
|
|
51
|
+
.map((log) => console.log(logger.defaultFormatter.format(log, 'console')));
|
|
52
|
+
};
|
package/docs/configuration.md
CHANGED
|
@@ -6,20 +6,81 @@ Our recommended way to configure the constants is to add a `.hyperwatchrc` at th
|
|
|
6
6
|
|
|
7
7
|
This file can be in either `JSON` (recommended) or `ini` format.
|
|
8
8
|
|
|
9
|
-
Here is an example of how this file can look like
|
|
9
|
+
Here is an example of how this file can look like:
|
|
10
10
|
|
|
11
11
|
```JSON
|
|
12
12
|
{
|
|
13
|
-
"port": 4000
|
|
13
|
+
"port": 4000,
|
|
14
|
+
"modules": {
|
|
15
|
+
"logs": { "active": true },
|
|
16
|
+
"agent": { "active": true },
|
|
17
|
+
"address": { "active": true }
|
|
18
|
+
},
|
|
19
|
+
"persistence": { "enabled": true }
|
|
14
20
|
}
|
|
15
21
|
```
|
|
16
22
|
|
|
17
|
-
This example would
|
|
23
|
+
This example would serve the app on port 4000, enable the live log streams, User-Agent parsing and the addresses aggregator, and keep aggregated data between restarts.
|
|
24
|
+
|
|
25
|
+
Values are merged with the defaults, so you only need to list what you change.
|
|
26
|
+
|
|
27
|
+
Constants can also be passed from a configuration file, as an argument of `hyperwatch.init()`:
|
|
28
|
+
|
|
29
|
+
```javascript
|
|
30
|
+
module.exports = function (hyperwatch) {
|
|
31
|
+
hyperwatch.init({ modules: { logs: { active: true } } });
|
|
32
|
+
// ...
|
|
33
|
+
};
|
|
34
|
+
```
|
|
18
35
|
|
|
19
36
|
You can find below the list of all configurable constants:
|
|
20
37
|
|
|
21
38
|
## Global
|
|
22
39
|
|
|
23
|
-
| Constant name
|
|
24
|
-
|
|
|
25
|
-
| port
|
|
40
|
+
| Constant name | Type | Default | Description |
|
|
41
|
+
| ----------------- | ------- | ------- | ------------------------------------------------------ |
|
|
42
|
+
| port | integer | `3000` | The port the app is running on |
|
|
43
|
+
| heartbeatInterval | integer | `30000` | Interval in ms between WebSocket pings sent to clients |
|
|
44
|
+
|
|
45
|
+
The `PORT` environment variable takes precedence over the `port` constant.
|
|
46
|
+
|
|
47
|
+
## Modules
|
|
48
|
+
|
|
49
|
+
Modules enrich logs and expose API endpoints. Each module is configured under `modules.<name>` with:
|
|
50
|
+
|
|
51
|
+
| Attribute | Type | Description |
|
|
52
|
+
| --------- | ------- | ------------------------------------------------------------ |
|
|
53
|
+
| active | boolean | Whether the module is loaded |
|
|
54
|
+
| priority | integer | Modules are loaded in ascending priority. Keep the defaults. |
|
|
55
|
+
|
|
56
|
+
Only `status` is active by default.
|
|
57
|
+
|
|
58
|
+
| Module | Description | Endpoints |
|
|
59
|
+
| ---------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------------------- |
|
|
60
|
+
| status | Status of inputs and pipeline nodes | `/status` |
|
|
61
|
+
| logs | Streams every pipeline node over HTTP and WebSocket | `/logs/<node>` |
|
|
62
|
+
| cloudflare | Uses Cloudflare headers (`cf-connecting-ip`, `cf-ipcountry`, `cf-ray`) for the client address and data | – |
|
|
63
|
+
| geoip | Geolocates addresses | – |
|
|
64
|
+
| agent | Parses User-Agents with [@hyperwatch/useragent](https://github.com/hyperwatch/useragent) | – |
|
|
65
|
+
| hostname | Reverse DNS lookup and forward verification of client addresses | – |
|
|
66
|
+
| language | Parses the `Accept-Language` header | – |
|
|
67
|
+
| dnsbl | Checks client addresses against DNS blocklists | – |
|
|
68
|
+
| address | Aggregates traffic per address | `/addresses` |
|
|
69
|
+
| signature | Aggregates traffic per request signature | `/signatures` |
|
|
70
|
+
| identity | Identifies known robots and crawlers. Depends on `agent`, `hostname`, `signature` and `address` | `/identities` |
|
|
71
|
+
| history | Keeps the latest logs of each pipeline node in memory (`capacity`, default `1000`), saved and restored when `persistence.enabled` | `/history/<node>.json` |
|
|
72
|
+
| sparkline | Adds activity sparklines to aggregator HTML tables | – |
|
|
73
|
+
|
|
74
|
+
Aggregator endpoints render an HTML table by default, or JSON and CSV with a `.json` or `.csv` extension. They accept `limit` (default `100`) and `sort` (default `count15m`) query parameters. A single entry is available at `/<aggregator>/<id>.json`, and `DELETE /<aggregator>` resets it.
|
|
75
|
+
|
|
76
|
+
To use a custom DNS server for the `hostname` module, set the `HYPERWATCH_DNS_SERVER` environment variable.
|
|
77
|
+
|
|
78
|
+
## Persistence
|
|
79
|
+
|
|
80
|
+
Aggregated data can be saved when Hyperwatch stops and loaded when it starts.
|
|
81
|
+
|
|
82
|
+
| Constant name | Type | Default | Description |
|
|
83
|
+
| --------------------- | ------- | ------------------ | ------------------------------------------------ |
|
|
84
|
+
| persistence.enabled | boolean | `false` | Whether to save and load aggregator data |
|
|
85
|
+
| persistence.path | string | `.hyperwatch-data` | Directory for the data, relative to the cwd |
|
|
86
|
+
| persistence.namespace | string | `null` | Optional sub-directory, to run several instances |
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Express embedding
|
|
2
|
+
|
|
3
|
+
Hyperwatch can run inside an existing Express 5 application, instead of running as a separate server. The app logs its own traffic and exposes the Hyperwatch API and live streams (HTTP and WebSocket) on its own port, under a path you choose.
|
|
4
|
+
|
|
5
|
+
```javascript
|
|
6
|
+
const http = require('node:http');
|
|
7
|
+
const express = require('express');
|
|
8
|
+
const basicAuth = require('express-basic-auth');
|
|
9
|
+
const hyperwatch = require('@hyperwatch/hyperwatch');
|
|
10
|
+
|
|
11
|
+
const {
|
|
12
|
+
PORT = 3000,
|
|
13
|
+
HYPERWATCH_USERNAME = 'hyperwatch',
|
|
14
|
+
HYPERWATCH_SECRET,
|
|
15
|
+
} = process.env;
|
|
16
|
+
|
|
17
|
+
const app = express();
|
|
18
|
+
const server = http.createServer(app);
|
|
19
|
+
|
|
20
|
+
hyperwatch.init({ modules: { logs: { active: true } } });
|
|
21
|
+
|
|
22
|
+
// Log every request: register the input before other middleware and routes
|
|
23
|
+
const input = hyperwatch.input.express.create();
|
|
24
|
+
app.use(input.middleware());
|
|
25
|
+
hyperwatch.pipeline.registerInput(input);
|
|
26
|
+
|
|
27
|
+
// Mount the Hyperwatch API and live streams, behind authentication
|
|
28
|
+
const auth = basicAuth({ users: { [HYPERWATCH_USERNAME]: HYPERWATCH_SECRET } });
|
|
29
|
+
|
|
30
|
+
hyperwatch.app.mount(app, {
|
|
31
|
+
server,
|
|
32
|
+
path: '/_hyperwatch',
|
|
33
|
+
middleware: auth,
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// Other middleware and application routes
|
|
37
|
+
|
|
38
|
+
hyperwatch.modules.start();
|
|
39
|
+
hyperwatch.pipeline.start();
|
|
40
|
+
server.listen(PORT);
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Don't call `hyperwatch.start()`: it would start the standalone Hyperwatch server.
|
|
44
|
+
|
|
45
|
+
## `hyperwatch.app.mount(app, options)`
|
|
46
|
+
|
|
47
|
+
| Option | Required | Description |
|
|
48
|
+
| ------------ | -------- | ---------------------------------------------------------------------------------------------------------------- |
|
|
49
|
+
| `server` | yes | The Node HTTP server of the app. `mount()` adds an `upgrade` listener to it, and never creates or starts one. |
|
|
50
|
+
| `path` | yes | The mount path, e.g. `'/_hyperwatch'`. It must start with `/` and not end with `/`. |
|
|
51
|
+
| `middleware` | no | A middleware function or an array of them, e.g. authentication. Applied to HTTP requests and WebSocket upgrades. |
|
|
52
|
+
| `fallback` | no | `(req, socket, head) => {}`, called with the WebSocket upgrades Hyperwatch doesn't own. |
|
|
53
|
+
|
|
54
|
+
### HTTP routes
|
|
55
|
+
|
|
56
|
+
`mount()` calls `app.use(path, ...middleware, router)`, at the point where it's called. The app's middleware order is kept:
|
|
57
|
+
|
|
58
|
+
- Middleware registered before `mount()` runs for Hyperwatch requests too. Mount Hyperwatch before a rate limiter if the watcher shouldn't be limited.
|
|
59
|
+
- Routes registered before `mount()` can answer first: mount Hyperwatch before a catch-all route (e.g. a Next.js request handler).
|
|
60
|
+
|
|
61
|
+
### WebSocket upgrades
|
|
62
|
+
|
|
63
|
+
`mount()` adds one `upgrade` listener to `server`:
|
|
64
|
+
|
|
65
|
+
- An upgrade with a malformed target (not an origin-form `/path?query`, e.g. `//[/` or an absolute URL) gets `400 Bad Request`, before reaching the app or `fallback`.
|
|
66
|
+
- An upgrade under the mount path (`/_hyperwatch` or `/_hyperwatch/…`) is sent through `app`, like an HTTP request. It goes through the same middleware, so authentication protects it, and Express answers errors as usual: a middleware can reject it with `401`, `next(error)` keeps the error's status, and an unknown Hyperwatch route gets `404`.
|
|
67
|
+
- Any other upgrade is passed to `fallback(req, socket, head)` when given. Otherwise it's left to the server's other `upgrade` listeners.
|
|
68
|
+
|
|
69
|
+
Hyperwatch only owns its mount path: `/other/logs/raw` is never handled by Hyperwatch, even though `/logs/raw` is a Hyperwatch route. The mount path is matched like Express matches `app.use()`: case-insensitively by default, and case-sensitively when the app enables `case sensitive routing`. So an HTTP request and a WebSocket upgrade to the same path always reach the same place.
|
|
70
|
+
|
|
71
|
+
`mount()` doesn't stop the server's other `upgrade` listeners from receiving Hyperwatch upgrades. A listener that closes the sockets of upgrades it doesn't serve would close the Hyperwatch WebSocket. Give such a component its own channel, and pass it the other upgrades with `fallback`, as below for Next.js.
|
|
72
|
+
|
|
73
|
+
### Mounting twice, and cleanup
|
|
74
|
+
|
|
75
|
+
`mount()` throws, before registering anything, when:
|
|
76
|
+
|
|
77
|
+
- Hyperwatch is already mounted on the same `server`.
|
|
78
|
+
- Hyperwatch was already mounted on the same app at the same path, matched like the app routes it (so `/_HYPERWATCH` is the same path by default). This holds even after `detachUpgrades()`: Express can't remove routes, so the first mount keeps answering at that path, with its original middleware. Mounting again couldn't change them, e.g. add authentication.
|
|
79
|
+
|
|
80
|
+
`mount()` returns `{ path, detachUpgrades }`. `detachUpgrades()` removes the `upgrade` listener, so Hyperwatch stops handling WebSocket upgrades, and releases the server for a mount at another path. It can be called several times. It doesn't unmount the HTTP routes, which stay mounted with their middleware.
|
|
81
|
+
|
|
82
|
+
## Next.js custom server (workaround for Next.js 16.3)
|
|
83
|
+
|
|
84
|
+
Next.js registers an `upgrade` listener for its own WebSockets (e.g. hot reload in development) on the server it runs behind: the `httpServer` option, or the server of the first request it handles. It ends the socket of any upgrade whose path matches one of its routes, e.g. a catch-all page, which includes `/_hyperwatch/logs/raw`.
|
|
85
|
+
|
|
86
|
+
So Next.js gets its own upgrade channel, and Hyperwatch passes it the upgrades it doesn't own:
|
|
87
|
+
|
|
88
|
+
```javascript
|
|
89
|
+
const { EventEmitter } = require('node:events');
|
|
90
|
+
const next = require('next');
|
|
91
|
+
|
|
92
|
+
// Next.js listens for upgrades on this channel, not on the server
|
|
93
|
+
const nextUpgrades = new EventEmitter();
|
|
94
|
+
const nextApp = next({ dev, httpServer: nextUpgrades });
|
|
95
|
+
const handle = nextApp.getRequestHandler();
|
|
96
|
+
|
|
97
|
+
hyperwatch.app.mount(app, {
|
|
98
|
+
server,
|
|
99
|
+
path: '/_hyperwatch',
|
|
100
|
+
middleware: auth,
|
|
101
|
+
fallback: (req, socket, head) =>
|
|
102
|
+
nextUpgrades.emit('upgrade', req, socket, head),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// Next.js renders the pages, after Hyperwatch
|
|
106
|
+
app.use((req, res) => handle(req, res));
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
This is a version-specific workaround, not a stable integration contract. Next.js documents `httpServer` as the HTTP server it runs behind, and doesn't offer a way to keep its upgrade listener off a custom server. In Next.js 16.3, `httpServer` is only used to listen for `upgrade` events, so an `EventEmitter` works: verified with Next.js 16.3.4 in development (hot reload) and production, with a catch-all page. Check it again when upgrading Next.js.
|
|
110
|
+
|
|
111
|
+
## Subscribing to the live logs
|
|
112
|
+
|
|
113
|
+
A watcher can subscribe to the logs with a WebSocket input:
|
|
114
|
+
|
|
115
|
+
```javascript
|
|
116
|
+
const { HYPERWATCH_USERNAME = 'hyperwatch', HYPERWATCH_SECRET } = process.env;
|
|
117
|
+
|
|
118
|
+
input.websocket.create({
|
|
119
|
+
type: 'client',
|
|
120
|
+
address: 'wss://example.org/_hyperwatch/logs/raw',
|
|
121
|
+
username: HYPERWATCH_USERNAME,
|
|
122
|
+
password: HYPERWATCH_SECRET,
|
|
123
|
+
reconnectOnClose: true,
|
|
124
|
+
});
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`config/websocket_client_example.js` is a complete watcher: it subscribes to the logs of an app running locally, enriches them and prints them.
|
|
128
|
+
|
|
129
|
+
```sh
|
|
130
|
+
HYPERWATCH_SECRET=… npm start config/websocket_client_example
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Upgrading from express-ws
|
|
134
|
+
|
|
135
|
+
With Hyperwatch 4.3 and Express 4, an app would call `expressWs(app)` and mount `hyperwatch.app.api` and `hyperwatch.app.websocket` itself. Express 5 is not compatible with express-ws, and Hyperwatch no longer uses it.
|
|
136
|
+
|
|
137
|
+
`hyperwatch.app.websocket` is still an Express middleware, so `app.use(path, hyperwatch.app.websocket)` doesn't throw, but on its own it doesn't handle any upgrade. Replace the express-ws setup and the `app.use()` calls with `hyperwatch.app.mount()`.
|
package/docs/input.md
CHANGED
|
@@ -3,15 +3,16 @@
|
|
|
3
3
|
Input configuration consists in 3 steps:
|
|
4
4
|
|
|
5
5
|
1. Instantiating a [type of input](#input-types) with the right configuration
|
|
6
|
-
2.
|
|
6
|
+
2. Optionally indicating to the input [how to parse the access logs](#other-formats)
|
|
7
7
|
3. Registering the input with the pipeline
|
|
8
8
|
|
|
9
|
-
You can configure and register as many inputs as you need. The
|
|
9
|
+
You can configure and register as many inputs as you need. The `/status` page shows the configured inputs, their status and how much traffic is going through them.
|
|
10
10
|
|
|
11
11
|
## Input Types
|
|
12
12
|
|
|
13
13
|
- All inputs support by default single logs in the [Hyperwatch JSON format](#json-format).
|
|
14
14
|
- All inputs support an optional `parse` parameter for other formats.
|
|
15
|
+
- All inputs accept an optional `name`, displayed on the `/status` page.
|
|
15
16
|
|
|
16
17
|
### Syslog
|
|
17
18
|
|
|
@@ -66,7 +67,7 @@ The input accepts the following options.
|
|
|
66
67
|
|
|
67
68
|
### WebSocket
|
|
68
69
|
|
|
69
|
-
The WebSocket input
|
|
70
|
+
The WebSocket input subscribes to a WebSocket server sending access logs (`client`), or listens for WebSocket connections sending access logs (`server`).
|
|
70
71
|
|
|
71
72
|
The input accepts the following options.
|
|
72
73
|
|
|
@@ -75,9 +76,29 @@ The input accepts the following options.
|
|
|
75
76
|
| type | string | no | Either 'client' or 'server' (default to 'client') |
|
|
76
77
|
| address | string | yes (if type is 'client') | The WebSocket address to connect to (e.g. 'wss://localhost:3000') |
|
|
77
78
|
| path | string | yes (if type is 'server') | The path where to listen for logs |
|
|
78
|
-
| parse | Parser | no | A function to parse the messages
|
|
79
|
+
| parse | Parser | no | A function to parse the messages (See [Formats](#other-formats) below) |
|
|
79
80
|
| sample | float | no | A sample rate, a float between 0 and 1. Will only send data this percentage of the time. |
|
|
80
81
|
|
|
82
|
+
### Express
|
|
83
|
+
|
|
84
|
+
The Express input receives logs from a middleware mounted in an Express application running in the same process as Hyperwatch.
|
|
85
|
+
|
|
86
|
+
Mount its middleware before the application routes, so every request is logged:
|
|
87
|
+
|
|
88
|
+
```javascript
|
|
89
|
+
const expressInput = input.express.create();
|
|
90
|
+
app.use(expressInput.middleware()); // Before application routes
|
|
91
|
+
pipeline.registerInput(expressInput);
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
| Attribute | Type | Required? | Description |
|
|
95
|
+
| --------- | ------- | --------- | --------------------------------------------------------------------------------------- |
|
|
96
|
+
| app | Express | no | An Express app to mount the middleware on when the pipeline starts. See the note below. |
|
|
97
|
+
|
|
98
|
+
**Note**: With the `app` option, the input calls `app.use()` only when the pipeline starts. Routes registered on the app before that run first, and their requests are not logged, even if they are declared after `input.express.create()`. Prefer mounting `middleware()` yourself, as above.
|
|
99
|
+
|
|
100
|
+
To log the traffic of an application running in a separate process, use the [Hyperwatch Express Logger](https://www.npmjs.com/package/@hyperwatch/express-logger) middleware with the HTTP, WebSocket or syslog input (see the [tutorial](./tutorials/express_input.md)).
|
|
101
|
+
|
|
81
102
|
## JSON Format
|
|
82
103
|
|
|
83
104
|
The JSON log parser parses access log in JSON that match the following schema.
|
|
@@ -143,7 +164,7 @@ If you are using Nginx, you can simply copy-and-paste the format specification f
|
|
|
143
164
|
parse: format.nginx.parser({
|
|
144
165
|
format:
|
|
145
166
|
'$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent"',
|
|
146
|
-
})
|
|
167
|
+
}),
|
|
147
168
|
}
|
|
148
169
|
```
|
|
149
170
|
|
|
@@ -167,7 +188,7 @@ If you are using Apache, you can simply copy-and-paste the format specification
|
|
|
167
188
|
{
|
|
168
189
|
parse: format.apache.parser({
|
|
169
190
|
format: '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
|
|
170
|
-
})
|
|
191
|
+
}),
|
|
171
192
|
}
|
|
172
193
|
```
|
|
173
194
|
|
|
@@ -202,21 +223,23 @@ If you're using a standard log format, do not hesitate to create a ticket in the
|
|
|
202
223
|
Simple real-time log processing of [Nginx's predefined combined](http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) log format with a log file located at `/var/log/nginx/access.log` can be achieved with the following configuration:
|
|
203
224
|
|
|
204
225
|
```javascript
|
|
205
|
-
|
|
206
|
-
const input =
|
|
207
|
-
const format = require('../format');
|
|
226
|
+
module.exports = function (hyperwatch) {
|
|
227
|
+
const { pipeline, input, format } = hyperwatch;
|
|
208
228
|
|
|
209
|
-
|
|
210
|
-
path: '/var/log/nginx/access.log',
|
|
211
|
-
parse: format.nginx.parser({ format: format.nginx.formats.combined }),
|
|
212
|
-
});
|
|
229
|
+
hyperwatch.init();
|
|
213
230
|
|
|
214
|
-
|
|
231
|
+
const nginxInput = input.file.create({
|
|
232
|
+
path: '/var/log/nginx/access.log',
|
|
233
|
+
parse: format.nginx.parser({ format: format.nginx.formats.combined }),
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
pipeline.registerInput(nginxInput);
|
|
237
|
+
};
|
|
215
238
|
```
|
|
216
239
|
|
|
217
240
|
When placed in `config/custom.js` it can be used by Hyperwatch with:
|
|
218
241
|
|
|
219
|
-
```
|
|
242
|
+
```bash
|
|
220
243
|
npm start config/custom
|
|
221
244
|
```
|
|
222
245
|
|
|
@@ -224,36 +247,39 @@ npm start config/custom
|
|
|
224
247
|
|
|
225
248
|
For more detailed log processing, it is recommended to use the _Hyperwatch combined_ log format:
|
|
226
249
|
|
|
227
|
-
```
|
|
250
|
+
```nginx
|
|
228
251
|
log_format hyperwatch_combined '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host"'
|
|
229
252
|
access_log /logs/access.log hyperwatch_combined;
|
|
230
253
|
```
|
|
231
254
|
|
|
232
255
|
With the following configuration for Hyperwatch:
|
|
233
256
|
|
|
234
|
-
```
|
|
257
|
+
```javascript
|
|
235
258
|
const defaultInput = input.file.create({
|
|
236
259
|
path: '/logs/access.log',
|
|
237
|
-
parse: format.nginx.parser({
|
|
238
|
-
|
|
260
|
+
parse: format.nginx.parser({
|
|
261
|
+
format: format.nginx.formats.hyperwatch_combined,
|
|
262
|
+
}),
|
|
263
|
+
});
|
|
239
264
|
```
|
|
240
265
|
|
|
241
266
|
### Behind a proxy
|
|
242
267
|
|
|
243
268
|
If behind a proxy, you might want to also report the `HTTP_X_FORWARDED_FOR` header to allow Hyperwatch to properly detect the client IP address.
|
|
244
269
|
|
|
245
|
-
```
|
|
270
|
+
```nginx
|
|
246
271
|
log_format hyperwatch_combined_with_x_forwarded_for '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host" "$http_x_forwarded_for"'
|
|
247
272
|
access_log /logs/access.log hyperwatch_combined_with_x_forwarded_for;
|
|
248
273
|
```
|
|
249
274
|
|
|
250
275
|
With the following configuration for Hyperwatch:
|
|
251
276
|
|
|
252
|
-
```
|
|
277
|
+
```javascript
|
|
253
278
|
const defaultInput = input.file.create({
|
|
254
279
|
path: '/logs/access.log',
|
|
255
280
|
parse: format.nginx.parser({
|
|
256
|
-
format:
|
|
257
|
-
|
|
258
|
-
})
|
|
281
|
+
format:
|
|
282
|
+
'$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host" "$http_x_forwarded_for"',
|
|
283
|
+
}),
|
|
284
|
+
});
|
|
259
285
|
```
|
|
@@ -10,10 +10,10 @@ Let's start.
|
|
|
10
10
|
|
|
11
11
|
On the same server where Apache is running, or on a server that is reachable by it, install the Hyperwatch processor.
|
|
12
12
|
|
|
13
|
-
As a
|
|
13
|
+
As a prerequisite, you'll need Node.js >= 24. We recommend [nvm](https://github.com/nvm-sh/nvm).
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
|
-
nvm install
|
|
16
|
+
nvm install 24
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
#### Install from npm
|
|
@@ -24,7 +24,7 @@ npm install -g @hyperwatch/hyperwatch
|
|
|
24
24
|
|
|
25
25
|
#### Install from Git
|
|
26
26
|
|
|
27
|
-
Alternatively, for
|
|
27
|
+
Alternatively, for development purpose, you can use Git and clone the public repository:
|
|
28
28
|
|
|
29
29
|
```bash
|
|
30
30
|
git clone https://github.com/hyperwatch/hyperwatch.git
|
|
@@ -34,13 +34,13 @@ npm install
|
|
|
34
34
|
|
|
35
35
|
### Configure Hyperwatch
|
|
36
36
|
|
|
37
|
-
In our suggested configuration, Hyperwatch will be listening for access logs in the
|
|
37
|
+
In our suggested configuration, Hyperwatch will be listening for access logs in the `hyperwatch_combined` format on port `1518`.
|
|
38
38
|
|
|
39
|
-
We always
|
|
39
|
+
We always recommend using the `hyperwatch_combined` format, which is logging more detailed information and allows for a much better analysis than the regular `combined` format.
|
|
40
40
|
|
|
41
|
-
To get more familiar, you can inspect default and example configurations in [`config/default.js`](
|
|
41
|
+
To get more familiar, you can inspect default and example configurations in [`config/default.js`](../../config/default.js) and [`config/example.js`](../../config/example.js) file.
|
|
42
42
|
|
|
43
|
-
Now, you can create your own configuration in
|
|
43
|
+
Now, you can create your own configuration in `apache_syslog_example.js` (a complete version is available in [`config/apache_syslog_example.js`](../../config/apache_syslog_example.js)):
|
|
44
44
|
|
|
45
45
|
```javascript
|
|
46
46
|
module.exports = function (hyperwatch) {
|
|
@@ -61,7 +61,7 @@ module.exports = function (hyperwatch) {
|
|
|
61
61
|
|
|
62
62
|
### Configure Apache
|
|
63
63
|
|
|
64
|
-
First, if you're following our recommendation and opted for the
|
|
64
|
+
First, if you're following our recommendation and opted for the `hyperwatch_combined` format, you need to define it in the Apache configuration. This will not replace the standard log format, just create an additional one.
|
|
65
65
|
|
|
66
66
|
```
|
|
67
67
|
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Accept}i\" \"%{Accept-Charset}i\" \"%{Accept-Encoding}i\" \"%{Accept-Language}i\" \"%{Connection}i\" \"%{Dnt}i\" \"%{From}i\" \"%{Host}i\"" hyperwatch_combined
|
|
@@ -79,13 +79,13 @@ Note: This is known to be working on _Ubuntu 16.04, 18.04_ with _logger 2.27.1,
|
|
|
79
79
|
|
|
80
80
|
In this example, there are 3 important things:
|
|
81
81
|
|
|
82
|
-
1. If Hyperwatch is running on the same server, we can use
|
|
83
|
-
If it's on a different server, replace
|
|
84
|
-
2. We configured Hyperwatch to listen for syslog messages in the
|
|
82
|
+
1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
|
|
83
|
+
If it's on a different server, replace `localhost` by the proper private or public IP address.
|
|
84
|
+
2. We configured Hyperwatch to listen for syslog messages in the `hyperwatch_combined` format on port `1518`.
|
|
85
85
|
We're properly passing that port in the configuration
|
|
86
|
-
3. Finally, we're asking Apache to use the
|
|
86
|
+
3. Finally, we're asking Apache to use the `hyperwatch_combined` log format we previously configured.
|
|
87
87
|
|
|
88
|
-
Don't forget to reload
|
|
88
|
+
Don't forget to reload Apache with the updated configuration. On Ubuntu, it would be:
|
|
89
89
|
|
|
90
90
|
```bash
|
|
91
91
|
service apache2 reload
|
|
@@ -101,4 +101,6 @@ hyperwatch apache_syslog_example.js
|
|
|
101
101
|
|
|
102
102
|
### Browse the interface
|
|
103
103
|
|
|
104
|
-
Now, you can point your browser to the IP/port where Hyperwatch is running. If you see
|
|
104
|
+
Now, you can point your browser to the `/status` page on the IP/port where Hyperwatch is running (e.g. `http://localhost:3000/status`). If you see traffic going through your input, congrats you made it!
|
|
105
|
+
|
|
106
|
+
To watch the logs live at `/logs/main` and explore aggregations such as `/addresses` or `/identities`, activate the corresponding modules. See [Global Configuration](../configuration.md#modules).
|