@agoric/telemetry 0.6.3-other-dev-3eb1a1d.0 → 0.6.3-other-dev-d15096d.0.d15096d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +29 -28
- package/src/context-aware-slog-file.js +7 -2
- package/src/context-aware-slog.js +11 -7
- package/src/flight-recorder.js +39 -19
- package/src/frcat-entrypoint.js +2 -2
- package/src/index.js +29 -26
- package/src/make-slog-sender.js +129 -109
- package/src/otel-and-flight-recorder.js +5 -1
- package/src/otel-context-aware-slog.js +12 -6
- package/src/otel-metrics.js +230 -0
- package/src/otel-trace.js +11 -1
- package/src/prometheus.js +22 -0
- package/src/serialize-slog-obj.js +32 -4
- package/src/slog-file.js +5 -1
- package/src/slog-sender-pipe-entrypoint.js +74 -69
- package/src/slog-sender-pipe.js +96 -111
- package/src/slog-to-otel.js +12 -6
- package/test/flight-recorder.test.js +42 -11
package/src/make-slog-sender.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import tmp from 'tmp';
|
|
3
|
-
import { PromiseAllOrErrors } from '@agoric/internal';
|
|
3
|
+
import { PromiseAllOrErrors, unprefixedProperties } from '@agoric/internal';
|
|
4
4
|
import { serializeSlogObj } from './serialize-slog-obj.js';
|
|
5
5
|
|
|
6
|
+
export const DEFAULT_SLOGSENDER_AGENT = 'self';
|
|
6
7
|
export const DEFAULT_SLOGSENDER_MODULE =
|
|
7
8
|
'@agoric/telemetry/src/flight-recorder.js';
|
|
8
9
|
export const SLOGFILE_SENDER_MODULE = '@agoric/telemetry/src/slog-file.js';
|
|
10
|
+
export const PROMETHEUS_SENDER_MODULE = '@agoric/telemetry/src/prometheus.js';
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
/**
|
|
13
|
+
* @import {SlogSender} from './index.js'
|
|
14
|
+
* @import {MakeSlogSender} from './index.js';
|
|
15
|
+
*/
|
|
13
16
|
|
|
14
17
|
/**
|
|
15
18
|
* @template T
|
|
@@ -19,102 +22,119 @@ export const DEFAULT_SLOGSENDER_AGENT = 'self';
|
|
|
19
22
|
const filterTruthy = arr => /** @type {any[]} */ (arr.filter(Boolean));
|
|
20
23
|
|
|
21
24
|
/**
|
|
22
|
-
*
|
|
25
|
+
* Create an aggregate slog sender that fans out inbound slog entries to modules
|
|
26
|
+
* as indicated by variables in the supplied `env` option. The SLOGSENDER value
|
|
27
|
+
* (or a default DEFAULT_SLOGSENDER_MODULE defined above) is split on commas
|
|
28
|
+
* into a list of module identifiers and adjusted by automatic insertions (a
|
|
29
|
+
* non-empty SLOGFILE value inserts DEFAULT_SLOGSENDER_AGENT defined above), and
|
|
30
|
+
* then each identifier is dynamically `import`ed for its own `makeSlogSender`
|
|
31
|
+
* export, which is invoked with a non-empty `stateDir` option and a modified
|
|
32
|
+
* `env` in which SLOGSENDER_AGENT_* variables have overridden their unprefixed
|
|
33
|
+
* equivalents to produce a subordinate slog sender.
|
|
34
|
+
* Subordinate slog senders remain isolated from each other, and any errors from
|
|
35
|
+
* them are caught and held until the next `forceFlush()` without disrupting
|
|
36
|
+
* any remaining slog entry fanout.
|
|
37
|
+
* If SLOGSENDER_AGENT is 'process', 'slog-sender-pipe.js' is used to load the
|
|
38
|
+
* subordinates in a child process rather than the main process.
|
|
39
|
+
* When there are no subordinates, the return value will be `undefined` rather
|
|
40
|
+
* than a slog sender function.
|
|
41
|
+
*
|
|
42
|
+
* @type {MakeSlogSender}
|
|
23
43
|
*/
|
|
24
44
|
export const makeSlogSender = async (opts = {}) => {
|
|
25
45
|
const { env = {}, stateDir: stateDirOption, ...otherOpts } = opts;
|
|
26
46
|
const {
|
|
27
47
|
SLOGSENDER = DEFAULT_SLOGSENDER_MODULE,
|
|
28
48
|
SLOGSENDER_AGENT = DEFAULT_SLOGSENDER_AGENT,
|
|
49
|
+
// While cosmic-swingset/kernel code includes its own Prometheus metrics
|
|
50
|
+
// export, that trumps a slog sender module doing so.
|
|
51
|
+
// This extraction can be removed when that changes, but in the meantime,
|
|
52
|
+
// opt-in is only by SLOGSENDER_AGENT_OTEL_EXPORTER_PROMETHEUS_PORT.
|
|
53
|
+
OTEL_EXPORTER_PROMETHEUS_PORT: _prometheusExportPort,
|
|
29
54
|
...otherEnv
|
|
30
55
|
} = env;
|
|
31
56
|
|
|
32
57
|
const agentEnv = {
|
|
33
58
|
...otherEnv,
|
|
34
|
-
...
|
|
35
|
-
Object.entries(otherEnv)
|
|
36
|
-
.filter(([k]) => k.match(/^(?:SLOGSENDER_AGENT_)+/)) // narrow to SLOGSENDER_AGENT_ prefixes.
|
|
37
|
-
.map(([k, v]) => [k.replace(/^(?:SLOGSENDER_AGENT_)+/, ''), v]), // Rewrite SLOGSENDER_AGENT_ to un-prefixed version.
|
|
38
|
-
),
|
|
59
|
+
...unprefixedProperties(otherEnv, 'SLOGSENDER_AGENT_'),
|
|
39
60
|
};
|
|
40
61
|
|
|
41
|
-
const slogSenderModules =
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
62
|
+
const slogSenderModules = new Set();
|
|
63
|
+
if (agentEnv.OTEL_EXPORTER_PROMETHEUS_PORT) {
|
|
64
|
+
slogSenderModules.add(PROMETHEUS_SENDER_MODULE);
|
|
65
|
+
}
|
|
66
|
+
if (agentEnv.SLOGFILE) {
|
|
67
|
+
slogSenderModules.add(SLOGFILE_SENDER_MODULE);
|
|
68
|
+
}
|
|
69
|
+
for (const moduleIdentifier of filterTruthy(SLOGSENDER.split(','))) {
|
|
70
|
+
if (moduleIdentifier.startsWith('-')) {
|
|
71
|
+
// Opt out of an automatically-included sender.
|
|
72
|
+
slogSenderModules.delete(moduleIdentifier.slice(1));
|
|
73
|
+
} else if (moduleIdentifier.startsWith('.')) {
|
|
74
|
+
// Resolve relative to the current working directory.
|
|
75
|
+
slogSenderModules.add(path.resolve(moduleIdentifier));
|
|
76
|
+
} else {
|
|
77
|
+
slogSenderModules.add(moduleIdentifier);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (!slogSenderModules.size) {
|
|
56
82
|
return undefined;
|
|
57
83
|
}
|
|
58
84
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
);
|
|
77
|
-
}
|
|
78
|
-
case 'worker':
|
|
79
|
-
default:
|
|
80
|
-
console.warn(`Unknown SLOGSENDER_AGENT=${SLOGSENDER_AGENT}`);
|
|
85
|
+
if (SLOGSENDER_AGENT === 'process') {
|
|
86
|
+
console.warn('Loading slog sender in subprocess');
|
|
87
|
+
return import('./slog-sender-pipe.js').then(async module =>
|
|
88
|
+
module.makeSlogSender({
|
|
89
|
+
env: {
|
|
90
|
+
...agentEnv,
|
|
91
|
+
SLOGSENDER,
|
|
92
|
+
SLOGSENDER_AGENT: 'self',
|
|
93
|
+
},
|
|
94
|
+
stateDir: stateDirOption,
|
|
95
|
+
...otherOpts,
|
|
96
|
+
}),
|
|
97
|
+
);
|
|
98
|
+
} else if (SLOGSENDER_AGENT && SLOGSENDER_AGENT !== 'self') {
|
|
99
|
+
console.warn(
|
|
100
|
+
`Unknown SLOGSENDER_AGENT=${SLOGSENDER_AGENT}; defaulting to 'self'`,
|
|
101
|
+
);
|
|
81
102
|
}
|
|
82
103
|
|
|
83
104
|
if (SLOGSENDER) {
|
|
84
105
|
console.warn('Loading slog sender modules:', ...slogSenderModules);
|
|
85
106
|
}
|
|
86
107
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
return /** @type {const} */ ([maker, moduleIdentifier]);
|
|
105
|
-
},
|
|
106
|
-
)
|
|
107
|
-
.catch(err => {
|
|
108
|
+
/** @type {Map<MakeSlogSender, string>} */
|
|
109
|
+
const makerMap = new Map();
|
|
110
|
+
await Promise.all(
|
|
111
|
+
[...slogSenderModules].map(async moduleIdentifier => {
|
|
112
|
+
await null;
|
|
113
|
+
try {
|
|
114
|
+
const module = await import(moduleIdentifier);
|
|
115
|
+
const { makeSlogSender: maker } = module;
|
|
116
|
+
if (typeof maker !== 'function') {
|
|
117
|
+
throw Error(`No 'makeSlogSender' function exported by module`);
|
|
118
|
+
} else if (maker === makeSlogSender) {
|
|
119
|
+
throw Error(`Cannot recursively load 'makeSlogSender' aggregator`);
|
|
120
|
+
}
|
|
121
|
+
const isReplacing = makerMap.get(maker);
|
|
122
|
+
if (isReplacing) {
|
|
108
123
|
console.warn(
|
|
109
|
-
`
|
|
110
|
-
err,
|
|
124
|
+
`The slog sender from ${moduleIdentifier} matches the one from ${isReplacing}.`,
|
|
111
125
|
);
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
126
|
+
}
|
|
127
|
+
makerMap.set(maker, moduleIdentifier);
|
|
128
|
+
} catch (err) {
|
|
129
|
+
console.warn(
|
|
130
|
+
`Failed to load slog sender from ${moduleIdentifier}.`,
|
|
131
|
+
err,
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
}),
|
|
135
|
+
);
|
|
116
136
|
|
|
117
|
-
if (!
|
|
137
|
+
if (!makerMap.size) {
|
|
118
138
|
return undefined;
|
|
119
139
|
}
|
|
120
140
|
|
|
@@ -122,11 +142,11 @@ export const makeSlogSender = async (opts = {}) => {
|
|
|
122
142
|
|
|
123
143
|
if (stateDir === undefined) {
|
|
124
144
|
stateDir = tmp.dirSync().name;
|
|
125
|
-
console.warn(`Using ${stateDir} for stateDir`);
|
|
145
|
+
console.warn(`Using ${stateDir} for slog sender stateDir`);
|
|
126
146
|
}
|
|
127
147
|
|
|
128
148
|
const senders = await Promise.all(
|
|
129
|
-
|
|
149
|
+
[...makerMap.entries()].map(async ([maker, moduleIdentifier]) =>
|
|
130
150
|
maker({
|
|
131
151
|
...otherOpts,
|
|
132
152
|
stateDir,
|
|
@@ -137,37 +157,37 @@ export const makeSlogSender = async (opts = {}) => {
|
|
|
137
157
|
|
|
138
158
|
if (!senders.length) {
|
|
139
159
|
return undefined;
|
|
140
|
-
} else {
|
|
141
|
-
// Optimize creating a JSON serialization only if needed
|
|
142
|
-
// by any of the sender modules
|
|
143
|
-
const hasSenderUsingJsonObj = senders.some(
|
|
144
|
-
({ usesJsonObject = true }) => usesJsonObject,
|
|
145
|
-
);
|
|
146
|
-
const getJsonObj = hasSenderUsingJsonObj
|
|
147
|
-
? serializeSlogObj
|
|
148
|
-
: () => undefined;
|
|
149
|
-
const sendErrors = [];
|
|
150
|
-
/** @type {SlogSender} */
|
|
151
|
-
const slogSender = (slogObj, jsonObj = getJsonObj(slogObj)) => {
|
|
152
|
-
for (const sender of senders) {
|
|
153
|
-
try {
|
|
154
|
-
sender(slogObj, jsonObj);
|
|
155
|
-
} catch (err) {
|
|
156
|
-
sendErrors.push(err);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
};
|
|
160
|
-
return Object.assign(slogSender, {
|
|
161
|
-
forceFlush: async () =>
|
|
162
|
-
PromiseAllOrErrors([
|
|
163
|
-
...senders.map(sender => sender.forceFlush?.()),
|
|
164
|
-
...sendErrors.splice(0).map(err => Promise.reject(err)),
|
|
165
|
-
]).then(() => {}),
|
|
166
|
-
shutdown: async () =>
|
|
167
|
-
PromiseAllOrErrors(senders.map(sender => sender.shutdown?.())).then(
|
|
168
|
-
() => {},
|
|
169
|
-
),
|
|
170
|
-
usesJsonObject: hasSenderUsingJsonObj,
|
|
171
|
-
});
|
|
172
160
|
}
|
|
161
|
+
|
|
162
|
+
// Optimize creating a JSON serialization only if needed
|
|
163
|
+
// by at least one of the senders.
|
|
164
|
+
const hasSenderUsingJsonObj = senders.some(
|
|
165
|
+
({ usesJsonObject = true }) => usesJsonObject,
|
|
166
|
+
);
|
|
167
|
+
const getJsonObj = hasSenderUsingJsonObj ? serializeSlogObj : () => undefined;
|
|
168
|
+
|
|
169
|
+
const sendErrors = [];
|
|
170
|
+
|
|
171
|
+
/** @type {SlogSender} */
|
|
172
|
+
const slogSender = (slogObj, jsonObj = getJsonObj(slogObj)) => {
|
|
173
|
+
for (const sender of senders) {
|
|
174
|
+
try {
|
|
175
|
+
sender(slogObj, jsonObj);
|
|
176
|
+
} catch (err) {
|
|
177
|
+
sendErrors.push(err);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
return Object.assign(slogSender, {
|
|
182
|
+
forceFlush: async () => {
|
|
183
|
+
await PromiseAllOrErrors([
|
|
184
|
+
...senders.map(sender => sender.forceFlush?.()),
|
|
185
|
+
...sendErrors.splice(0).map(err => Promise.reject(err)),
|
|
186
|
+
]);
|
|
187
|
+
},
|
|
188
|
+
shutdown: async () => {
|
|
189
|
+
await PromiseAllOrErrors(senders.map(sender => sender.shutdown?.()));
|
|
190
|
+
},
|
|
191
|
+
usesJsonObject: hasSenderUsingJsonObj,
|
|
192
|
+
});
|
|
173
193
|
};
|
|
@@ -2,7 +2,11 @@ import { NonNullish } from '@agoric/internal';
|
|
|
2
2
|
import { makeSlogSender as makeSlogSenderFromEnv } from './make-slog-sender.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* @
|
|
5
|
+
* @import {MakeSlogSenderOptions} from './index.js';
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* @param {MakeSlogSenderOptions} opts
|
|
6
10
|
*/
|
|
7
11
|
export const makeSlogSender = async opts => {
|
|
8
12
|
const { SLOGFILE: _1, SLOGSENDER: _2, ...otherEnv } = opts.env || {};
|
|
@@ -11,6 +11,12 @@ import { makeContextualSlogProcessor } from './context-aware-slog.js';
|
|
|
11
11
|
import { getResourceAttributes } from './index.js';
|
|
12
12
|
import { serializeSlogObj } from './serialize-slog-obj.js';
|
|
13
13
|
|
|
14
|
+
/**
|
|
15
|
+
* @import {Context} from './context-aware-slog.js';
|
|
16
|
+
* @import {MakeSlogSenderOptions} from './index.js';
|
|
17
|
+
* @import {Slog} from './context-aware-slog.js';
|
|
18
|
+
*/
|
|
19
|
+
|
|
14
20
|
const DEFAULT_CONTEXT_FILE = 'slog-context.json';
|
|
15
21
|
const FILE_ENCODING = 'utf8';
|
|
16
22
|
|
|
@@ -22,7 +28,7 @@ export const getContextFilePersistenceUtils = filePath => {
|
|
|
22
28
|
|
|
23
29
|
return {
|
|
24
30
|
/**
|
|
25
|
-
* @param {
|
|
31
|
+
* @param {Context} context
|
|
26
32
|
*/
|
|
27
33
|
persistContext: context => {
|
|
28
34
|
try {
|
|
@@ -33,7 +39,7 @@ export const getContextFilePersistenceUtils = filePath => {
|
|
|
33
39
|
},
|
|
34
40
|
|
|
35
41
|
/**
|
|
36
|
-
* @returns {
|
|
42
|
+
* @returns {Context | null}
|
|
37
43
|
*/
|
|
38
44
|
restoreContext: () => {
|
|
39
45
|
try {
|
|
@@ -47,7 +53,7 @@ export const getContextFilePersistenceUtils = filePath => {
|
|
|
47
53
|
};
|
|
48
54
|
|
|
49
55
|
/**
|
|
50
|
-
* @param {
|
|
56
|
+
* @param {MakeSlogSenderOptions} options
|
|
51
57
|
*/
|
|
52
58
|
export const makeSlogSender = async options => {
|
|
53
59
|
const { CHAIN_ID, OTEL_EXPORTER_OTLP_ENDPOINT } = options.env || {};
|
|
@@ -79,12 +85,12 @@ export const makeSlogSender = async options => {
|
|
|
79
85
|
);
|
|
80
86
|
|
|
81
87
|
/**
|
|
82
|
-
* @param {
|
|
88
|
+
* @param {Slog} slog
|
|
83
89
|
*/
|
|
84
90
|
const slogSender = slog => {
|
|
85
|
-
const {
|
|
91
|
+
const { time, ...logRecord } = contextualSlogProcessor(slog);
|
|
86
92
|
|
|
87
|
-
const [secondsStr, fractionStr] = String(
|
|
93
|
+
const [secondsStr, fractionStr] = String(time).split('.');
|
|
88
94
|
const seconds = parseInt(secondsStr, 10);
|
|
89
95
|
const nanoSeconds = parseInt(
|
|
90
96
|
(fractionStr || String(0)).padEnd(9, String(0)).slice(0, 9),
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { q, Fail } from '@endo/errors';
|
|
2
|
+
|
|
3
|
+
import * as ActionType from '@agoric/internal/src/action-types.js';
|
|
4
|
+
import { objectMapMutable } from '@agoric/internal/src/js-utils.js';
|
|
5
|
+
import {
|
|
6
|
+
HISTOGRAM_METRICS,
|
|
7
|
+
BLOCK_HISTOGRAM_METRICS,
|
|
8
|
+
KERNEL_STATS_METRICS,
|
|
9
|
+
makeQueueMetrics,
|
|
10
|
+
} from '@agoric/internal/src/metrics.js';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @import {MetricOptions, ObservableCounter, ObservableUpDownCounter} from '@opentelemetry/api';
|
|
14
|
+
* @import {MeterProvider} from '@opentelemetry/sdk-metrics';
|
|
15
|
+
* @import {TotalMap} from '@agoric/internal';
|
|
16
|
+
* @import {MakeSlogSenderOptions} from './index.js';
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const knownActionTypes = new Set(Object.values(ActionType.QueuedActionType));
|
|
20
|
+
|
|
21
|
+
/** @param {MakeSlogSenderOptions & {otelMeterName: string, otelMeterProvider?: MeterProvider}} opts */
|
|
22
|
+
export const makeSlogSender = async (opts = /** @type {any} */ ({})) => {
|
|
23
|
+
const { otelMeterName, otelMeterProvider } = opts;
|
|
24
|
+
if (!otelMeterName) throw Fail`OTel meter name is required`;
|
|
25
|
+
if (!otelMeterProvider) return;
|
|
26
|
+
|
|
27
|
+
const shutdown = async () => {
|
|
28
|
+
await otelMeterProvider.shutdown();
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const otelMeter = otelMeterProvider.getMeter(otelMeterName);
|
|
32
|
+
|
|
33
|
+
const processedInboundActionCounter = otelMeter.createCounter(
|
|
34
|
+
'cosmic_swingset_inbound_actions',
|
|
35
|
+
{ description: 'Processed inbound action counts by type' },
|
|
36
|
+
);
|
|
37
|
+
const histograms = {
|
|
38
|
+
...objectMapMutable(HISTOGRAM_METRICS, (desc, name) => {
|
|
39
|
+
const { boundaries, ...options } = desc;
|
|
40
|
+
const advice = boundaries && { explicitBucketBoundaries: boundaries };
|
|
41
|
+
return otelMeter.createHistogram(name, { ...options, advice });
|
|
42
|
+
}),
|
|
43
|
+
...objectMapMutable(BLOCK_HISTOGRAM_METRICS, (desc, name) =>
|
|
44
|
+
otelMeter.createHistogram(name, desc),
|
|
45
|
+
),
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const inboundQueueMetrics = makeQueueMetrics({
|
|
49
|
+
otelMeter,
|
|
50
|
+
namePrefix: 'cosmic_swingset_inbound_queue',
|
|
51
|
+
descPrefix: 'inbound queue',
|
|
52
|
+
console,
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
// Values for KERNEL_STATS_METRICS could be built up locally by observing slog
|
|
56
|
+
// entries, but they are all collectively reported in "kernel-stats"
|
|
57
|
+
// (@see {@link ../../cosmic-swingset/src/kernel-stats.js exportKernelStats})
|
|
58
|
+
// and for now we just reflect that, which requires implementation as async
|
|
59
|
+
// ("observable") instruments rather than synchronous ones.
|
|
60
|
+
/** @typedef {string} KernelStatsKey */
|
|
61
|
+
/** @typedef {string} KernelMetricName */
|
|
62
|
+
/** @type {TotalMap<KernelStatsKey, number>} */
|
|
63
|
+
const kernelStats = new Map();
|
|
64
|
+
/** @type {Map<KernelMetricName, ObservableCounter | ObservableUpDownCounter>} */
|
|
65
|
+
const kernelStatsCounters = new Map();
|
|
66
|
+
for (const meta of KERNEL_STATS_METRICS) {
|
|
67
|
+
const { key, name, sub, metricType, ...options } = meta;
|
|
68
|
+
kernelStats.set(key, 0);
|
|
69
|
+
if (metricType === 'gauge') {
|
|
70
|
+
kernelStats.set(`${key}Up`, 0);
|
|
71
|
+
kernelStats.set(`${key}Down`, 0);
|
|
72
|
+
kernelStats.set(`${key}Max`, 0);
|
|
73
|
+
} else if (metricType !== 'counter') {
|
|
74
|
+
Fail`Unknown metric type ${q(metricType)} for key ${q(key)} name ${q(name)}`;
|
|
75
|
+
}
|
|
76
|
+
let counter = kernelStatsCounters.get(name);
|
|
77
|
+
if (!counter) {
|
|
78
|
+
counter =
|
|
79
|
+
metricType === 'counter'
|
|
80
|
+
? otelMeter.createObservableCounter(name, options)
|
|
81
|
+
: otelMeter.createObservableUpDownCounter(name, options);
|
|
82
|
+
kernelStatsCounters.set(name, counter);
|
|
83
|
+
}
|
|
84
|
+
const attributes = sub ? { [sub.dimension]: sub.value } : {};
|
|
85
|
+
counter.addCallback(observer => {
|
|
86
|
+
observer.observe(kernelStats.get(key), attributes);
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
const expectedKernelStats = new Set(kernelStats.keys());
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* @typedef {object} LazyStats
|
|
93
|
+
* @property {string} namePrefix
|
|
94
|
+
* @property {MetricOptions} options
|
|
95
|
+
* @property {Set<string>} keys
|
|
96
|
+
* @property {Record<string, number>} data
|
|
97
|
+
*/
|
|
98
|
+
/** @type {(namePrefix: string, description: string) => LazyStats} */
|
|
99
|
+
const makeLazyStats = (namePrefix, description) => {
|
|
100
|
+
return { namePrefix, options: { description }, keys: new Set(), data: {} };
|
|
101
|
+
};
|
|
102
|
+
const dynamicAfterCommitStatsCounters = {
|
|
103
|
+
memoryUsage: makeLazyStats(
|
|
104
|
+
'memoryUsage_',
|
|
105
|
+
'kernel process memory statistic',
|
|
106
|
+
),
|
|
107
|
+
heapStats: makeLazyStats('heapStats_', 'v8 kernel heap statistic'),
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
const slogSender = ({ type: slogType, ...slogObj }) => {
|
|
111
|
+
switch (slogType) {
|
|
112
|
+
// Consume cosmic-swingset block lifecycle slog entries.
|
|
113
|
+
case 'cosmic-swingset-init': {
|
|
114
|
+
const { inboundQueueInitialLengths: lengths } = slogObj;
|
|
115
|
+
inboundQueueMetrics.initLengths(lengths);
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
case 'cosmic-swingset-begin-block': {
|
|
119
|
+
const {
|
|
120
|
+
interBlockSeconds,
|
|
121
|
+
afterCommitHangoverSeconds,
|
|
122
|
+
blockLagSeconds,
|
|
123
|
+
} = slogObj;
|
|
124
|
+
|
|
125
|
+
Number.isFinite(interBlockSeconds) &&
|
|
126
|
+
histograms.interBlockSeconds.record(interBlockSeconds);
|
|
127
|
+
histograms.afterCommitHangoverSeconds.record(
|
|
128
|
+
afterCommitHangoverSeconds,
|
|
129
|
+
);
|
|
130
|
+
Number.isFinite(blockLagSeconds) &&
|
|
131
|
+
histograms.blockLagSeconds.record(blockLagSeconds);
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
case 'cosmic-swingset-run-finish': {
|
|
135
|
+
histograms.swingset_block_processing_seconds.record(slogObj.seconds);
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
case 'cosmic-swingset-end-block-finish': {
|
|
139
|
+
const { inboundQueueStartLengths, processedActionCounts } = slogObj;
|
|
140
|
+
inboundQueueMetrics.updateLengths(inboundQueueStartLengths);
|
|
141
|
+
for (const processedActionRecord of processedActionCounts) {
|
|
142
|
+
const { count, phase, type: actionType } = processedActionRecord;
|
|
143
|
+
if (!knownActionTypes.has(actionType)) {
|
|
144
|
+
console.warn('Unknown inbound action type', actionType);
|
|
145
|
+
}
|
|
146
|
+
processedInboundActionCounter.add(count, { actionType });
|
|
147
|
+
inboundQueueMetrics.decLength(phase);
|
|
148
|
+
}
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
case 'cosmic-swingset-commit-block-finish': {
|
|
152
|
+
const {
|
|
153
|
+
runSeconds,
|
|
154
|
+
chainTime,
|
|
155
|
+
saveTime,
|
|
156
|
+
cosmosCommitSeconds,
|
|
157
|
+
fullSaveTime,
|
|
158
|
+
} = slogObj;
|
|
159
|
+
histograms.swingsetRunSeconds.record(runSeconds);
|
|
160
|
+
histograms.swingsetChainSaveSeconds.record(chainTime);
|
|
161
|
+
histograms.swingsetCommitSeconds.record(saveTime);
|
|
162
|
+
histograms.cosmosCommitSeconds.record(cosmosCommitSeconds);
|
|
163
|
+
histograms.fullCommitSeconds.record(fullSaveTime);
|
|
164
|
+
break;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Consume Swingset kernel slog entries.
|
|
168
|
+
case 'vat-startup-finish': {
|
|
169
|
+
histograms.swingset_vat_startup.record(slogObj.seconds * 1000);
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
case 'crank-finish': {
|
|
173
|
+
const { crankType, messageType, seconds } = slogObj;
|
|
174
|
+
// TODO: Reflect crankType/messageType as proper dimensional attributes.
|
|
175
|
+
// For now, we're going for parity with direct metrics.
|
|
176
|
+
if (crankType !== 'routing' && messageType !== 'create-vat') {
|
|
177
|
+
histograms.swingset_crank_processing_time.record(seconds * 1000);
|
|
178
|
+
}
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Consume miscellaneous slog entries.
|
|
183
|
+
case 'kernel-stats': {
|
|
184
|
+
const { stats } = slogObj;
|
|
185
|
+
const notYetFoundKernelStats = new Set(expectedKernelStats);
|
|
186
|
+
for (const [key, value] of Object.entries(stats)) {
|
|
187
|
+
notYetFoundKernelStats.delete(key);
|
|
188
|
+
if (!kernelStats.has(key)) {
|
|
189
|
+
console.warn('Unexpected SwingSet kernel statistic', key);
|
|
190
|
+
}
|
|
191
|
+
kernelStats.set(key, value);
|
|
192
|
+
}
|
|
193
|
+
if (notYetFoundKernelStats.size) {
|
|
194
|
+
console.warn('Expected SwingSet kernel statistics not found', [
|
|
195
|
+
...notYetFoundKernelStats,
|
|
196
|
+
]);
|
|
197
|
+
}
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
case 'cosmic-swingset-after-commit-stats': {
|
|
201
|
+
const dynamicCounterEntries = Object.entries(
|
|
202
|
+
dynamicAfterCommitStatsCounters,
|
|
203
|
+
);
|
|
204
|
+
for (const [slogKey, meta] of dynamicCounterEntries) {
|
|
205
|
+
const { namePrefix, options, keys } = meta;
|
|
206
|
+
meta.data = slogObj[slogKey] || {};
|
|
207
|
+
const newKeys = Object.keys(meta.data).filter(key => !keys.has(key));
|
|
208
|
+
for (const key of newKeys) {
|
|
209
|
+
keys.add(key);
|
|
210
|
+
const name = `${namePrefix}${key}`;
|
|
211
|
+
const gauge = otelMeter.createObservableUpDownCounter(
|
|
212
|
+
name,
|
|
213
|
+
options,
|
|
214
|
+
);
|
|
215
|
+
gauge.addCallback(observer => {
|
|
216
|
+
observer.observe(meta.data[key]);
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
break;
|
|
221
|
+
}
|
|
222
|
+
default:
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
return Object.assign(slogSender, {
|
|
227
|
+
shutdown,
|
|
228
|
+
usesJsonObject: false,
|
|
229
|
+
});
|
|
230
|
+
};
|
package/src/otel-trace.js
CHANGED
|
@@ -22,16 +22,26 @@ export const SPAN_EXPORT_DELAY_MS = 1_000;
|
|
|
22
22
|
export const makeOtelTracingProvider = opts => {
|
|
23
23
|
const { env = process.env } = opts || {};
|
|
24
24
|
|
|
25
|
+
// https://opentelemetry.io/docs/concepts/signals/
|
|
26
|
+
// https://opentelemetry.io/docs/specs/otel/protocol/exporter/#endpoint-urls-for-otlphttp
|
|
27
|
+
// https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/experimental/packages/exporter-trace-otlp-http/README.md#configuration-options-as-environment-variables
|
|
25
28
|
const { OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT } =
|
|
26
29
|
env;
|
|
27
30
|
if (!OTEL_EXPORTER_OTLP_ENDPOINT && !OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) {
|
|
31
|
+
console.debug(
|
|
32
|
+
'Not enabling OTLP Traces Exporter; enable with OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=<target URL> or OTEL_EXPORTER_OTLP_ENDPOINT=<target URL prefix>',
|
|
33
|
+
);
|
|
28
34
|
return undefined;
|
|
29
35
|
}
|
|
30
36
|
|
|
31
37
|
const resource = new Resource(getResourceAttributes(opts));
|
|
32
38
|
|
|
33
39
|
const exporter = new OTLPTraceExporter();
|
|
34
|
-
console.info(
|
|
40
|
+
console.info(
|
|
41
|
+
'Enabling OTLP Traces Exporter to',
|
|
42
|
+
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||
|
|
43
|
+
`${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`,
|
|
44
|
+
);
|
|
35
45
|
|
|
36
46
|
const provider = new BasicTracerProvider({ resource });
|
|
37
47
|
provider.addSpanProcessor(
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Fail } from '@endo/errors';
|
|
2
|
+
|
|
3
|
+
import { getPrometheusMeterProvider } from './index.js';
|
|
4
|
+
import { makeSlogSender as makeOtelMetricsSender } from './otel-metrics.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @import {MakeSlogSenderOptions} from './index.js';
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/** @param {MakeSlogSenderOptions & {otelMeterName?: string}} opts */
|
|
11
|
+
export const makeSlogSender = async (opts = {}) => {
|
|
12
|
+
const { env, otelMeterName, serviceName } = opts;
|
|
13
|
+
if (!otelMeterName) throw Fail`OTel meter name is required`;
|
|
14
|
+
const otelMeterProvider = getPrometheusMeterProvider({
|
|
15
|
+
console,
|
|
16
|
+
env,
|
|
17
|
+
serviceName,
|
|
18
|
+
});
|
|
19
|
+
if (!otelMeterProvider) return;
|
|
20
|
+
|
|
21
|
+
return makeOtelMetricsSender({ ...opts, otelMeterName, otelMeterProvider });
|
|
22
|
+
};
|