@agoric/telemetry 0.6.3-upgrade-14-dev-c8f9e7b.0 → 0.6.3-upgrade-16-dev-8879538.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +0 -33
- package/package.json +22 -17
- package/scripts/ingest.sh +2 -2
- package/src/flight-recorder.js +151 -86
- package/src/frcat-entrypoint.js +12 -8
- package/src/index.js +9 -7
- package/src/ingest-slog-entrypoint.js +11 -2
- package/src/make-slog-sender.js +3 -3
- package/src/otel-and-flight-recorder.js +3 -0
- package/src/slog-sender-pipe-entrypoint.js +11 -6
- package/src/slog-sender-pipe.js +3 -4
- package/src/slog-to-otel.js +9 -9
- package/test/flight-recorder.test.js +83 -0
- package/test/prepare-test-env-ava.js +0 -2
- package/{jsconfig.json → tsconfig.json} +1 -0
- package/test/test-flight-recorder.js +0 -53
- /package/test/{test-import.js → import.test.js} +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -3,39 +3,6 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
-
### [0.6.3-u13.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.3-u12.0...@agoric/telemetry@0.6.3-u13.0) (2023-12-07)
|
|
7
|
-
|
|
8
|
-
**Note:** Version bump only for package @agoric/telemetry
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
### [0.6.3-u12.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.3-u11wf.0...@agoric/telemetry@0.6.3-u12.0) (2023-11-10)
|
|
15
|
-
|
|
16
|
-
**Note:** Version bump only for package @agoric/telemetry
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
### [0.6.3-u11wf.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.3-u11.0...@agoric/telemetry@0.6.3-u11wf.0) (2023-09-23)
|
|
23
|
-
|
|
24
|
-
**Note:** Version bump only for package @agoric/telemetry
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
### [0.6.3-u11.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.2...@agoric/telemetry@0.6.3-u11.0) (2023-08-24)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
### Features
|
|
34
|
-
|
|
35
|
-
* **cosmic-swingset:** add JS upgrade plan handler stub ([7803d3d](https://github.com/Agoric/agoric-sdk/commit/7803d3de8e0cba681dfd27dacfc3577eed0bf2f8))
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
6
|
### [0.6.2](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.1...@agoric/telemetry@0.6.2) (2023-06-02)
|
|
40
7
|
|
|
41
8
|
**Note:** Version bump only for package @agoric/telemetry
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agoric/telemetry",
|
|
3
|
-
"version": "0.6.3-upgrade-
|
|
3
|
+
"version": "0.6.3-upgrade-16-dev-8879538.0+8879538",
|
|
4
4
|
"description": "Agoric's telemetry implementation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": "https://github.com/Agoric/agoric-sdk",
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"test:xs": "exit 0",
|
|
13
13
|
"lint-fix": "yarn lint:eslint --fix",
|
|
14
14
|
"lint": "run-s --continue-on-error lint:*",
|
|
15
|
-
"lint:types": "tsc
|
|
15
|
+
"lint:types": "tsc",
|
|
16
16
|
"lint:eslint": "eslint ."
|
|
17
17
|
},
|
|
18
18
|
"bin": {
|
|
@@ -22,12 +22,12 @@
|
|
|
22
22
|
"author": "Agoric",
|
|
23
23
|
"license": "Apache-2.0",
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@agoric/assert": "0.6.1-upgrade-
|
|
26
|
-
"@agoric/internal": "0.
|
|
27
|
-
"@agoric/store": "0.9.3-upgrade-
|
|
28
|
-
"@endo/init": "
|
|
29
|
-
"@endo/marshal": "
|
|
30
|
-
"@endo/stream": "
|
|
25
|
+
"@agoric/assert": "0.6.1-upgrade-16-dev-8879538.0+8879538",
|
|
26
|
+
"@agoric/internal": "0.3.3-upgrade-16-dev-8879538.0+8879538",
|
|
27
|
+
"@agoric/store": "0.9.3-upgrade-16-dev-8879538.0+8879538",
|
|
28
|
+
"@endo/init": "^1.1.2",
|
|
29
|
+
"@endo/marshal": "^1.5.0",
|
|
30
|
+
"@endo/stream": "^1.2.2",
|
|
31
31
|
"@opentelemetry/api": "~1.3.0",
|
|
32
32
|
"@opentelemetry/exporter-prometheus": "~0.35.0",
|
|
33
33
|
"@opentelemetry/exporter-trace-otlp-http": "~0.35.0",
|
|
@@ -36,29 +36,34 @@
|
|
|
36
36
|
"@opentelemetry/sdk-trace-base": "~1.9.0",
|
|
37
37
|
"@opentelemetry/semantic-conventions": "~1.9.0",
|
|
38
38
|
"anylogger": "^0.21.0",
|
|
39
|
-
"better-sqlite3": "^
|
|
40
|
-
"bufferfromfile": "agoric-labs/BufferFromFile#Agoric-built",
|
|
39
|
+
"better-sqlite3": "^9.1.1",
|
|
41
40
|
"tmp": "^0.2.1"
|
|
42
41
|
},
|
|
43
42
|
"devDependencies": {
|
|
44
|
-
"@endo/lockdown": "0.
|
|
45
|
-
"@endo/ses-ava": "
|
|
46
|
-
"ava": "^5.
|
|
47
|
-
"c8": "^
|
|
43
|
+
"@endo/lockdown": "^1.0.7",
|
|
44
|
+
"@endo/ses-ava": "^1.2.2",
|
|
45
|
+
"ava": "^5.3.0",
|
|
46
|
+
"c8": "^9.1.0",
|
|
48
47
|
"tmp": "^0.2.1"
|
|
49
48
|
},
|
|
50
49
|
"publishConfig": {
|
|
51
50
|
"access": "public"
|
|
52
51
|
},
|
|
53
52
|
"engines": {
|
|
54
|
-
"node": "
|
|
53
|
+
"node": "^18.12 || ^20.9"
|
|
55
54
|
},
|
|
56
55
|
"ava": {
|
|
57
56
|
"files": [
|
|
58
|
-
"test
|
|
57
|
+
"test/**/*.test.*"
|
|
58
|
+
],
|
|
59
|
+
"require": [
|
|
60
|
+
"@endo/init/debug.js"
|
|
59
61
|
],
|
|
60
62
|
"timeout": "20m",
|
|
61
63
|
"workerThreads": false
|
|
62
64
|
},
|
|
63
|
-
"
|
|
65
|
+
"typeCoverage": {
|
|
66
|
+
"atLeast": 87.03
|
|
67
|
+
},
|
|
68
|
+
"gitHead": "8879538cd1d125a08346f02dd5701d0d70c90bb8"
|
|
64
69
|
}
|
package/scripts/ingest.sh
CHANGED
|
@@ -18,8 +18,8 @@ if [[ ! -s "$PROGRESS" ]]; then
|
|
|
18
18
|
fi
|
|
19
19
|
if [[ -n "$INGEST_START" ]]; then
|
|
20
20
|
case "$1" in
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
*.Z | *.gz) firstline=$(zcat "$1" | head -1) ;;
|
|
22
|
+
*) firstline=$(head -1 "$1") ;;
|
|
23
23
|
esac
|
|
24
24
|
echo "$firstline" | jq --arg targetStart "$INGEST_START" \
|
|
25
25
|
'{virtualTimeOffset: (($targetStart | fromdate) - .time), lastSlogTime: 0}' \
|
package/src/flight-recorder.js
CHANGED
|
@@ -1,20 +1,10 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/* global Buffer */
|
|
1
3
|
/// <reference types="ses" />
|
|
2
4
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
// keep writing new events into the RAM window and updating the start/end
|
|
7
|
-
// pointers, with some sequencing to make sure the record gets written before
|
|
8
|
-
// the pointer is updated. Then, no mattter how abruptly the process is
|
|
9
|
-
// terminated, as long as the host computer itself is still running, the on-disk
|
|
10
|
-
// file would contain the most recent state, and anybody who reads the file will
|
|
11
|
-
// get the most recent state. The host kernel (linux) is under no obligation to
|
|
12
|
-
// flush it to disk any particular time, but knows when reads happen, so there's
|
|
13
|
-
// no coherency problem, and the speed is unaffected by disk write speeds.
|
|
14
|
-
|
|
15
|
-
import BufferFromFile from 'bufferfromfile';
|
|
16
|
-
import { promises as fsPromises } from 'fs';
|
|
17
|
-
import path from 'path';
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import fsp from 'node:fs/promises';
|
|
7
|
+
import path from 'node:path';
|
|
18
8
|
import { serializeSlogObj } from './serialize-slog-obj.js';
|
|
19
9
|
|
|
20
10
|
const { Fail } = assert;
|
|
@@ -31,12 +21,16 @@ const I_ARENA_START = 4 * BigUint64Array.BYTES_PER_ELEMENT;
|
|
|
31
21
|
|
|
32
22
|
const RECORD_HEADER_SIZE = BigUint64Array.BYTES_PER_ELEMENT;
|
|
33
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Initializes a circular buffer with the given size, creating the buffer file if it doesn't exist or is not large enough.
|
|
26
|
+
*
|
|
27
|
+
* @param {string} bufferFile - the file path for the circular buffer
|
|
28
|
+
* @param {number} circularBufferSize - the size of the circular buffer
|
|
29
|
+
* @returns {Promise<bigint>} the size of the initialized circular buffer
|
|
30
|
+
*/
|
|
34
31
|
const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
|
|
35
|
-
if (!circularBufferSize) {
|
|
36
|
-
return undefined;
|
|
37
|
-
}
|
|
38
32
|
// If the file doesn't exist, or is not large enough, create it.
|
|
39
|
-
const stbuf = await
|
|
33
|
+
const stbuf = await fsp.stat(bufferFile).catch(e => {
|
|
40
34
|
if (e.code === 'ENOENT') {
|
|
41
35
|
return undefined;
|
|
42
36
|
}
|
|
@@ -57,8 +51,8 @@ const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
|
|
|
57
51
|
header.setBigUint64(I_CIRC_START, 0n);
|
|
58
52
|
header.setBigUint64(I_CIRC_END, 0n);
|
|
59
53
|
|
|
60
|
-
await
|
|
61
|
-
await
|
|
54
|
+
await fsp.mkdir(path.dirname(bufferFile), { recursive: true });
|
|
55
|
+
await fsp.writeFile(bufferFile, headerBuf);
|
|
62
56
|
|
|
63
57
|
if (stbuf && stbuf.size >= circularBufferSize) {
|
|
64
58
|
// File is big enough.
|
|
@@ -66,50 +60,20 @@ const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
|
|
|
66
60
|
}
|
|
67
61
|
|
|
68
62
|
// Increase the file size.
|
|
69
|
-
await
|
|
63
|
+
await fsp.truncate(bufferFile, circularBufferSize);
|
|
70
64
|
return arenaSize;
|
|
71
65
|
};
|
|
72
66
|
|
|
73
|
-
|
|
74
|
-
circularBufferSize = DEFAULT_CBUF_SIZE,
|
|
75
|
-
stateDir = '/tmp',
|
|
76
|
-
circularBufferFilename,
|
|
77
|
-
}) => {
|
|
78
|
-
const filename = circularBufferFilename || `${stateDir}/${DEFAULT_CBUF_FILE}`;
|
|
79
|
-
// console.log({ circularBufferFilename, filename });
|
|
80
|
-
|
|
81
|
-
const newArenaSize = await initializeCircularBuffer(
|
|
82
|
-
filename,
|
|
83
|
-
circularBufferSize,
|
|
84
|
-
);
|
|
67
|
+
/** @typedef {Awaited<ReturnType<typeof makeSimpleCircularBuffer>>} CircularBuffer */
|
|
85
68
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const hdrArenaSize = header.getBigUint64(I_ARENA_SIZE);
|
|
95
|
-
const arenaSize = newArenaSize || hdrArenaSize;
|
|
96
|
-
|
|
97
|
-
const hdrMagic = header.getBigUint64(I_MAGIC);
|
|
98
|
-
SLOG_MAGIC === hdrMagic ||
|
|
99
|
-
Fail`${filename} is not a slog buffer; wanted magic ${SLOG_MAGIC}, got ${hdrMagic}`;
|
|
100
|
-
arenaSize === hdrArenaSize ||
|
|
101
|
-
Fail`${filename} arena size mismatch; wanted ${arenaSize}, got ${hdrArenaSize}`;
|
|
102
|
-
const arena = new Uint8Array(
|
|
103
|
-
fileBuf.buffer,
|
|
104
|
-
header.byteLength,
|
|
105
|
-
Number(arenaSize),
|
|
106
|
-
);
|
|
107
|
-
|
|
108
|
-
/**
|
|
109
|
-
* @param {Uint8Array} outbuf
|
|
110
|
-
* @param {number} [offset] offset relative to the current trailing edge (circStart) of the data
|
|
111
|
-
* @returns {IteratorResult<Uint8Array, void>}
|
|
112
|
-
*/
|
|
69
|
+
/**
|
|
70
|
+
*
|
|
71
|
+
* @param {bigint} arenaSize
|
|
72
|
+
* @param {DataView} header
|
|
73
|
+
* @param {(outbuf: Uint8Array, readStart: number, firstReadLength: number) => void} readRecord
|
|
74
|
+
* @param {(record: Uint8Array, firstWriteLength: number, circEnd: bigint) => Promise<void>} writeRecord
|
|
75
|
+
*/
|
|
76
|
+
function finishCircularBuffer(arenaSize, header, readRecord, writeRecord) {
|
|
113
77
|
const readCircBuf = (outbuf, offset = 0) => {
|
|
114
78
|
offset + outbuf.byteLength <= arenaSize ||
|
|
115
79
|
Fail`Reading past end of circular buffer`;
|
|
@@ -132,19 +96,13 @@ export const makeMemoryMappedCircularBuffer = async ({
|
|
|
132
96
|
// The data is contiguous, like ---AAABBB---
|
|
133
97
|
return { done: true, value: undefined };
|
|
134
98
|
}
|
|
135
|
-
outbuf
|
|
136
|
-
if (firstReadLength < outbuf.byteLength) {
|
|
137
|
-
outbuf.set(
|
|
138
|
-
arena.subarray(0, outbuf.byteLength - firstReadLength),
|
|
139
|
-
firstReadLength,
|
|
140
|
-
);
|
|
141
|
-
}
|
|
99
|
+
readRecord(outbuf, readStart, firstReadLength);
|
|
142
100
|
return { done: false, value: outbuf };
|
|
143
101
|
};
|
|
144
102
|
|
|
145
|
-
/** @
|
|
146
|
-
const writeCircBuf = data => {
|
|
147
|
-
if (RECORD_HEADER_SIZE + data.byteLength >
|
|
103
|
+
/** @type {(data: Uint8Array) => Promise<void>} */
|
|
104
|
+
const writeCircBuf = async data => {
|
|
105
|
+
if (RECORD_HEADER_SIZE + data.byteLength > arenaSize) {
|
|
148
106
|
// The data is too big to fit in the arena, so skip it.
|
|
149
107
|
const tooBigRecord = JSON.stringify({
|
|
150
108
|
type: 'slog-record-too-big',
|
|
@@ -153,14 +111,17 @@ export const makeMemoryMappedCircularBuffer = async ({
|
|
|
153
111
|
data = new TextEncoder().encode(tooBigRecord);
|
|
154
112
|
}
|
|
155
113
|
|
|
156
|
-
if (RECORD_HEADER_SIZE + data.byteLength >
|
|
114
|
+
if (RECORD_HEADER_SIZE + data.byteLength > arenaSize) {
|
|
157
115
|
// Silently drop, it just doesn't fit.
|
|
158
116
|
return;
|
|
159
117
|
}
|
|
160
118
|
|
|
119
|
+
// Allocate for the data and a header
|
|
161
120
|
const record = new Uint8Array(RECORD_HEADER_SIZE + data.byteLength);
|
|
121
|
+
// Set the data, after the header
|
|
162
122
|
record.set(data, RECORD_HEADER_SIZE);
|
|
163
123
|
|
|
124
|
+
// Set the size in the header
|
|
164
125
|
const lengthPrefix = new DataView(record.buffer);
|
|
165
126
|
lengthPrefix.setBigUint64(0, BigInt(data.byteLength));
|
|
166
127
|
|
|
@@ -206,31 +167,135 @@ export const makeMemoryMappedCircularBuffer = async ({
|
|
|
206
167
|
);
|
|
207
168
|
}
|
|
208
169
|
|
|
209
|
-
arena.set(record.subarray(0, firstWriteLength), Number(circEnd));
|
|
210
|
-
if (firstWriteLength < record.byteLength) {
|
|
211
|
-
// Write to the beginning of the arena.
|
|
212
|
-
arena.set(record.subarray(firstWriteLength, record.byteLength), 0);
|
|
213
|
-
}
|
|
214
170
|
header.setBigUint64(
|
|
215
171
|
I_CIRC_END,
|
|
216
172
|
(circEnd + BigInt(record.byteLength)) % arenaSize,
|
|
217
173
|
);
|
|
174
|
+
|
|
175
|
+
return writeRecord(record, firstWriteLength, circEnd);
|
|
218
176
|
};
|
|
219
177
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
178
|
+
return { readCircBuf, writeCircBuf };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* @param {{
|
|
183
|
+
* circularBufferSize?: number,
|
|
184
|
+
* stateDir?: string,
|
|
185
|
+
* circularBufferFilename?: string
|
|
186
|
+
* }} opts
|
|
187
|
+
*/
|
|
188
|
+
export const makeSimpleCircularBuffer = async ({
|
|
189
|
+
circularBufferSize = DEFAULT_CBUF_SIZE,
|
|
190
|
+
stateDir = '/tmp',
|
|
191
|
+
circularBufferFilename,
|
|
192
|
+
}) => {
|
|
193
|
+
const filename = circularBufferFilename || `${stateDir}/${DEFAULT_CBUF_FILE}`;
|
|
194
|
+
|
|
195
|
+
const newArenaSize = await initializeCircularBuffer(
|
|
196
|
+
filename,
|
|
197
|
+
circularBufferSize,
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const file = await fsp.open(filename, 'r+');
|
|
201
|
+
|
|
202
|
+
const headerBuffer = Buffer.alloc(I_ARENA_START);
|
|
203
|
+
|
|
204
|
+
await file.read({
|
|
205
|
+
buffer: headerBuffer,
|
|
206
|
+
length: I_ARENA_START,
|
|
207
|
+
position: 0,
|
|
208
|
+
});
|
|
209
|
+
const header = new DataView(headerBuffer.buffer);
|
|
210
|
+
|
|
211
|
+
// Detect the arena size from the header, if not initialized.
|
|
212
|
+
const hdrArenaSize = header.getBigUint64(I_ARENA_SIZE);
|
|
213
|
+
const arenaSize = newArenaSize || hdrArenaSize;
|
|
214
|
+
|
|
215
|
+
const hdrMagic = header.getBigUint64(I_MAGIC);
|
|
216
|
+
SLOG_MAGIC === hdrMagic ||
|
|
217
|
+
Fail`${filename} is not a slog buffer; wanted magic ${SLOG_MAGIC}, got ${hdrMagic}`;
|
|
218
|
+
arenaSize === hdrArenaSize ||
|
|
219
|
+
Fail`${filename} arena size mismatch; wanted ${arenaSize}, got ${hdrArenaSize}`;
|
|
220
|
+
|
|
221
|
+
/** @type {(outbuf: Uint8Array, readStart: number, firstReadLength: number) => void} */
|
|
222
|
+
const readRecord = (outbuf, readStart, firstReadLength) => {
|
|
223
|
+
const bytesRead = fs.readSync(file.fd, outbuf, {
|
|
224
|
+
length: firstReadLength,
|
|
225
|
+
position: Number(readStart) + I_ARENA_START,
|
|
226
|
+
});
|
|
227
|
+
assert.equal(bytesRead, firstReadLength, 'Too few bytes read');
|
|
228
|
+
|
|
229
|
+
if (bytesRead < outbuf.byteLength) {
|
|
230
|
+
fs.readSync(file.fd, outbuf, {
|
|
231
|
+
offset: firstReadLength,
|
|
232
|
+
length: outbuf.byteLength - firstReadLength,
|
|
233
|
+
position: I_ARENA_START,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Writes to the file, offset by the header size. Also updates the file header.
|
|
240
|
+
*
|
|
241
|
+
* @param {Uint8Array} record
|
|
242
|
+
* @param {number} firstWriteLength
|
|
243
|
+
* @param {bigint} circEnd
|
|
244
|
+
*/
|
|
245
|
+
const writeRecord = async (record, firstWriteLength, circEnd) => {
|
|
246
|
+
await file.write(
|
|
247
|
+
record,
|
|
248
|
+
// TS saying options bag not available
|
|
249
|
+
0,
|
|
250
|
+
firstWriteLength,
|
|
251
|
+
I_ARENA_START + Number(circEnd),
|
|
252
|
+
);
|
|
253
|
+
if (firstWriteLength < record.byteLength) {
|
|
254
|
+
// Write to the beginning of the arena.
|
|
255
|
+
await file.write(
|
|
256
|
+
record,
|
|
257
|
+
firstWriteLength,
|
|
258
|
+
record.byteLength - firstWriteLength,
|
|
259
|
+
I_ARENA_START,
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Write out the updated file header.
|
|
264
|
+
// This is somewhat independent of writing the record itself, but it needs
|
|
265
|
+
// updating each time a record is written.
|
|
266
|
+
await file.write(headerBuffer, undefined, undefined, 0);
|
|
225
267
|
};
|
|
226
268
|
|
|
227
|
-
return
|
|
269
|
+
return finishCircularBuffer(arenaSize, header, readRecord, writeRecord);
|
|
228
270
|
};
|
|
229
271
|
|
|
230
|
-
|
|
231
|
-
|
|
272
|
+
/**
|
|
273
|
+
*
|
|
274
|
+
* @param {Pick<Awaited<ReturnType<typeof makeSimpleCircularBuffer>>, 'writeCircBuf'>} circBuf
|
|
275
|
+
*/
|
|
276
|
+
export const makeSlogSenderFromBuffer = ({ writeCircBuf }) => {
|
|
277
|
+
/** @type {Promise<void>} */
|
|
278
|
+
let toWrite = Promise.resolve();
|
|
279
|
+
const writeJSON = (obj, serialized = serializeSlogObj(obj)) => {
|
|
280
|
+
// Prepend a newline so that the file can be more easily manipulated.
|
|
281
|
+
const data = new TextEncoder().encode(`\n${serialized}`);
|
|
282
|
+
// console.log('have obj', obj, data);
|
|
283
|
+
toWrite = toWrite.then(() => writeCircBuf(data));
|
|
284
|
+
};
|
|
232
285
|
return Object.assign(writeJSON, {
|
|
233
|
-
forceFlush: async () => {
|
|
286
|
+
forceFlush: async () => {
|
|
287
|
+
await toWrite;
|
|
288
|
+
},
|
|
234
289
|
usesJsonObject: true,
|
|
235
290
|
});
|
|
236
291
|
};
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Loaded dynamically by makeSlogSender()
|
|
295
|
+
*
|
|
296
|
+
* @type {import('./index.js').MakeSlogSender}
|
|
297
|
+
*/
|
|
298
|
+
export const makeSlogSender = async opts => {
|
|
299
|
+
const { writeCircBuf } = await makeSimpleCircularBuffer(opts);
|
|
300
|
+
return makeSlogSenderFromBuffer({ writeCircBuf });
|
|
301
|
+
};
|
package/src/frcat-entrypoint.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import '@endo/init';
|
|
7
7
|
|
|
8
|
-
import {
|
|
8
|
+
import { makeSimpleCircularBuffer } from './flight-recorder.js';
|
|
9
9
|
|
|
10
10
|
const main = async () => {
|
|
11
11
|
const files = process.argv.slice(2);
|
|
@@ -14,8 +14,7 @@ const main = async () => {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
for await (const file of files) {
|
|
17
|
-
|
|
18
|
-
const { readCircBuf } = await makeMemoryMappedCircularBuffer({
|
|
17
|
+
const { readCircBuf } = await makeSimpleCircularBuffer({
|
|
19
18
|
circularBufferFilename: file,
|
|
20
19
|
circularBufferSize: 0,
|
|
21
20
|
});
|
|
@@ -50,13 +49,18 @@ const main = async () => {
|
|
|
50
49
|
}
|
|
51
50
|
|
|
52
51
|
// If the buffer is full, wait for stdout to drain.
|
|
53
|
-
// eslint-disable-next-line no-await-in-loop, @jessie.js/no-nested-await
|
|
54
52
|
await new Promise(resolve => process.stdout.once('drain', resolve));
|
|
55
53
|
}
|
|
56
54
|
}
|
|
57
55
|
};
|
|
58
56
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
process.exitCode = 1;
|
|
58
|
+
main().then(
|
|
59
|
+
() => {
|
|
60
|
+
process.exitCode = 0;
|
|
61
|
+
},
|
|
62
|
+
err => {
|
|
63
|
+
console.error('Failed with', err);
|
|
64
|
+
process.exit(process.exitCode || 1);
|
|
65
|
+
},
|
|
66
|
+
);
|
package/src/index.js
CHANGED
|
@@ -13,6 +13,9 @@ export * from './make-slog-sender.js';
|
|
|
13
13
|
* shutdown?: () => Promise<void>;
|
|
14
14
|
* }} SlogSender
|
|
15
15
|
*/
|
|
16
|
+
/**
|
|
17
|
+
* @typedef {(opts: import('./index.js').MakeSlogSenderOptions) => SlogSender | undefined} MakeSlogSender
|
|
18
|
+
*/
|
|
16
19
|
/**
|
|
17
20
|
* @typedef {MakeSlogSenderCommonOptions & Record<string, unknown>} MakeSlogSenderOptions
|
|
18
21
|
* @typedef {object} MakeSlogSenderCommonOptions
|
|
@@ -34,9 +37,9 @@ export const tryFlushSlogSender = async (
|
|
|
34
37
|
await Promise.resolve(slogSender?.forceFlush?.()).catch(err => {
|
|
35
38
|
log?.('Failed to flush slog sender', err);
|
|
36
39
|
if (err.errors) {
|
|
37
|
-
err.errors
|
|
40
|
+
for (const error of err.errors) {
|
|
38
41
|
log?.('nested error:', error);
|
|
39
|
-
}
|
|
42
|
+
}
|
|
40
43
|
}
|
|
41
44
|
if (env.SLOGSENDER_FAIL_ON_ERROR) {
|
|
42
45
|
throw err;
|
|
@@ -58,21 +61,20 @@ export const getResourceAttributes = ({
|
|
|
58
61
|
SDK_REVISION;
|
|
59
62
|
}
|
|
60
63
|
if (!resourceAttributes[SemanticResourceAttributes.SERVICE_INSTANCE_ID]) {
|
|
61
|
-
resourceAttributes[
|
|
62
|
-
|
|
63
|
-
] = `${Math.random()}`;
|
|
64
|
+
resourceAttributes[SemanticResourceAttributes.SERVICE_INSTANCE_ID] =
|
|
65
|
+
`${Math.random()}`;
|
|
64
66
|
}
|
|
65
67
|
if (serviceName) {
|
|
66
68
|
resourceAttributes[SemanticResourceAttributes.SERVICE_NAME] = serviceName;
|
|
67
69
|
}
|
|
68
70
|
if (OTEL_RESOURCE_ATTRIBUTES) {
|
|
69
71
|
// Allow overriding resource attributes.
|
|
70
|
-
OTEL_RESOURCE_ATTRIBUTES.split(',')
|
|
72
|
+
for (const kv of OTEL_RESOURCE_ATTRIBUTES.split(',')) {
|
|
71
73
|
const match = kv.match(/^([^=]*)=(.*)$/);
|
|
72
74
|
if (match) {
|
|
73
75
|
resourceAttributes[match[1]] = match[2];
|
|
74
76
|
}
|
|
75
|
-
}
|
|
77
|
+
}
|
|
76
78
|
}
|
|
77
79
|
return resourceAttributes;
|
|
78
80
|
};
|
|
@@ -74,7 +74,7 @@ async function run() {
|
|
|
74
74
|
if (!flush) {
|
|
75
75
|
return;
|
|
76
76
|
}
|
|
77
|
-
await slogSender.forceFlush();
|
|
77
|
+
await slogSender.forceFlush?.();
|
|
78
78
|
fs.writeFileSync(progressFileName, JSON.stringify(progress));
|
|
79
79
|
};
|
|
80
80
|
|
|
@@ -143,4 +143,13 @@ async function run() {
|
|
|
143
143
|
);
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
|
|
146
|
+
process.exitCode = 1;
|
|
147
|
+
run().then(
|
|
148
|
+
() => {
|
|
149
|
+
process.exitCode = 0;
|
|
150
|
+
},
|
|
151
|
+
err => {
|
|
152
|
+
console.error('Failed with', err);
|
|
153
|
+
process.exit(process.exitCode || 1);
|
|
154
|
+
},
|
|
155
|
+
);
|
package/src/make-slog-sender.js
CHANGED
|
@@ -9,7 +9,7 @@ export const SLOGFILE_SENDER_MODULE = '@agoric/telemetry/src/slog-file.js';
|
|
|
9
9
|
|
|
10
10
|
export const DEFAULT_SLOGSENDER_AGENT = 'self';
|
|
11
11
|
|
|
12
|
-
/** @
|
|
12
|
+
/** @import {SlogSender} from './index.js' */
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* @template T
|
|
@@ -19,7 +19,7 @@ export const DEFAULT_SLOGSENDER_AGENT = 'self';
|
|
|
19
19
|
const filterTruthy = arr => /** @type {any[]} */ (arr.filter(Boolean));
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
|
-
* @
|
|
22
|
+
* @type {import('./index.js').MakeSlogSender}
|
|
23
23
|
*/
|
|
24
24
|
export const makeSlogSender = async (opts = {}) => {
|
|
25
25
|
const { env = {}, stateDir: stateDirOption, ...otherOpts } = opts;
|
|
@@ -88,7 +88,7 @@ export const makeSlogSender = async (opts = {}) => {
|
|
|
88
88
|
slogSenderModules.map(async moduleIdentifier =>
|
|
89
89
|
import(moduleIdentifier)
|
|
90
90
|
.then(
|
|
91
|
-
/** @param {{makeSlogSender: (
|
|
91
|
+
/** @param {{makeSlogSender: import('./index.js').MakeSlogSender}} module */ ({
|
|
92
92
|
makeSlogSender: maker,
|
|
93
93
|
}) => {
|
|
94
94
|
if (typeof maker !== 'function') {
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { NonNullish } from '@agoric/assert';
|
|
2
2
|
import { makeSlogSender as makeSlogSenderFromEnv } from './make-slog-sender.js';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* @param {import('./index.js').MakeSlogSenderOptions} opts
|
|
6
|
+
*/
|
|
4
7
|
export const makeSlogSender = async opts => {
|
|
5
8
|
const { SLOGFILE: _1, SLOGSENDER: _2, ...otherEnv } = opts.env || {};
|
|
6
9
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/* global process */
|
|
2
2
|
import '@endo/init';
|
|
3
3
|
|
|
4
|
-
import { makeAggregateError } from '@agoric/internal';
|
|
5
4
|
import anylogger from 'anylogger';
|
|
6
5
|
import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
|
|
7
6
|
|
|
@@ -75,7 +74,7 @@ const main = async () => {
|
|
|
75
74
|
sendErrors.unshift(actualFlushError);
|
|
76
75
|
}
|
|
77
76
|
|
|
78
|
-
return
|
|
77
|
+
return AggregateError(sendErrors.splice(0));
|
|
79
78
|
};
|
|
80
79
|
|
|
81
80
|
process.on(
|
|
@@ -130,7 +129,13 @@ const main = async () => {
|
|
|
130
129
|
);
|
|
131
130
|
};
|
|
132
131
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
132
|
+
process.exitCode = 1;
|
|
133
|
+
main().then(
|
|
134
|
+
() => {
|
|
135
|
+
process.exitCode = 0;
|
|
136
|
+
},
|
|
137
|
+
err => {
|
|
138
|
+
logger.error('Failed with', err);
|
|
139
|
+
process.exit(process.exitCode || 1);
|
|
140
|
+
},
|
|
141
|
+
);
|
package/src/slog-sender-pipe.js
CHANGED
|
@@ -6,8 +6,7 @@ import { makeQueue } from '@endo/stream';
|
|
|
6
6
|
|
|
7
7
|
import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
|
|
8
8
|
|
|
9
|
-
const
|
|
10
|
-
const dirname = path.dirname(filename);
|
|
9
|
+
const dirname = path.dirname(new URL(import.meta.url).pathname);
|
|
11
10
|
|
|
12
11
|
const logger = anylogger('slog-sender-pipe');
|
|
13
12
|
|
|
@@ -78,11 +77,11 @@ export const makeSlogSender = async opts => {
|
|
|
78
77
|
/**
|
|
79
78
|
* @typedef {{
|
|
80
79
|
* init: {
|
|
81
|
-
* message: import('./slog-sender-pipe-entrypoint').InitMessage;
|
|
80
|
+
* message: import('./slog-sender-pipe-entrypoint.js').InitMessage;
|
|
82
81
|
* reply: SlogSenderInitReply;
|
|
83
82
|
* };
|
|
84
83
|
* flush: {
|
|
85
|
-
* message: import('./slog-sender-pipe-entrypoint').FlushMessage;
|
|
84
|
+
* message: import('./slog-sender-pipe-entrypoint.js').FlushMessage;
|
|
86
85
|
* reply: SlogSenderFlushReply;
|
|
87
86
|
* };
|
|
88
87
|
* }} SlogSenderWaitMessagesAndReplies
|
package/src/slog-to-otel.js
CHANGED
|
@@ -14,11 +14,8 @@ import {
|
|
|
14
14
|
|
|
15
15
|
// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.VERBOSE);
|
|
16
16
|
|
|
17
|
-
/** @
|
|
18
|
-
/** @
|
|
19
|
-
/** @typedef {import('@opentelemetry/api').SpanContext} SpanContext */
|
|
20
|
-
/** @typedef {import('@opentelemetry/api').SpanOptions} SpanOptions */
|
|
21
|
-
/** @typedef {import('@opentelemetry/api').SpanAttributes} SpanAttributes */
|
|
17
|
+
/** @import {Span, Link as SpanLink} from '@opentelemetry/api' */
|
|
18
|
+
/** @import {SpanContext, SpanOptions} from '@opentelemetry/api' */
|
|
22
19
|
|
|
23
20
|
const { assign } = Object;
|
|
24
21
|
|
|
@@ -54,9 +51,9 @@ const serializeInto = (value, prefix, target = {}, depth = 3) => {
|
|
|
54
51
|
} else {
|
|
55
52
|
const proto = Object.getPrototypeOf(value);
|
|
56
53
|
if (proto == null || proto === Object.prototype) {
|
|
57
|
-
|
|
58
|
-
serializeInto(nested, `${prefix}.${key}`, target, depth)
|
|
59
|
-
|
|
54
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
55
|
+
serializeInto(nested, `${prefix}.${key}`, target, depth);
|
|
56
|
+
}
|
|
60
57
|
return target;
|
|
61
58
|
}
|
|
62
59
|
}
|
|
@@ -142,7 +139,10 @@ export const makeSlogToOtelKit = (tracer, overrideAttrs = {}) => {
|
|
|
142
139
|
serializeBodyFormat: 'smallcaps',
|
|
143
140
|
});
|
|
144
141
|
|
|
145
|
-
/**
|
|
142
|
+
/**
|
|
143
|
+
* @param {import('@agoric/swingset-vat').SwingSetCapData} data
|
|
144
|
+
* @returns {any}
|
|
145
|
+
*/
|
|
146
146
|
const unserialize = data => {
|
|
147
147
|
try {
|
|
148
148
|
const body = rawUnserialize(data);
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import tmp from 'tmp';
|
|
3
|
+
import { test } from './prepare-test-env-ava.js';
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
makeSimpleCircularBuffer,
|
|
7
|
+
makeSlogSenderFromBuffer,
|
|
8
|
+
} from '../src/flight-recorder.js';
|
|
9
|
+
|
|
10
|
+
// Factored this way to support multiple implementations, which at one point there were
|
|
11
|
+
const bufferTests = test.macro(
|
|
12
|
+
/**
|
|
13
|
+
*
|
|
14
|
+
* @param {*} t
|
|
15
|
+
* @param {{makeBuffer: Function}} input
|
|
16
|
+
*/
|
|
17
|
+
async (t, input) => {
|
|
18
|
+
const BUFFER_SIZE = 512;
|
|
19
|
+
|
|
20
|
+
const { name: tmpFile, removeCallback } = tmp.fileSync();
|
|
21
|
+
const { readCircBuf, writeCircBuf } = await input.makeBuffer({
|
|
22
|
+
circularBufferSize: BUFFER_SIZE,
|
|
23
|
+
circularBufferFilename: tmpFile,
|
|
24
|
+
});
|
|
25
|
+
const slogSender = makeSlogSenderFromBuffer({ writeCircBuf });
|
|
26
|
+
slogSender({ type: 'start' });
|
|
27
|
+
await slogSender.forceFlush();
|
|
28
|
+
t.is(fs.readFileSync(tmpFile, { encoding: 'utf8' }).length, BUFFER_SIZE);
|
|
29
|
+
|
|
30
|
+
const len0 = new Uint8Array(BigUint64Array.BYTES_PER_ELEMENT);
|
|
31
|
+
const { done: done0 } = readCircBuf(len0);
|
|
32
|
+
t.false(done0, 'readCircBuf should not be done');
|
|
33
|
+
const dv0 = new DataView(len0.buffer);
|
|
34
|
+
const buf0 = new Uint8Array(Number(dv0.getBigUint64(0)));
|
|
35
|
+
const { done: done0b } = readCircBuf(buf0, len0.byteLength);
|
|
36
|
+
t.false(done0b, 'readCircBuf should not be done');
|
|
37
|
+
const buf0Str = new TextDecoder().decode(buf0);
|
|
38
|
+
t.is(buf0Str, `\n{"type":"start"}`, `start compare failed`);
|
|
39
|
+
|
|
40
|
+
const last = 500;
|
|
41
|
+
for (let i = 0; i < last; i += 1) {
|
|
42
|
+
slogSender({ type: 'iteration', iteration: i });
|
|
43
|
+
await slogSender.forceFlush();
|
|
44
|
+
t.is(
|
|
45
|
+
fs.readFileSync(tmpFile, { encoding: 'utf8' }).length,
|
|
46
|
+
BUFFER_SIZE,
|
|
47
|
+
`iteration ${i} length mismatch`,
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let offset = 0;
|
|
52
|
+
const len1 = new Uint8Array(BigUint64Array.BYTES_PER_ELEMENT);
|
|
53
|
+
for (let i = 490; i < last; i += 1) {
|
|
54
|
+
const { done: done1 } = readCircBuf(len1, offset);
|
|
55
|
+
offset += len1.byteLength;
|
|
56
|
+
t.false(done1, `readCircBuf ${i} should not be done`);
|
|
57
|
+
const dv1 = new DataView(len1.buffer);
|
|
58
|
+
const buf1 = new Uint8Array(Number(dv1.getBigUint64(0)));
|
|
59
|
+
const { done: done1b } = readCircBuf(buf1, offset);
|
|
60
|
+
offset += buf1.byteLength;
|
|
61
|
+
t.false(done1b, `readCircBuf ${i} should not be done`);
|
|
62
|
+
const buf1Str = new TextDecoder().decode(buf1);
|
|
63
|
+
t.is(
|
|
64
|
+
buf1Str,
|
|
65
|
+
`\n{"type":"iteration","iteration":${i}}`,
|
|
66
|
+
`iteration ${i} compare failed`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const { done: done2 } = readCircBuf(len1, offset);
|
|
71
|
+
t.assert(done2, `readCircBuf ${last} should be done`);
|
|
72
|
+
|
|
73
|
+
slogSender(null, 'PRE-SERIALIZED');
|
|
74
|
+
await slogSender.forceFlush();
|
|
75
|
+
t.truthy(fs.readFileSync(tmpFile).includes('PRE-SERIALIZED'));
|
|
76
|
+
// console.log({ tmpFile });
|
|
77
|
+
removeCallback();
|
|
78
|
+
},
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
test('simple', bufferTests, {
|
|
82
|
+
makeBuffer: makeSimpleCircularBuffer,
|
|
83
|
+
});
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import tmp from 'tmp';
|
|
2
|
-
import { test } from './prepare-test-env-ava.js';
|
|
3
|
-
|
|
4
|
-
import { makeMemoryMappedCircularBuffer } from '../src/flight-recorder.js';
|
|
5
|
-
|
|
6
|
-
test('flight-recorder sanity', async t => {
|
|
7
|
-
const { name: tmpFile, removeCallback } = tmp.fileSync();
|
|
8
|
-
const { writeJSON: slogSender, readCircBuf } =
|
|
9
|
-
await makeMemoryMappedCircularBuffer({
|
|
10
|
-
circularBufferSize: 512,
|
|
11
|
-
circularBufferFilename: tmpFile,
|
|
12
|
-
});
|
|
13
|
-
slogSender({ type: 'start' });
|
|
14
|
-
|
|
15
|
-
const len0 = new Uint8Array(BigUint64Array.BYTES_PER_ELEMENT);
|
|
16
|
-
const { done: done0 } = readCircBuf(len0);
|
|
17
|
-
t.false(done0, 'readCircBuf should not be done');
|
|
18
|
-
const dv0 = new DataView(len0.buffer);
|
|
19
|
-
const buf0 = new Uint8Array(Number(dv0.getBigUint64(0)));
|
|
20
|
-
const { done: done0b } = readCircBuf(buf0, len0.byteLength);
|
|
21
|
-
t.false(done0b, 'readCircBuf should not be done');
|
|
22
|
-
const buf0Str = new TextDecoder().decode(buf0);
|
|
23
|
-
t.is(buf0Str, `\n{"type":"start"}`, `start compare failed`);
|
|
24
|
-
|
|
25
|
-
const last = 500;
|
|
26
|
-
for (let i = 0; i < last; i += 1) {
|
|
27
|
-
slogSender({ type: 'iteration', iteration: i });
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
let offset = 0;
|
|
31
|
-
const len1 = new Uint8Array(BigUint64Array.BYTES_PER_ELEMENT);
|
|
32
|
-
for (let i = 490; i < last; i += 1) {
|
|
33
|
-
const { done: done1 } = readCircBuf(len1, offset);
|
|
34
|
-
offset += len1.byteLength;
|
|
35
|
-
t.false(done1, `readCircBuf ${i} should not be done`);
|
|
36
|
-
const dv1 = new DataView(len1.buffer);
|
|
37
|
-
const buf1 = new Uint8Array(Number(dv1.getBigUint64(0)));
|
|
38
|
-
const { done: done1b } = readCircBuf(buf1, offset);
|
|
39
|
-
offset += buf1.byteLength;
|
|
40
|
-
t.false(done1b, `readCircBuf ${i} should not be done`);
|
|
41
|
-
const buf1Str = new TextDecoder().decode(buf1);
|
|
42
|
-
t.is(
|
|
43
|
-
buf1Str,
|
|
44
|
-
`\n{"type":"iteration","iteration":${i}}`,
|
|
45
|
-
`iteration ${i} compare failed`,
|
|
46
|
-
);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const { done: done2 } = readCircBuf(len1, offset);
|
|
50
|
-
t.assert(done2, `readCircBuf ${last} should be done`);
|
|
51
|
-
// console.log({ tmpFile });
|
|
52
|
-
removeCallback();
|
|
53
|
-
});
|
|
File without changes
|