@oxy.so/telemetry 1.2.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/cjs/browser.js +18 -8
- package/dist/cjs/collector.js +111 -26
- package/dist/cjs/metrics.js +208 -0
- package/dist/cjs/server.js +3 -1
- package/dist/cjs/socket.js +38 -9
- package/dist/esm/browser.js +18 -8
- package/dist/esm/collector.js +111 -26
- package/dist/esm/metrics.js +203 -0
- package/dist/esm/server.js +3 -1
- package/dist/esm/socket.js +38 -9
- package/dist/types/metrics.d.ts +79 -0
- package/dist/types/package.json +1 -0
- package/package.json +11 -12
- package/src/__tests__/browser.test.ts +22 -6
- package/src/__tests__/collector.test.ts +132 -27
- package/src/__tests__/metrics.test.ts +45 -0
- package/src/__tests__/server.test.ts +7 -5
- package/src/__tests__/socket.test.ts +80 -29
- package/src/browser.ts +35 -11
- package/src/collector.ts +152 -35
- package/src/metrics.ts +286 -0
- package/src/server.ts +8 -2
- package/src/socket.ts +43 -11
- package/dist/cjs/edge.js +0 -137
- package/dist/esm/edge.js +0 -134
- package/dist/types/edge.d.ts +0 -32
- package/src/__tests__/edge.test.ts +0 -110
- package/src/edge.ts +0 -117
package/dist/esm/browser.js
CHANGED
|
@@ -22,9 +22,9 @@ function createUuid(cryptoApi) {
|
|
|
22
22
|
function eligibleLocation(location) {
|
|
23
23
|
if (!location || (location.protocol !== 'https:' && location.protocol !== 'http:'))
|
|
24
24
|
return false;
|
|
25
|
-
return location.hostname !== 'localhost'
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
return (location.hostname !== 'localhost' &&
|
|
26
|
+
location.hostname !== '127.0.0.1' &&
|
|
27
|
+
location.hostname !== '[::1]');
|
|
28
28
|
}
|
|
29
29
|
export function createBrowserTelemetry(options = {}) {
|
|
30
30
|
let activityId = null;
|
|
@@ -37,7 +37,9 @@ export function createBrowserTelemetry(options = {}) {
|
|
|
37
37
|
if (!hasInjectedBrowser && !hasBrowserRuntime)
|
|
38
38
|
return {};
|
|
39
39
|
const now = (options.now ?? Date.now)();
|
|
40
|
-
if (!activityId ||
|
|
40
|
+
if (!activityId ||
|
|
41
|
+
now < activityIdCreatedAt ||
|
|
42
|
+
now - activityIdCreatedAt >= (options.activityIdRotationMs ?? DEFAULT_ACTIVITY_ID_ROTATION_MS)) {
|
|
41
43
|
activityId = createUuid(options.crypto ?? globalThis.crypto);
|
|
42
44
|
if (!activityId)
|
|
43
45
|
return {};
|
|
@@ -54,7 +56,11 @@ export function createBrowserTelemetry(options = {}) {
|
|
|
54
56
|
try {
|
|
55
57
|
const response = await (options.fetchTrace
|
|
56
58
|
? options.fetchTrace(controller.signal)
|
|
57
|
-
: fetch('/cdn-cgi/trace', {
|
|
59
|
+
: fetch('/cdn-cgi/trace', {
|
|
60
|
+
cache: 'no-store',
|
|
61
|
+
credentials: 'omit',
|
|
62
|
+
signal: controller.signal,
|
|
63
|
+
}));
|
|
58
64
|
if (!response.ok)
|
|
59
65
|
return null;
|
|
60
66
|
const match = (await response.text()).match(/^colo=([a-z]{3})$/im);
|
|
@@ -69,7 +75,9 @@ export function createBrowserTelemetry(options = {}) {
|
|
|
69
75
|
};
|
|
70
76
|
const getEdgeRegionHeader = async () => {
|
|
71
77
|
const now = (options.now ?? Date.now)();
|
|
72
|
-
if (!edgePopPromise ||
|
|
78
|
+
if (!edgePopPromise ||
|
|
79
|
+
now < edgePopReadAt ||
|
|
80
|
+
now - edgePopReadAt >= (options.edgeCacheMs ?? DEFAULT_EDGE_CACHE_MS)) {
|
|
73
81
|
edgePopReadAt = now;
|
|
74
82
|
edgePopPromise = discoverEdgePop();
|
|
75
83
|
}
|
|
@@ -81,7 +89,9 @@ export function createBrowserTelemetry(options = {}) {
|
|
|
81
89
|
let peekInFlight = false;
|
|
82
90
|
const peekEdgeRegionHeader = () => {
|
|
83
91
|
const now = (options.now ?? Date.now)();
|
|
84
|
-
if (!peekInFlight &&
|
|
92
|
+
if (!peekInFlight &&
|
|
93
|
+
(now < peekRefreshAt ||
|
|
94
|
+
now - peekRefreshAt >= (options.edgePeekRefreshMs ?? DEFAULT_EDGE_PEEK_REFRESH_MS))) {
|
|
85
95
|
peekRefreshAt = now;
|
|
86
96
|
peekInFlight = true;
|
|
87
97
|
void discoverEdgePop()
|
|
@@ -100,7 +110,7 @@ export function createBrowserTelemetry(options = {}) {
|
|
|
100
110
|
getEdgeRegionHeader,
|
|
101
111
|
peekEdgeRegionHeader,
|
|
102
112
|
async getHeaders() {
|
|
103
|
-
return { ...await getEdgeRegionHeader(), ...getActivityIdHeader() };
|
|
113
|
+
return { ...(await getEdgeRegionHeader()), ...getActivityIdHeader() };
|
|
104
114
|
},
|
|
105
115
|
};
|
|
106
116
|
}
|
package/dist/esm/collector.js
CHANGED
|
@@ -1,10 +1,33 @@
|
|
|
1
1
|
import { metadataFromHeaders } from './server.js';
|
|
2
2
|
const TYPES = {
|
|
3
|
-
auth: 'identity',
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
auth: 'identity',
|
|
4
|
+
session: 'identity',
|
|
5
|
+
users: 'identity',
|
|
6
|
+
profiles: 'identity',
|
|
7
|
+
accounts: 'identity',
|
|
8
|
+
ai: 'ai',
|
|
9
|
+
inference: 'ai',
|
|
10
|
+
models: 'ai',
|
|
11
|
+
chat: 'ai',
|
|
12
|
+
completions: 'ai',
|
|
13
|
+
stream: 'media',
|
|
14
|
+
tracks: 'media',
|
|
15
|
+
episodes: 'media',
|
|
16
|
+
recordings: 'media',
|
|
17
|
+
rooms: 'media',
|
|
18
|
+
messages: 'communication',
|
|
19
|
+
notifications: 'communication',
|
|
20
|
+
mail: 'communication',
|
|
21
|
+
media: 'media',
|
|
22
|
+
files: 'media',
|
|
23
|
+
assets: 'media',
|
|
24
|
+
upload: 'media',
|
|
25
|
+
uploads: 'media',
|
|
26
|
+
cdn: 'media',
|
|
27
|
+
storage: 'media',
|
|
28
|
+
images: 'media',
|
|
29
|
+
audio: 'media',
|
|
30
|
+
videos: 'media',
|
|
8
31
|
};
|
|
9
32
|
export function trafficType(path) {
|
|
10
33
|
const parts = path.split('?')[0].split('/').filter(Boolean);
|
|
@@ -34,7 +57,12 @@ export function createTrafficCollector(publish, now = Date.now) {
|
|
|
34
57
|
if (pending.size === 0)
|
|
35
58
|
return;
|
|
36
59
|
const emittedAt = new Date(now()).toISOString();
|
|
37
|
-
const events = [...pending.values()].map(({ flow, requests, startedAt }) => ({
|
|
60
|
+
const events = [...pending.values()].map(({ flow, requests, startedAt }) => ({
|
|
61
|
+
...flow,
|
|
62
|
+
requests,
|
|
63
|
+
windowStartedAt: new Date(startedAt).toISOString(),
|
|
64
|
+
emittedAt,
|
|
65
|
+
}));
|
|
38
66
|
pending.clear();
|
|
39
67
|
flushing = (async () => {
|
|
40
68
|
for (let offset = 0; offset < events.length; offset += 256)
|
|
@@ -54,7 +82,10 @@ export function trafficMiddleware(collector, service, region) {
|
|
|
54
82
|
response.setHeader?.('X-Oxy-Region', region);
|
|
55
83
|
const path = request.path ?? request.url ?? '/';
|
|
56
84
|
// Never report collection itself, probes or the dashboard transport.
|
|
57
|
-
if (/^\/(?:api\/)?(?:health|ready|live|platform-stats|platform-activity|platform-infrastructure|infra-status)(?:\/|$)/.test(path) ||
|
|
85
|
+
if (/^\/(?:api\/)?(?:health|ready|live|platform-stats|platform-activity|platform-infrastructure|infra-status)(?:\/|$)/.test(path) ||
|
|
86
|
+
path.startsWith('/internal/activity') ||
|
|
87
|
+
path.startsWith('/auth/service-token') ||
|
|
88
|
+
path.startsWith('/cdn-cgi/')) {
|
|
58
89
|
next();
|
|
59
90
|
return;
|
|
60
91
|
}
|
|
@@ -63,11 +94,28 @@ export function trafficMiddleware(collector, service, region) {
|
|
|
63
94
|
let finished = false;
|
|
64
95
|
const inboundFlow = () => {
|
|
65
96
|
// Read service identity AFTER the route's shared auth middleware resolved it.
|
|
66
|
-
const name = request.serviceApp?.appName
|
|
97
|
+
const name = request.serviceApp?.appName
|
|
98
|
+
?.toLowerCase()
|
|
99
|
+
.replace(/[^a-z0-9-]/g, '-')
|
|
100
|
+
.slice(0, 40);
|
|
67
101
|
const internal = Boolean(name);
|
|
68
102
|
const forwardedRegion = request.headers['x-oxy-source-region'];
|
|
69
|
-
const peerRegion = internal
|
|
70
|
-
|
|
103
|
+
const peerRegion = internal
|
|
104
|
+
? normalizeInfrastructureRegion(forwardedRegion)
|
|
105
|
+
: edgePop
|
|
106
|
+
? `edge-${edgePop}`
|
|
107
|
+
: undefined;
|
|
108
|
+
return {
|
|
109
|
+
region,
|
|
110
|
+
service,
|
|
111
|
+
activityType,
|
|
112
|
+
scope: internal ? 'internal' : 'external',
|
|
113
|
+
direction: 'inbound',
|
|
114
|
+
sourceRegion: peerRegion,
|
|
115
|
+
targetRegion: region,
|
|
116
|
+
sourceService: name,
|
|
117
|
+
targetService: service,
|
|
118
|
+
};
|
|
71
119
|
};
|
|
72
120
|
const finish = () => {
|
|
73
121
|
if (finished)
|
|
@@ -75,7 +123,14 @@ export function trafficMiddleware(collector, service, region) {
|
|
|
75
123
|
finished = true;
|
|
76
124
|
const flow = inboundFlow();
|
|
77
125
|
collector.record(flow);
|
|
78
|
-
collector.record({
|
|
126
|
+
collector.record({
|
|
127
|
+
...flow,
|
|
128
|
+
direction: 'outbound',
|
|
129
|
+
sourceRegion: region,
|
|
130
|
+
targetRegion: flow.sourceRegion,
|
|
131
|
+
sourceService: service,
|
|
132
|
+
targetService: flow.sourceService,
|
|
133
|
+
});
|
|
79
134
|
};
|
|
80
135
|
response.once('finish', finish);
|
|
81
136
|
// Failed/aborted requests are still real inbound activity. Do not invent a
|
|
@@ -93,13 +148,21 @@ export function instrumentTrafficFetch(fetcher, collector, service, region, reso
|
|
|
93
148
|
return Object.assign(async (input, init) => {
|
|
94
149
|
const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url);
|
|
95
150
|
// Collection/auth control traffic cannot generate a feedback loop.
|
|
96
|
-
if (url.pathname.startsWith('/internal/activity') ||
|
|
151
|
+
if (url.pathname.startsWith('/internal/activity') ||
|
|
152
|
+
url.pathname.startsWith('/auth/service-token') ||
|
|
153
|
+
url.pathname === '/platform-infrastructure')
|
|
97
154
|
return fetcher(input, init);
|
|
98
155
|
const peer = resolveEndpoint(url);
|
|
99
156
|
const flow = {
|
|
100
|
-
service,
|
|
101
|
-
|
|
102
|
-
|
|
157
|
+
service,
|
|
158
|
+
region,
|
|
159
|
+
scope: peer ? 'internal' : 'external',
|
|
160
|
+
direction: 'outbound',
|
|
161
|
+
activityType: trafficType(url.pathname),
|
|
162
|
+
sourceRegion: region,
|
|
163
|
+
targetRegion: peer?.region,
|
|
164
|
+
sourceService: service,
|
|
165
|
+
targetService: peer?.service,
|
|
103
166
|
};
|
|
104
167
|
const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined));
|
|
105
168
|
if (peer)
|
|
@@ -112,23 +175,43 @@ export function instrumentTrafficFetch(fetcher, collector, service, region, reso
|
|
|
112
175
|
collector.record({ ...flow, targetRegion: undefined });
|
|
113
176
|
throw error;
|
|
114
177
|
}
|
|
115
|
-
const targetRegion = peer
|
|
178
|
+
const targetRegion = peer
|
|
179
|
+
? normalizeInfrastructureRegion(response.headers.get('X-Oxy-Region') ?? undefined)
|
|
180
|
+
: undefined;
|
|
116
181
|
collector.record({ ...flow, targetRegion });
|
|
117
|
-
collector.record({
|
|
182
|
+
collector.record({
|
|
183
|
+
...flow,
|
|
184
|
+
direction: 'inbound',
|
|
185
|
+
sourceRegion: targetRegion,
|
|
186
|
+
targetRegion: region,
|
|
187
|
+
sourceService: peer?.service,
|
|
188
|
+
targetService: service,
|
|
189
|
+
});
|
|
118
190
|
return response;
|
|
119
191
|
}, fetcher);
|
|
120
192
|
}
|
|
121
193
|
// Exact deployed API hosts; an arbitrary caller-supplied hostname cannot claim
|
|
122
194
|
// to be an internal service. Region is infrastructure metadata, never IP geo.
|
|
123
195
|
const OXY_API_SERVICES = {
|
|
124
|
-
'api.oxy.so': 'oxy-api',
|
|
125
|
-
'
|
|
126
|
-
'api.
|
|
127
|
-
'api.
|
|
128
|
-
'
|
|
129
|
-
'api.
|
|
130
|
-
'
|
|
131
|
-
'api.
|
|
196
|
+
'api.oxy.so': 'oxy-api',
|
|
197
|
+
'api.website.oxy.so': 'website',
|
|
198
|
+
'website-api.oxy.so': 'website',
|
|
199
|
+
'api.mention.earth': 'mention',
|
|
200
|
+
'api.mercaria.co': 'mercaria',
|
|
201
|
+
'api.alia.onl': 'alia',
|
|
202
|
+
'kaana.ai': 'kaana',
|
|
203
|
+
'api.homiio.com': 'homiio',
|
|
204
|
+
'api.syra.fm': 'syra',
|
|
205
|
+
'mention.earth': 'mention',
|
|
206
|
+
'mcp.mention.earth': 'mention-mcp',
|
|
207
|
+
'api.clarity.surf': 'clarity',
|
|
208
|
+
'api.peable.to': 'peable',
|
|
209
|
+
'api.crowdsource.oxy.so': 'crowdsource',
|
|
210
|
+
'api.moovo.now': 'moovo',
|
|
211
|
+
'api.allo.you': 'allo',
|
|
212
|
+
'api.noted.oxy.so': 'noted',
|
|
213
|
+
'api.schedio.app': 'schedio',
|
|
214
|
+
'api.tnp.network': 'tnp-api',
|
|
132
215
|
};
|
|
133
216
|
export function resolveOxyServiceEndpoint(url) {
|
|
134
217
|
const service = OXY_API_SERVICES[url.hostname];
|
|
@@ -165,7 +248,9 @@ const REGION_LOCATIONS = {
|
|
|
165
248
|
'me-central-1': { label: 'UAE', coordinates: [54.37, 24.45] },
|
|
166
249
|
'il-central-1': { label: 'Tel Aviv', coordinates: [34.78, 32.09] },
|
|
167
250
|
};
|
|
168
|
-
export function infrastructureLocation(region) {
|
|
251
|
+
export function infrastructureLocation(region) {
|
|
252
|
+
return REGION_LOCATIONS[region];
|
|
253
|
+
}
|
|
169
254
|
export function normalizeInfrastructureRegion(value) {
|
|
170
255
|
return typeof value === 'string' && /^[a-z]{2}(?:-[a-z]+)+-\d$/.test(value) ? value : undefined;
|
|
171
256
|
}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operational metrics for Oxy services, published as CloudWatch Embedded Metric
|
|
3
|
+
* Format (EMF) log lines.
|
|
4
|
+
*
|
|
5
|
+
* WHY EMF ON STDOUT: every Oxy backend runs on ECS with the `awslogs` driver, and
|
|
6
|
+
* CloudWatch Logs extracts metrics from any log event shaped as EMF — no agent,
|
|
7
|
+
* no sidecar, no scraper, no IAM grant beyond the logging the task already has.
|
|
8
|
+
* A registry that only an HTTP scrape can read (Prometheus) measures into a void
|
|
9
|
+
* when nothing scrapes it, which is how Mention's counters lost every value on
|
|
10
|
+
* every deploy.
|
|
11
|
+
*
|
|
12
|
+
* WHY AGGREGATED: values are folded in memory and flushed once per interval as
|
|
13
|
+
* ONE line per (metric, label set), not one line per observation. A latency is
|
|
14
|
+
* kept as a distribution (`Values`/`Counts`), so CloudWatch still answers
|
|
15
|
+
* SampleCount, p50 and p99 for it — the request COUNT is the distribution's
|
|
16
|
+
* SampleCount, so a service needs no separate counter for it.
|
|
17
|
+
*
|
|
18
|
+
* WHY MOST LABELS ARE NOT DIMENSIONS: every distinct combination of dimension
|
|
19
|
+
* values is a billed custom metric. Route x method as dimensions once measured
|
|
20
|
+
* 405 metrics and ~80 % of the account's CloudWatch bill (Mention #1334). So a
|
|
21
|
+
* label becomes a DIMENSION only when a metric definition names it in
|
|
22
|
+
* `dimensionSets`, and only labels with a small, closed value set belong there.
|
|
23
|
+
* Every other label is written as a plain property of the line: free to store,
|
|
24
|
+
* and grouped in Logs Insights (`stats sum(RequestLatency.Count) by Route`).
|
|
25
|
+
*/
|
|
26
|
+
/** One HTTP request: SampleCount is the request count, percentiles the latency. */
|
|
27
|
+
export const HTTP_REQUEST_LATENCY = {
|
|
28
|
+
name: 'RequestLatency',
|
|
29
|
+
unit: 'Milliseconds',
|
|
30
|
+
kind: 'distribution',
|
|
31
|
+
dimensionSets: [['StatusClass'], ['Caller']],
|
|
32
|
+
};
|
|
33
|
+
/** One request a rate limiter refused (429), per calling application. */
|
|
34
|
+
export const HTTP_RATE_LIMITED = {
|
|
35
|
+
name: 'RateLimited',
|
|
36
|
+
unit: 'Count',
|
|
37
|
+
kind: 'sum',
|
|
38
|
+
dimensionSets: [['Caller']],
|
|
39
|
+
};
|
|
40
|
+
/** `2xx`, `4xx`, … — a closed set, safe as a dimension. */
|
|
41
|
+
export function statusClass(status) {
|
|
42
|
+
return Number.isInteger(status) && status >= 100 && status <= 599
|
|
43
|
+
? `${Math.floor(status / 100)}xx`
|
|
44
|
+
: 'other';
|
|
45
|
+
}
|
|
46
|
+
/** EMF allows at most 100 distinct values per metric on one line. */
|
|
47
|
+
const MAX_DISTRIBUTION_VALUES = 100;
|
|
48
|
+
/** Neighbouring values within ~5 % share a bucket, which bounds the set. */
|
|
49
|
+
const BUCKET_BASE = 1.05;
|
|
50
|
+
const LABEL_NAME = /^[A-Za-z][A-Za-z0-9_]{0,63}$/;
|
|
51
|
+
const MAX_LABEL_VALUE_LENGTH = 128;
|
|
52
|
+
const RESERVED = new Set(['_aws', 'Service']);
|
|
53
|
+
/** Representative value of a bucket: values within ~5 % collapse together. */
|
|
54
|
+
function bucketOf(value) {
|
|
55
|
+
if (value <= 0)
|
|
56
|
+
return 0;
|
|
57
|
+
const exponent = Math.round(Math.log(value) / Math.log(BUCKET_BASE));
|
|
58
|
+
return Number((BUCKET_BASE ** exponent).toPrecision(3));
|
|
59
|
+
}
|
|
60
|
+
/** Merge adjacent buckets (keeping the larger value) until EMF accepts them. */
|
|
61
|
+
function boundedDistribution(buckets) {
|
|
62
|
+
let entries = [...buckets.entries()].sort(([left], [right]) => left - right);
|
|
63
|
+
while (entries.length > MAX_DISTRIBUTION_VALUES) {
|
|
64
|
+
const merged = [];
|
|
65
|
+
for (let index = 0; index < entries.length; index += 2) {
|
|
66
|
+
const [value, count] = entries[index];
|
|
67
|
+
const next = entries[index + 1];
|
|
68
|
+
merged.push(next ? [next[0], count + next[1]] : [value, count]);
|
|
69
|
+
}
|
|
70
|
+
entries = merged;
|
|
71
|
+
}
|
|
72
|
+
return { values: entries.map(([value]) => value), counts: entries.map(([, count]) => count) };
|
|
73
|
+
}
|
|
74
|
+
function cleanLabels(metric, labels) {
|
|
75
|
+
const clean = {};
|
|
76
|
+
for (const [name, raw] of Object.entries(labels ?? {})) {
|
|
77
|
+
if (!LABEL_NAME.test(name) || RESERVED.has(name) || name === metric.name)
|
|
78
|
+
continue;
|
|
79
|
+
const value = String(raw).slice(0, MAX_LABEL_VALUE_LENGTH);
|
|
80
|
+
clean[name] = value.length > 0 ? value : 'unknown';
|
|
81
|
+
}
|
|
82
|
+
// A dimension named by the definition must be present on every line, or
|
|
83
|
+
// CloudWatch drops the metric from that line.
|
|
84
|
+
for (const set of metric.dimensionSets ?? []) {
|
|
85
|
+
for (const name of set)
|
|
86
|
+
clean[name] ?? (clean[name] = 'unknown');
|
|
87
|
+
}
|
|
88
|
+
return clean;
|
|
89
|
+
}
|
|
90
|
+
function seriesKey(metric, labels) {
|
|
91
|
+
const pairs = Object.keys(labels)
|
|
92
|
+
.sort()
|
|
93
|
+
.map((name) => `${name}=${labels[name]}`);
|
|
94
|
+
return `${metric.name}\u0000${pairs.join('\u0000')}`;
|
|
95
|
+
}
|
|
96
|
+
/** One line on stdout. Reached through `globalThis`: this package carries no Node types. */
|
|
97
|
+
function defaultWrite(line) {
|
|
98
|
+
globalThis.process?.stdout?.write?.(`${line}\n`);
|
|
99
|
+
}
|
|
100
|
+
export function createEmfPublisher(options) {
|
|
101
|
+
const flushIntervalMs = options.flushIntervalMs ?? 60000;
|
|
102
|
+
const maxSeries = options.maxSeries ?? 2000;
|
|
103
|
+
const write = options.write ?? defaultWrite;
|
|
104
|
+
const now = options.now ?? Date.now;
|
|
105
|
+
let series = new Map();
|
|
106
|
+
const line = (entry, timestamp) => {
|
|
107
|
+
const { metric } = entry;
|
|
108
|
+
const dimensions = [
|
|
109
|
+
['Service'],
|
|
110
|
+
...(metric.dimensionSets ?? []).map((set) => ['Service', ...set]),
|
|
111
|
+
];
|
|
112
|
+
let value;
|
|
113
|
+
if (metric.kind === 'sum')
|
|
114
|
+
value = entry.sum;
|
|
115
|
+
else if (metric.kind === 'gauge')
|
|
116
|
+
value = entry.last;
|
|
117
|
+
else {
|
|
118
|
+
const { values, counts } = boundedDistribution(entry.buckets);
|
|
119
|
+
value = {
|
|
120
|
+
Values: values,
|
|
121
|
+
Counts: counts,
|
|
122
|
+
Max: entry.max,
|
|
123
|
+
Min: entry.min,
|
|
124
|
+
Count: entry.count,
|
|
125
|
+
Sum: entry.sum,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
return JSON.stringify({
|
|
129
|
+
_aws: {
|
|
130
|
+
Timestamp: timestamp,
|
|
131
|
+
CloudWatchMetrics: [
|
|
132
|
+
{
|
|
133
|
+
Namespace: options.namespace,
|
|
134
|
+
Dimensions: dimensions,
|
|
135
|
+
Metrics: [{ Name: metric.name, Unit: metric.unit }],
|
|
136
|
+
},
|
|
137
|
+
],
|
|
138
|
+
},
|
|
139
|
+
...entry.labels,
|
|
140
|
+
Service: options.service,
|
|
141
|
+
[metric.name]: value,
|
|
142
|
+
});
|
|
143
|
+
};
|
|
144
|
+
const flush = () => {
|
|
145
|
+
if (series.size === 0)
|
|
146
|
+
return;
|
|
147
|
+
const current = series;
|
|
148
|
+
series = new Map();
|
|
149
|
+
const timestamp = now();
|
|
150
|
+
for (const entry of current.values()) {
|
|
151
|
+
try {
|
|
152
|
+
write(line(entry, timestamp));
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
// Metrics are best effort; a failed write must never reach a request.
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
const timer = setInterval(flush, flushIntervalMs);
|
|
160
|
+
timer.unref?.();
|
|
161
|
+
return {
|
|
162
|
+
record(metric, value, labels) {
|
|
163
|
+
if (!Number.isFinite(value))
|
|
164
|
+
return;
|
|
165
|
+
if (metric.kind !== 'gauge' && value < 0)
|
|
166
|
+
return;
|
|
167
|
+
let clean = cleanLabels(metric, labels);
|
|
168
|
+
let key = seriesKey(metric, clean);
|
|
169
|
+
if (!series.has(key) && series.size >= maxSeries) {
|
|
170
|
+
clean = Object.fromEntries(Object.keys(clean).map((name) => [name, 'other']));
|
|
171
|
+
key = seriesKey(metric, clean);
|
|
172
|
+
}
|
|
173
|
+
let entry = series.get(key);
|
|
174
|
+
if (!entry) {
|
|
175
|
+
entry = {
|
|
176
|
+
metric,
|
|
177
|
+
labels: clean,
|
|
178
|
+
sum: 0,
|
|
179
|
+
last: 0,
|
|
180
|
+
count: 0,
|
|
181
|
+
min: Number.POSITIVE_INFINITY,
|
|
182
|
+
max: Number.NEGATIVE_INFINITY,
|
|
183
|
+
buckets: new Map(),
|
|
184
|
+
};
|
|
185
|
+
series.set(key, entry);
|
|
186
|
+
}
|
|
187
|
+
entry.sum += value;
|
|
188
|
+
entry.last = value;
|
|
189
|
+
entry.count += 1;
|
|
190
|
+
entry.min = Math.min(entry.min, value);
|
|
191
|
+
entry.max = Math.max(entry.max, value);
|
|
192
|
+
if (metric.kind === 'distribution') {
|
|
193
|
+
const bucket = bucketOf(value);
|
|
194
|
+
entry.buckets.set(bucket, (entry.buckets.get(bucket) ?? 0) + 1);
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
flush,
|
|
198
|
+
stop() {
|
|
199
|
+
clearInterval(timer);
|
|
200
|
+
flush();
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
package/dist/esm/server.js
CHANGED
|
@@ -7,7 +7,9 @@ function firstHeaderValue(value) {
|
|
|
7
7
|
return typeof first === 'string' ? first : undefined;
|
|
8
8
|
}
|
|
9
9
|
export function normalizeEdgePop(value) {
|
|
10
|
-
return firstHeaderValue(value)
|
|
10
|
+
return firstHeaderValue(value)
|
|
11
|
+
?.match(/^[a-z]{3}$/i)?.[0]
|
|
12
|
+
?.toLowerCase();
|
|
11
13
|
}
|
|
12
14
|
export function normalizeActivityId(value) {
|
|
13
15
|
const candidate = firstHeaderValue(value);
|
package/dist/esm/socket.js
CHANGED
|
@@ -1,11 +1,26 @@
|
|
|
1
1
|
import { normalizeInfrastructureRegion } from './collector.js';
|
|
2
2
|
import { metadataFromHeaders } from './server.js';
|
|
3
3
|
const CATEGORIES = {
|
|
4
|
-
auth: 'identity',
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
auth: 'identity',
|
|
5
|
+
session: 'identity',
|
|
6
|
+
user: 'identity',
|
|
7
|
+
profile: 'identity',
|
|
8
|
+
chat: 'ai',
|
|
9
|
+
agent: 'ai',
|
|
10
|
+
inference: 'ai',
|
|
11
|
+
completion: 'ai',
|
|
12
|
+
message: 'communication',
|
|
13
|
+
notification: 'communication',
|
|
14
|
+
typing: 'communication',
|
|
15
|
+
media: 'media',
|
|
16
|
+
audio: 'media',
|
|
17
|
+
video: 'media',
|
|
18
|
+
stream: 'media',
|
|
19
|
+
player: 'media',
|
|
20
|
+
track: 'media',
|
|
21
|
+
playback: 'media',
|
|
22
|
+
playlist: 'media',
|
|
23
|
+
room: 'media',
|
|
9
24
|
};
|
|
10
25
|
/** Event names are reduced locally to five categories; payloads are never read. */
|
|
11
26
|
export function socketTrafficType(event) {
|
|
@@ -27,9 +42,16 @@ export function observeTrafficSocket(socket, collector, service, region, options
|
|
|
27
42
|
const peer = options.getPeer?.();
|
|
28
43
|
const peerService = peer && /^[a-z][a-z0-9-]{0,39}$/.test(peer.service) ? peer.service : undefined;
|
|
29
44
|
const edgePop = metadataFromHeaders(socket.handshake.headers).edgePop;
|
|
30
|
-
const peerRegion = peerService
|
|
45
|
+
const peerRegion = peerService
|
|
46
|
+
? normalizeInfrastructureRegion(peer?.region)
|
|
47
|
+
: edgePop
|
|
48
|
+
? `edge-${edgePop}`
|
|
49
|
+
: undefined;
|
|
31
50
|
collector.record({
|
|
32
|
-
region,
|
|
51
|
+
region,
|
|
52
|
+
service,
|
|
53
|
+
scope: peerService ? 'internal' : 'external',
|
|
54
|
+
direction,
|
|
33
55
|
activityType: socketTrafficType(event),
|
|
34
56
|
sourceRegion: direction === 'inbound' ? peerRegion : region,
|
|
35
57
|
targetRegion: direction === 'inbound' ? region : peerRegion,
|
|
@@ -61,9 +83,16 @@ export function observeTrafficWebSocket(socket, collector, service, region, opti
|
|
|
61
83
|
const peer = options.getPeer?.();
|
|
62
84
|
const peerService = peer && /^[a-z][a-z0-9-]{0,39}$/.test(peer.service) ? peer.service : undefined;
|
|
63
85
|
const edgePop = metadataFromHeaders(options.headers ?? {}).edgePop;
|
|
64
|
-
const peerRegion = peerService
|
|
86
|
+
const peerRegion = peerService
|
|
87
|
+
? normalizeInfrastructureRegion(peer?.region)
|
|
88
|
+
: edgePop
|
|
89
|
+
? `edge-${edgePop}`
|
|
90
|
+
: undefined;
|
|
65
91
|
collector.record({
|
|
66
|
-
service,
|
|
92
|
+
service,
|
|
93
|
+
region,
|
|
94
|
+
scope: peerService ? 'internal' : 'external',
|
|
95
|
+
direction,
|
|
67
96
|
activityType: options.activityType ?? 'communication',
|
|
68
97
|
sourceRegion: direction === 'inbound' ? peerRegion : region,
|
|
69
98
|
targetRegion: direction === 'inbound' ? region : peerRegion,
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operational metrics for Oxy services, published as CloudWatch Embedded Metric
|
|
3
|
+
* Format (EMF) log lines.
|
|
4
|
+
*
|
|
5
|
+
* WHY EMF ON STDOUT: every Oxy backend runs on ECS with the `awslogs` driver, and
|
|
6
|
+
* CloudWatch Logs extracts metrics from any log event shaped as EMF — no agent,
|
|
7
|
+
* no sidecar, no scraper, no IAM grant beyond the logging the task already has.
|
|
8
|
+
* A registry that only an HTTP scrape can read (Prometheus) measures into a void
|
|
9
|
+
* when nothing scrapes it, which is how Mention's counters lost every value on
|
|
10
|
+
* every deploy.
|
|
11
|
+
*
|
|
12
|
+
* WHY AGGREGATED: values are folded in memory and flushed once per interval as
|
|
13
|
+
* ONE line per (metric, label set), not one line per observation. A latency is
|
|
14
|
+
* kept as a distribution (`Values`/`Counts`), so CloudWatch still answers
|
|
15
|
+
* SampleCount, p50 and p99 for it — the request COUNT is the distribution's
|
|
16
|
+
* SampleCount, so a service needs no separate counter for it.
|
|
17
|
+
*
|
|
18
|
+
* WHY MOST LABELS ARE NOT DIMENSIONS: every distinct combination of dimension
|
|
19
|
+
* values is a billed custom metric. Route x method as dimensions once measured
|
|
20
|
+
* 405 metrics and ~80 % of the account's CloudWatch bill (Mention #1334). So a
|
|
21
|
+
* label becomes a DIMENSION only when a metric definition names it in
|
|
22
|
+
* `dimensionSets`, and only labels with a small, closed value set belong there.
|
|
23
|
+
* Every other label is written as a plain property of the line: free to store,
|
|
24
|
+
* and grouped in Logs Insights (`stats sum(RequestLatency.Count) by Route`).
|
|
25
|
+
*/
|
|
26
|
+
export type MetricUnit = 'Count' | 'Milliseconds' | 'Seconds' | 'Bytes' | 'Percent' | 'Count/Second' | 'None';
|
|
27
|
+
/**
|
|
28
|
+
* `sum` adds every value in the interval (a counter), `distribution` keeps the
|
|
29
|
+
* values for percentiles (a latency or a size), and `gauge` keeps the last one.
|
|
30
|
+
*/
|
|
31
|
+
export type MetricKind = 'sum' | 'distribution' | 'gauge';
|
|
32
|
+
export interface MetricDefinition {
|
|
33
|
+
/** CloudWatch metric name; also the field holding its value on the line. */
|
|
34
|
+
readonly name: string;
|
|
35
|
+
readonly unit: MetricUnit;
|
|
36
|
+
readonly kind: MetricKind;
|
|
37
|
+
/**
|
|
38
|
+
* Label names promoted to CloudWatch dimensions. Each inner list is one
|
|
39
|
+
* dimension set, published IN ADDITION to the always-present `[Service]`
|
|
40
|
+
* set. Only closed, small value sets belong here: each distinct combination
|
|
41
|
+
* is a billed custom metric.
|
|
42
|
+
*/
|
|
43
|
+
readonly dimensionSets?: readonly (readonly string[])[];
|
|
44
|
+
}
|
|
45
|
+
export type MetricLabels = Readonly<Record<string, string>>;
|
|
46
|
+
export interface EmfPublisherOptions {
|
|
47
|
+
/** CloudWatch namespace, e.g. `Oxy/Api`. */
|
|
48
|
+
readonly namespace: string;
|
|
49
|
+
/** Value of the `Service` dimension every metric carries. */
|
|
50
|
+
readonly service: string;
|
|
51
|
+
/** Flush period. Default 60 s, CloudWatch's standard resolution. */
|
|
52
|
+
readonly flushIntervalMs?: number;
|
|
53
|
+
/**
|
|
54
|
+
* Distinct (metric, label set) series held per interval. A label value that
|
|
55
|
+
* traffic can mint must not be able to grow memory or log volume without
|
|
56
|
+
* bound, so past this many every NEW series is folded into one whose labels
|
|
57
|
+
* all read `other`. Default 2000.
|
|
58
|
+
*/
|
|
59
|
+
readonly maxSeries?: number;
|
|
60
|
+
/** Where each line goes. Default: one line on stdout. */
|
|
61
|
+
readonly write?: (line: string) => void;
|
|
62
|
+
/** Clock, for tests. */
|
|
63
|
+
readonly now?: () => number;
|
|
64
|
+
}
|
|
65
|
+
export interface EmfPublisher {
|
|
66
|
+
/** Fold one observation into the current interval. Never throws. */
|
|
67
|
+
record(metric: MetricDefinition, value: number, labels?: MetricLabels): void;
|
|
68
|
+
/** Write the current interval now and start a new one. */
|
|
69
|
+
flush(): void;
|
|
70
|
+
/** Flush and stop the timer, for graceful shutdown. */
|
|
71
|
+
stop(): void;
|
|
72
|
+
}
|
|
73
|
+
/** One HTTP request: SampleCount is the request count, percentiles the latency. */
|
|
74
|
+
export declare const HTTP_REQUEST_LATENCY: MetricDefinition;
|
|
75
|
+
/** One request a rate limiter refused (429), per calling application. */
|
|
76
|
+
export declare const HTTP_RATE_LIMITED: MetricDefinition;
|
|
77
|
+
/** `2xx`, `4xx`, … — a closed set, safe as a dimension. */
|
|
78
|
+
export declare function statusClass(status: number): string;
|
|
79
|
+
export declare function createEmfPublisher(options: EmfPublisherOptions): EmfPublisher;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"type":"commonjs"}
|