@sentry/roach 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +561 -2
- package/package.json +74 -3
- package/src/action.ts +81 -0
- package/src/addresses.ts +103 -0
- package/src/certificates.ts +169 -0
- package/src/cli.ts +35 -0
- package/src/client.ts +211 -0
- package/src/config.ts +141 -0
- package/src/gcs.ts +145 -0
- package/src/recorder.ts +464 -0
- package/src/recordings.ts +42 -0
- package/src/report.ts +55 -0
- package/src/request-key.ts +165 -0
- package/src/secrets.ts +66 -0
- package/src/server.ts +544 -0
- package/src/service.ts +572 -0
- package/src/store.ts +47 -0
- package/src/streams.ts +115 -0
- package/src/types.ts +130 -0
- package/src/usage.ts +120 -0
- package/src/values.ts +229 -0
- package/src/vitest-setup.ts +66 -0
- package/src/vitest.ts +108 -0
package/src/streams.ts
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recorded event streams (`text/event-stream`).
|
|
3
|
+
*
|
|
4
|
+
* A model stream sends a tool call or a text in many small deltas, so an id
|
|
5
|
+
* that the model repeats can be split over two events. Before the proxy
|
|
6
|
+
* writes changing values as placeholders (`values.ts`), it merges the
|
|
7
|
+
* deltas of each Anthropic Messages content block into one event. A client
|
|
8
|
+
* joins the deltas anyway, so it gets the same content.
|
|
9
|
+
*
|
|
10
|
+
* Thinking blocks keep their recorded text. A signature covers that text,
|
|
11
|
+
* and the provider refuses a changed thinking block in a later request.
|
|
12
|
+
* So a later request sends back the values of the recording run in its
|
|
13
|
+
* thinking blocks, and `request-key.ts` must not fill placeholders from
|
|
14
|
+
* them.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** The text field of each delta type that the proxy merges. */
|
|
18
|
+
const DELTA_FIELDS: Record<string, string> = {
|
|
19
|
+
input_json_delta: "partial_json",
|
|
20
|
+
text_delta: "text",
|
|
21
|
+
thinking_delta: "thinking",
|
|
22
|
+
};
|
|
23
|
+
/** The content block types of thinking, in a stream and in a request. */
|
|
24
|
+
export const THINKING_BLOCK_TYPES = new Set(["redacted_thinking", "thinking"]);
|
|
25
|
+
const THINKING_TYPES = new Set([
|
|
26
|
+
...THINKING_BLOCK_TYPES,
|
|
27
|
+
"signature_delta",
|
|
28
|
+
"thinking_delta",
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
interface StreamEvent {
|
|
32
|
+
text: string;
|
|
33
|
+
data?: Record<string, unknown>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Whether a content type is an event stream. */
|
|
37
|
+
export const isEventStream = (contentType: string | undefined) =>
|
|
38
|
+
/^text\/event-stream\b/i.test(contentType ?? "");
|
|
39
|
+
|
|
40
|
+
function parseEvent(text: string): StreamEvent {
|
|
41
|
+
const line = text.split("\n").find((entry) => entry.startsWith("data:"));
|
|
42
|
+
if (!line) return { text };
|
|
43
|
+
try {
|
|
44
|
+
const data: unknown = JSON.parse(line.slice(5).trim());
|
|
45
|
+
return data && typeof data === "object" && !Array.isArray(data)
|
|
46
|
+
? { text, data: data as Record<string, unknown> }
|
|
47
|
+
: { text };
|
|
48
|
+
} catch {
|
|
49
|
+
return { text };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The type and text field of a delta event that the proxy can merge. */
|
|
54
|
+
function mergeableDelta(event: StreamEvent) {
|
|
55
|
+
if (event.data?.type !== "content_block_delta") return undefined;
|
|
56
|
+
const delta = event.data.delta as Record<string, unknown> | undefined;
|
|
57
|
+
const field = DELTA_FIELDS[String(delta?.type)];
|
|
58
|
+
if (!delta || !field || typeof delta[field] !== "string") return undefined;
|
|
59
|
+
return { delta, field, index: event.data.index };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function splitEvents(body: string): StreamEvent[] {
|
|
63
|
+
return body
|
|
64
|
+
.replace(/\r\n/g, "\n")
|
|
65
|
+
.split(/\n\n+/)
|
|
66
|
+
.filter((text) => text.trim() !== "")
|
|
67
|
+
.map(parseEvent);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const joinEvents = (events: StreamEvent[]) =>
|
|
71
|
+
events.map((event) => `${event.text}\n\n`).join("");
|
|
72
|
+
|
|
73
|
+
/** Merge the deltas of each content block into one event. */
|
|
74
|
+
export function mergeStreamDeltas(body: string): string {
|
|
75
|
+
const events: StreamEvent[] = [];
|
|
76
|
+
for (const event of splitEvents(body)) {
|
|
77
|
+
const current = mergeableDelta(event);
|
|
78
|
+
const previousEvent = events.at(-1);
|
|
79
|
+
const previous = previousEvent && mergeableDelta(previousEvent);
|
|
80
|
+
if (
|
|
81
|
+
current &&
|
|
82
|
+
previous &&
|
|
83
|
+
previous.index === current.index &&
|
|
84
|
+
previous.delta.type === current.delta.type
|
|
85
|
+
) {
|
|
86
|
+
previous.delta[previous.field] =
|
|
87
|
+
`${previous.delta[previous.field] as string}${current.delta[current.field] as string}`;
|
|
88
|
+
previousEvent.text = `event: content_block_delta\ndata: ${JSON.stringify(previousEvent.data)}`;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
events.push(event);
|
|
92
|
+
}
|
|
93
|
+
return joinEvents(events);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function isThinking(event: StreamEvent): boolean {
|
|
97
|
+
const delta = event.data?.delta as { type?: unknown } | undefined;
|
|
98
|
+
const block = event.data?.content_block as { type?: unknown } | undefined;
|
|
99
|
+
return (
|
|
100
|
+
THINKING_TYPES.has(String(delta?.type)) ||
|
|
101
|
+
THINKING_TYPES.has(String(block?.type))
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Change each event of a stream, but keep thinking events as they are. */
|
|
106
|
+
export function mapStreamEvents(
|
|
107
|
+
body: string,
|
|
108
|
+
change: (text: string) => string,
|
|
109
|
+
): string {
|
|
110
|
+
return joinEvents(
|
|
111
|
+
splitEvents(body).map((event) =>
|
|
112
|
+
isThinking(event) ? event : { text: change(event.text) },
|
|
113
|
+
),
|
|
114
|
+
);
|
|
115
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The public contract of Roach: rules, modes, runs, sessions, and the
|
|
3
|
+
* results of a run. The client, the service, and the vitest integration
|
|
4
|
+
* share it. See `README.md`.
|
|
5
|
+
*/
|
|
6
|
+
import type { ValuePatterns } from "./values.ts";
|
|
7
|
+
|
|
8
|
+
/** The path prefix of the control API. */
|
|
9
|
+
export const CONTROL_PATH = "/__roach";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* - `auto`: replay a request that has a recording. Send other requests live
|
|
13
|
+
* and record them.
|
|
14
|
+
* - `replay`: replay a request that has a recording. Fail other requests
|
|
15
|
+
* with HTTP 412. Nothing goes live, and nothing is recorded.
|
|
16
|
+
* - `record`: send every request live and record it again.
|
|
17
|
+
* - `off`: record and replay nothing.
|
|
18
|
+
*/
|
|
19
|
+
export type RecordingMode = "auto" | "off" | "record" | "replay";
|
|
20
|
+
|
|
21
|
+
/** One kind of traffic that the proxy records. */
|
|
22
|
+
export interface RecordingRule {
|
|
23
|
+
/**
|
|
24
|
+
* The name of the rule, such as `model`. Its keys start with
|
|
25
|
+
* `<name>/`. Use letters, digits, `_`, `.`, and `-`.
|
|
26
|
+
*/
|
|
27
|
+
name: string;
|
|
28
|
+
/** The requests of the rule. Each field that is set must match. */
|
|
29
|
+
match: {
|
|
30
|
+
/** The method, such as `POST`. */
|
|
31
|
+
method?: string;
|
|
32
|
+
/** The start of the URL, such as `https://ai-gateway.vercel.sh/`. */
|
|
33
|
+
url?: string;
|
|
34
|
+
/** Header values. Names are lowercase. */
|
|
35
|
+
headers?: Record<string, string>;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Request headers that are in the key, with the method, the URL, and the
|
|
39
|
+
* body. Names are lowercase.
|
|
40
|
+
*/
|
|
41
|
+
keyHeaders?: string[];
|
|
42
|
+
/**
|
|
43
|
+
* Values in the request body that change from run to run, such as ids and
|
|
44
|
+
* times. See `values.ts`.
|
|
45
|
+
*/
|
|
46
|
+
values?: ValuePatterns;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** A request that a rule matched, but that had no recording. */
|
|
50
|
+
export interface RecordingMiss {
|
|
51
|
+
rule: string;
|
|
52
|
+
/** The session of the request. */
|
|
53
|
+
session?: string | undefined;
|
|
54
|
+
/** The key of the recording that the request needed. */
|
|
55
|
+
file: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The totals of a run. */
|
|
59
|
+
export interface RecordingStats {
|
|
60
|
+
/**
|
|
61
|
+
* Requests by rule name. `missed` counts the requests that `replay` mode
|
|
62
|
+
* failed, because they had no recording.
|
|
63
|
+
*/
|
|
64
|
+
counts: Record<string, { live: number; missed: number; replayed: number }>;
|
|
65
|
+
/** The requests without a recording, in order. The first 500 are kept. */
|
|
66
|
+
misses: RecordingMiss[];
|
|
67
|
+
/** Recordings that the proxy wrote and that were new or changed. */
|
|
68
|
+
written: number;
|
|
69
|
+
/** New recordings that the proxy dropped, because their session failed. */
|
|
70
|
+
discarded: number;
|
|
71
|
+
/**
|
|
72
|
+
* Requests that matched no rule, by origin. They went live and were not
|
|
73
|
+
* recorded. Use this to find traffic that a rule misses.
|
|
74
|
+
*/
|
|
75
|
+
passthrough: Record<string, number>;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The configuration of one run. The client sends it. */
|
|
79
|
+
export interface RunConfig {
|
|
80
|
+
/**
|
|
81
|
+
* The tenant of the run, as `owner/repo`, such as
|
|
82
|
+
* `process.env.GITHUB_REPOSITORY`.
|
|
83
|
+
*/
|
|
84
|
+
tenant: string;
|
|
85
|
+
mode: RecordingMode;
|
|
86
|
+
rules: RecordingRule[];
|
|
87
|
+
/**
|
|
88
|
+
* The only origins that the run can reach, such as
|
|
89
|
+
* `https://ai-gateway.vercel.sh`. The service sends requests only to
|
|
90
|
+
* their public addresses.
|
|
91
|
+
*/
|
|
92
|
+
allow: string[];
|
|
93
|
+
/**
|
|
94
|
+
* The name of the run in Sentry, such as
|
|
95
|
+
* `${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}`. Default: the run id.
|
|
96
|
+
*/
|
|
97
|
+
name?: string;
|
|
98
|
+
/**
|
|
99
|
+
* Credentials that this run must never write, such as the API keys of
|
|
100
|
+
* its CI job. The service redacts them with the `secrets` of the service
|
|
101
|
+
* config, keeps them only in memory, and forgets them when the run ends.
|
|
102
|
+
*/
|
|
103
|
+
secrets?: string[];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** A run that the service created. */
|
|
107
|
+
export interface RemoteRun {
|
|
108
|
+
/** The id of the run. */
|
|
109
|
+
id: string;
|
|
110
|
+
/**
|
|
111
|
+
* The proxy URL, with the run id and token as its user and password. A
|
|
112
|
+
* request through it belongs to the open session when only one is open.
|
|
113
|
+
*/
|
|
114
|
+
url: string;
|
|
115
|
+
/** The run token. Session and stats calls use it. */
|
|
116
|
+
token: string;
|
|
117
|
+
/** The base URL of the control API of the run. */
|
|
118
|
+
controlUrl: string;
|
|
119
|
+
/** The PEM certificate of the authority that signs intercepted hosts. */
|
|
120
|
+
caCert: string;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** A session that the service opened. */
|
|
124
|
+
export interface OpenedSession {
|
|
125
|
+
/**
|
|
126
|
+
* The proxy URL of the session. Each request through it belongs to the
|
|
127
|
+
* session, also when other sessions are open.
|
|
128
|
+
*/
|
|
129
|
+
url: string;
|
|
130
|
+
}
|
package/src/usage.ts
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The token usage of a model response, as `gen_ai` span attributes.
|
|
3
|
+
*
|
|
4
|
+
* Sentry calculates the cost of a span in USD (`gen_ai.cost.*`) from these
|
|
5
|
+
* attributes and its own price of the model. So Roach needs no prices, and
|
|
6
|
+
* the client sends nothing extra. A replayed response has the usage of its
|
|
7
|
+
* recording: the cost that the replay saved.
|
|
8
|
+
*
|
|
9
|
+
* The proxy reads the usage fields of two model APIs: Anthropic Messages
|
|
10
|
+
* (`input_tokens`, `cache_read_input_tokens`) and OpenAI Chat Completions
|
|
11
|
+
* (`prompt_tokens`, `prompt_tokens_details`). It reads the whole body, or
|
|
12
|
+
* each event of a stream. A later event gives the total so far, so the last
|
|
13
|
+
* value of each field wins.
|
|
14
|
+
*
|
|
15
|
+
* As in Sentry, `gen_ai.usage.input_tokens` includes the cache tokens.
|
|
16
|
+
* Anthropic counts them apart, so the proxy adds them.
|
|
17
|
+
*/
|
|
18
|
+
import { isEventStream } from "./streams.ts";
|
|
19
|
+
|
|
20
|
+
type Fields = Record<string, unknown>;
|
|
21
|
+
|
|
22
|
+
const isObject = (value: unknown): value is Fields =>
|
|
23
|
+
typeof value === "object" && value !== null && !Array.isArray(value);
|
|
24
|
+
|
|
25
|
+
const count = (value: unknown) =>
|
|
26
|
+
typeof value === "number" && Number.isFinite(value) && value >= 0
|
|
27
|
+
? value
|
|
28
|
+
: undefined;
|
|
29
|
+
|
|
30
|
+
function parseJson(text: string): unknown {
|
|
31
|
+
try {
|
|
32
|
+
return JSON.parse(text);
|
|
33
|
+
} catch {
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** The JSON objects of a body: the body itself, or each stream event. */
|
|
39
|
+
function bodyObjects(contentType: string | undefined, body: string): Fields[] {
|
|
40
|
+
const values = isEventStream(contentType)
|
|
41
|
+
? body
|
|
42
|
+
.split("\n")
|
|
43
|
+
.filter((line) => line.startsWith("data:"))
|
|
44
|
+
.map((line) => parseJson(line.slice(5).trim()))
|
|
45
|
+
: /json/i.test(contentType ?? "")
|
|
46
|
+
? [parseJson(body)]
|
|
47
|
+
: [];
|
|
48
|
+
return values.filter(isObject);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** The model that a request asks for, from its JSON body. */
|
|
52
|
+
function requestModel(body: string): string | undefined {
|
|
53
|
+
const parsed = parseJson(body);
|
|
54
|
+
return isObject(parsed) && typeof parsed.model === "string"
|
|
55
|
+
? parsed.model
|
|
56
|
+
: undefined;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The `gen_ai` attributes of one model call, or `undefined` when the
|
|
61
|
+
* response has no usage.
|
|
62
|
+
*/
|
|
63
|
+
export function usageAttributes(
|
|
64
|
+
requestBody: string,
|
|
65
|
+
contentType: string | undefined,
|
|
66
|
+
responseBody: string,
|
|
67
|
+
): Record<string, string | number> | undefined {
|
|
68
|
+
let model: string | undefined;
|
|
69
|
+
const usage: Fields = {};
|
|
70
|
+
for (const object of bodyObjects(contentType, responseBody)) {
|
|
71
|
+
// An Anthropic stream sends the model and the first usage in
|
|
72
|
+
// `message_start`.
|
|
73
|
+
const message = isObject(object.message) ? object.message : object;
|
|
74
|
+
if (typeof message.model === "string") model = message.model;
|
|
75
|
+
for (const source of [message.usage, object.usage]) {
|
|
76
|
+
if (!isObject(source)) continue;
|
|
77
|
+
for (const [name, value] of Object.entries(source)) {
|
|
78
|
+
if (value !== null && value !== undefined) usage[name] = value;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const details = (name: string, field: string) =>
|
|
84
|
+
isObject(usage[name]) ? count(usage[name][field]) : undefined;
|
|
85
|
+
const cacheRead =
|
|
86
|
+
count(usage.cache_read_input_tokens) ??
|
|
87
|
+
details("prompt_tokens_details", "cached_tokens");
|
|
88
|
+
const cacheWrite = count(usage.cache_creation_input_tokens);
|
|
89
|
+
const input =
|
|
90
|
+
count(usage.prompt_tokens) ??
|
|
91
|
+
addCounts(count(usage.input_tokens), cacheRead, cacheWrite);
|
|
92
|
+
const output = count(usage.completion_tokens) ?? count(usage.output_tokens);
|
|
93
|
+
const reasoning = details("completion_tokens_details", "reasoning_tokens");
|
|
94
|
+
if (input === undefined && output === undefined) return undefined;
|
|
95
|
+
|
|
96
|
+
const attributes: Record<string, string | number> = {
|
|
97
|
+
"gen_ai.operation.name": "chat",
|
|
98
|
+
};
|
|
99
|
+
// Sentry prices the response model, or the request model without one. A
|
|
100
|
+
// gateway can name the model in the response in another way than its
|
|
101
|
+
// price list does, so the requested name wins.
|
|
102
|
+
const requested = requestModel(requestBody);
|
|
103
|
+
if (requested) attributes["gen_ai.request.model"] = requested;
|
|
104
|
+
else if (model) attributes["gen_ai.response.model"] = model;
|
|
105
|
+
const set = (name: string, value: number | undefined) => {
|
|
106
|
+
if (value !== undefined) attributes[`gen_ai.usage.${name}`] = value;
|
|
107
|
+
};
|
|
108
|
+
set("input_tokens", input);
|
|
109
|
+
set("output_tokens", output);
|
|
110
|
+
set("cache_read.input_tokens", cacheRead);
|
|
111
|
+
set("cache_creation.input_tokens", cacheWrite);
|
|
112
|
+
set("reasoning.output_tokens", reasoning);
|
|
113
|
+
return attributes;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** The sum of the counts that are set, or `undefined` when none is. */
|
|
117
|
+
function addCounts(...values: Array<number | undefined>): number | undefined {
|
|
118
|
+
const set = values.filter((value) => value !== undefined);
|
|
119
|
+
return set.length === 0 ? undefined : set.reduce((sum, v) => sum + v, 0);
|
|
120
|
+
}
|
package/src/values.ts
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Changing values in recorded requests, such as ids and clock times.
|
|
3
|
+
*
|
|
4
|
+
* A rule names the values that change from run to run. The proxy handles
|
|
5
|
+
* them in two directions:
|
|
6
|
+
*
|
|
7
|
+
* - The key sees each value as `<<name>>`, so two runs with other ids or
|
|
8
|
+
* times send the same request.
|
|
9
|
+
* - A response often repeats a value of its request, for example when the
|
|
10
|
+
* model archives the memory with id `<<uuid>>`. When the proxy records a
|
|
11
|
+
* response, it writes each value of the request as `<<name:n>>`, the n-th
|
|
12
|
+
* value of that name in the request. On replay, it writes back the n-th
|
|
13
|
+
* value of the current request. So a replayed response uses the ids of
|
|
14
|
+
* this run, not the ids of the recording run.
|
|
15
|
+
*
|
|
16
|
+
* A value that the response makes itself, such as a date that the model
|
|
17
|
+
* calculates, stays as it was recorded.
|
|
18
|
+
*/
|
|
19
|
+
import vm from "node:vm";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Lookbehind: the value does not continue a word or number. The proxy
|
|
23
|
+
* matches values in JSON text, where `\nab12` is a newline and then
|
|
24
|
+
* `ab12`. So a letter after a backslash, such as the `n` of `\n`, does not
|
|
25
|
+
* count as a word. Use it in a pattern that starts with a letter or digit.
|
|
26
|
+
*/
|
|
27
|
+
export const NOT_AFTER_WORD = String.raw`(?<!(?:^|[^\\])[0-9A-Za-z])`;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Patterns for common changing values. Use them in `values` of a rule. Each
|
|
31
|
+
* pattern is a regular expression source.
|
|
32
|
+
*/
|
|
33
|
+
export const VALUE_PATTERNS = {
|
|
34
|
+
/** A UUID, also inside a name such as `junior-ws-<uuid>`. */
|
|
35
|
+
uuid: String.raw`${NOT_AFTER_WORD}[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(?![0-9A-Za-z])`,
|
|
36
|
+
/**
|
|
37
|
+
* An ISO time, with `T` or a space, with or without seconds, and in UTC,
|
|
38
|
+
* with an offset, or in local time.
|
|
39
|
+
*/
|
|
40
|
+
isoTime: String.raw`(?<![\d-])\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2}| ?UTC)?(?![\d:])`,
|
|
41
|
+
/** An ISO date, such as `2026-10-07`. */
|
|
42
|
+
date: String.raw`(?<![\d-])\d{4}-\d{2}-\d{2}(?![\d-])`,
|
|
43
|
+
/**
|
|
44
|
+
* Unix milliseconds from 2023 to 2033. Unix seconds, such as Slack
|
|
45
|
+
* timestamps, do not match.
|
|
46
|
+
*/
|
|
47
|
+
epochMs: String.raw`(?<!\d)1[789]\d{11}(?!\d)`,
|
|
48
|
+
/**
|
|
49
|
+
* A SHA-256 hex digest. Ids that the product hashes from other changing
|
|
50
|
+
* values, such as attachment ids, change with them.
|
|
51
|
+
*/
|
|
52
|
+
sha256: String.raw`${NOT_AFTER_WORD}[0-9a-f]{64}(?![0-9A-Za-z])`,
|
|
53
|
+
/**
|
|
54
|
+
* A git commit id. Its hash covers the commit time, so it changes on each
|
|
55
|
+
* run. A full id has 40 hex digits. A short id only counts in the output
|
|
56
|
+
* of `git commit`, such as `[main (root-commit) a7a9f1d]`, and in the range
|
|
57
|
+
* of `git push`, such as `a5fd79f..a73e959`, because seven hex digits can
|
|
58
|
+
* also be a word.
|
|
59
|
+
*/
|
|
60
|
+
gitCommit: String.raw`${NOT_AFTER_WORD}[0-9a-f]{40}(?![0-9A-Za-z])|(?<=\[[^\]\s]+ (?:\(root-commit\) )?)[0-9a-f]{7,12}(?=\])|${NOT_AFTER_WORD}[0-9a-f]{7,12}(?=\.\.\.?[0-9a-f]{7,12}(?![0-9A-Za-z]))|(?<=${NOT_AFTER_WORD}[0-9a-f]{7,12}\.\.\.?)[0-9a-f]{7,12}(?![0-9A-Za-z])`,
|
|
61
|
+
} as const;
|
|
62
|
+
|
|
63
|
+
/** Changing values by name. Each value is a regular expression source. */
|
|
64
|
+
export type ValuePatterns = Record<string, string>;
|
|
65
|
+
|
|
66
|
+
/** The values of one request, by name, in the order of the request. */
|
|
67
|
+
export type RequestValues = Map<string, string[]>;
|
|
68
|
+
|
|
69
|
+
interface CompiledValues {
|
|
70
|
+
names: string[];
|
|
71
|
+
/** One expression for all patterns, with one group per name. */
|
|
72
|
+
pattern: RegExp;
|
|
73
|
+
/** The index of the group of each name. */
|
|
74
|
+
groups: number[];
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const compiled = new WeakMap<ValuePatterns, CompiledValues>();
|
|
78
|
+
|
|
79
|
+
/** The number of capturing groups in a pattern. */
|
|
80
|
+
const groupCount = (source: string) =>
|
|
81
|
+
new RegExp(`${source}|`).exec("")!.length - 1;
|
|
82
|
+
|
|
83
|
+
function compile(patterns: ValuePatterns): CompiledValues {
|
|
84
|
+
const cached = compiled.get(patterns);
|
|
85
|
+
if (cached) return cached;
|
|
86
|
+
const names = Object.keys(patterns);
|
|
87
|
+
const groups: number[] = [];
|
|
88
|
+
let next = 1;
|
|
89
|
+
for (const name of names) {
|
|
90
|
+
groups.push(next);
|
|
91
|
+
next += 1 + groupCount(patterns[name]!);
|
|
92
|
+
}
|
|
93
|
+
const result = {
|
|
94
|
+
names,
|
|
95
|
+
// Earlier names win when two patterns match at the same place.
|
|
96
|
+
pattern: new RegExp(
|
|
97
|
+
names.map((name) => `(${patterns[name]})`).join("|"),
|
|
98
|
+
"g",
|
|
99
|
+
),
|
|
100
|
+
groups,
|
|
101
|
+
};
|
|
102
|
+
compiled.set(patterns, result);
|
|
103
|
+
return result;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Run `fn` with a time limit. A slow pattern cannot stop itself, but the
|
|
108
|
+
* `timeout` of `vm` stops any code that runs too long, also code outside
|
|
109
|
+
* the context that the script calls. A timeout throws an error with HTTP
|
|
110
|
+
* status 400.
|
|
111
|
+
*/
|
|
112
|
+
export function withTimeLimit<T>(ms: number, fn: () => T): T {
|
|
113
|
+
try {
|
|
114
|
+
return vm.runInContext("fn()", vm.createContext({ fn }), {
|
|
115
|
+
timeout: ms,
|
|
116
|
+
}) as T;
|
|
117
|
+
} catch (error) {
|
|
118
|
+
if ((error as { code?: string }).code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
|
|
119
|
+
throw Object.assign(
|
|
120
|
+
new Error(`the value patterns took over ${ms} ms. Make them simpler.`),
|
|
121
|
+
{ status: 400 },
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
throw error;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Check that the patterns of each rule compile. Returns the error, or
|
|
130
|
+
* `undefined`. A pattern can be slow to compile, so all of them together
|
|
131
|
+
* have 1 s.
|
|
132
|
+
*/
|
|
133
|
+
export function invalidValuePatterns(
|
|
134
|
+
list: ValuePatterns[],
|
|
135
|
+
): string | undefined {
|
|
136
|
+
try {
|
|
137
|
+
withTimeLimit(1000, () => list.forEach((patterns) => compile(patterns)));
|
|
138
|
+
return undefined;
|
|
139
|
+
} catch (error) {
|
|
140
|
+
return (error as Error).message;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Values that an earlier request of the same session had, by value. */
|
|
145
|
+
export type KnownValues = ReadonlyMap<string, string>;
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Replace each changing value in `text` with `<<name>>`. Returns the text
|
|
149
|
+
* and the values in their order. With no patterns, nothing changes.
|
|
150
|
+
*
|
|
151
|
+
* `known` maps values of earlier requests in the session to their names. A
|
|
152
|
+
* known value also counts where no pattern finds it. For example, the
|
|
153
|
+
* model repeats the short commit id of a `git push` range in its reply,
|
|
154
|
+
* and a later request sends that reply back. A pattern wins over a known
|
|
155
|
+
* value at the same place.
|
|
156
|
+
*/
|
|
157
|
+
export function extractValues(
|
|
158
|
+
text: string,
|
|
159
|
+
patterns: ValuePatterns | undefined,
|
|
160
|
+
known?: KnownValues,
|
|
161
|
+
): { text: string; values: RequestValues } {
|
|
162
|
+
const values: RequestValues = new Map();
|
|
163
|
+
if (!patterns || Object.keys(patterns).length === 0) {
|
|
164
|
+
return { text, values };
|
|
165
|
+
}
|
|
166
|
+
const { names, pattern, groups } = compile(patterns);
|
|
167
|
+
const hasKnown = known !== undefined && known.size > 0;
|
|
168
|
+
// Known values are the last group, after the groups of the patterns.
|
|
169
|
+
const knownGroup = groupCount(pattern.source) + 1;
|
|
170
|
+
const search = hasKnown
|
|
171
|
+
? new RegExp(`${pattern.source}|(${literals([...known.keys()])})`, "g")
|
|
172
|
+
: pattern;
|
|
173
|
+
const result = text.replace(search, (...match: unknown[]) => {
|
|
174
|
+
const value = match[0] as string;
|
|
175
|
+
const name =
|
|
176
|
+
hasKnown && match[knownGroup] !== undefined
|
|
177
|
+
? known.get(value)!
|
|
178
|
+
: names[groups.findIndex((group) => match[group] !== undefined)]!;
|
|
179
|
+
const list = values.get(name) ?? [];
|
|
180
|
+
list.push(value);
|
|
181
|
+
values.set(name, list);
|
|
182
|
+
return `<<${name}>>`;
|
|
183
|
+
});
|
|
184
|
+
return { text: result, values };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* A pattern source for any of `values`, where it is not part of a longer
|
|
191
|
+
* word or number. Longer values come first, so a date never replaces part
|
|
192
|
+
* of a time.
|
|
193
|
+
*/
|
|
194
|
+
const literals = (values: string[]) =>
|
|
195
|
+
`${NOT_AFTER_WORD}(?:${values
|
|
196
|
+
.toSorted((a, b) => b.length - a.length)
|
|
197
|
+
.map(escape)
|
|
198
|
+
.join("|")})(?![0-9A-Za-z])`;
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Write each value of the request that a recorded response repeats as
|
|
202
|
+
* `<<name:n>>`. A value is replaced only where it is not part of a longer
|
|
203
|
+
* word or number.
|
|
204
|
+
*/
|
|
205
|
+
export function templateValues(text: string, values: RequestValues): string {
|
|
206
|
+
const placeholders = new Map<string, string>();
|
|
207
|
+
for (const [name, list] of values) {
|
|
208
|
+
list.forEach((value, index) => {
|
|
209
|
+
if (!placeholders.has(value)) {
|
|
210
|
+
placeholders.set(value, `<<${name}:${index + 1}>>`);
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
if (placeholders.size === 0) return text;
|
|
215
|
+
const pattern = new RegExp(literals([...placeholders.keys()]), "g");
|
|
216
|
+
return text.replace(pattern, (value) => placeholders.get(value)!);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Write back the values of the current request into a recorded response.
|
|
221
|
+
* This undoes `templateValues()` with the values of this run.
|
|
222
|
+
*/
|
|
223
|
+
export function fillValues(text: string, values: RequestValues): string {
|
|
224
|
+
return text.replace(
|
|
225
|
+
/<<([^<>:\s]+):(\d+)>>/g,
|
|
226
|
+
(placeholder, name: string, index: string) =>
|
|
227
|
+
values.get(name)?.[Number(index) - 1] ?? placeholder,
|
|
228
|
+
);
|
|
229
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The vitest setup file of Roach (`@sentry/roach/vitest/setup`). See "Use
|
|
3
|
+
* with vitest" in `README.md`.
|
|
4
|
+
*
|
|
5
|
+
* It opens one session for each test of the run that `vitest.ts` started.
|
|
6
|
+
* Requests of the test go through the proxy URL of its session, so tests
|
|
7
|
+
* in other workers record at the same time. Its child processes get that
|
|
8
|
+
* URL too. A passed test writes its new recordings, and a failed test
|
|
9
|
+
* drops them. A test with a request that `replay` mode failed fails.
|
|
10
|
+
*
|
|
11
|
+
* The tests of one file share one process, so they must run one at a
|
|
12
|
+
* time. Do not use `test.concurrent` with Roach.
|
|
13
|
+
*/
|
|
14
|
+
import http from "node:http";
|
|
15
|
+
import {
|
|
16
|
+
beforeEach,
|
|
17
|
+
inject,
|
|
18
|
+
onTestFinished,
|
|
19
|
+
type ProvidedContext,
|
|
20
|
+
} from "vitest";
|
|
21
|
+
import { connectRoach } from "./client.ts";
|
|
22
|
+
|
|
23
|
+
const provided = inject("roach");
|
|
24
|
+
if (provided === undefined) {
|
|
25
|
+
throw new Error(
|
|
26
|
+
"Roach: add @sentry/roach/vitest to globalSetup to use the setup file",
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
// `null` is `off` mode: no run, so requests go direct.
|
|
30
|
+
if (provided) useRun(provided);
|
|
31
|
+
|
|
32
|
+
/** Open a session for each test of the run. */
|
|
33
|
+
function useRun(run: NonNullable<ProvidedContext["roach"]>): void {
|
|
34
|
+
const control = connectRoach(run);
|
|
35
|
+
beforeEach(async ({ task }) => {
|
|
36
|
+
// The full name starts with the file, so tests of two files never share
|
|
37
|
+
// a session.
|
|
38
|
+
const session = await control.startSession(task.fullName);
|
|
39
|
+
const previous = {
|
|
40
|
+
HTTP_PROXY: process.env.HTTP_PROXY,
|
|
41
|
+
HTTPS_PROXY: process.env.HTTPS_PROXY,
|
|
42
|
+
};
|
|
43
|
+
process.env.HTTP_PROXY = session.url;
|
|
44
|
+
process.env.HTTPS_PROXY = session.url;
|
|
45
|
+
const restoreProxy = http.setGlobalProxyFromEnv({
|
|
46
|
+
HTTP_PROXY: session.url,
|
|
47
|
+
HTTPS_PROXY: session.url,
|
|
48
|
+
NO_PROXY: run.noProxy,
|
|
49
|
+
});
|
|
50
|
+
onTestFinished(async ({ task: finished }) => {
|
|
51
|
+
restoreProxy();
|
|
52
|
+
for (const [name, value] of Object.entries(previous)) {
|
|
53
|
+
if (value === undefined) delete process.env[name];
|
|
54
|
+
else process.env[name] = value;
|
|
55
|
+
}
|
|
56
|
+
const { missed } = await session.end(finished.result?.state === "pass");
|
|
57
|
+
// `replay` mode failed these requests, so nothing went live. The test
|
|
58
|
+
// must fail, also when it recovered from the error.
|
|
59
|
+
if (missed > 0) {
|
|
60
|
+
throw new Error(
|
|
61
|
+
`Roach: ${missed} request(s) had no recording in replay mode. Run with ROACH_TOKEN to record them.`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
}
|