@watchtowerobservability/node 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +186 -3
  2. package/dist/cjs/appsec.d.ts +95 -0
  3. package/dist/cjs/appsec.js +206 -0
  4. package/dist/cjs/index.d.ts +3 -0
  5. package/dist/cjs/index.js +14 -1
  6. package/dist/cjs/integrations/amqp.d.ts +13 -0
  7. package/dist/cjs/integrations/amqp.js +96 -0
  8. package/dist/cjs/integrations/bullmq.d.ts +19 -0
  9. package/dist/cjs/integrations/bullmq.js +83 -0
  10. package/dist/cjs/integrations/cache.d.ts +13 -0
  11. package/dist/cjs/integrations/cache.js +51 -0
  12. package/dist/cjs/integrations/db.d.ts +62 -0
  13. package/dist/cjs/integrations/db.js +288 -0
  14. package/dist/cjs/integrations/duckdb.d.ts +22 -0
  15. package/dist/cjs/integrations/duckdb.js +66 -0
  16. package/dist/cjs/integrations/graphile-worker.d.ts +22 -0
  17. package/dist/cjs/integrations/graphile-worker.js +50 -0
  18. package/dist/cjs/integrations/level.d.ts +18 -0
  19. package/dist/cjs/integrations/level.js +71 -0
  20. package/dist/cjs/integrations/lmdb.d.ts +26 -0
  21. package/dist/cjs/integrations/lmdb.js +78 -0
  22. package/dist/cjs/integrations/messaging.d.ts +17 -0
  23. package/dist/cjs/integrations/messaging.js +73 -0
  24. package/dist/cjs/integrations/mqtt.d.ts +7 -0
  25. package/dist/cjs/integrations/mqtt.js +41 -0
  26. package/dist/cjs/integrations/nats.d.ts +17 -0
  27. package/dist/cjs/integrations/nats.js +73 -0
  28. package/dist/cjs/integrations/orm.d.ts +47 -0
  29. package/dist/cjs/integrations/orm.js +93 -0
  30. package/dist/cjs/integrations/pg-boss.d.ts +8 -0
  31. package/dist/cjs/integrations/pg-boss.js +63 -0
  32. package/dist/cjs/integrations/pools.d.ts +41 -0
  33. package/dist/cjs/integrations/pools.js +94 -0
  34. package/dist/cjs/integrations/postgres.d.ts +12 -0
  35. package/dist/cjs/integrations/postgres.js +92 -0
  36. package/dist/cjs/integrations/sqlite.d.ts +36 -0
  37. package/dist/cjs/integrations/sqlite.js +125 -0
  38. package/dist/cjs/pprof.d.ts +6 -0
  39. package/dist/cjs/pprof.js +88 -0
  40. package/dist/cjs/profiling.d.ts +4 -1
  41. package/dist/cjs/profiling.js +18 -8
  42. package/dist/cjs/sql.d.ts +19 -0
  43. package/dist/cjs/sql.js +332 -0
  44. package/dist/cjs/test-plan.d.ts +10 -1
  45. package/dist/cjs/test-plan.js +91 -1
  46. package/dist/esm/appsec.d.ts +95 -0
  47. package/dist/esm/appsec.js +193 -0
  48. package/dist/esm/index.d.ts +3 -0
  49. package/dist/esm/index.js +14 -1
  50. package/dist/esm/integrations/amqp.d.ts +13 -0
  51. package/dist/esm/integrations/amqp.js +92 -0
  52. package/dist/esm/integrations/bullmq.d.ts +19 -0
  53. package/dist/esm/integrations/bullmq.js +79 -0
  54. package/dist/esm/integrations/cache.d.ts +13 -0
  55. package/dist/esm/integrations/cache.js +46 -0
  56. package/dist/esm/integrations/db.d.ts +62 -0
  57. package/dist/esm/integrations/db.js +239 -0
  58. package/dist/esm/integrations/duckdb.d.ts +22 -0
  59. package/dist/esm/integrations/duckdb.js +29 -0
  60. package/dist/esm/integrations/graphile-worker.d.ts +22 -0
  61. package/dist/esm/integrations/graphile-worker.js +46 -0
  62. package/dist/esm/integrations/level.d.ts +18 -0
  63. package/dist/esm/integrations/level.js +35 -0
  64. package/dist/esm/integrations/lmdb.d.ts +26 -0
  65. package/dist/esm/integrations/lmdb.js +42 -0
  66. package/dist/esm/integrations/messaging.d.ts +17 -0
  67. package/dist/esm/integrations/messaging.js +64 -0
  68. package/dist/esm/integrations/mqtt.d.ts +7 -0
  69. package/dist/esm/integrations/mqtt.js +38 -0
  70. package/dist/esm/integrations/nats.d.ts +17 -0
  71. package/dist/esm/integrations/nats.js +70 -0
  72. package/dist/esm/integrations/orm.d.ts +47 -0
  73. package/dist/esm/integrations/orm.js +86 -0
  74. package/dist/esm/integrations/pg-boss.d.ts +8 -0
  75. package/dist/esm/integrations/pg-boss.js +60 -0
  76. package/dist/esm/integrations/pools.d.ts +41 -0
  77. package/dist/esm/integrations/pools.js +89 -0
  78. package/dist/esm/integrations/postgres.d.ts +12 -0
  79. package/dist/esm/integrations/postgres.js +55 -0
  80. package/dist/esm/integrations/sqlite.d.ts +36 -0
  81. package/dist/esm/integrations/sqlite.js +84 -0
  82. package/dist/esm/pprof.d.ts +6 -0
  83. package/dist/esm/pprof.js +85 -0
  84. package/dist/esm/profiling.d.ts +4 -1
  85. package/dist/esm/profiling.js +17 -8
  86. package/dist/esm/sql.d.ts +19 -0
  87. package/dist/esm/sql.js +324 -0
  88. package/dist/esm/test-plan.d.ts +10 -1
  89. package/dist/esm/test-plan.js +89 -1
  90. package/package.json +161 -1
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Folded stacks (`[span_id=…,trace_id=…\t]a;b;c` root first, as `foldProfile` makes them) to a
3
+ * gzipped pprof with sample types samples/count and cpu/nanoseconds. The `k=v` prefix becomes
4
+ * sample labels. Frame names keep their `(file:line)` suffix; one function per distinct frame.
5
+ */
6
+ export declare function foldedToPprof(counts: Map<string, number>, periodNs: number, startMs: number, durationMs: number): Uint8Array;
@@ -0,0 +1,85 @@
1
+ // Minimal pprof (profile.proto) encoder, no dependency: folded stacks in, gzipped protobuf out.
2
+ import { gzipSync } from 'node:zlib';
3
+ /** Protobuf writer for the handful of field shapes profile.proto needs. */
4
+ class Pb {
5
+ b = [];
6
+ varint(x) {
7
+ let n = BigInt(x);
8
+ while (n > 127n) {
9
+ this.b.push(Number(n & 127n) | 128);
10
+ n >>= 7n;
11
+ }
12
+ this.b.push(Number(n));
13
+ }
14
+ int(field, v) {
15
+ if (v) {
16
+ this.varint(field << 3);
17
+ this.varint(v);
18
+ }
19
+ return this;
20
+ }
21
+ bytes(field, data) {
22
+ this.varint((field << 3) | 2);
23
+ this.varint(data.length);
24
+ for (let i = 0; i < data.length; i++)
25
+ this.b.push(data[i]);
26
+ return this;
27
+ }
28
+ msg(field, m) {
29
+ return this.bytes(field, m.b);
30
+ }
31
+ packed(field, vs) {
32
+ const p = new Pb();
33
+ for (const v of vs)
34
+ p.varint(v);
35
+ return this.bytes(field, p.b);
36
+ }
37
+ }
38
+ /**
39
+ * Folded stacks (`[span_id=…,trace_id=…\t]a;b;c` root first, as `foldProfile` makes them) to a
40
+ * gzipped pprof with sample types samples/count and cpu/nanoseconds. The `k=v` prefix becomes
41
+ * sample labels. Frame names keep their `(file:line)` suffix; one function per distinct frame.
42
+ */
43
+ export function foldedToPprof(counts, periodNs, startMs, durationMs) {
44
+ const strings = [''];
45
+ const index = new Map([['', 0]]);
46
+ const s = (x) => {
47
+ let i = index.get(x);
48
+ if (i === undefined) {
49
+ i = strings.length;
50
+ strings.push(x);
51
+ index.set(x, i);
52
+ }
53
+ return i;
54
+ };
55
+ const vt = (type, unit) => new Pb().int(1, s(type)).int(2, s(unit));
56
+ const frames = new Map();
57
+ const out = new Pb().msg(1, vt('samples', 'count')).msg(1, vt('cpu', 'nanoseconds'));
58
+ for (const [key, n] of counts) {
59
+ const tab = key.indexOf('\t');
60
+ const labels = tab >= 0 ? key.slice(0, tab) : '';
61
+ // pprof location ids are leaf first
62
+ const ids = key.slice(tab + 1).split(';').reverse().map((f) => {
63
+ let id = frames.get(f);
64
+ if (id === undefined)
65
+ frames.set(f, (id = frames.size + 1));
66
+ return id;
67
+ });
68
+ const sample = new Pb().packed(1, ids).packed(2, [n, n * periodNs]);
69
+ for (const kv of labels ? labels.split(',') : []) {
70
+ const eq = kv.indexOf('=');
71
+ if (eq > 0)
72
+ sample.msg(3, new Pb().int(1, s(kv.slice(0, eq))).int(2, s(kv.slice(eq + 1))));
73
+ }
74
+ out.msg(2, sample);
75
+ }
76
+ for (const [f, id] of frames) {
77
+ out.msg(4, new Pb().int(1, id).msg(4, new Pb().int(1, id))); // Location{id, line[{function_id}]}
78
+ out.msg(5, new Pb().int(1, id).int(2, s(f))); // Function{id, name}
79
+ }
80
+ out.int(9, BigInt(Math.round(startMs)) * 1000000n).int(10, BigInt(Math.round(durationMs)) * 1000000n);
81
+ out.msg(11, vt('cpu', 'nanoseconds')).int(12, periodNs);
82
+ for (const str of strings)
83
+ out.bytes(6, Buffer.from(str));
84
+ return new Uint8Array(gzipSync(Uint8Array.from(out.b)));
85
+ }
@@ -48,7 +48,7 @@ export interface ProfilerOptions {
48
48
  tags?: Record<string, string | undefined>;
49
49
  /** Sampling interval in microseconds (default 10 000 = 100 Hz). */
50
50
  intervalUs?: number;
51
- /** Upload period (default 15 000 ms). */
51
+ /** Upload period (default 60 000 ms). */
52
52
  periodMs?: number;
53
53
  /** Label `withSpan` code after `await` too (see [[enableAsyncSpanLabels]]). Default false. */
54
54
  asyncSpans?: boolean;
@@ -56,6 +56,9 @@ export interface ProfilerOptions {
56
56
  /** Root-first folded stacks with their sample counts; V8's synthetic `(root)` is left out. */
57
57
  export declare function foldProfile(p: CpuProfile, spans?: SpanInterval[]): Map<string, number>;
58
58
  export declare function foldedBody(counts: Map<string, number>): string;
59
+ /** `/v1/profiles?format=pprof&service=…&version=…` on the gateway. */
60
+ export declare function profilesUrl(endpoint: string, service: string, version?: string): string;
61
+ /** Pyroscope-compatible `/ingest` URL for folded uploads (kept for callers; the Profiler sends pprof). */
59
62
  export declare function ingestUrl(endpoint: string, service: string, tags: Record<string, string | undefined>, fromS: number, untilS: number, hz: number): string;
60
63
  export declare class Profiler {
61
64
  uploaded: number;
@@ -1,12 +1,13 @@
1
1
  // Continuous CPU profiler on V8's own sampler (`node:inspector`, no native addon): every
2
- // `periodMs` the CPU profile is folded into stacks and posted to the gateway's
3
- // Pyroscope-compatible `/ingest` as `<service>.cpu` (format `folded`). Best effort: a failed
4
- // upload is counted and dropped. `withSpan` labels samples taken while its function runs, joined
2
+ // `periodMs` the CPU profile is folded into stacks, encoded as pprof and posted to the gateway's
3
+ // `/v1/profiles?format=pprof` with `service` and `version`. Best effort: a failed upload is
4
+ // counted and dropped. `withSpan` labels samples taken while its function runs, joined
5
5
  // on V8's sample timestamps, so a trace links to its exact profile.
6
6
  import { AsyncLocalStorage } from 'node:async_hooks';
7
7
  import { Session } from 'node:inspector';
8
8
  import { promiseHooks } from 'node:v8';
9
9
  import { trace } from '@opentelemetry/api';
10
+ import { foldedToPprof } from './pprof.js';
10
11
  const intervals = [];
11
12
  const MAX_INTERVALS = 100_000;
12
13
  function nowUs() {
@@ -140,6 +141,14 @@ export function foldedBody(counts) {
140
141
  out += `${k} ${n}\n`;
141
142
  return out;
142
143
  }
144
+ /** `/v1/profiles?format=pprof&service=…&version=…` on the gateway. */
145
+ export function profilesUrl(endpoint, service, version) {
146
+ const q = new URLSearchParams({ format: 'pprof', service });
147
+ if (version)
148
+ q.set('version', version);
149
+ return `${endpoint.replace(/\/+$/, '')}/v1/profiles?${q}`;
150
+ }
151
+ /** Pyroscope-compatible `/ingest` URL for folded uploads (kept for callers; the Profiler sends pprof). */
143
152
  export function ingestUrl(endpoint, service, tags, fromS, untilS, hz) {
144
153
  const labels = Object.entries(tags).filter(([, v]) => v).sort(([a], [b]) => a.localeCompare(b))
145
154
  .map(([k, v]) => `${k}=${v}`).join(',');
@@ -170,7 +179,7 @@ export class Profiler {
170
179
  this.service = o.service ?? process.env.WT_SERVICE ?? 'app';
171
180
  this.tags = { env: process.env.WT_ENVIRONMENT, version: process.env.WT_RELEASE, ...o.tags };
172
181
  this.intervalUs = Math.min(Math.max(o.intervalUs ?? 10_000, 1_000), 1_000_000);
173
- this.periodMs = Math.max(o.periodMs ?? 15_000, 1_000);
182
+ this.periodMs = Math.max(o.periodMs ?? 60_000, 1_000);
174
183
  if (o.asyncSpans)
175
184
  enableAsyncSpanLabels();
176
185
  }
@@ -213,10 +222,10 @@ export class Profiler {
213
222
  }
214
223
  async upload(counts, fromS, untilS) {
215
224
  try {
216
- const hz = Math.round(1_000_000 / this.intervalUs);
217
- const r = await fetch(ingestUrl(this.endpoint, this.service, this.tags, fromS, untilS, hz), {
218
- method: 'POST', body: foldedBody(counts),
219
- headers: { authorization: `Bearer ${this.key}`, 'content-type': 'text/plain' },
225
+ const body = foldedToPprof(counts, this.intervalUs * 1000, fromS * 1000, (untilS - fromS) * 1000);
226
+ const r = await fetch(profilesUrl(this.endpoint, this.service, this.tags.version), {
227
+ method: 'POST', body,
228
+ headers: { authorization: `Bearer ${this.key}`, 'content-type': 'application/octet-stream' },
220
229
  signal: AbortSignal.timeout(10_000),
221
230
  });
222
231
  if (r.ok)
@@ -0,0 +1,19 @@
1
+ /** Longest statement looked at, in UTF-8 bytes; the rest is dropped. */
2
+ export declare const MAX_STATEMENT_BYTES: number;
3
+ /**
4
+ * Literal-free statement text: strings, numbers and `$1` become `?`, comments go, whitespace
5
+ * collapses, `IN (?, ?)` becomes `IN (?+)`. `pg` keeps short `"identifiers"` and the `#` operator.
6
+ */
7
+ export declare function normalizeSql(stmt: string, pg?: boolean): string;
8
+ /** `[normalized text, decimal xxh3_64 of its UTF-8 bytes]`: the `db.query.hash` spans carry. */
9
+ export declare function normalizeAndHash(stmt: string, pg?: boolean): [string, string];
10
+ /** Systems whose dialect keeps `"identifier"` quotes and the `#` operator (as the worker decides). */
11
+ export declare const isPgDialect: (system: string | undefined) => system is "postgresql" | "cockroachdb" | "redshift";
12
+ /** `SELECT users`-style summary: the operation and first table, never values or the statement text. */
13
+ export declare function querySummary(text: string | null | undefined, fallback?: string): {
14
+ operation?: string;
15
+ target?: string;
16
+ summary: string;
17
+ };
18
+ /** XXH3_64bits(input), seed 0. */
19
+ export declare function xxh3(p: Uint8Array): bigint;
@@ -0,0 +1,324 @@
1
+ // Shared SQL normaliser and statement hash. A byte-for-byte port of the worker's
2
+ // services/worker/src/sqlnorm.rs (`normalize_with`, xxh3_64), so `db.query.hash` from the SDK equals
3
+ // the `wt.db.query_hash` the server stamps. Algorithm: docs/product/instrumentation/sql-normalization.md;
4
+ // vectors: testdata/sql_normalize_vectors.json. Change both sides together.
5
+ /** Longest statement looked at, in UTF-8 bytes; the rest is dropped. */
6
+ export const MAX_STATEMENT_BYTES = 8 * 1024;
7
+ const MAX_DOLLAR_TAG = 32;
8
+ const ALNUM = /^[\p{Alphabetic}\p{N}]$/u; // Rust char::is_alphanumeric
9
+ const SPACE = /^\p{White_Space}$/u; // Rust char::is_whitespace
10
+ const isAlnum = (c) => ALNUM.test(c);
11
+ const isWord = (c) => c === '_' || c === '$' || ALNUM.test(c);
12
+ const isDigit = (c) => c !== undefined && c >= '0' && c <= '9';
13
+ /** Code points of the first 8 KiB of UTF-8, cut back to a whole character. */
14
+ function cap(s) {
15
+ const out = [];
16
+ let bytes = 0;
17
+ for (const c of s) {
18
+ const cp = c.codePointAt(0);
19
+ bytes += cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4;
20
+ if (bytes > MAX_STATEMENT_BYTES)
21
+ break;
22
+ out.push(c);
23
+ }
24
+ return out;
25
+ }
26
+ const startsWith = (cs, at, tag) => tag.every((t, k) => cs[at + k] === t);
27
+ function dollarQuoted(cs, i) {
28
+ let tagLen = -1;
29
+ for (let k = 0; k < MAX_DOLLAR_TAG + 1 && i + 1 + k < cs.length; k++) {
30
+ if (cs[i + 1 + k] === '$') {
31
+ tagLen = k;
32
+ break;
33
+ }
34
+ }
35
+ if (tagLen < 0)
36
+ return undefined;
37
+ const tag = cs.slice(i, i + tagLen + 2);
38
+ if (!tag.slice(1, tagLen + 1).every((c) => isAlnum(c) || c === '_') || isDigit(tag[1]))
39
+ return undefined;
40
+ for (let j = i + tag.length; j < cs.length; j++)
41
+ if (startsWith(cs, j, tag))
42
+ return j + tag.length;
43
+ return cs.length;
44
+ }
45
+ function token(cs, i, pg, out) {
46
+ const c = cs[i];
47
+ if (c === "'") {
48
+ i++;
49
+ while (i < cs.length) {
50
+ if (cs[i] === "'") {
51
+ if (cs[i + 1] !== "'")
52
+ break;
53
+ i++;
54
+ }
55
+ i++;
56
+ }
57
+ out.push('?');
58
+ // a quote right after a backslash is ambiguous (MySQL escapes): mask to the end
59
+ return i < cs.length && cs[i - 1] === '\\' ? cs.length : i + 1;
60
+ }
61
+ if (c === '"' || c === '`') {
62
+ let close = cs.indexOf(c, i + 1);
63
+ if (close < 0)
64
+ close = cs.length;
65
+ const body = cs.slice(i + 1, close);
66
+ const ident = c === '"' && pg && close < cs.length && body.length >= 1 && body.length <= 64
67
+ && body.every((b) => isAlnum(b) || b === '_' || b === '$');
68
+ if (ident)
69
+ out.push(...cs.slice(i, close + 1));
70
+ else
71
+ out.push('?');
72
+ return close + 1;
73
+ }
74
+ if (c === '$') {
75
+ if (isDigit(cs[i + 1])) {
76
+ i++;
77
+ while (isDigit(cs[i]))
78
+ i++;
79
+ out.push('?');
80
+ return i;
81
+ }
82
+ const end = dollarQuoted(cs, i);
83
+ if (end !== undefined) {
84
+ out.push('?');
85
+ return end;
86
+ }
87
+ }
88
+ if (isDigit(c)) {
89
+ while (i < cs.length && (isWord(cs[i]) || cs[i] === '.'))
90
+ i++;
91
+ out.push('?');
92
+ }
93
+ else if (isWord(c)) {
94
+ while (i < cs.length && isWord(cs[i]))
95
+ out.push(cs[i++]);
96
+ }
97
+ else {
98
+ out.push(c);
99
+ i++;
100
+ }
101
+ return i;
102
+ }
103
+ const enc = new TextEncoder();
104
+ const dec = new TextDecoder();
105
+ const SP = 0x20;
106
+ /** After `in`: optional spaces, `(`, `?` items split by commas, `)`. Returns [open, end]. */
107
+ function inList(b, i) {
108
+ const skip = () => { while (b[i] === SP)
109
+ i++; };
110
+ skip();
111
+ const open = i;
112
+ if (b[i] !== 0x28)
113
+ return undefined;
114
+ i++;
115
+ for (;;) {
116
+ skip();
117
+ if (b[i] !== 0x3f)
118
+ return undefined;
119
+ i++;
120
+ skip();
121
+ if (b[i] === 0x2c)
122
+ i++;
123
+ else if (b[i] === 0x29)
124
+ return [open, i + 1];
125
+ else
126
+ return undefined;
127
+ }
128
+ }
129
+ /** `in (?, ?, ?)` -> `in (?+)`. On UTF-8 bytes like the Rust: the word check reads the previous byte as Latin-1. */
130
+ function collapseIn(s) {
131
+ const out = [];
132
+ let i = 0;
133
+ while (i < s.length) {
134
+ const atWord = i === 0 || !isWord(String.fromCharCode(s[i - 1]));
135
+ const span = atWord && i + 2 <= s.length && (s[i] | 0x20) === 0x69 && (s[i + 1] | 0x20) === 0x6e
136
+ ? inList(s, i + 2) : undefined;
137
+ if (span) {
138
+ for (let k = i; k < span[0]; k++)
139
+ out.push(s[k]);
140
+ out.push(0x28, 0x3f, 0x2b, 0x29);
141
+ i = span[1];
142
+ }
143
+ else {
144
+ out.push(s[i++]);
145
+ }
146
+ }
147
+ return Uint8Array.from(out);
148
+ }
149
+ function normalizeBytes(stmt, pg) {
150
+ const cs = cap(stmt);
151
+ const out = [];
152
+ let space = false;
153
+ let i = 0;
154
+ while (i < cs.length) {
155
+ const c = cs[i];
156
+ const next = cs[i + 1];
157
+ if (SPACE.test(c)) {
158
+ space = true;
159
+ i++;
160
+ }
161
+ else if (c === '/' && next === '*') {
162
+ let depth = 1;
163
+ i += 2;
164
+ while (i < cs.length && depth > 0) {
165
+ if (cs[i] === '/' && cs[i + 1] === '*') {
166
+ depth++;
167
+ i += 2;
168
+ }
169
+ else if (cs[i] === '*' && cs[i + 1] === '/') {
170
+ depth--;
171
+ i += 2;
172
+ }
173
+ else
174
+ i++;
175
+ }
176
+ space = true;
177
+ }
178
+ else if ((c === '-' && next === '-') || (c === '#' && !pg)) {
179
+ while (i < cs.length && cs[i] !== '\n')
180
+ i++;
181
+ space = true;
182
+ }
183
+ else {
184
+ if (space && out.length)
185
+ out.push(' ');
186
+ space = false;
187
+ i = token(cs, i, pg, out);
188
+ }
189
+ }
190
+ return collapseIn(enc.encode(out.join('')));
191
+ }
192
+ /**
193
+ * Literal-free statement text: strings, numbers and `$1` become `?`, comments go, whitespace
194
+ * collapses, `IN (?, ?)` becomes `IN (?+)`. `pg` keeps short `"identifiers"` and the `#` operator.
195
+ */
196
+ export function normalizeSql(stmt, pg = false) {
197
+ return dec.decode(normalizeBytes(stmt, pg));
198
+ }
199
+ /** `[normalized text, decimal xxh3_64 of its UTF-8 bytes]`: the `db.query.hash` spans carry. */
200
+ export function normalizeAndHash(stmt, pg = false) {
201
+ const b = normalizeBytes(stmt, pg);
202
+ return [dec.decode(b), xxh3(b).toString()];
203
+ }
204
+ /** Systems whose dialect keeps `"identifier"` quotes and the `#` operator (as the worker decides). */
205
+ export const isPgDialect = (system) => system === 'postgresql' || system === 'cockroachdb' || system === 'redshift';
206
+ const OP = /^\s*(?:with\b[\s\S]*?\)\s*)?([a-z]+)/i;
207
+ const TARGET = /\b(?:from|into|update|join|table)\s+((?:"[^"]+"|[\w$]+)(?:\.(?:"[^"]+"|[\w$]+))?)/i;
208
+ /** `SELECT users`-style summary: the operation and first table, never values or the statement text. */
209
+ export function querySummary(text, fallback = 'postgres') {
210
+ const operation = text?.match(OP)?.[1]?.toUpperCase();
211
+ const target = text?.match(TARGET)?.[1]?.replace(/"/g, '');
212
+ return { operation, target, summary: [operation ?? fallback, target].filter(Boolean).join(' ') };
213
+ }
214
+ // ---- XXH3-64, seed 0, default secret (port of the xxHash reference; BigInt is fine at <= 8 KiB) ----
215
+ const M64 = (1n << 64n) - 1n;
216
+ const P32_1 = 0x9e3779b1n, P32_2 = 0x85ebca77n, P32_3 = 0xc2b2ae3dn;
217
+ const P64_1 = 0x9e3779b185ebca87n, P64_2 = 0xc2b2ae3d27d4eb4fn, P64_3 = 0x165667b19e3779f9n;
218
+ const P64_4 = 0x85ebca77c2b2ae63n, P64_5 = 0x27d4eb2f165667c5n;
219
+ const SECRET = Uint8Array.from(Buffer.from('b8fe6c3923a44bbe7c01812cf721ad1cded46de9839097db7240a4a4b7b3671fcb79e64eccc0e578825ad07dccff7221'
220
+ + 'b8084674f743248ee03590e6813a264c3c2852bb91c300cb88d0658b1b532ea371644897a20df94e3819ef46a9deacd8'
221
+ + 'a8fa763fe39c343ff9dcbbc7c70b4f1d8a51e04bcdb45931c89f7ec9d9787364eac5ac8334d3ebc3c581a0fffa1363eb'
222
+ + '170ddd51b7f0da49d316552629d4689e2b16be587d47a1fc8ff8b8d17ad031ce45cb3a8f95160428afd7fbcabb4b407e', 'hex'));
223
+ const r64 = (b, o) => new DataView(b.buffer, b.byteOffset + o, 8).getBigUint64(0, true);
224
+ const r32 = (b, o) => BigInt(new DataView(b.buffer, b.byteOffset + o, 4).getUint32(0, true));
225
+ const mul = (a, b) => (a * b) & M64;
226
+ const add = (a, b) => (a + b) & M64;
227
+ const rotl = (x, r) => ((x << r) | (x >> (64n - r))) & M64;
228
+ const swap64 = (x) => {
229
+ let r = 0n;
230
+ for (let k = 0; k < 8; k++) {
231
+ r = (r << 8n) | (x & 0xffn);
232
+ x >>= 8n;
233
+ }
234
+ return r;
235
+ };
236
+ const fold = (a, b) => { const p = a * b; return (p ^ (p >> 64n)) & M64; };
237
+ const avalanche = (h) => { h ^= h >> 37n; h = mul(h, 0x165667919e3779f9n); return h ^ (h >> 32n); };
238
+ const avalanche64 = (h) => {
239
+ h ^= h >> 33n;
240
+ h = mul(h, P64_2);
241
+ h ^= h >> 29n;
242
+ h = mul(h, P64_3);
243
+ return h ^ (h >> 32n);
244
+ };
245
+ const mix16 = (p, o, s) => fold(r64(p, o) ^ r64(SECRET, s), r64(p, o + 8) ^ r64(SECRET, s + 8));
246
+ function accumulate512(acc, p, o, s) {
247
+ for (let i = 0; i < 8; i++) {
248
+ const v = r64(p, o + 8 * i);
249
+ const k = v ^ r64(SECRET, s + 8 * i);
250
+ acc[i ^ 1] = add(acc[i ^ 1], v);
251
+ acc[i] = add(acc[i], (k & 0xffffffffn) * (k >> 32n));
252
+ }
253
+ }
254
+ function long(p) {
255
+ const len = p.length;
256
+ const acc = [P32_3, P64_1, P64_2, P64_3, P64_4, P32_2, P64_5, P32_1];
257
+ const stripesPerBlock = (SECRET.length - 64) / 8;
258
+ const block = 64 * stripesPerBlock;
259
+ const blocks = Math.floor((len - 1) / block);
260
+ for (let n = 0; n < blocks; n++) {
261
+ for (let s = 0; s < stripesPerBlock; s++)
262
+ accumulate512(acc, p, n * block + s * 64, s * 8);
263
+ for (let i = 0; i < 8; i++) {
264
+ let a = acc[i];
265
+ a ^= a >> 47n;
266
+ a ^= r64(SECRET, SECRET.length - 64 + 8 * i);
267
+ acc[i] = mul(a, P32_1);
268
+ }
269
+ }
270
+ const stripes = Math.floor((len - 1 - block * blocks) / 64);
271
+ for (let s = 0; s < stripes; s++)
272
+ accumulate512(acc, p, blocks * block + s * 64, s * 8);
273
+ accumulate512(acc, p, len - 64, SECRET.length - 64 - 7);
274
+ let r = mul(BigInt(len), P64_1);
275
+ for (let i = 0; i < 4; i++)
276
+ r = add(r, fold(acc[2 * i] ^ r64(SECRET, 11 + 16 * i), acc[2 * i + 1] ^ r64(SECRET, 19 + 16 * i)));
277
+ return avalanche(r);
278
+ }
279
+ /** XXH3_64bits(input), seed 0. */
280
+ export function xxh3(p) {
281
+ const len = p.length;
282
+ const L = BigInt(len);
283
+ if (len === 0)
284
+ return avalanche64(r64(SECRET, 56) ^ r64(SECRET, 64));
285
+ if (len <= 3) {
286
+ const combined = (BigInt(p[0]) << 16n) | (BigInt(p[len >> 1]) << 24n) | BigInt(p[len - 1]) | (L << 8n);
287
+ return avalanche64(combined ^ (r32(SECRET, 0) ^ r32(SECRET, 4)));
288
+ }
289
+ if (len <= 8) {
290
+ const keyed = (r32(p, len - 4) + (r32(p, 0) << 32n)) ^ (r64(SECRET, 8) ^ r64(SECRET, 16));
291
+ let h = keyed ^ rotl(keyed, 49n) ^ rotl(keyed, 24n);
292
+ h = mul(h, 0x9fb21c651e98df25n);
293
+ h ^= add(h >> 35n, L);
294
+ h = mul(h, 0x9fb21c651e98df25n);
295
+ return h ^ (h >> 28n);
296
+ }
297
+ if (len <= 16) {
298
+ const lo = r64(p, 0) ^ (r64(SECRET, 24) ^ r64(SECRET, 32));
299
+ const hi = r64(p, len - 8) ^ (r64(SECRET, 40) ^ r64(SECRET, 48));
300
+ return avalanche(add(add(add(L, swap64(lo)), hi), fold(lo, hi)));
301
+ }
302
+ if (len <= 128) {
303
+ let acc = mul(L, P64_1);
304
+ if (len > 32) {
305
+ if (len > 64) {
306
+ if (len > 96)
307
+ acc = add(add(acc, mix16(p, 48, 96)), mix16(p, len - 64, 112));
308
+ acc = add(add(acc, mix16(p, 32, 64)), mix16(p, len - 48, 80));
309
+ }
310
+ acc = add(add(acc, mix16(p, 16, 32)), mix16(p, len - 32, 48));
311
+ }
312
+ return avalanche(add(add(acc, mix16(p, 0, 0)), mix16(p, len - 16, 16)));
313
+ }
314
+ if (len <= 240) {
315
+ let acc = mul(L, P64_1);
316
+ for (let i = 0; i < 8; i++)
317
+ acc = add(acc, mix16(p, 16 * i, 16 * i));
318
+ acc = avalanche(acc);
319
+ for (let i = 8; i < Math.floor(len / 16); i++)
320
+ acc = add(acc, mix16(p, 16 * i, 16 * (i - 8) + 3));
321
+ return avalanche(add(acc, mix16(p, len - 16, 136 - 17)));
322
+ }
323
+ return long(p);
324
+ }
@@ -46,13 +46,15 @@ export interface PlanTask {
46
46
  }[];
47
47
  };
48
48
  }
49
- /** Vitest's `beforeAll`/`afterEach`, passed in so this package does not import Vitest. */
49
+ /** Vitest's `beforeAll`/`afterEach`/`afterAll`, passed in so this package does not import Vitest. */
50
50
  export interface VitestHooks {
51
51
  beforeAll(fn: (suite: PlanTask) => unknown): void;
52
52
  afterEach(fn: (ctx: {
53
53
  task: PlanTask;
54
54
  skip(note?: string): void;
55
55
  }) => unknown): void;
56
+ /** needed for WT_TIA_RECORD=1 */
57
+ afterAll?(fn: (suite: PlanTask) => unknown): void;
56
58
  }
57
59
  /** Applies the plan in a Vitest setup file. Decisions go to `task.meta.wt` for the reporter. */
58
60
  export declare function vitestSetup(hooks: VitestHooks): void;
@@ -90,4 +92,11 @@ export declare function jestEnvironment<B extends JestEnvBase>(Base: B): B;
90
92
  export declare function toJunit(suite: string, tests: TestRecord[]): string;
91
93
  /** Posts the results as JUnit when WT_CI_RUN_ID is set; never throws. */
92
94
  export declare function uploadResults(client: Client | null, suite: string, tests: TestRecord[]): Promise<boolean>;
95
+ /** Repository-relative files with an executed statement in an istanbul coverage object. */
96
+ export declare function coveredFiles(cov: unknown, root?: string): string[];
97
+ /**
98
+ * Uploads the coverage of one test file for its tests (`suite::rest` ids); never throws.
99
+ * Needs WT_API_URL, WT_PROJECT_ID and WT_API_KEY (the coverage map needs a project API key).
100
+ */
101
+ export declare function uploadCoverage(file: string | undefined, ids: string[], cov: unknown, fetchImpl?: typeof globalThis.fetch): Promise<boolean>;
93
102
  export {};
@@ -10,13 +10,17 @@
10
10
  // - a test the project has never seen is retried up to `efd_retries` times (early flake detection).
11
11
  // The reporters upload the results as JUnit (`POST /v1/projects/{prj}/test-runs`) when
12
12
  // WT_CI_RUN_ID is set, which feeds flaky-test detection and auto-quarantine.
13
+ // With WT_TIA_RECORD=1 (and WT_API_KEY) each test file's istanbul coverage (Vitest
14
+ // `coverage.provider: 'istanbul'`, Jest `--coverage`) is uploaded as the per-test coverage map
15
+ // (`PUT /v1/projects/{prj}/tests/coverage-map`): every test of the file gets the files the file
16
+ // executed plus the test file itself, which Test Impact Analysis then uses to skip tests.
13
17
  //
14
18
  // Ids are `<suite>::<file relative to cwd> > <describe>... > <test>`, suite = WT_TEST_SUITE or
15
19
  // the framework name. Vitest:
16
20
  // globalSetup: ['./wt-global.mjs'] // export { default } from '@watchtowerobservability/node/test-plan'
17
21
  // setupFiles: ['./wt-setup.ts'] // import { beforeAll, afterEach } from 'vitest';
18
22
  // // import { vitestSetup } from '@watchtowerobservability/node/test-plan';
19
- // // vitestSetup({ beforeAll, afterEach });
23
+ // // vitestSetup({ beforeAll, afterEach, afterAll });
20
24
  // Jest: globalSetup as above, and testEnvironment: './wt-env.js' with
21
25
  // module.exports = require('@watchtowerobservability/node/test-plan').jestEnvironment(require('jest-environment-node').TestEnvironment);
22
26
  import { execFileSync } from 'node:child_process';
@@ -120,8 +124,24 @@ export function decider(plan) {
120
124
  return d;
121
125
  };
122
126
  }
127
+ /** Ids of the tests in `t` that ran (a skipped test executed nothing, so it gets no coverage). */
128
+ function ranIds(t, suite, file, names, out) {
129
+ for (const c of t.tasks || []) {
130
+ const ran = c.mode !== 'skip' && c.mode !== 'todo' && !!c.result?.state && c.result.state !== 'skip';
131
+ if (c.tasks)
132
+ ranIds(c, suite, file, [...names, c.name], out);
133
+ else if (ran)
134
+ out.push(testId(suite, file, [...names, c.name]));
135
+ }
136
+ return out;
137
+ }
123
138
  /** Applies the plan in a Vitest setup file. Decisions go to `task.meta.wt` for the reporter. */
124
139
  export function vitestSetup(hooks) {
140
+ if (process.env.WT_TIA_RECORD === '1' && hooks.afterAll) {
141
+ const suite = suiteName('vitest');
142
+ const g = globalThis;
143
+ hooks.afterAll((file) => uploadCoverage(file.filepath, ranIds(file, suite, file.filepath, [], []), g.__VITEST_COVERAGE__ || g.__coverage__));
144
+ }
125
145
  const plan = loadPlan();
126
146
  if (!plan)
127
147
  return;
@@ -165,6 +185,7 @@ export function jestEnvironment(Base) {
165
185
  wtDecide = decider(loadPlan());
166
186
  wtSuite = suiteName('jest');
167
187
  wtRetrySaved = [];
188
+ wtRan = [];
168
189
  // any[]: see JestEnvBase
169
190
  constructor(...args) {
170
191
  super(...args);
@@ -180,6 +201,12 @@ export function jestEnvironment(Base) {
180
201
  if (super.handleTestEvent)
181
202
  await super.handleTestEvent(event, state);
182
203
  const t = event.test;
204
+ if (event.name === 'test_done' && t && process.env.WT_TIA_RECORD === '1')
205
+ this.wtRan.push(this.wtId(t));
206
+ if (event.name === 'run_finish' && process.env.WT_TIA_RECORD === '1') {
207
+ await uploadCoverage(this.wtFile, this.wtRan, this.global.__coverage__);
208
+ return;
209
+ }
183
210
  if (event.name === 'test_start' && t) {
184
211
  if (this.wtDecide(this.wtId(t)).skip)
185
212
  t.mode = 'skip';
@@ -246,3 +273,64 @@ export async function uploadResults(client, suite, tests) {
246
273
  return false;
247
274
  }
248
275
  }
276
+ // ---------------------------------------------------------------- Test Impact Analysis recording
277
+ let rootCache;
278
+ /** The repository top level (git diff paths are relative to it), else the working directory. */
279
+ function repoRoot() {
280
+ if (rootCache === undefined) {
281
+ try {
282
+ rootCache = execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 10_000 }).trim() || process.cwd();
283
+ }
284
+ catch (_e) {
285
+ rootCache = process.cwd();
286
+ }
287
+ }
288
+ return rootCache;
289
+ }
290
+ const repoRel = (f, root) => relative(root, f).split(sep).join('/');
291
+ /** Repository-relative files with an executed statement in an istanbul coverage object. */
292
+ export function coveredFiles(cov, root = repoRoot()) {
293
+ const out = new Set();
294
+ if (!cov || typeof cov !== 'object')
295
+ return [];
296
+ for (const [key, fc] of Object.entries(cov)) {
297
+ if (!fc || !fc.s || !Object.values(fc.s).some((n) => n > 0))
298
+ continue;
299
+ const f = repoRel(fc.path || key, root);
300
+ if (f && !f.startsWith('..'))
301
+ out.add(f);
302
+ }
303
+ return [...out].sort();
304
+ }
305
+ /**
306
+ * Uploads the coverage of one test file for its tests (`suite::rest` ids); never throws.
307
+ * Needs WT_API_URL, WT_PROJECT_ID and WT_API_KEY (the coverage map needs a project API key).
308
+ */
309
+ export async function uploadCoverage(file, ids, cov, fetchImpl = globalThis.fetch) {
310
+ const env = process.env;
311
+ const [api, project, key] = [env.WT_API_URL, env.WT_PROJECT_ID, env.WT_API_KEY];
312
+ if (!api || !project || !key || ids.length === 0)
313
+ return false;
314
+ const root = repoRoot();
315
+ const files = coveredFiles(cov, root);
316
+ if (file) {
317
+ const self = repoRel(file, root);
318
+ if (self && !self.startsWith('..') && !files.includes(self))
319
+ files.push(self);
320
+ }
321
+ const tests = [...new Set(ids)].slice(0, 20_000).map((id) => {
322
+ const i = id.indexOf('::');
323
+ return { suite: i < 0 ? '' : id.slice(0, i), test: i < 0 ? id : id.slice(i + 2), files: files.slice(0, 2000) };
324
+ });
325
+ try {
326
+ const res = await fetchImpl(`${api.replace(/\/+$/, '')}/v1/projects/${encodeURIComponent(project)}/tests/coverage-map`, {
327
+ method: 'PUT',
328
+ headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
329
+ body: JSON.stringify({ sha: (env.WT_SHA || env.GITHUB_SHA || env.CI_COMMIT_SHA || '').slice(0, 64), tests }),
330
+ });
331
+ return res.ok;
332
+ }
333
+ catch (_e) {
334
+ return false;
335
+ }
336
+ }