@steve31415/baselib 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/check-test-owners.js +0 -0
- package/dist/http.js +5 -2
- package/dist/log-browser.js +8 -1
- package/dist/log-core.d.ts +60 -4
- package/dist/log-core.js +258 -23
- package/dist/log.d.ts +19 -4
- package/dist/log.js +75 -21
- package/package.json +1 -1
|
File without changes
|
package/dist/http.js
CHANGED
|
@@ -57,12 +57,15 @@ export function startServer(opts) {
|
|
|
57
57
|
});
|
|
58
58
|
process.once('SIGTERM', () => {
|
|
59
59
|
opts.logger.info('SIGTERM received, shutting down');
|
|
60
|
+
// final: last-chance delivery + drop markers, time-boxed. The two racing
|
|
61
|
+
// callers below join the same single-flight finalFlush, so whichever
|
|
62
|
+
// fires second waits for the real work instead of exiting under it.
|
|
60
63
|
server.close(() => {
|
|
61
|
-
void flushAllLoggers().finally(() => process.exit(0));
|
|
64
|
+
void flushAllLoggers({ final: true }).finally(() => process.exit(0));
|
|
62
65
|
});
|
|
63
66
|
// Cloud Run's grace period is 10s; don't rely on every socket closing.
|
|
64
67
|
setTimeout(() => {
|
|
65
|
-
void flushAllLoggers().finally(() => process.exit(0));
|
|
68
|
+
void flushAllLoggers({ final: true }).finally(() => process.exit(0));
|
|
66
69
|
}, 5000).unref();
|
|
67
70
|
});
|
|
68
71
|
return server;
|
package/dist/log-browser.js
CHANGED
|
@@ -23,9 +23,16 @@ export function createBrowserLogger(opts = {}) {
|
|
|
23
23
|
? new LogShipper({
|
|
24
24
|
url: `${baseUrl}/v1/datasets/${dataset}/ingest`,
|
|
25
25
|
token,
|
|
26
|
+
// Chromium rejects keepalive request bodies over 64 KiB outright, so
|
|
27
|
+
// keep batches small; keepalive lets a flush started during page
|
|
28
|
+
// unload complete. Retries run while the page lives; page-death
|
|
29
|
+
// remains best-effort, so a browser give-up is only console-noted.
|
|
26
30
|
batchSize: 20,
|
|
27
|
-
// keepalive lets a flush started during page unload complete.
|
|
28
31
|
fetchInit: { keepalive: true },
|
|
32
|
+
onGiveUp: (batch, error, reason) => console.warn(`log ship dropped a batch (${reason}): ${String(error)}`, {
|
|
33
|
+
batch_id: batch.id,
|
|
34
|
+
events: batch.events.length,
|
|
35
|
+
}),
|
|
29
36
|
})
|
|
30
37
|
: undefined;
|
|
31
38
|
function build(context) {
|
package/dist/log-core.d.ts
CHANGED
|
@@ -28,29 +28,85 @@ export interface SerializedError {
|
|
|
28
28
|
}
|
|
29
29
|
export declare function serializeError(err: unknown): SerializedError;
|
|
30
30
|
export declare function truncate(s: string, max: number): string;
|
|
31
|
+
export type ShipFailureReason = 'give-up' | 'buffer-overflow' | 'shutdown';
|
|
32
|
+
/** What failure callbacks see of a batch. `attempt` is attempts made so far. */
|
|
33
|
+
export interface ShippedBatch {
|
|
34
|
+
id: string;
|
|
35
|
+
events: LogEvent[];
|
|
36
|
+
bytes: number;
|
|
37
|
+
firstTime: string;
|
|
38
|
+
lastTime: string;
|
|
39
|
+
attempt: number;
|
|
40
|
+
maxAttempts: number;
|
|
41
|
+
}
|
|
31
42
|
export interface ShipperOptions {
|
|
32
43
|
/** Full ingest URL, e.g. https://api.axiom.co/v1/datasets/apps/ingest */
|
|
33
44
|
url: string;
|
|
34
45
|
token: string;
|
|
35
46
|
batchSize?: number;
|
|
36
47
|
intervalMs?: number;
|
|
48
|
+
/** Total delivery attempts per batch (default 4). */
|
|
49
|
+
maxAttempts?: number;
|
|
50
|
+
/** Backoff before attempt 2, 3, … (default [5s, 15s, 40s]). On an idle
|
|
51
|
+
* Cloud Run instance the CPU is throttled between requests, so these
|
|
52
|
+
* fire at the next CPU window (next request or shutdown), not on the
|
|
53
|
+
* wall clock. */
|
|
54
|
+
retryDelaysMs?: number[];
|
|
55
|
+
/** Per-attempt fetch timeout (default 3s). */
|
|
56
|
+
attemptTimeoutMs?: number;
|
|
57
|
+
/** Caps across ALL undelivered batches; overflow drops oldest with an
|
|
58
|
+
* onGiveUp('buffer-overflow'). Defaults 5000 events / 5 MB. */
|
|
59
|
+
maxBufferedEvents?: number;
|
|
60
|
+
maxBufferedBytes?: number;
|
|
37
61
|
fetchFn?: typeof fetch;
|
|
38
62
|
/** Extra fetch options (browser sets keepalive). */
|
|
39
63
|
fetchInit?: RequestInit;
|
|
40
|
-
/**
|
|
41
|
-
|
|
64
|
+
/** A delivery attempt failed; the batch will be retried. */
|
|
65
|
+
onAttemptFailure?: (batch: ShippedBatch, error: unknown) => void;
|
|
66
|
+
/** The batch is abandoned — its events will never reach Axiom. */
|
|
67
|
+
onGiveUp?: (batch: ShippedBatch, error: unknown, reason: ShipFailureReason) => void;
|
|
42
68
|
}
|
|
43
69
|
export declare class LogShipper {
|
|
44
70
|
private readonly opts;
|
|
45
71
|
private queue;
|
|
46
|
-
private
|
|
47
|
-
private
|
|
72
|
+
private pending;
|
|
73
|
+
private totalEvents;
|
|
74
|
+
private totalBytes;
|
|
75
|
+
private batchTimer;
|
|
76
|
+
private pumpTimer;
|
|
77
|
+
private pumping;
|
|
78
|
+
private closed;
|
|
79
|
+
private finalPromise;
|
|
80
|
+
private readonly inflight;
|
|
48
81
|
private readonly batchSize;
|
|
49
82
|
private readonly intervalMs;
|
|
83
|
+
private readonly maxAttempts;
|
|
84
|
+
private readonly retryDelaysMs;
|
|
85
|
+
private readonly attemptTimeoutMs;
|
|
86
|
+
private readonly maxBufferedEvents;
|
|
87
|
+
private readonly maxBufferedBytes;
|
|
50
88
|
constructor(opts: ShipperOptions);
|
|
51
89
|
enqueue(event: LogEvent): void;
|
|
90
|
+
/** Form a batch from the queue and give it (and any mature retries) an
|
|
91
|
+
* attempt. Resolves once this call's batch has its first attempt settled
|
|
92
|
+
* — raced with one attempt timeout, so it is bounded even when the pump
|
|
93
|
+
* is busy with older batches. */
|
|
52
94
|
flush(): Promise<void>;
|
|
95
|
+
/** Shutdown path: one parallel last-chance attempt for everything
|
|
96
|
+
* undelivered, then a sweep that marks the rest dropped. Single-flight
|
|
97
|
+
* while in flight (racing SIGTERM callers join one promise), but re-arms
|
|
98
|
+
* after settling — a mid-run caller that logs afterward must not find a
|
|
99
|
+
* stale settled promise that no-ops its real shutdown later. */
|
|
100
|
+
finalFlush(deadlineMs?: number): Promise<void>;
|
|
101
|
+
private formBatch;
|
|
102
|
+
private remove;
|
|
103
|
+
private giveUp;
|
|
104
|
+
private pump;
|
|
105
|
+
private schedulePump;
|
|
106
|
+
private attempt;
|
|
107
|
+
/** Returns the failure, or undefined on success. Never throws. */
|
|
53
108
|
private ship;
|
|
109
|
+
private runFinal;
|
|
54
110
|
}
|
|
55
111
|
export interface EventFactoryOptions {
|
|
56
112
|
app: string;
|
package/dist/log-core.js
CHANGED
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
// serialization. Used by both the server logger (log.ts) and the browser
|
|
3
3
|
// logger (log-browser.ts). Ships to Axiom's native JSON ingest — chosen over
|
|
4
4
|
// OTLP for its synchronous commit and clean top-level field names (design:
|
|
5
|
-
// ~/migration/research/base-services-design.md §4).
|
|
5
|
+
// ~/migration/research/base-services-design.md §4). Delivery is hardened with
|
|
6
|
+
// bounded, backed-off retries and per-batch identity
|
|
7
|
+
// (~/migration/research/logging-reliability-design.md): every undelivered
|
|
8
|
+
// batch either lands in Axiom or ends in an onGiveUp callback — the server
|
|
9
|
+
// logger turns those into stdout drop markers that watchdog2's daily
|
|
10
|
+
// ship-failure check counts.
|
|
6
11
|
export function serializeError(err) {
|
|
7
12
|
if (err instanceof Error) {
|
|
8
13
|
const out = { name: err.name, message: err.message, stack: err.stack };
|
|
@@ -19,45 +24,224 @@ export function serializeError(err) {
|
|
|
19
24
|
export function truncate(s, max) {
|
|
20
25
|
return s.length <= max ? s : s.slice(0, max) + `…[+${s.length - max}]`;
|
|
21
26
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
27
|
+
function randomId() {
|
|
28
|
+
const c = globalThis.crypto;
|
|
29
|
+
return c?.randomUUID?.() ?? Math.random().toString(36).slice(2) + Date.now().toString(36);
|
|
30
|
+
}
|
|
31
|
+
function unrefTimer(t) {
|
|
32
|
+
;
|
|
33
|
+
t.unref?.();
|
|
34
|
+
}
|
|
35
|
+
/** Resolve when `p` does, or after `ms` (unref'd — never holds the process). */
|
|
36
|
+
function raceTimeout(p, ms) {
|
|
37
|
+
return new Promise((resolve) => {
|
|
38
|
+
const t = setTimeout(resolve, ms);
|
|
39
|
+
unrefTimer(t);
|
|
40
|
+
void p.then(() => { clearTimeout(t); resolve(); }, () => { clearTimeout(t); resolve(); });
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
// Batches events and ships them with plain fetch. enqueue() never throws.
|
|
44
|
+
// One capped `pending` store holds every undelivered batch; a single serial
|
|
45
|
+
// pump attempts one at a time; finalFlush() is the time-boxed, single-flight
|
|
46
|
+
// shutdown path (parallel last-chance attempts, then a sweep that marks
|
|
47
|
+
// everything undelivered as dropped).
|
|
25
48
|
export class LogShipper {
|
|
26
49
|
opts;
|
|
27
50
|
queue = [];
|
|
28
|
-
|
|
29
|
-
|
|
51
|
+
pending = [];
|
|
52
|
+
totalEvents = 0;
|
|
53
|
+
totalBytes = 0;
|
|
54
|
+
batchTimer;
|
|
55
|
+
pumpTimer;
|
|
56
|
+
pumping = false;
|
|
57
|
+
closed = false;
|
|
58
|
+
finalPromise;
|
|
59
|
+
inflight = new Set();
|
|
30
60
|
batchSize;
|
|
31
61
|
intervalMs;
|
|
62
|
+
maxAttempts;
|
|
63
|
+
retryDelaysMs;
|
|
64
|
+
attemptTimeoutMs;
|
|
65
|
+
maxBufferedEvents;
|
|
66
|
+
maxBufferedBytes;
|
|
32
67
|
constructor(opts) {
|
|
33
68
|
this.opts = opts;
|
|
34
69
|
this.batchSize = opts.batchSize ?? 50;
|
|
35
70
|
this.intervalMs = opts.intervalMs ?? 2000;
|
|
71
|
+
this.maxAttempts = opts.maxAttempts ?? 4;
|
|
72
|
+
this.retryDelaysMs = opts.retryDelaysMs ?? [5_000, 15_000, 40_000];
|
|
73
|
+
this.attemptTimeoutMs = opts.attemptTimeoutMs ?? 3_000;
|
|
74
|
+
this.maxBufferedEvents = opts.maxBufferedEvents ?? 5_000;
|
|
75
|
+
this.maxBufferedBytes = opts.maxBufferedBytes ?? 5 * 1024 * 1024;
|
|
36
76
|
}
|
|
37
77
|
enqueue(event) {
|
|
38
78
|
this.queue.push(event);
|
|
39
79
|
if (this.queue.length >= this.batchSize) {
|
|
40
80
|
void this.flush();
|
|
41
81
|
}
|
|
42
|
-
else if (!this.
|
|
43
|
-
this.
|
|
44
|
-
|
|
82
|
+
else if (!this.batchTimer) {
|
|
83
|
+
this.batchTimer = setTimeout(() => void this.flush(), this.intervalMs);
|
|
84
|
+
// In Node, don't hold the process open just to flush logs.
|
|
85
|
+
unrefTimer(this.batchTimer);
|
|
45
86
|
}
|
|
46
87
|
}
|
|
88
|
+
/** Form a batch from the queue and give it (and any mature retries) an
|
|
89
|
+
* attempt. Resolves once this call's batch has its first attempt settled
|
|
90
|
+
* — raced with one attempt timeout, so it is bounded even when the pump
|
|
91
|
+
* is busy with older batches. */
|
|
47
92
|
flush() {
|
|
48
|
-
if (this.
|
|
49
|
-
clearTimeout(this.
|
|
50
|
-
this.
|
|
93
|
+
if (this.batchTimer) {
|
|
94
|
+
clearTimeout(this.batchTimer);
|
|
95
|
+
this.batchTimer = undefined;
|
|
51
96
|
}
|
|
97
|
+
const batch = this.formBatch();
|
|
98
|
+
void this.pump();
|
|
99
|
+
const cap = this.attemptTimeoutMs + 500;
|
|
100
|
+
if (batch)
|
|
101
|
+
return raceTimeout(batch.firstAttempt, cap);
|
|
102
|
+
if (this.inflight.size > 0)
|
|
103
|
+
return raceTimeout(Promise.allSettled([...this.inflight]), cap);
|
|
104
|
+
return Promise.resolve();
|
|
105
|
+
}
|
|
106
|
+
/** Shutdown path: one parallel last-chance attempt for everything
|
|
107
|
+
* undelivered, then a sweep that marks the rest dropped. Single-flight
|
|
108
|
+
* while in flight (racing SIGTERM callers join one promise), but re-arms
|
|
109
|
+
* after settling — a mid-run caller that logs afterward must not find a
|
|
110
|
+
* stale settled promise that no-ops its real shutdown later. */
|
|
111
|
+
finalFlush(deadlineMs = 3_000) {
|
|
112
|
+
this.finalPromise ??= this.runFinal(deadlineMs).finally(() => {
|
|
113
|
+
this.finalPromise = undefined;
|
|
114
|
+
this.closed = false;
|
|
115
|
+
});
|
|
116
|
+
return this.finalPromise;
|
|
117
|
+
}
|
|
118
|
+
formBatch() {
|
|
52
119
|
if (this.queue.length === 0)
|
|
53
|
-
return
|
|
54
|
-
const
|
|
120
|
+
return undefined;
|
|
121
|
+
const events = this.queue;
|
|
55
122
|
this.queue = [];
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
123
|
+
const id = randomId();
|
|
124
|
+
events.forEach((e, i) => {
|
|
125
|
+
e.batch_id = id;
|
|
126
|
+
e.batch_seq = i;
|
|
127
|
+
});
|
|
128
|
+
const body = JSON.stringify(events);
|
|
129
|
+
let settleFirst;
|
|
130
|
+
const firstAttempt = new Promise((r) => (settleFirst = r));
|
|
131
|
+
const batch = {
|
|
132
|
+
id,
|
|
133
|
+
events,
|
|
134
|
+
body,
|
|
135
|
+
bytes: new TextEncoder().encode(body).length,
|
|
136
|
+
firstTime: events.reduce((m, e) => (e._time < m ? e._time : m), events[0]._time),
|
|
137
|
+
lastTime: events.reduce((m, e) => (e._time > m ? e._time : m), events[0]._time),
|
|
138
|
+
attempt: 0,
|
|
139
|
+
maxAttempts: this.maxAttempts,
|
|
140
|
+
nextAttemptAt: 0,
|
|
141
|
+
attempting: false,
|
|
142
|
+
terminal: false,
|
|
143
|
+
firstAttempt,
|
|
144
|
+
settleFirst,
|
|
145
|
+
};
|
|
146
|
+
// Enforce the caps over everything undelivered: evict oldest first; if
|
|
147
|
+
// nothing evictable remains, the new batch itself is the casualty.
|
|
148
|
+
while (this.totalEvents + events.length > this.maxBufferedEvents ||
|
|
149
|
+
this.totalBytes + batch.bytes > this.maxBufferedBytes) {
|
|
150
|
+
const victim = this.pending.find((b) => !b.attempting && !b.terminal);
|
|
151
|
+
if (!victim) {
|
|
152
|
+
batch.terminal = true;
|
|
153
|
+
settleFirst();
|
|
154
|
+
this.opts.onGiveUp?.(batch, new Error('retry buffer overflow'), 'buffer-overflow');
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
this.giveUp(victim, new Error('retry buffer overflow'), 'buffer-overflow');
|
|
158
|
+
}
|
|
159
|
+
this.pending.push(batch);
|
|
160
|
+
this.totalEvents += events.length;
|
|
161
|
+
this.totalBytes += batch.bytes;
|
|
162
|
+
return batch;
|
|
163
|
+
}
|
|
164
|
+
remove(batch) {
|
|
165
|
+
batch.terminal = true;
|
|
166
|
+
batch.settleFirst();
|
|
167
|
+
const i = this.pending.indexOf(batch);
|
|
168
|
+
if (i >= 0) {
|
|
169
|
+
this.pending.splice(i, 1);
|
|
170
|
+
this.totalEvents -= batch.events.length;
|
|
171
|
+
this.totalBytes -= batch.bytes;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
giveUp(batch, error, reason) {
|
|
175
|
+
if (batch.terminal)
|
|
176
|
+
return;
|
|
177
|
+
this.remove(batch);
|
|
178
|
+
this.opts.onGiveUp?.(batch, error, reason);
|
|
179
|
+
}
|
|
180
|
+
// Serial drainer: one attempt in flight at a time. Reentrant-safe (the
|
|
181
|
+
// running loop picks up batches formed while it was awaiting).
|
|
182
|
+
async pump() {
|
|
183
|
+
if (this.pumping || this.closed)
|
|
184
|
+
return;
|
|
185
|
+
this.pumping = true;
|
|
186
|
+
try {
|
|
187
|
+
for (;;) {
|
|
188
|
+
const now = Date.now();
|
|
189
|
+
const batch = this.pending.find((b) => !b.attempting && !b.terminal && b.nextAttemptAt <= now);
|
|
190
|
+
if (!batch || this.closed)
|
|
191
|
+
break;
|
|
192
|
+
await this.attempt(batch, this.attemptTimeoutMs);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
finally {
|
|
196
|
+
this.pumping = false;
|
|
197
|
+
this.schedulePump();
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
schedulePump() {
|
|
201
|
+
if (this.pumpTimer) {
|
|
202
|
+
clearTimeout(this.pumpTimer);
|
|
203
|
+
this.pumpTimer = undefined;
|
|
204
|
+
}
|
|
205
|
+
if (this.closed)
|
|
206
|
+
return;
|
|
207
|
+
const times = this.pending.filter((b) => !b.attempting && !b.terminal).map((b) => b.nextAttemptAt);
|
|
208
|
+
if (times.length === 0)
|
|
209
|
+
return;
|
|
210
|
+
const delay = Math.max(0, Math.min(...times) - Date.now());
|
|
211
|
+
this.pumpTimer = setTimeout(() => void this.pump(), delay);
|
|
212
|
+
unrefTimer(this.pumpTimer);
|
|
59
213
|
}
|
|
60
|
-
|
|
214
|
+
// One delivery attempt. All state transitions are synchronous around the
|
|
215
|
+
// single await; a batch marked terminal meanwhile (shutdown sweep) is left
|
|
216
|
+
// alone — never double-marked, never re-buffered.
|
|
217
|
+
attempt(batch, timeoutMs) {
|
|
218
|
+
batch.attempting = true;
|
|
219
|
+
const p = (async () => {
|
|
220
|
+
const error = await this.ship(batch.body, timeoutMs);
|
|
221
|
+
batch.attempting = false;
|
|
222
|
+
if (batch.terminal)
|
|
223
|
+
return;
|
|
224
|
+
batch.attempt++;
|
|
225
|
+
batch.settleFirst();
|
|
226
|
+
if (error === undefined) {
|
|
227
|
+
this.remove(batch);
|
|
228
|
+
}
|
|
229
|
+
else if (batch.attempt >= this.maxAttempts) {
|
|
230
|
+
this.giveUp(batch, error, 'give-up');
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
this.opts.onAttemptFailure?.(batch, error);
|
|
234
|
+
batch.nextAttemptAt =
|
|
235
|
+
Date.now() + this.retryDelaysMs[Math.min(batch.attempt - 1, this.retryDelaysMs.length - 1)];
|
|
236
|
+
this.schedulePump();
|
|
237
|
+
}
|
|
238
|
+
})();
|
|
239
|
+
this.inflight.add(p);
|
|
240
|
+
void p.finally(() => this.inflight.delete(p));
|
|
241
|
+
return p;
|
|
242
|
+
}
|
|
243
|
+
/** Returns the failure, or undefined on success. Never throws. */
|
|
244
|
+
async ship(body, timeoutMs) {
|
|
61
245
|
const fetchFn = this.opts.fetchFn ?? fetch;
|
|
62
246
|
try {
|
|
63
247
|
const res = await fetchFn(this.opts.url, {
|
|
@@ -67,14 +251,65 @@ export class LogShipper {
|
|
|
67
251
|
authorization: `Bearer ${this.opts.token}`,
|
|
68
252
|
'content-type': 'application/json',
|
|
69
253
|
},
|
|
70
|
-
body
|
|
254
|
+
body,
|
|
255
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
71
256
|
});
|
|
72
|
-
|
|
73
|
-
this.opts.onShipFailure?.(batch, new Error(`ingest returned ${res.status}`));
|
|
74
|
-
}
|
|
257
|
+
return res.ok ? undefined : new Error(`ingest returned ${res.status}`);
|
|
75
258
|
}
|
|
76
259
|
catch (err) {
|
|
77
|
-
|
|
260
|
+
return err ?? new Error('ship failed');
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
async runFinal(deadlineMs) {
|
|
264
|
+
this.closed = true; // stops the pump loop and its timer
|
|
265
|
+
if (this.batchTimer)
|
|
266
|
+
clearTimeout(this.batchTimer);
|
|
267
|
+
if (this.pumpTimer)
|
|
268
|
+
clearTimeout(this.pumpTimer);
|
|
269
|
+
const deadline = Date.now() + deadlineMs;
|
|
270
|
+
const attempted = new Set();
|
|
271
|
+
// Last-chance attempts run in parallel (the serial-pump bound is
|
|
272
|
+
// deliberately waived at shutdown so late batches aren't starved), each
|
|
273
|
+
// at most once, time-boxed by the remaining deadline.
|
|
274
|
+
const attemptAll = () => {
|
|
275
|
+
this.formBatch();
|
|
276
|
+
for (const b of [...this.pending]) {
|
|
277
|
+
const remaining = deadline - Date.now();
|
|
278
|
+
if (remaining <= 0)
|
|
279
|
+
break;
|
|
280
|
+
if (!b.attempting && !b.terminal && !attempted.has(b)) {
|
|
281
|
+
attempted.add(b);
|
|
282
|
+
void this.attempt(b, Math.min(this.attemptTimeoutMs, remaining));
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
};
|
|
286
|
+
// Await in-flight work against a REF'd deadline timer (an unref'd one
|
|
287
|
+
// would let a draining process exit before the sweep below writes the
|
|
288
|
+
// drop markers), re-running attemptAll after each wake so a batch whose
|
|
289
|
+
// pump attempt was in flight at entry — typically the newest — still
|
|
290
|
+
// gets its once-only last chance when that attempt fails early.
|
|
291
|
+
for (;;) {
|
|
292
|
+
attemptAll();
|
|
293
|
+
if (Date.now() >= deadline)
|
|
294
|
+
break;
|
|
295
|
+
const moreToStart = this.pending.some((b) => !b.attempting && !b.terminal && !attempted.has(b));
|
|
296
|
+
if (this.inflight.size === 0 && this.queue.length === 0 && !moreToStart)
|
|
297
|
+
break;
|
|
298
|
+
let timer;
|
|
299
|
+
const sleep = new Promise((r) => (timer = setTimeout(r, Math.max(1, deadline - Date.now()))));
|
|
300
|
+
await Promise.race([Promise.allSettled([...this.inflight]), sleep]);
|
|
301
|
+
clearTimeout(timer);
|
|
302
|
+
}
|
|
303
|
+
// Sweep: everything still undelivered — in flight or never attempted —
|
|
304
|
+
// is dropped with a marker. (An attempt still in flight may yet land:
|
|
305
|
+
// the rare phantom drop, accepted and documented for triage.) No awaits
|
|
306
|
+
// between here and resolution: settlement must reach the callers'
|
|
307
|
+
// exit handlers through microtasks alone, so nothing can be enqueued
|
|
308
|
+
// marker-less after the sweep. Note the cap inversion during this
|
|
309
|
+
// phase: with every batch attempting, overflow drops the NEWEST batch
|
|
310
|
+
// (nothing older is evictable) — bounded and marked, accepted.
|
|
311
|
+
for (const b of [...this.pending]) {
|
|
312
|
+
this.giveUp(b, new Error('undelivered at shutdown'), 'shutdown');
|
|
78
313
|
}
|
|
79
314
|
}
|
|
80
315
|
}
|
package/dist/log.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { Logger } from './log-core.js';
|
|
1
|
+
import { Logger, LogShipper, ShipperOptions } from './log-core.js';
|
|
2
2
|
export { LogShipper, serializeError, truncate } from './log-core.js';
|
|
3
|
-
export type { Logger, LogEvent, LogLevel, ShipperOptions } from './log-core.js';
|
|
3
|
+
export type { Logger, LogEvent, LogLevel, ShipperOptions, ShippedBatch, ShipFailureReason, } from './log-core.js';
|
|
4
4
|
export interface LoggerOptions {
|
|
5
5
|
/** App name, e.g. 'auth'. Becomes the `app` field queried in Axiom. */
|
|
6
6
|
app: string;
|
|
@@ -15,6 +15,21 @@ export interface LoggerOptions {
|
|
|
15
15
|
context?: Record<string, unknown>;
|
|
16
16
|
fetchFn?: typeof fetch;
|
|
17
17
|
}
|
|
18
|
-
/**
|
|
19
|
-
|
|
18
|
+
/** Register a directly-constructed shipper (e.g. watchdog2's platform-dataset
|
|
19
|
+
* shipper) so flushAllLoggers and shutdown cover it too. */
|
|
20
|
+
export declare function registerShipper(shipper: LogShipper): void;
|
|
21
|
+
/** Flush every registered shipper. `final` is the shutdown path: time-boxed
|
|
22
|
+
* last-chance delivery, then drop markers for whatever remains (see
|
|
23
|
+
* LogShipper.finalFlush) — racing callers join the same underlying work. */
|
|
24
|
+
export declare function flushAllLoggers(opts?: {
|
|
25
|
+
final?: boolean;
|
|
26
|
+
}): Promise<void>;
|
|
27
|
+
/** Ship-failure marker callbacks, writing one JSON line per event to stdout.
|
|
28
|
+
* writeSync, not process.stdout.write: the markers must survive an
|
|
29
|
+
* immediately following process.exit(), which can truncate async pipe
|
|
30
|
+
* writes — they are the only record that a batch never reached Axiom.
|
|
31
|
+
* first_time/last_time are the batch's own event-time range; the marker
|
|
32
|
+
* line itself can be written minutes later (next CPU window), so triage
|
|
33
|
+
* windows on these fields, never on the marker's timestamp. */
|
|
34
|
+
export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp'>;
|
|
20
35
|
export declare function createLogger(opts: LoggerOptions): Logger;
|
package/dist/log.js
CHANGED
|
@@ -6,12 +6,74 @@
|
|
|
6
6
|
// a real problem; log entry points, outbound calls, and DB writes with ids;
|
|
7
7
|
// rich meta is encouraged, but keep meta keys disciplined — every distinct
|
|
8
8
|
// flattened key becomes an Axiom field (dataset cap 1024).
|
|
9
|
+
import { writeSync } from 'node:fs';
|
|
9
10
|
import { LogShipper, makeEvent, } from './log-core.js';
|
|
10
11
|
export { LogShipper, serializeError, truncate } from './log-core.js';
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
// Registry of every shipper created here (plus any registered explicitly),
|
|
13
|
+
// used by flushAllLoggers and graceful shutdown. Shipper-level on purpose:
|
|
14
|
+
// with() children share their root's shipper, so registering loggers leaked
|
|
15
|
+
// one entry per child.
|
|
16
|
+
const allShippers = new Set();
|
|
17
|
+
/** Register a directly-constructed shipper (e.g. watchdog2's platform-dataset
|
|
18
|
+
* shipper) so flushAllLoggers and shutdown cover it too. */
|
|
19
|
+
export function registerShipper(shipper) {
|
|
20
|
+
allShippers.add(shipper);
|
|
21
|
+
}
|
|
22
|
+
/** Flush every registered shipper. `final` is the shutdown path: time-boxed
|
|
23
|
+
* last-chance delivery, then drop markers for whatever remains (see
|
|
24
|
+
* LogShipper.finalFlush) — racing callers join the same underlying work. */
|
|
25
|
+
export async function flushAllLoggers(opts = {}) {
|
|
26
|
+
await Promise.all([...allShippers].map((s) => (opts.final ? s.finalFlush() : s.flush())));
|
|
27
|
+
}
|
|
28
|
+
/** Ship-failure marker callbacks, writing one JSON line per event to stdout.
|
|
29
|
+
* writeSync, not process.stdout.write: the markers must survive an
|
|
30
|
+
* immediately following process.exit(), which can truncate async pipe
|
|
31
|
+
* writes — they are the only record that a batch never reached Axiom.
|
|
32
|
+
* first_time/last_time are the batch's own event-time range; the marker
|
|
33
|
+
* line itself can be written minutes later (next CPU window), so triage
|
|
34
|
+
* windows on these fields, never on the marker's timestamp. */
|
|
35
|
+
export function stdoutShipMarkers(app, dataset) {
|
|
36
|
+
const write = (line) => {
|
|
37
|
+
try {
|
|
38
|
+
writeSync(1, JSON.stringify(line) + '\n');
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// stdout is gone; nothing further to do.
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
return {
|
|
45
|
+
onAttemptFailure: (batch, error) => write({
|
|
46
|
+
severity: 'WARNING',
|
|
47
|
+
app,
|
|
48
|
+
dataset,
|
|
49
|
+
kind: 'log-ship-failure',
|
|
50
|
+
batch_id: batch.id,
|
|
51
|
+
events: batch.events.length,
|
|
52
|
+
bytes: batch.bytes,
|
|
53
|
+
first_time: batch.firstTime,
|
|
54
|
+
last_time: batch.lastTime,
|
|
55
|
+
attempt: batch.attempt,
|
|
56
|
+
max_attempts: batch.maxAttempts,
|
|
57
|
+
error: String(error),
|
|
58
|
+
message: `axiom ship attempt ${batch.attempt}/${batch.maxAttempts} failed: ${String(error)} ` +
|
|
59
|
+
`(${batch.events.length} events, ${batch.bytes} bytes; will retry)`,
|
|
60
|
+
}),
|
|
61
|
+
onGiveUp: (batch, error, reason) => write({
|
|
62
|
+
severity: 'ERROR',
|
|
63
|
+
app,
|
|
64
|
+
dataset,
|
|
65
|
+
kind: 'log-ship-drop',
|
|
66
|
+
batch_id: batch.id,
|
|
67
|
+
events: batch.events.length,
|
|
68
|
+
bytes: batch.bytes,
|
|
69
|
+
first_time: batch.firstTime,
|
|
70
|
+
last_time: batch.lastTime,
|
|
71
|
+
reason,
|
|
72
|
+
error: String(error),
|
|
73
|
+
message: `axiom ship dropped a batch (${reason}): ${String(error)} ` +
|
|
74
|
+
`(${batch.events.length} events, ${batch.bytes} bytes; stdout mirror has them)`,
|
|
75
|
+
}),
|
|
76
|
+
};
|
|
15
77
|
}
|
|
16
78
|
export function createLogger(opts) {
|
|
17
79
|
const token = opts.token ?? process.env.AXIOM_INGEST_TOKEN;
|
|
@@ -23,18 +85,15 @@ export function createLogger(opts) {
|
|
|
23
85
|
url: `${baseUrl}/v1/datasets/${dataset}/ingest`,
|
|
24
86
|
token,
|
|
25
87
|
fetchFn: opts.fetchFn,
|
|
26
|
-
|
|
27
|
-
// The stdout mirror already carried these events; record the
|
|
28
|
-
// delivery failure itself so Axiom gaps are explainable.
|
|
29
|
-
process.stdout.write(JSON.stringify({
|
|
30
|
-
severity: 'WARNING',
|
|
31
|
-
message: `axiom ship failed: ${String(error)} (${events.length} events; stdout mirror has them)`,
|
|
32
|
-
app: opts.app,
|
|
33
|
-
kind: 'log-ship-failure',
|
|
34
|
-
}) + '\n');
|
|
35
|
-
},
|
|
88
|
+
...stdoutShipMarkers(opts.app, dataset),
|
|
36
89
|
})
|
|
37
90
|
: undefined;
|
|
91
|
+
if (shipper) {
|
|
92
|
+
registerShipper(shipper);
|
|
93
|
+
// Scripts/jobs: when the event loop drains, make the last-chance
|
|
94
|
+
// attempt and write drop markers for anything undelivered.
|
|
95
|
+
process.once('beforeExit', () => void shipper.finalFlush());
|
|
96
|
+
}
|
|
38
97
|
function build(context) {
|
|
39
98
|
const factory = { app: opts.app, source: 'server', context };
|
|
40
99
|
const emit = (level, message, meta) => {
|
|
@@ -43,7 +102,7 @@ export function createLogger(opts) {
|
|
|
43
102
|
process.stdout.write(JSON.stringify(event) + '\n');
|
|
44
103
|
shipper?.enqueue(event);
|
|
45
104
|
};
|
|
46
|
-
|
|
105
|
+
return {
|
|
47
106
|
debug: (m, meta) => emit('debug', m, meta),
|
|
48
107
|
info: (m, meta) => emit('info', m, meta),
|
|
49
108
|
warn: (m, meta) => emit('warn', m, meta),
|
|
@@ -51,11 +110,6 @@ export function createLogger(opts) {
|
|
|
51
110
|
with: (extra) => build({ ...context, ...extra }),
|
|
52
111
|
flush: () => shipper?.flush() ?? Promise.resolve(),
|
|
53
112
|
};
|
|
54
|
-
allLoggers.add(logger);
|
|
55
|
-
return logger;
|
|
56
113
|
}
|
|
57
|
-
|
|
58
|
-
// Best-effort flush when the event loop drains (covers scripts/jobs).
|
|
59
|
-
process.once('beforeExit', () => void root.flush());
|
|
60
|
-
return root;
|
|
114
|
+
return build(opts.context);
|
|
61
115
|
}
|