@steve31415/baselib 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/deploy/deploy.js +1 -0
- package/dist/deploy/gcs.d.ts +5 -0
- package/dist/deploy/gcs.js +55 -7
- package/dist/log-browser.js +8 -3
- package/dist/log-core.d.ts +40 -2
- package/dist/log-core.js +152 -11
- package/dist/log.d.ts +1 -1
- package/dist/log.js +29 -7
- package/dist/s2s.d.ts +1 -1
- package/package.json +1 -1
package/dist/deploy/deploy.js
CHANGED
|
@@ -228,6 +228,7 @@ export async function runDeploy(options) {
|
|
|
228
228
|
throw new Error('deploy exceeded its 25-minute deadline; lock may be breakable — aborting before traffic');
|
|
229
229
|
}
|
|
230
230
|
if (!(await verifyLock(runner, lock))) {
|
|
231
|
+
lock.stopHeartbeat();
|
|
231
232
|
lock = null; // someone broke and superseded us; never touch their lock
|
|
232
233
|
throw new Error('deploy lock lost (superseded by another deploy or rollback); aborting before traffic');
|
|
233
234
|
}
|
package/dist/deploy/gcs.d.ts
CHANGED
|
@@ -2,11 +2,16 @@ import type { HoldRecord, ReleaseRecord, Runner } from './types.js';
|
|
|
2
2
|
export interface LockHandle {
|
|
3
3
|
uri: string;
|
|
4
4
|
generation: string;
|
|
5
|
+
/** Ends the heartbeat; releaseLock calls it, and so must anyone who gives
|
|
6
|
+
* the lock up without releasing it. */
|
|
7
|
+
stopHeartbeat(): void;
|
|
5
8
|
}
|
|
6
9
|
export interface AcquireOptions {
|
|
7
10
|
waitMs?: number;
|
|
8
11
|
staleMs?: number;
|
|
9
12
|
pollMs?: number;
|
|
13
|
+
/** Refresh interval; 0 disables the heartbeat (tests). */
|
|
14
|
+
heartbeatMs?: number;
|
|
10
15
|
now?: () => number;
|
|
11
16
|
sleep?: (ms: number) => Promise<void>;
|
|
12
17
|
log?: (message: string) => void;
|
package/dist/deploy/gcs.js
CHANGED
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
// GCS-backed coordination for pw-deploy/pw-rollback: the per-app deploy lock
|
|
2
|
-
// (create-only, generation-fenced, server-time
|
|
3
|
-
// release records, and the default
|
|
2
|
+
// (create-only, generation-fenced, heartbeat-refreshed, server-time
|
|
3
|
+
// staleness), the rollback hold, release records, and the default
|
|
4
|
+
// retained-asset publisher.
|
|
4
5
|
import { readdir, stat, writeFile } from 'node:fs/promises';
|
|
5
6
|
import { join } from 'node:path';
|
|
6
7
|
import { tmpdir } from 'node:os';
|
|
7
8
|
import { CommandFailure, must } from './exec.js';
|
|
8
9
|
import { contentTypeFor, lockIsStale, SHA40 } from './plan.js';
|
|
9
|
-
|
|
10
|
+
/** A holder refreshes its lock every HEARTBEAT_MS; a lock not refreshed for
|
|
11
|
+
* STALE_MS is a dead holder's and may be broken — well inside the WAIT_MS a
|
|
12
|
+
* waiter is prepared to spend, so a killed deploy no longer needs a human
|
|
13
|
+
* to delete its lock. (Until 2026-09-04 staleness was judged from creation
|
|
14
|
+
* time with a 30-minute threshold, read from a key gcloud does not emit:
|
|
15
|
+
* the automatic break never fired.) */
|
|
16
|
+
const HEARTBEAT_MS = 30_000;
|
|
17
|
+
const STALE_MS = 3 * 60_000;
|
|
10
18
|
const WAIT_MS = 5 * 60_000;
|
|
11
19
|
const POLL_MS = 10_000;
|
|
12
20
|
async function describeObject(runner, uri) {
|
|
@@ -45,17 +53,23 @@ export async function acquireLock(runner, uri, body, opts = {}) {
|
|
|
45
53
|
const meta = await describeObject(runner, uri);
|
|
46
54
|
if (!meta?.generation)
|
|
47
55
|
throw new Error(`lock created but not describable: ${uri}`);
|
|
48
|
-
|
|
56
|
+
const generation = String(meta.generation);
|
|
57
|
+
const stopHeartbeat = startHeartbeat(runner, uri, generation, {
|
|
58
|
+
everyMs: opts.heartbeatMs ?? HEARTBEAT_MS,
|
|
59
|
+
now,
|
|
60
|
+
log,
|
|
61
|
+
});
|
|
62
|
+
return { uri, generation, stopHeartbeat };
|
|
49
63
|
}
|
|
50
64
|
const meta = await describeObject(runner, uri);
|
|
51
65
|
if (meta?.generation) {
|
|
52
66
|
const stale = opts.breakExisting ||
|
|
53
|
-
(meta.
|
|
54
|
-
lockIsStale(meta.
|
|
67
|
+
(meta.update_time !== undefined &&
|
|
68
|
+
lockIsStale(meta.update_time, now(), opts.staleMs ?? STALE_MS));
|
|
55
69
|
if (stale) {
|
|
56
70
|
log(opts.breakExisting
|
|
57
71
|
? `breaking existing lock (rollback takes priority): ${uri}`
|
|
58
|
-
: `breaking stale lock (
|
|
72
|
+
: `breaking stale lock (last refreshed ${meta.update_time}): ${uri}`);
|
|
59
73
|
// Conditional on the observed generation: two breakers cannot both
|
|
60
74
|
// win, and a just-released-and-reacquired lock is not clobbered.
|
|
61
75
|
await conditionalDelete(runner, uri, String(meta.generation));
|
|
@@ -77,12 +91,46 @@ export async function acquireLock(runner, uri, body, opts = {}) {
|
|
|
77
91
|
await sleep(opts.pollMs ?? POLL_MS);
|
|
78
92
|
}
|
|
79
93
|
}
|
|
94
|
+
/** Refresh the lock's custom metadata, fenced on our generation: a metadata
|
|
95
|
+
* update advances the object's server-side update_time (what staleness is
|
|
96
|
+
* judged from) without changing its generation (what the fence is). A
|
|
97
|
+
* refresh that fails the fence means a successor broke us — stop, and let
|
|
98
|
+
* verifyLock report it before traffic. Refreshes run one at a time. */
|
|
99
|
+
function startHeartbeat(runner, uri, generation, opts) {
|
|
100
|
+
if (opts.everyMs <= 0)
|
|
101
|
+
return () => { };
|
|
102
|
+
let stopped = false;
|
|
103
|
+
let chain = Promise.resolve();
|
|
104
|
+
const stop = () => {
|
|
105
|
+
stopped = true;
|
|
106
|
+
clearInterval(timer);
|
|
107
|
+
};
|
|
108
|
+
const refresh = async () => {
|
|
109
|
+
if (stopped)
|
|
110
|
+
return;
|
|
111
|
+
const result = await runner('gcloud', [
|
|
112
|
+
'storage', 'objects', 'update', uri,
|
|
113
|
+
`--custom-metadata=heartbeat=${new Date(opts.now()).toISOString()}`,
|
|
114
|
+
`--if-generation-match=${generation}`, '--quiet',
|
|
115
|
+
]);
|
|
116
|
+
if (result.code !== 0) {
|
|
117
|
+
opts.log(`lock heartbeat failed (generation ${generation} gone?): ${result.stderr.trim()}`);
|
|
118
|
+
stop();
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
const timer = setInterval(() => {
|
|
122
|
+
chain = chain.then(refresh);
|
|
123
|
+
}, opts.everyMs);
|
|
124
|
+
timer.unref?.();
|
|
125
|
+
return stop;
|
|
126
|
+
}
|
|
80
127
|
/** The fence: true only while our exact generation still exists. */
|
|
81
128
|
export async function verifyLock(runner, handle) {
|
|
82
129
|
const meta = await describeObject(runner, handle.uri);
|
|
83
130
|
return meta !== null && String(meta.generation) === handle.generation;
|
|
84
131
|
}
|
|
85
132
|
export async function releaseLock(runner, handle) {
|
|
133
|
+
handle.stopHeartbeat();
|
|
86
134
|
const ok = await conditionalDelete(runner, handle.uri, handle.generation);
|
|
87
135
|
if (!ok) {
|
|
88
136
|
// A successor broke us; their lock must survive. Nothing to clean up.
|
package/dist/log-browser.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
//
|
|
11
11
|
// The token is ingest-only and dataset-scoped by design — visible to any
|
|
12
12
|
// signed-in user, able to do nothing but add log events.
|
|
13
|
-
import { LogShipper,
|
|
13
|
+
import { LogShipper, makeEvents } from './log-core.js';
|
|
14
14
|
export { serializeError } from './log-core.js';
|
|
15
15
|
function metaContent(name) {
|
|
16
16
|
return document.querySelector(`meta[name="${name}"]`)?.content || undefined;
|
|
@@ -35,18 +35,23 @@ export function createBrowserLogger(opts = {}) {
|
|
|
35
35
|
batch_id: batch.id,
|
|
36
36
|
events: batch.events.length,
|
|
37
37
|
}),
|
|
38
|
+
onPartialFailure: (batch, failed, failures) => console.warn(`log ship: Axiom rejected ${failed} of ${batch.events.length} events`, {
|
|
39
|
+
batch_id: batch.id,
|
|
40
|
+
failures,
|
|
41
|
+
}),
|
|
38
42
|
})
|
|
39
43
|
: undefined;
|
|
40
44
|
function build(context) {
|
|
41
45
|
const factory = { app, source: 'browser', context };
|
|
42
46
|
const emit = (level, message, meta) => {
|
|
43
|
-
const
|
|
47
|
+
const events = makeEvents(factory, level, message, {
|
|
44
48
|
...meta,
|
|
45
49
|
page: location.pathname,
|
|
46
50
|
});
|
|
47
51
|
if (opts.console)
|
|
48
52
|
console[level === 'debug' ? 'debug' : level === 'error' ? 'error' : 'log'](message, meta ?? '');
|
|
49
|
-
|
|
53
|
+
for (const event of events)
|
|
54
|
+
shipper?.enqueue(event);
|
|
50
55
|
};
|
|
51
56
|
return {
|
|
52
57
|
debug: (m, meta) => emit('debug', m, meta),
|
package/dist/log-core.d.ts
CHANGED
|
@@ -28,7 +28,11 @@ export interface SerializedError {
|
|
|
28
28
|
}
|
|
29
29
|
export declare function serializeError(err: unknown): SerializedError;
|
|
30
30
|
export declare function truncate(s: string, max: number): string;
|
|
31
|
-
|
|
31
|
+
/** give-up: every attempt failed; buffer-overflow: evicted by the cap;
|
|
32
|
+
* shutdown: undelivered at the final flush; rejected: Axiom refused the
|
|
33
|
+
* bytes outright (a 4xx other than 408/429 — e.g. the dataset column limit),
|
|
34
|
+
* so the batch was dropped on its first attempt with Axiom's error text. */
|
|
35
|
+
export type ShipFailureReason = 'give-up' | 'buffer-overflow' | 'shutdown' | 'rejected';
|
|
32
36
|
/** What failure callbacks see of a batch. `attempt` is attempts made so far. */
|
|
33
37
|
export interface ShippedBatch {
|
|
34
38
|
id: string;
|
|
@@ -66,6 +70,9 @@ export interface ShipperOptions {
|
|
|
66
70
|
onAttemptFailure?: (batch: ShippedBatch, error: unknown) => void;
|
|
67
71
|
/** The batch is abandoned — its events will never reach Axiom. */
|
|
68
72
|
onGiveUp?: (batch: ShippedBatch, error: unknown, reason: ShipFailureReason) => void;
|
|
73
|
+
/** Axiom accepted the batch (2xx) but reported `failed` events it did not
|
|
74
|
+
* keep, listed in `failures`. Those events are lost; the rest landed. */
|
|
75
|
+
onPartialFailure?: (batch: ShippedBatch, failed: number, failures: unknown[]) => void;
|
|
69
76
|
}
|
|
70
77
|
export declare class LogShipper {
|
|
71
78
|
private readonly opts;
|
|
@@ -105,7 +112,7 @@ export declare class LogShipper {
|
|
|
105
112
|
private pump;
|
|
106
113
|
private schedulePump;
|
|
107
114
|
private attempt;
|
|
108
|
-
/**
|
|
115
|
+
/** One POST. Never throws. */
|
|
109
116
|
private ship;
|
|
110
117
|
private runFinal;
|
|
111
118
|
}
|
|
@@ -115,3 +122,34 @@ export interface EventFactoryOptions {
|
|
|
115
122
|
context?: Record<string, unknown>;
|
|
116
123
|
}
|
|
117
124
|
export declare function makeEvent(opts: EventFactoryOptions, level: LogLevel, message: string, meta?: Record<string, unknown>): LogEvent;
|
|
125
|
+
/** The event for one logger call plus, the first time a site's meta breaks
|
|
126
|
+
* the shape rules in this process, a WARN `kind=log-shape` naming the site. */
|
|
127
|
+
export declare function makeEvents(opts: EventFactoryOptions, level: LogLevel, message: string, meta?: Record<string, unknown>): LogEvent[];
|
|
128
|
+
/** Grammar for one key segment under `meta`. */
|
|
129
|
+
export declare const META_KEY: RegExp;
|
|
130
|
+
/** Deepest leaf allowed: `meta.a.b`. */
|
|
131
|
+
export declare const META_MAX_DEPTH = 2;
|
|
132
|
+
/** Flattened leaves per event above which the site is flagged as too wide. */
|
|
133
|
+
export declare const META_LEAF_BUDGET = 40;
|
|
134
|
+
export interface ShapeIssue {
|
|
135
|
+
/** rows: data-shaped keys rewritten as rows; stringified: an object below
|
|
136
|
+
* the depth cap became a JSON string; wide: over the leaf budget. */
|
|
137
|
+
kind: 'rows' | 'stringified' | 'wide';
|
|
138
|
+
/** Dotted path of the object concerned, e.g. `meta.phases`. */
|
|
139
|
+
path: string;
|
|
140
|
+
/** rows: the first offending key. */
|
|
141
|
+
sample?: string;
|
|
142
|
+
/** wide: the flattened leaf count. */
|
|
143
|
+
leaves?: number;
|
|
144
|
+
}
|
|
145
|
+
/** Apply the key grammar and depth cap to `meta`. Returns the input object
|
|
146
|
+
* itself when nothing needed rewriting. Arrays are never walked — an array
|
|
147
|
+
* is one column whatever its elements look like, which is what makes rows
|
|
148
|
+
* the escape hatch. */
|
|
149
|
+
export declare function shapeMeta(meta: Record<string, unknown>): {
|
|
150
|
+
meta: Record<string, unknown>;
|
|
151
|
+
issues: ShapeIssue[];
|
|
152
|
+
leaves: number;
|
|
153
|
+
};
|
|
154
|
+
/** Forget which sites have been flagged (tests). */
|
|
155
|
+
export declare function resetLogShapeWarnings(): void;
|
package/dist/log-core.js
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
// (~/migration/research/logging-reliability-design.md): every undelivered
|
|
8
8
|
// batch either lands in Axiom or ends in an onGiveUp callback — the server
|
|
9
9
|
// logger turns those into stdout drop markers that watchdog2's daily
|
|
10
|
-
// ship-failure check counts.
|
|
10
|
+
// ship-failure check counts. makeEvent also guards the shape of `meta`: every
|
|
11
|
+
// distinct key is an Axiom column (see "Key-shape guard" below).
|
|
11
12
|
export function serializeError(err) {
|
|
12
13
|
if (err instanceof Error) {
|
|
13
14
|
const out = { name: err.name, message: err.message, stack: err.stack };
|
|
@@ -24,6 +25,12 @@ export function serializeError(err) {
|
|
|
24
25
|
export function truncate(s, max) {
|
|
25
26
|
return s.length <= max ? s : s.slice(0, max) + `…[+${s.length - max}]`;
|
|
26
27
|
}
|
|
28
|
+
const RESPONSE_BODY_MAX = 300;
|
|
29
|
+
/** A 4xx is a verdict on the bytes, not the moment: retrying the identical
|
|
30
|
+
* body cannot succeed. 408 (timeout) and 429 (rate limit) are the exceptions. */
|
|
31
|
+
function isTerminalStatus(status) {
|
|
32
|
+
return status >= 400 && status < 500 && status !== 408 && status !== 429;
|
|
33
|
+
}
|
|
27
34
|
function randomId() {
|
|
28
35
|
const c = globalThis.crypto;
|
|
29
36
|
return c?.randomUUID?.() ?? Math.random().toString(36).slice(2) + Date.now().toString(36);
|
|
@@ -217,20 +224,25 @@ export class LogShipper {
|
|
|
217
224
|
attempt(batch, timeoutMs) {
|
|
218
225
|
batch.attempting = true;
|
|
219
226
|
const p = (async () => {
|
|
220
|
-
const
|
|
227
|
+
const outcome = await this.ship(batch.body, timeoutMs);
|
|
221
228
|
batch.attempting = false;
|
|
222
229
|
if (batch.terminal)
|
|
223
230
|
return;
|
|
224
231
|
batch.attempt++;
|
|
225
232
|
batch.settleFirst();
|
|
226
|
-
if (
|
|
233
|
+
if (outcome.ok) {
|
|
227
234
|
this.remove(batch);
|
|
235
|
+
if (outcome.failed > 0)
|
|
236
|
+
this.opts.onPartialFailure?.(batch, outcome.failed, outcome.failures);
|
|
237
|
+
}
|
|
238
|
+
else if (outcome.terminal) {
|
|
239
|
+
this.giveUp(batch, outcome.error, 'rejected');
|
|
228
240
|
}
|
|
229
241
|
else if (batch.attempt >= this.maxAttempts) {
|
|
230
|
-
this.giveUp(batch, error, 'give-up');
|
|
242
|
+
this.giveUp(batch, outcome.error, 'give-up');
|
|
231
243
|
}
|
|
232
244
|
else {
|
|
233
|
-
this.opts.onAttemptFailure?.(batch, error);
|
|
245
|
+
this.opts.onAttemptFailure?.(batch, outcome.error);
|
|
234
246
|
batch.nextAttemptAt =
|
|
235
247
|
Date.now() + this.retryDelaysMs[Math.min(batch.attempt - 1, this.retryDelaysMs.length - 1)];
|
|
236
248
|
this.schedulePump();
|
|
@@ -240,7 +252,7 @@ export class LogShipper {
|
|
|
240
252
|
void p.finally(() => this.inflight.delete(p));
|
|
241
253
|
return p;
|
|
242
254
|
}
|
|
243
|
-
/**
|
|
255
|
+
/** One POST. Never throws. */
|
|
244
256
|
async ship(body, timeoutMs) {
|
|
245
257
|
const fetchFn = this.opts.fetchFn ?? fetch;
|
|
246
258
|
try {
|
|
@@ -254,10 +266,26 @@ export class LogShipper {
|
|
|
254
266
|
body,
|
|
255
267
|
signal: AbortSignal.timeout(timeoutMs),
|
|
256
268
|
});
|
|
257
|
-
|
|
269
|
+
if (res.ok) {
|
|
270
|
+
const parsed = (await res.json().catch(() => undefined));
|
|
271
|
+
return {
|
|
272
|
+
ok: true,
|
|
273
|
+
failed: typeof parsed?.failed === 'number' ? parsed.failed : 0,
|
|
274
|
+
failures: Array.isArray(parsed?.failures) ? parsed.failures : [],
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
// Axiom's error body names the cause (e.g. the field that would exceed
|
|
278
|
+
// the column limit); quote it so the drop marker can be acted on.
|
|
279
|
+
const text = (await res.text().catch(() => '')).replace(/\s+/g, ' ').trim();
|
|
280
|
+
const detail = text ? `: ${truncate(text, RESPONSE_BODY_MAX)}` : '';
|
|
281
|
+
return {
|
|
282
|
+
ok: false,
|
|
283
|
+
error: new Error(`ingest returned ${res.status}${detail}`),
|
|
284
|
+
terminal: isTerminalStatus(res.status),
|
|
285
|
+
};
|
|
258
286
|
}
|
|
259
287
|
catch (err) {
|
|
260
|
-
return err ?? new Error('ship failed');
|
|
288
|
+
return { ok: false, error: err ?? new Error('ship failed'), terminal: false };
|
|
261
289
|
}
|
|
262
290
|
}
|
|
263
291
|
async runFinal(deadlineMs) {
|
|
@@ -314,11 +342,25 @@ export class LogShipper {
|
|
|
314
342
|
}
|
|
315
343
|
}
|
|
316
344
|
export function makeEvent(opts, level, message, meta) {
|
|
345
|
+
return buildEvent(opts, level, message, meta).event;
|
|
346
|
+
}
|
|
347
|
+
function buildEvent(opts, level, message, meta) {
|
|
317
348
|
let m = meta;
|
|
318
|
-
|
|
319
|
-
|
|
349
|
+
const issues = [];
|
|
350
|
+
if (m) {
|
|
351
|
+
if (m.error !== undefined && !isPlainSerialized(m.error)) {
|
|
352
|
+
m = { ...m, error: serializeError(m.error) };
|
|
353
|
+
}
|
|
354
|
+
const shaped = shapeMeta(m);
|
|
355
|
+
issues.push(...shaped.issues);
|
|
356
|
+
if (shaped.leaves > META_LEAF_BUDGET)
|
|
357
|
+
issues.push({ kind: 'wide', path: 'meta', leaves: shaped.leaves });
|
|
358
|
+
m = shaped.meta;
|
|
359
|
+
const rewrites = [...new Set(shaped.issues.map((i) => i.kind))].sort();
|
|
360
|
+
if (rewrites.length > 0)
|
|
361
|
+
m = { ...m, logShape: rewrites.join('+') };
|
|
320
362
|
}
|
|
321
|
-
|
|
363
|
+
const event = {
|
|
322
364
|
_time: new Date().toISOString(),
|
|
323
365
|
app: opts.app,
|
|
324
366
|
level,
|
|
@@ -328,6 +370,105 @@ export function makeEvent(opts, level, message, meta) {
|
|
|
328
370
|
...opts.context,
|
|
329
371
|
...(m ? { meta: m } : {}),
|
|
330
372
|
};
|
|
373
|
+
return { event, issues };
|
|
374
|
+
}
|
|
375
|
+
/** The event for one logger call plus, the first time a site's meta breaks
|
|
376
|
+
* the shape rules in this process, a WARN `kind=log-shape` naming the site. */
|
|
377
|
+
export function makeEvents(opts, level, message, meta) {
|
|
378
|
+
const { event, issues } = buildEvent(opts, level, message, meta);
|
|
379
|
+
const fresh = issues.filter((i) => {
|
|
380
|
+
const key = `${i.kind}\n${message}`;
|
|
381
|
+
if (flagged.has(key))
|
|
382
|
+
return false;
|
|
383
|
+
flagged.add(key);
|
|
384
|
+
return true;
|
|
385
|
+
});
|
|
386
|
+
if (fresh.length === 0)
|
|
387
|
+
return [event];
|
|
388
|
+
const warning = makeEvent(opts, 'warn', `log-shape: "${message}" — ${fresh.map(describeIssue).join('; ')}. ` +
|
|
389
|
+
'Fix the logging site: keys are columns (OPERATIONS.md).', { sourceMessage: message, sourceLevel: level, issues: fresh });
|
|
390
|
+
return [event, { ...warning, kind: 'log-shape' }];
|
|
391
|
+
}
|
|
392
|
+
// ---- Key-shape guard -------------------------------------------------------
|
|
393
|
+
// Every distinct flattened key under `meta` becomes an Axiom column — one
|
|
394
|
+
// schema per dataset, fleet-wide, kept for the retention year, capped at
|
|
395
|
+
// 1025 — and at the cap Axiom rejects whole batches. Keys must therefore be
|
|
396
|
+
// a fixed vocabulary; identity belongs in values. The guard rewrites the two
|
|
397
|
+
// shapes that mint columns from data (an object keyed by data → rows; an
|
|
398
|
+
// object nested deeper than two levels → a JSON string), marks the event
|
|
399
|
+
// with `meta.logShape`, and flags the site once per process so it gets fixed
|
|
400
|
+
// at the source. Rules: ~/plasticine-way/docs/OPERATIONS.md → "Keys are
|
|
401
|
+
// columns".
|
|
402
|
+
/** Grammar for one key segment under `meta`. */
|
|
403
|
+
export const META_KEY = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
|
|
404
|
+
/** Deepest leaf allowed: `meta.a.b`. */
|
|
405
|
+
export const META_MAX_DEPTH = 2;
|
|
406
|
+
/** Flattened leaves per event above which the site is flagged as too wide. */
|
|
407
|
+
export const META_LEAF_BUDGET = 40;
|
|
408
|
+
const STRINGIFIED_MAX = 2000;
|
|
409
|
+
const SAMPLE_KEY_MAX = 60;
|
|
410
|
+
function isPlainObject(v) {
|
|
411
|
+
if (typeof v !== 'object' || v === null || Array.isArray(v))
|
|
412
|
+
return false;
|
|
413
|
+
const proto = Object.getPrototypeOf(v);
|
|
414
|
+
return proto === Object.prototype || proto === null;
|
|
415
|
+
}
|
|
416
|
+
/** Apply the key grammar and depth cap to `meta`. Returns the input object
|
|
417
|
+
* itself when nothing needed rewriting. Arrays are never walked — an array
|
|
418
|
+
* is one column whatever its elements look like, which is what makes rows
|
|
419
|
+
* the escape hatch. */
|
|
420
|
+
export function shapeMeta(meta) {
|
|
421
|
+
const issues = [];
|
|
422
|
+
let leaves = 0;
|
|
423
|
+
const toRows = (obj) => Object.entries(obj).map(([key, value]) => (isPlainObject(value) ? { key, ...value } : { key, value }));
|
|
424
|
+
const walk = (obj, path, depth) => {
|
|
425
|
+
const bad = Object.keys(obj).find((k) => !META_KEY.test(k));
|
|
426
|
+
if (bad !== undefined) {
|
|
427
|
+
issues.push({ kind: 'rows', path, sample: truncate(bad, SAMPLE_KEY_MAX) });
|
|
428
|
+
leaves++;
|
|
429
|
+
// meta itself keyed by data: keep it an object so the marker fits.
|
|
430
|
+
return depth === 1 ? { rows: toRows(obj) } : toRows(obj);
|
|
431
|
+
}
|
|
432
|
+
const out = {};
|
|
433
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
434
|
+
// The serialized error is baselib's own fixed shape (its `cause` sits
|
|
435
|
+
// at level 3) and is exempt from the depth cap.
|
|
436
|
+
if (isPlainObject(v) && !(depth === 1 && k === 'error' && isPlainSerialized(v))) {
|
|
437
|
+
if (depth >= META_MAX_DEPTH) {
|
|
438
|
+
issues.push({ kind: 'stringified', path: `${path}.${k}` });
|
|
439
|
+
out[k] = truncate(JSON.stringify(v), STRINGIFIED_MAX);
|
|
440
|
+
leaves++;
|
|
441
|
+
}
|
|
442
|
+
else {
|
|
443
|
+
out[k] = walk(v, `${path}.${k}`, depth + 1);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
else {
|
|
447
|
+
out[k] = v;
|
|
448
|
+
leaves++;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
return out;
|
|
452
|
+
};
|
|
453
|
+
const shaped = walk(meta, 'meta', 1);
|
|
454
|
+
return { meta: issues.length > 0 ? shaped : meta, issues, leaves };
|
|
455
|
+
}
|
|
456
|
+
// Sites already flagged in this process, by issue kind and message: the
|
|
457
|
+
// first offending event is worth a WARN, the ten-thousandth is not.
|
|
458
|
+
const flagged = new Set();
|
|
459
|
+
/** Forget which sites have been flagged (tests). */
|
|
460
|
+
export function resetLogShapeWarnings() {
|
|
461
|
+
flagged.clear();
|
|
462
|
+
}
|
|
463
|
+
function describeIssue(i) {
|
|
464
|
+
switch (i.kind) {
|
|
465
|
+
case 'rows':
|
|
466
|
+
return `${i.path} is keyed by data (e.g. "${i.sample}") → logged as rows`;
|
|
467
|
+
case 'stringified':
|
|
468
|
+
return `${i.path} is nested deeper than two levels → logged as a JSON string`;
|
|
469
|
+
case 'wide':
|
|
470
|
+
return `${i.path} flattens to ${i.leaves} leaves (budget ${META_LEAF_BUDGET})`;
|
|
471
|
+
}
|
|
331
472
|
}
|
|
332
473
|
function isPlainSerialized(v) {
|
|
333
474
|
return typeof v === 'object' && v !== null && !(v instanceof Error) && 'message' in v && 'name' in v;
|
package/dist/log.d.ts
CHANGED
|
@@ -31,5 +31,5 @@ export declare function flushAllLoggers(opts?: {
|
|
|
31
31
|
* first_time/last_time are the batch's own event-time range; the marker
|
|
32
32
|
* line itself can be written minutes later (next CPU window), so triage
|
|
33
33
|
* windows on these fields, never on the marker's timestamp. */
|
|
34
|
-
export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp'>;
|
|
34
|
+
export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp' | 'onPartialFailure'>;
|
|
35
35
|
export declare function createLogger(opts: LoggerOptions): Logger;
|
package/dist/log.js
CHANGED
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
//
|
|
5
5
|
// Conventions (new PW OPERATIONS.md): ERROR for anything that could reflect
|
|
6
6
|
// a real problem; log entry points, outbound calls, and DB writes with ids;
|
|
7
|
-
// rich meta is encouraged, but
|
|
8
|
-
//
|
|
7
|
+
// rich meta is encouraged, but keys are columns — every distinct flattened
|
|
8
|
+
// key becomes an Axiom field (dataset cap 1025), so identity goes in values.
|
|
9
|
+
// log-core's key-shape guard rewrites offenders and WARNs once per site.
|
|
9
10
|
import { writeSync } from 'node:fs';
|
|
10
|
-
import { LogShipper,
|
|
11
|
+
import { LogShipper, makeEvents, truncate, } from './log-core.js';
|
|
11
12
|
export { LogShipper, serializeError, truncate } from './log-core.js';
|
|
12
13
|
// Registry of every shipper created here (plus any registered explicitly),
|
|
13
14
|
// used by flushAllLoggers and graceful shutdown. Shipper-level on purpose:
|
|
@@ -73,6 +74,26 @@ export function stdoutShipMarkers(app, dataset) {
|
|
|
73
74
|
message: `axiom ship dropped a batch (${reason}): ${String(error)} ` +
|
|
74
75
|
`(${batch.events.length} events, ${batch.bytes} bytes; stdout mirror has them)`,
|
|
75
76
|
}),
|
|
77
|
+
// Same marker kind, sized to what was actually lost, so the daily
|
|
78
|
+
// ship-failure report's dropped-event total stays exact.
|
|
79
|
+
onPartialFailure: (batch, failed, failures) => {
|
|
80
|
+
const detail = truncate(JSON.stringify(failures), 300);
|
|
81
|
+
write({
|
|
82
|
+
severity: 'ERROR',
|
|
83
|
+
app,
|
|
84
|
+
dataset,
|
|
85
|
+
kind: 'log-ship-drop',
|
|
86
|
+
batch_id: batch.id,
|
|
87
|
+
events: failed,
|
|
88
|
+
bytes: batch.bytes,
|
|
89
|
+
first_time: batch.firstTime,
|
|
90
|
+
last_time: batch.lastTime,
|
|
91
|
+
reason: 'partial',
|
|
92
|
+
error: detail,
|
|
93
|
+
message: `axiom accepted a batch but rejected ${failed} of ${batch.events.length} events: ${detail} ` +
|
|
94
|
+
`(stdout mirror has them)`,
|
|
95
|
+
});
|
|
96
|
+
},
|
|
76
97
|
};
|
|
77
98
|
}
|
|
78
99
|
export function createLogger(opts) {
|
|
@@ -97,10 +118,11 @@ export function createLogger(opts) {
|
|
|
97
118
|
function build(context) {
|
|
98
119
|
const factory = { app: opts.app, source: 'server', context };
|
|
99
120
|
const emit = (level, message, meta) => {
|
|
100
|
-
const event
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
121
|
+
for (const event of makeEvents(factory, level, message, meta)) {
|
|
122
|
+
if (stdout)
|
|
123
|
+
process.stdout.write(JSON.stringify(event) + '\n');
|
|
124
|
+
shipper?.enqueue(event);
|
|
125
|
+
}
|
|
104
126
|
};
|
|
105
127
|
return {
|
|
106
128
|
debug: (m, meta) => emit('debug', m, meta),
|
package/dist/s2s.d.ts
CHANGED
|
@@ -61,7 +61,7 @@ export declare function resolveS2sCaller(opts: S2sOptions & {
|
|
|
61
61
|
* sessions) — e.g. scheduler pokes, Pub/Sub push, admin APIs. */
|
|
62
62
|
export declare function s2sAuth(opts: S2sOptions): import("hono").MiddlewareHandler<any, string, {}, Response | (Response & import("hono").TypedResponse<{
|
|
63
63
|
error: string;
|
|
64
|
-
},
|
|
64
|
+
}, 429 | 401 | 403, "json">)>;
|
|
65
65
|
/** fetch() with an OIDC identity token for the target's canonical audience.
|
|
66
66
|
* In test mode, sends the synthetic X-Test-S2S-Caller header instead
|
|
67
67
|
* (identity from TEST_S2S_IDENTITY, default 'test-service@test'). */
|
package/package.json
CHANGED