@steve31415/baselib 3.0.2 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/email.d.ts +34 -0
- package/dist/email.js +80 -0
- package/dist/log-browser.js +8 -3
- package/dist/log-core.d.ts +40 -2
- package/dist/log-core.js +152 -11
- package/dist/log.d.ts +1 -1
- package/dist/log.js +29 -7
- package/dist/s2s.d.ts +1 -1
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ What it provides and why: `docs/SPEC.md`. How it's put together:
|
|
|
6
6
|
`~/migration/research/base-services-design.md` (step 5).
|
|
7
7
|
|
|
8
8
|
Server subpath exports: `config`, `log`, `auth`, `s2s`, `db`, `http`, `sync`,
|
|
9
|
-
`app-update`, and `
|
|
9
|
+
`app-update`, `llm`, and `email`. Browser exports: `log-browser`, `rum`, `sync-browser`, and
|
|
10
10
|
`app-update-browser`.
|
|
11
11
|
|
|
12
12
|
`sync` provides the Postgres event-log and server protocol primitives;
|
|
@@ -23,6 +23,10 @@ generic Yjs or service-worker coordinator.
|
|
|
23
23
|
registry, automatic cross-provider failover, portable JSON-schema output.
|
|
24
24
|
Usage guide: `~/plasticine-way/docs/LLM.md`.
|
|
25
25
|
|
|
26
|
+
`email` is the fleet's transactional email path: `sendEmail` through Resend
|
|
27
|
+
from a named `NAME@snewman.net` sender, no outbox — a failed send is an
|
|
28
|
+
ERROR log (`~/plasticine-way/docs/SECURITY.md`, "Resend API key").
|
|
29
|
+
|
|
26
30
|
Bins: `check-test-owners` — the fleet's structural test-coverage gate; every
|
|
27
31
|
app runs it from `npm run verify`.
|
|
28
32
|
|
package/dist/email.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type Logger } from './log-core.js';
|
|
2
|
+
export declare const RESEND_API_URL = "https://api.resend.com/emails";
|
|
3
|
+
/** An email to send. At least one of `text` / `html` must be present. */
|
|
4
|
+
export interface EmailMessage {
|
|
5
|
+
/** RFC 5322 from address, e.g. `Watchdog <watchdog@snewman.net>`. */
|
|
6
|
+
from: string;
|
|
7
|
+
to: string;
|
|
8
|
+
subject: string;
|
|
9
|
+
text?: string;
|
|
10
|
+
html?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface SendEmailResult {
|
|
13
|
+
ok: boolean;
|
|
14
|
+
/** Resend's message id, when the send succeeded and the response parsed. */
|
|
15
|
+
emailId?: string;
|
|
16
|
+
/** Human-readable failure description, present when `ok` is false. */
|
|
17
|
+
error?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface EmailDeps {
|
|
20
|
+
/** Resend API key. Falsy means "email not configured": logged at ERROR with
|
|
21
|
+
* the message, never sent. */
|
|
22
|
+
apiKey: string | undefined;
|
|
23
|
+
logger: Logger;
|
|
24
|
+
/** Defaults to the global `fetch`. Injectable for tests. */
|
|
25
|
+
fetchImpl?: typeof fetch;
|
|
26
|
+
/** Merged into every log line so callers can attribute the send. */
|
|
27
|
+
context?: Record<string, unknown>;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Send one email through Resend. Never throws: every failure is an ERROR
|
|
31
|
+
* log (with enough of the message to redo the send by hand) and an
|
|
32
|
+
* `{ ok: false, error }` result the caller surfaces where it matters.
|
|
33
|
+
*/
|
|
34
|
+
export declare function sendEmail(msg: EmailMessage, deps: EmailDeps): Promise<SendEmailResult>;
|
package/dist/email.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Transactional email through Resend — the fleet's one email path for
|
|
2
|
+
// system-generated mail (Steve's ruling 2026-09-03, migration decision log):
|
|
3
|
+
// named senders on the verified snewman.net domain (`Watchdog
|
|
4
|
+
// <watchdog@snewman.net>`, `Lurch <steve@snewman.net>`, …), key from Secret
|
|
5
|
+
// Manager `shared--resend-api-key`. Gmail via Mirror2's token broker stays
|
|
6
|
+
// only for mail that is genuinely Steve-to-Steve (Digest2).
|
|
7
|
+
//
|
|
8
|
+
// Ported from the old-world package @steve31415/resend-mailer 1.0.0
|
|
9
|
+
// (plasticine-apps/resend-mailer@5148649) minus its D1 outbox: the new-world
|
|
10
|
+
// rule is no outbox — a failed send is an ERROR log carrying the message,
|
|
11
|
+
// so the daily triage sees it and the send can be redone by hand. The
|
|
12
|
+
// request shape and error handling are the package's, kept verbatim: they
|
|
13
|
+
// ran against the live Resend API for months (TESTING.md, "Assumptions
|
|
14
|
+
// about external systems"): `POST https://api.resend.com/emails` with
|
|
15
|
+
// `{from, to: [to], subject, text?, html?}` answers 200 `{id}`; failures are
|
|
16
|
+
// 4xx/5xx with a JSON body.
|
|
17
|
+
import { serializeError, truncate } from './log-core.js';
|
|
18
|
+
export const RESEND_API_URL = 'https://api.resend.com/emails';
|
|
19
|
+
/**
|
|
20
|
+
* Send one email through Resend. Never throws: every failure is an ERROR
|
|
21
|
+
* log (with enough of the message to redo the send by hand) and an
|
|
22
|
+
* `{ ok: false, error }` result the caller surfaces where it matters.
|
|
23
|
+
*/
|
|
24
|
+
export async function sendEmail(msg, deps) {
|
|
25
|
+
const { logger } = deps;
|
|
26
|
+
const fetchImpl = deps.fetchImpl ?? fetch;
|
|
27
|
+
const base = {
|
|
28
|
+
to: msg.to,
|
|
29
|
+
subject: truncate(msg.subject, 200),
|
|
30
|
+
textLength: msg.text?.length ?? 0,
|
|
31
|
+
htmlLength: msg.html?.length ?? 0,
|
|
32
|
+
...deps.context,
|
|
33
|
+
};
|
|
34
|
+
const preview = { text: truncate(msg.text ?? '', 500), html: truncate(msg.html ?? '', 500) };
|
|
35
|
+
if (!msg.text && !msg.html) {
|
|
36
|
+
logger.error('email_rejected_no_body', base);
|
|
37
|
+
return { ok: false, error: 'sendEmail requires a non-empty `text` or `html` body' };
|
|
38
|
+
}
|
|
39
|
+
if (!deps.apiKey) {
|
|
40
|
+
// Nothing else will record what we failed to send: log the whole message.
|
|
41
|
+
logger.error('email_not_configured', { ...base, hasKey: false, ...preview });
|
|
42
|
+
return { ok: false, error: 'Email not configured: missing Resend API key' };
|
|
43
|
+
}
|
|
44
|
+
logger.info('email_sending', base);
|
|
45
|
+
const start = Date.now();
|
|
46
|
+
try {
|
|
47
|
+
const res = await fetchImpl(RESEND_API_URL, {
|
|
48
|
+
method: 'POST',
|
|
49
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${deps.apiKey}` },
|
|
50
|
+
body: JSON.stringify({
|
|
51
|
+
from: msg.from,
|
|
52
|
+
to: [msg.to],
|
|
53
|
+
subject: msg.subject,
|
|
54
|
+
...(msg.text !== undefined ? { text: msg.text } : {}),
|
|
55
|
+
...(msg.html !== undefined ? { html: msg.html } : {}),
|
|
56
|
+
}),
|
|
57
|
+
});
|
|
58
|
+
const elapsedMs = Date.now() - start;
|
|
59
|
+
const body = await res.text().catch(() => '');
|
|
60
|
+
if (!res.ok) {
|
|
61
|
+
const error = `Resend API ${res.status}: ${truncate(body, 500)}`;
|
|
62
|
+
logger.error('email_send_failed', { ...base, status: res.status, body: truncate(body, 500), elapsedMs, ...preview });
|
|
63
|
+
return { ok: false, error };
|
|
64
|
+
}
|
|
65
|
+
let emailId;
|
|
66
|
+
try {
|
|
67
|
+
emailId = JSON.parse(body).id;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
// Non-JSON success body: unusual, but the send did succeed.
|
|
71
|
+
}
|
|
72
|
+
logger.info('email_sent', { ...base, status: res.status, emailId, elapsedMs });
|
|
73
|
+
return { ok: true, emailId };
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
const elapsedMs = Date.now() - start;
|
|
77
|
+
logger.error('email_send_error', { ...base, error: serializeError(err), elapsedMs, ...preview });
|
|
78
|
+
return { ok: false, error: `Network error: ${err instanceof Error ? err.message : String(err)}` };
|
|
79
|
+
}
|
|
80
|
+
}
|
package/dist/log-browser.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
//
|
|
11
11
|
// The token is ingest-only and dataset-scoped by design — visible to any
|
|
12
12
|
// signed-in user, able to do nothing but add log events.
|
|
13
|
-
import { LogShipper,
|
|
13
|
+
import { LogShipper, makeEvents } from './log-core.js';
|
|
14
14
|
export { serializeError } from './log-core.js';
|
|
15
15
|
function metaContent(name) {
|
|
16
16
|
return document.querySelector(`meta[name="${name}"]`)?.content || undefined;
|
|
@@ -35,18 +35,23 @@ export function createBrowserLogger(opts = {}) {
|
|
|
35
35
|
batch_id: batch.id,
|
|
36
36
|
events: batch.events.length,
|
|
37
37
|
}),
|
|
38
|
+
onPartialFailure: (batch, failed, failures) => console.warn(`log ship: Axiom rejected ${failed} of ${batch.events.length} events`, {
|
|
39
|
+
batch_id: batch.id,
|
|
40
|
+
failures,
|
|
41
|
+
}),
|
|
38
42
|
})
|
|
39
43
|
: undefined;
|
|
40
44
|
function build(context) {
|
|
41
45
|
const factory = { app, source: 'browser', context };
|
|
42
46
|
const emit = (level, message, meta) => {
|
|
43
|
-
const
|
|
47
|
+
const events = makeEvents(factory, level, message, {
|
|
44
48
|
...meta,
|
|
45
49
|
page: location.pathname,
|
|
46
50
|
});
|
|
47
51
|
if (opts.console)
|
|
48
52
|
console[level === 'debug' ? 'debug' : level === 'error' ? 'error' : 'log'](message, meta ?? '');
|
|
49
|
-
|
|
53
|
+
for (const event of events)
|
|
54
|
+
shipper?.enqueue(event);
|
|
50
55
|
};
|
|
51
56
|
return {
|
|
52
57
|
debug: (m, meta) => emit('debug', m, meta),
|
package/dist/log-core.d.ts
CHANGED
|
@@ -28,7 +28,11 @@ export interface SerializedError {
|
|
|
28
28
|
}
|
|
29
29
|
export declare function serializeError(err: unknown): SerializedError;
|
|
30
30
|
export declare function truncate(s: string, max: number): string;
|
|
31
|
-
|
|
31
|
+
/** give-up: every attempt failed; buffer-overflow: evicted by the cap;
|
|
32
|
+
* shutdown: undelivered at the final flush; rejected: Axiom refused the
|
|
33
|
+
* bytes outright (a 4xx other than 408/429 — e.g. the dataset column limit),
|
|
34
|
+
* so the batch was dropped on its first attempt with Axiom's error text. */
|
|
35
|
+
export type ShipFailureReason = 'give-up' | 'buffer-overflow' | 'shutdown' | 'rejected';
|
|
32
36
|
/** What failure callbacks see of a batch. `attempt` is attempts made so far. */
|
|
33
37
|
export interface ShippedBatch {
|
|
34
38
|
id: string;
|
|
@@ -66,6 +70,9 @@ export interface ShipperOptions {
|
|
|
66
70
|
onAttemptFailure?: (batch: ShippedBatch, error: unknown) => void;
|
|
67
71
|
/** The batch is abandoned — its events will never reach Axiom. */
|
|
68
72
|
onGiveUp?: (batch: ShippedBatch, error: unknown, reason: ShipFailureReason) => void;
|
|
73
|
+
/** Axiom accepted the batch (2xx) but reported `failed` events it did not
|
|
74
|
+
* keep, listed in `failures`. Those events are lost; the rest landed. */
|
|
75
|
+
onPartialFailure?: (batch: ShippedBatch, failed: number, failures: unknown[]) => void;
|
|
69
76
|
}
|
|
70
77
|
export declare class LogShipper {
|
|
71
78
|
private readonly opts;
|
|
@@ -105,7 +112,7 @@ export declare class LogShipper {
|
|
|
105
112
|
private pump;
|
|
106
113
|
private schedulePump;
|
|
107
114
|
private attempt;
|
|
108
|
-
/**
|
|
115
|
+
/** One POST. Never throws. */
|
|
109
116
|
private ship;
|
|
110
117
|
private runFinal;
|
|
111
118
|
}
|
|
@@ -115,3 +122,34 @@ export interface EventFactoryOptions {
|
|
|
115
122
|
context?: Record<string, unknown>;
|
|
116
123
|
}
|
|
117
124
|
export declare function makeEvent(opts: EventFactoryOptions, level: LogLevel, message: string, meta?: Record<string, unknown>): LogEvent;
|
|
125
|
+
/** The event for one logger call plus, the first time a site's meta breaks
|
|
126
|
+
* the shape rules in this process, a WARN `kind=log-shape` naming the site. */
|
|
127
|
+
export declare function makeEvents(opts: EventFactoryOptions, level: LogLevel, message: string, meta?: Record<string, unknown>): LogEvent[];
|
|
128
|
+
/** Grammar for one key segment under `meta`. */
|
|
129
|
+
export declare const META_KEY: RegExp;
|
|
130
|
+
/** Deepest leaf allowed: `meta.a.b`. */
|
|
131
|
+
export declare const META_MAX_DEPTH = 2;
|
|
132
|
+
/** Flattened leaves per event above which the site is flagged as too wide. */
|
|
133
|
+
export declare const META_LEAF_BUDGET = 40;
|
|
134
|
+
export interface ShapeIssue {
|
|
135
|
+
/** rows: data-shaped keys rewritten as rows; stringified: an object below
|
|
136
|
+
* the depth cap became a JSON string; wide: over the leaf budget. */
|
|
137
|
+
kind: 'rows' | 'stringified' | 'wide';
|
|
138
|
+
/** Dotted path of the object concerned, e.g. `meta.phases`. */
|
|
139
|
+
path: string;
|
|
140
|
+
/** rows: the first offending key. */
|
|
141
|
+
sample?: string;
|
|
142
|
+
/** wide: the flattened leaf count. */
|
|
143
|
+
leaves?: number;
|
|
144
|
+
}
|
|
145
|
+
/** Apply the key grammar and depth cap to `meta`. Returns the input object
|
|
146
|
+
* itself when nothing needed rewriting. Arrays are never walked — an array
|
|
147
|
+
* is one column whatever its elements look like, which is what makes rows
|
|
148
|
+
* the escape hatch. */
|
|
149
|
+
export declare function shapeMeta(meta: Record<string, unknown>): {
|
|
150
|
+
meta: Record<string, unknown>;
|
|
151
|
+
issues: ShapeIssue[];
|
|
152
|
+
leaves: number;
|
|
153
|
+
};
|
|
154
|
+
/** Forget which sites have been flagged (tests). */
|
|
155
|
+
export declare function resetLogShapeWarnings(): void;
|
package/dist/log-core.js
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
// (~/migration/research/logging-reliability-design.md): every undelivered
|
|
8
8
|
// batch either lands in Axiom or ends in an onGiveUp callback — the server
|
|
9
9
|
// logger turns those into stdout drop markers that watchdog2's daily
|
|
10
|
-
// ship-failure check counts.
|
|
10
|
+
// ship-failure check counts. makeEvent also guards the shape of `meta`: every
|
|
11
|
+
// distinct key is an Axiom column (see "Key-shape guard" below).
|
|
11
12
|
export function serializeError(err) {
|
|
12
13
|
if (err instanceof Error) {
|
|
13
14
|
const out = { name: err.name, message: err.message, stack: err.stack };
|
|
@@ -24,6 +25,12 @@ export function serializeError(err) {
|
|
|
24
25
|
export function truncate(s, max) {
|
|
25
26
|
return s.length <= max ? s : s.slice(0, max) + `…[+${s.length - max}]`;
|
|
26
27
|
}
|
|
28
|
+
const RESPONSE_BODY_MAX = 300;
|
|
29
|
+
/** A 4xx is a verdict on the bytes, not the moment: retrying the identical
|
|
30
|
+
* body cannot succeed. 408 (timeout) and 429 (rate limit) are the exceptions. */
|
|
31
|
+
function isTerminalStatus(status) {
|
|
32
|
+
return status >= 400 && status < 500 && status !== 408 && status !== 429;
|
|
33
|
+
}
|
|
27
34
|
function randomId() {
|
|
28
35
|
const c = globalThis.crypto;
|
|
29
36
|
return c?.randomUUID?.() ?? Math.random().toString(36).slice(2) + Date.now().toString(36);
|
|
@@ -217,20 +224,25 @@ export class LogShipper {
|
|
|
217
224
|
attempt(batch, timeoutMs) {
|
|
218
225
|
batch.attempting = true;
|
|
219
226
|
const p = (async () => {
|
|
220
|
-
const
|
|
227
|
+
const outcome = await this.ship(batch.body, timeoutMs);
|
|
221
228
|
batch.attempting = false;
|
|
222
229
|
if (batch.terminal)
|
|
223
230
|
return;
|
|
224
231
|
batch.attempt++;
|
|
225
232
|
batch.settleFirst();
|
|
226
|
-
if (
|
|
233
|
+
if (outcome.ok) {
|
|
227
234
|
this.remove(batch);
|
|
235
|
+
if (outcome.failed > 0)
|
|
236
|
+
this.opts.onPartialFailure?.(batch, outcome.failed, outcome.failures);
|
|
237
|
+
}
|
|
238
|
+
else if (outcome.terminal) {
|
|
239
|
+
this.giveUp(batch, outcome.error, 'rejected');
|
|
228
240
|
}
|
|
229
241
|
else if (batch.attempt >= this.maxAttempts) {
|
|
230
|
-
this.giveUp(batch, error, 'give-up');
|
|
242
|
+
this.giveUp(batch, outcome.error, 'give-up');
|
|
231
243
|
}
|
|
232
244
|
else {
|
|
233
|
-
this.opts.onAttemptFailure?.(batch, error);
|
|
245
|
+
this.opts.onAttemptFailure?.(batch, outcome.error);
|
|
234
246
|
batch.nextAttemptAt =
|
|
235
247
|
Date.now() + this.retryDelaysMs[Math.min(batch.attempt - 1, this.retryDelaysMs.length - 1)];
|
|
236
248
|
this.schedulePump();
|
|
@@ -240,7 +252,7 @@ export class LogShipper {
|
|
|
240
252
|
void p.finally(() => this.inflight.delete(p));
|
|
241
253
|
return p;
|
|
242
254
|
}
|
|
243
|
-
/**
|
|
255
|
+
/** One POST. Never throws. */
|
|
244
256
|
async ship(body, timeoutMs) {
|
|
245
257
|
const fetchFn = this.opts.fetchFn ?? fetch;
|
|
246
258
|
try {
|
|
@@ -254,10 +266,26 @@ export class LogShipper {
|
|
|
254
266
|
body,
|
|
255
267
|
signal: AbortSignal.timeout(timeoutMs),
|
|
256
268
|
});
|
|
257
|
-
|
|
269
|
+
if (res.ok) {
|
|
270
|
+
const parsed = (await res.json().catch(() => undefined));
|
|
271
|
+
return {
|
|
272
|
+
ok: true,
|
|
273
|
+
failed: typeof parsed?.failed === 'number' ? parsed.failed : 0,
|
|
274
|
+
failures: Array.isArray(parsed?.failures) ? parsed.failures : [],
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
// Axiom's error body names the cause (e.g. the field that would exceed
|
|
278
|
+
// the column limit); quote it so the drop marker can be acted on.
|
|
279
|
+
const text = (await res.text().catch(() => '')).replace(/\s+/g, ' ').trim();
|
|
280
|
+
const detail = text ? `: ${truncate(text, RESPONSE_BODY_MAX)}` : '';
|
|
281
|
+
return {
|
|
282
|
+
ok: false,
|
|
283
|
+
error: new Error(`ingest returned ${res.status}${detail}`),
|
|
284
|
+
terminal: isTerminalStatus(res.status),
|
|
285
|
+
};
|
|
258
286
|
}
|
|
259
287
|
catch (err) {
|
|
260
|
-
return err ?? new Error('ship failed');
|
|
288
|
+
return { ok: false, error: err ?? new Error('ship failed'), terminal: false };
|
|
261
289
|
}
|
|
262
290
|
}
|
|
263
291
|
async runFinal(deadlineMs) {
|
|
@@ -314,11 +342,25 @@ export class LogShipper {
|
|
|
314
342
|
}
|
|
315
343
|
}
|
|
316
344
|
export function makeEvent(opts, level, message, meta) {
|
|
345
|
+
return buildEvent(opts, level, message, meta).event;
|
|
346
|
+
}
|
|
347
|
+
function buildEvent(opts, level, message, meta) {
|
|
317
348
|
let m = meta;
|
|
318
|
-
|
|
319
|
-
|
|
349
|
+
const issues = [];
|
|
350
|
+
if (m) {
|
|
351
|
+
if (m.error !== undefined && !isPlainSerialized(m.error)) {
|
|
352
|
+
m = { ...m, error: serializeError(m.error) };
|
|
353
|
+
}
|
|
354
|
+
const shaped = shapeMeta(m);
|
|
355
|
+
issues.push(...shaped.issues);
|
|
356
|
+
if (shaped.leaves > META_LEAF_BUDGET)
|
|
357
|
+
issues.push({ kind: 'wide', path: 'meta', leaves: shaped.leaves });
|
|
358
|
+
m = shaped.meta;
|
|
359
|
+
const rewrites = [...new Set(shaped.issues.map((i) => i.kind))].sort();
|
|
360
|
+
if (rewrites.length > 0)
|
|
361
|
+
m = { ...m, logShape: rewrites.join('+') };
|
|
320
362
|
}
|
|
321
|
-
|
|
363
|
+
const event = {
|
|
322
364
|
_time: new Date().toISOString(),
|
|
323
365
|
app: opts.app,
|
|
324
366
|
level,
|
|
@@ -328,6 +370,105 @@ export function makeEvent(opts, level, message, meta) {
|
|
|
328
370
|
...opts.context,
|
|
329
371
|
...(m ? { meta: m } : {}),
|
|
330
372
|
};
|
|
373
|
+
return { event, issues };
|
|
374
|
+
}
|
|
375
|
+
/** The event for one logger call plus, the first time a site's meta breaks
|
|
376
|
+
* the shape rules in this process, a WARN `kind=log-shape` naming the site. */
|
|
377
|
+
export function makeEvents(opts, level, message, meta) {
|
|
378
|
+
const { event, issues } = buildEvent(opts, level, message, meta);
|
|
379
|
+
const fresh = issues.filter((i) => {
|
|
380
|
+
const key = `${i.kind}\n${message}`;
|
|
381
|
+
if (flagged.has(key))
|
|
382
|
+
return false;
|
|
383
|
+
flagged.add(key);
|
|
384
|
+
return true;
|
|
385
|
+
});
|
|
386
|
+
if (fresh.length === 0)
|
|
387
|
+
return [event];
|
|
388
|
+
const warning = makeEvent(opts, 'warn', `log-shape: "${message}" — ${fresh.map(describeIssue).join('; ')}. ` +
|
|
389
|
+
'Fix the logging site: keys are columns (OPERATIONS.md).', { sourceMessage: message, sourceLevel: level, issues: fresh });
|
|
390
|
+
return [event, { ...warning, kind: 'log-shape' }];
|
|
391
|
+
}
|
|
392
|
+
// ---- Key-shape guard -------------------------------------------------------
|
|
393
|
+
// Every distinct flattened key under `meta` becomes an Axiom column — one
|
|
394
|
+
// schema per dataset, fleet-wide, kept for the retention year, capped at
|
|
395
|
+
// 1025 — and at the cap Axiom rejects whole batches. Keys must therefore be
|
|
396
|
+
// a fixed vocabulary; identity belongs in values. The guard rewrites the two
|
|
397
|
+
// shapes that mint columns from data (an object keyed by data → rows; an
|
|
398
|
+
// object nested deeper than two levels → a JSON string), marks the event
|
|
399
|
+
// with `meta.logShape`, and flags the site once per process so it gets fixed
|
|
400
|
+
// at the source. Rules: ~/plasticine-way/docs/OPERATIONS.md → "Keys are
|
|
401
|
+
// columns".
|
|
402
|
+
/** Grammar for one key segment under `meta`. */
|
|
403
|
+
export const META_KEY = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
|
|
404
|
+
/** Deepest leaf allowed: `meta.a.b`. */
|
|
405
|
+
export const META_MAX_DEPTH = 2;
|
|
406
|
+
/** Flattened leaves per event above which the site is flagged as too wide. */
|
|
407
|
+
export const META_LEAF_BUDGET = 40;
|
|
408
|
+
const STRINGIFIED_MAX = 2000;
|
|
409
|
+
const SAMPLE_KEY_MAX = 60;
|
|
410
|
+
function isPlainObject(v) {
|
|
411
|
+
if (typeof v !== 'object' || v === null || Array.isArray(v))
|
|
412
|
+
return false;
|
|
413
|
+
const proto = Object.getPrototypeOf(v);
|
|
414
|
+
return proto === Object.prototype || proto === null;
|
|
415
|
+
}
|
|
416
|
+
/** Apply the key grammar and depth cap to `meta`. Returns the input object
|
|
417
|
+
* itself when nothing needed rewriting. Arrays are never walked — an array
|
|
418
|
+
* is one column whatever its elements look like, which is what makes rows
|
|
419
|
+
* the escape hatch. */
|
|
420
|
+
export function shapeMeta(meta) {
|
|
421
|
+
const issues = [];
|
|
422
|
+
let leaves = 0;
|
|
423
|
+
const toRows = (obj) => Object.entries(obj).map(([key, value]) => (isPlainObject(value) ? { key, ...value } : { key, value }));
|
|
424
|
+
const walk = (obj, path, depth) => {
|
|
425
|
+
const bad = Object.keys(obj).find((k) => !META_KEY.test(k));
|
|
426
|
+
if (bad !== undefined) {
|
|
427
|
+
issues.push({ kind: 'rows', path, sample: truncate(bad, SAMPLE_KEY_MAX) });
|
|
428
|
+
leaves++;
|
|
429
|
+
// meta itself keyed by data: keep it an object so the marker fits.
|
|
430
|
+
return depth === 1 ? { rows: toRows(obj) } : toRows(obj);
|
|
431
|
+
}
|
|
432
|
+
const out = {};
|
|
433
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
434
|
+
// The serialized error is baselib's own fixed shape (its `cause` sits
|
|
435
|
+
// at level 3) and is exempt from the depth cap.
|
|
436
|
+
if (isPlainObject(v) && !(depth === 1 && k === 'error' && isPlainSerialized(v))) {
|
|
437
|
+
if (depth >= META_MAX_DEPTH) {
|
|
438
|
+
issues.push({ kind: 'stringified', path: `${path}.${k}` });
|
|
439
|
+
out[k] = truncate(JSON.stringify(v), STRINGIFIED_MAX);
|
|
440
|
+
leaves++;
|
|
441
|
+
}
|
|
442
|
+
else {
|
|
443
|
+
out[k] = walk(v, `${path}.${k}`, depth + 1);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
else {
|
|
447
|
+
out[k] = v;
|
|
448
|
+
leaves++;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
return out;
|
|
452
|
+
};
|
|
453
|
+
const shaped = walk(meta, 'meta', 1);
|
|
454
|
+
return { meta: issues.length > 0 ? shaped : meta, issues, leaves };
|
|
455
|
+
}
|
|
456
|
+
// Sites already flagged in this process, by issue kind and message: the
|
|
457
|
+
// first offending event is worth a WARN, the ten-thousandth is not.
|
|
458
|
+
const flagged = new Set();
|
|
459
|
+
/** Forget which sites have been flagged (tests). */
|
|
460
|
+
export function resetLogShapeWarnings() {
|
|
461
|
+
flagged.clear();
|
|
462
|
+
}
|
|
463
|
+
function describeIssue(i) {
|
|
464
|
+
switch (i.kind) {
|
|
465
|
+
case 'rows':
|
|
466
|
+
return `${i.path} is keyed by data (e.g. "${i.sample}") → logged as rows`;
|
|
467
|
+
case 'stringified':
|
|
468
|
+
return `${i.path} is nested deeper than two levels → logged as a JSON string`;
|
|
469
|
+
case 'wide':
|
|
470
|
+
return `${i.path} flattens to ${i.leaves} leaves (budget ${META_LEAF_BUDGET})`;
|
|
471
|
+
}
|
|
331
472
|
}
|
|
332
473
|
function isPlainSerialized(v) {
|
|
333
474
|
return typeof v === 'object' && v !== null && !(v instanceof Error) && 'message' in v && 'name' in v;
|
package/dist/log.d.ts
CHANGED
|
@@ -31,5 +31,5 @@ export declare function flushAllLoggers(opts?: {
|
|
|
31
31
|
* first_time/last_time are the batch's own event-time range; the marker
|
|
32
32
|
* line itself can be written minutes later (next CPU window), so triage
|
|
33
33
|
* windows on these fields, never on the marker's timestamp. */
|
|
34
|
-
export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp'>;
|
|
34
|
+
export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp' | 'onPartialFailure'>;
|
|
35
35
|
export declare function createLogger(opts: LoggerOptions): Logger;
|
package/dist/log.js
CHANGED
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
//
|
|
5
5
|
// Conventions (new PW OPERATIONS.md): ERROR for anything that could reflect
|
|
6
6
|
// a real problem; log entry points, outbound calls, and DB writes with ids;
|
|
7
|
-
// rich meta is encouraged, but
|
|
8
|
-
//
|
|
7
|
+
// rich meta is encouraged, but keys are columns — every distinct flattened
|
|
8
|
+
// key becomes an Axiom field (dataset cap 1025), so identity goes in values.
|
|
9
|
+
// log-core's key-shape guard rewrites offenders and WARNs once per site.
|
|
9
10
|
import { writeSync } from 'node:fs';
|
|
10
|
-
import { LogShipper,
|
|
11
|
+
import { LogShipper, makeEvents, truncate, } from './log-core.js';
|
|
11
12
|
export { LogShipper, serializeError, truncate } from './log-core.js';
|
|
12
13
|
// Registry of every shipper created here (plus any registered explicitly),
|
|
13
14
|
// used by flushAllLoggers and graceful shutdown. Shipper-level on purpose:
|
|
@@ -73,6 +74,26 @@ export function stdoutShipMarkers(app, dataset) {
|
|
|
73
74
|
message: `axiom ship dropped a batch (${reason}): ${String(error)} ` +
|
|
74
75
|
`(${batch.events.length} events, ${batch.bytes} bytes; stdout mirror has them)`,
|
|
75
76
|
}),
|
|
77
|
+
// Same marker kind, sized to what was actually lost, so the daily
|
|
78
|
+
// ship-failure report's dropped-event total stays exact.
|
|
79
|
+
onPartialFailure: (batch, failed, failures) => {
|
|
80
|
+
const detail = truncate(JSON.stringify(failures), 300);
|
|
81
|
+
write({
|
|
82
|
+
severity: 'ERROR',
|
|
83
|
+
app,
|
|
84
|
+
dataset,
|
|
85
|
+
kind: 'log-ship-drop',
|
|
86
|
+
batch_id: batch.id,
|
|
87
|
+
events: failed,
|
|
88
|
+
bytes: batch.bytes,
|
|
89
|
+
first_time: batch.firstTime,
|
|
90
|
+
last_time: batch.lastTime,
|
|
91
|
+
reason: 'partial',
|
|
92
|
+
error: detail,
|
|
93
|
+
message: `axiom accepted a batch but rejected ${failed} of ${batch.events.length} events: ${detail} ` +
|
|
94
|
+
`(stdout mirror has them)`,
|
|
95
|
+
});
|
|
96
|
+
},
|
|
76
97
|
};
|
|
77
98
|
}
|
|
78
99
|
export function createLogger(opts) {
|
|
@@ -97,10 +118,11 @@ export function createLogger(opts) {
|
|
|
97
118
|
function build(context) {
|
|
98
119
|
const factory = { app: opts.app, source: 'server', context };
|
|
99
120
|
const emit = (level, message, meta) => {
|
|
100
|
-
const event
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
121
|
+
for (const event of makeEvents(factory, level, message, meta)) {
|
|
122
|
+
if (stdout)
|
|
123
|
+
process.stdout.write(JSON.stringify(event) + '\n');
|
|
124
|
+
shipper?.enqueue(event);
|
|
125
|
+
}
|
|
104
126
|
};
|
|
105
127
|
return {
|
|
106
128
|
debug: (m, meta) => emit('debug', m, meta),
|
package/dist/s2s.d.ts
CHANGED
|
@@ -61,7 +61,7 @@ export declare function resolveS2sCaller(opts: S2sOptions & {
|
|
|
61
61
|
* sessions) — e.g. scheduler pokes, Pub/Sub push, admin APIs. */
|
|
62
62
|
export declare function s2sAuth(opts: S2sOptions): import("hono").MiddlewareHandler<any, string, {}, Response | (Response & import("hono").TypedResponse<{
|
|
63
63
|
error: string;
|
|
64
|
-
},
|
|
64
|
+
}, 429 | 401 | 403, "json">)>;
|
|
65
65
|
/** fetch() with an OIDC identity token for the target's canonical audience.
|
|
66
66
|
* In test mode, sends the synthetic X-Test-S2S-Caller header instead
|
|
67
67
|
* (identity from TEST_S2S_IDENTITY, default 'test-service@test'). */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@steve31415/baselib",
|
|
3
|
-
"version": "3.0
|
|
3
|
+
"version": "3.2.0",
|
|
4
4
|
"description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,6 +64,10 @@
|
|
|
64
64
|
"./llm": {
|
|
65
65
|
"types": "./dist/llm/index.d.ts",
|
|
66
66
|
"default": "./dist/llm/index.js"
|
|
67
|
+
},
|
|
68
|
+
"./email": {
|
|
69
|
+
"types": "./dist/email.d.ts",
|
|
70
|
+
"default": "./dist/email.js"
|
|
67
71
|
}
|
|
68
72
|
},
|
|
69
73
|
"bin": {
|