@steve31415/baselib 3.1.0 → 3.2.0

package/dist/log-browser.js

@@ -10,7 +10,7 @@
 //
 // The token is ingest-only and dataset-scoped by design — visible to any
 // signed-in user, able to do nothing but add log events.
-import { LogShipper, makeEvent } from './log-core.js';
+import { LogShipper, makeEvents } from './log-core.js';
 export { serializeError } from './log-core.js';
 function metaContent(name) {
     return document.querySelector(`meta[name="${name}"]`)?.content || undefined;
@@ -35,18 +35,23 @@ export function createBrowserLogger(opts = {}) {
                 batch_id: batch.id,
                 events: batch.events.length,
             }),
+            onPartialFailure: (batch, failed, failures) => console.warn(`log ship: Axiom rejected ${failed} of ${batch.events.length} events`, {
+                batch_id: batch.id,
+                failures,
+            }),
         })
         : undefined;
     function build(context) {
         const factory = { app, source: 'browser', context };
         const emit = (level, message, meta) => {
-            const event = makeEvent(factory, level, message, {
+            const events = makeEvents(factory, level, message, {
                 ...meta,
                 page: location.pathname,
             });
             if (opts.console)
                 console[level === 'debug' ? 'debug' : level === 'error' ? 'error' : 'log'](message, meta ?? '');
-            shipper?.enqueue(event);
+            for (const event of events)
+                shipper?.enqueue(event);
         };
         return {
             debug: (m, meta) => emit('debug', m, meta),

package/dist/log-core.d.ts

@@ -28,7 +28,11 @@ export interface SerializedError {
 }
 export declare function serializeError(err: unknown): SerializedError;
 export declare function truncate(s: string, max: number): string;
-export type ShipFailureReason = 'give-up' | 'buffer-overflow' | 'shutdown';
+/** give-up: every attempt failed; buffer-overflow: evicted by the cap;
+ *  shutdown: undelivered at the final flush; rejected: Axiom refused the
+ *  bytes outright (a 4xx other than 408/429 — e.g. the dataset column limit),
+ *  so the batch was dropped on its first attempt with Axiom's error text. */
+export type ShipFailureReason = 'give-up' | 'buffer-overflow' | 'shutdown' | 'rejected';
 /** What failure callbacks see of a batch. `attempt` is attempts made so far. */
 export interface ShippedBatch {
     id: string;
@@ -66,6 +70,9 @@ export interface ShipperOptions {
     onAttemptFailure?: (batch: ShippedBatch, error: unknown) => void;
     /** The batch is abandoned — its events will never reach Axiom. */
     onGiveUp?: (batch: ShippedBatch, error: unknown, reason: ShipFailureReason) => void;
+    /** Axiom accepted the batch (2xx) but reported `failed` events it did not
+     *  keep, listed in `failures`. Those events are lost; the rest landed. */
+    onPartialFailure?: (batch: ShippedBatch, failed: number, failures: unknown[]) => void;
 }
 export declare class LogShipper {
     private readonly opts;
@@ -105,7 +112,7 @@ export declare class LogShipper {
     private pump;
     private schedulePump;
     private attempt;
-    /** Returns the failure, or undefined on success. Never throws. */
+    /** One POST. Never throws. */
     private ship;
     private runFinal;
 }
@@ -115,3 +122,34 @@ export interface EventFactoryOptions {
     context?: Record<string, unknown>;
 }
 export declare function makeEvent(opts: EventFactoryOptions, level: LogLevel, message: string, meta?: Record<string, unknown>): LogEvent;
+/** The event for one logger call plus, the first time a site's meta breaks
+ *  the shape rules in this process, a WARN `kind=log-shape` naming the site. */
+export declare function makeEvents(opts: EventFactoryOptions, level: LogLevel, message: string, meta?: Record<string, unknown>): LogEvent[];
+/** Grammar for one key segment under `meta`. */
+export declare const META_KEY: RegExp;
+/** Deepest leaf allowed: `meta.a.b`. */
+export declare const META_MAX_DEPTH = 2;
+/** Flattened leaves per event above which the site is flagged as too wide. */
+export declare const META_LEAF_BUDGET = 40;
+export interface ShapeIssue {
+    /** rows: data-shaped keys rewritten as rows; stringified: an object below
+     *  the depth cap became a JSON string; wide: over the leaf budget. */
+    kind: 'rows' | 'stringified' | 'wide';
+    /** Dotted path of the object concerned, e.g. `meta.phases`. */
+    path: string;
+    /** rows: the first offending key. */
+    sample?: string;
+    /** wide: the flattened leaf count. */
+    leaves?: number;
+}
+/** Apply the key grammar and depth cap to `meta`. Returns the input object
+ *  itself when nothing needed rewriting. Arrays are never walked — an array
+ *  is one column whatever its elements look like, which is what makes rows
+ *  the escape hatch. */
+export declare function shapeMeta(meta: Record<string, unknown>): {
+    meta: Record<string, unknown>;
+    issues: ShapeIssue[];
+    leaves: number;
+};
+/** Forget which sites have been flagged (tests). */
+export declare function resetLogShapeWarnings(): void;

package/dist/log-core.js

@@ -7,7 +7,8 @@
 // (~/migration/research/logging-reliability-design.md): every undelivered
 // batch either lands in Axiom or ends in an onGiveUp callback — the server
 // logger turns those into stdout drop markers that watchdog2's daily
-// ship-failure check counts.
+// ship-failure check counts. makeEvent also guards the shape of `meta`: every
+// distinct key is an Axiom column (see "Key-shape guard" below).
 export function serializeError(err) {
     if (err instanceof Error) {
         const out = { name: err.name, message: err.message, stack: err.stack };
@@ -24,6 +25,12 @@ export function serializeError(err) {
 export function truncate(s, max) {
     return s.length <= max ? s : s.slice(0, max) + `…[+${s.length - max}]`;
 }
+const RESPONSE_BODY_MAX = 300;
+/** A 4xx is a verdict on the bytes, not the moment: retrying the identical
+ *  body cannot succeed. 408 (timeout) and 429 (rate limit) are the exceptions. */
+function isTerminalStatus(status) {
+    return status >= 400 && status < 500 && status !== 408 && status !== 429;
+}
 function randomId() {
     const c = globalThis.crypto;
     return c?.randomUUID?.() ?? Math.random().toString(36).slice(2) + Date.now().toString(36);
@@ -217,20 +224,25 @@ export class LogShipper {
     attempt(batch, timeoutMs) {
         batch.attempting = true;
         const p = (async () => {
-            const error = await this.ship(batch.body, timeoutMs);
+            const outcome = await this.ship(batch.body, timeoutMs);
             batch.attempting = false;
             if (batch.terminal)
                 return;
             batch.attempt++;
             batch.settleFirst();
-            if (error === undefined) {
+            if (outcome.ok) {
                 this.remove(batch);
+                if (outcome.failed > 0)
+                    this.opts.onPartialFailure?.(batch, outcome.failed, outcome.failures);
+            }
+            else if (outcome.terminal) {
+                this.giveUp(batch, outcome.error, 'rejected');
             }
             else if (batch.attempt >= this.maxAttempts) {
-                this.giveUp(batch, error, 'give-up');
+                this.giveUp(batch, outcome.error, 'give-up');
             }
             else {
-                this.opts.onAttemptFailure?.(batch, error);
+                this.opts.onAttemptFailure?.(batch, outcome.error);
                 batch.nextAttemptAt =
                     Date.now() + this.retryDelaysMs[Math.min(batch.attempt - 1, this.retryDelaysMs.length - 1)];
                 this.schedulePump();
@@ -240,7 +252,7 @@ export class LogShipper {
         void p.finally(() => this.inflight.delete(p));
         return p;
     }
-    /** Returns the failure, or undefined on success. Never throws. */
+    /** One POST. Never throws. */
     async ship(body, timeoutMs) {
         const fetchFn = this.opts.fetchFn ?? fetch;
         try {
@@ -254,10 +266,26 @@ export class LogShipper {
                 body,
                 signal: AbortSignal.timeout(timeoutMs),
             });
-            return res.ok ? undefined : new Error(`ingest returned ${res.status}`);
+            if (res.ok) {
+                const parsed = (await res.json().catch(() => undefined));
+                return {
+                    ok: true,
+                    failed: typeof parsed?.failed === 'number' ? parsed.failed : 0,
+                    failures: Array.isArray(parsed?.failures) ? parsed.failures : [],
+                };
+            }
+            // Axiom's error body names the cause (e.g. the field that would exceed
+            // the column limit); quote it so the drop marker can be acted on.
+            const text = (await res.text().catch(() => '')).replace(/\s+/g, ' ').trim();
+            const detail = text ? `: ${truncate(text, RESPONSE_BODY_MAX)}` : '';
+            return {
+                ok: false,
+                error: new Error(`ingest returned ${res.status}${detail}`),
+                terminal: isTerminalStatus(res.status),
+            };
         }
         catch (err) {
-            return err ?? new Error('ship failed');
+            return { ok: false, error: err ?? new Error('ship failed'), terminal: false };
         }
     }
     async runFinal(deadlineMs) {
@@ -314,11 +342,25 @@ export class LogShipper {
     }
 }
 export function makeEvent(opts, level, message, meta) {
+    return buildEvent(opts, level, message, meta).event;
+}
+function buildEvent(opts, level, message, meta) {
     let m = meta;
-    if (m && m.error !== undefined && !isPlainSerialized(m.error)) {
-        m = { ...m, error: serializeError(m.error) };
+    const issues = [];
+    if (m) {
+        if (m.error !== undefined && !isPlainSerialized(m.error)) {
+            m = { ...m, error: serializeError(m.error) };
+        }
+        const shaped = shapeMeta(m);
+        issues.push(...shaped.issues);
+        if (shaped.leaves > META_LEAF_BUDGET)
+            issues.push({ kind: 'wide', path: 'meta', leaves: shaped.leaves });
+        m = shaped.meta;
+        const rewrites = [...new Set(shaped.issues.map((i) => i.kind))].sort();
+        if (rewrites.length > 0)
+            m = { ...m, logShape: rewrites.join('+') };
     }
-    return {
+    const event = {
         _time: new Date().toISOString(),
         app: opts.app,
         level,
@@ -328,6 +370,105 @@ export function makeEvent(opts, level, message, meta) {
         ...opts.context,
         ...(m ? { meta: m } : {}),
     };
+    return { event, issues };
+}
+/** The event for one logger call plus, the first time a site's meta breaks
+ *  the shape rules in this process, a WARN `kind=log-shape` naming the site. */
+export function makeEvents(opts, level, message, meta) {
+    const { event, issues } = buildEvent(opts, level, message, meta);
+    const fresh = issues.filter((i) => {
+        const key = `${i.kind}\n${message}`;
+        if (flagged.has(key))
+            return false;
+        flagged.add(key);
+        return true;
+    });
+    if (fresh.length === 0)
+        return [event];
+    const warning = makeEvent(opts, 'warn', `log-shape: "${message}" — ${fresh.map(describeIssue).join('; ')}. ` +
+        'Fix the logging site: keys are columns (OPERATIONS.md).', { sourceMessage: message, sourceLevel: level, issues: fresh });
+    return [event, { ...warning, kind: 'log-shape' }];
+}
+// ---- Key-shape guard -------------------------------------------------------
+// Every distinct flattened key under `meta` becomes an Axiom column — one
+// schema per dataset, fleet-wide, kept for the retention year, capped at
+// 1025 — and at the cap Axiom rejects whole batches. Keys must therefore be
+// a fixed vocabulary; identity belongs in values. The guard rewrites the two
+// shapes that mint columns from data (an object keyed by data → rows; an
+// object nested deeper than two levels → a JSON string), marks the event
+// with `meta.logShape`, and flags the site once per process so it gets fixed
+// at the source. Rules: ~/plasticine-way/docs/OPERATIONS.md → "Keys are
+// columns".
+/** Grammar for one key segment under `meta`. */
+export const META_KEY = /^[A-Za-z][A-Za-z0-9_]{0,39}$/;
+/** Deepest leaf allowed: `meta.a.b`. */
+export const META_MAX_DEPTH = 2;
+/** Flattened leaves per event above which the site is flagged as too wide. */
+export const META_LEAF_BUDGET = 40;
+const STRINGIFIED_MAX = 2000;
+const SAMPLE_KEY_MAX = 60;
+function isPlainObject(v) {
+    if (typeof v !== 'object' || v === null || Array.isArray(v))
+        return false;
+    const proto = Object.getPrototypeOf(v);
+    return proto === Object.prototype || proto === null;
+}
+/** Apply the key grammar and depth cap to `meta`. Returns the input object
+ *  itself when nothing needed rewriting. Arrays are never walked — an array
+ *  is one column whatever its elements look like, which is what makes rows
+ *  the escape hatch. */
+export function shapeMeta(meta) {
+    const issues = [];
+    let leaves = 0;
+    const toRows = (obj) => Object.entries(obj).map(([key, value]) => (isPlainObject(value) ? { key, ...value } : { key, value }));
+    const walk = (obj, path, depth) => {
+        const bad = Object.keys(obj).find((k) => !META_KEY.test(k));
+        if (bad !== undefined) {
+            issues.push({ kind: 'rows', path, sample: truncate(bad, SAMPLE_KEY_MAX) });
+            leaves++;
+            // meta itself keyed by data: keep it an object so the marker fits.
+            return depth === 1 ? { rows: toRows(obj) } : toRows(obj);
+        }
+        const out = {};
+        for (const [k, v] of Object.entries(obj)) {
+            // The serialized error is baselib's own fixed shape (its `cause` sits
+            // at level 3) and is exempt from the depth cap.
+            if (isPlainObject(v) && !(depth === 1 && k === 'error' && isPlainSerialized(v))) {
+                if (depth >= META_MAX_DEPTH) {
+                    issues.push({ kind: 'stringified', path: `${path}.${k}` });
+                    out[k] = truncate(JSON.stringify(v), STRINGIFIED_MAX);
+                    leaves++;
+                }
+                else {
+                    out[k] = walk(v, `${path}.${k}`, depth + 1);
+                }
+            }
+            else {
+                out[k] = v;
+                leaves++;
+            }
+        }
+        return out;
+    };
+    const shaped = walk(meta, 'meta', 1);
+    return { meta: issues.length > 0 ? shaped : meta, issues, leaves };
+}
+// Sites already flagged in this process, by issue kind and message: the
+// first offending event is worth a WARN, the ten-thousandth is not.
+const flagged = new Set();
+/** Forget which sites have been flagged (tests). */
+export function resetLogShapeWarnings() {
+    flagged.clear();
+}
+function describeIssue(i) {
+    switch (i.kind) {
+        case 'rows':
+            return `${i.path} is keyed by data (e.g. "${i.sample}") → logged as rows`;
+        case 'stringified':
+            return `${i.path} is nested deeper than two levels → logged as a JSON string`;
+        case 'wide':
+            return `${i.path} flattens to ${i.leaves} leaves (budget ${META_LEAF_BUDGET})`;
+    }
 }
 function isPlainSerialized(v) {
     return typeof v === 'object' && v !== null && !(v instanceof Error) && 'message' in v && 'name' in v;

package/dist/log.d.ts

@@ -31,5 +31,5 @@ export declare function flushAllLoggers(opts?: {
  *  first_time/last_time are the batch's own event-time range; the marker
  *  line itself can be written minutes later (next CPU window), so triage
  *  windows on these fields, never on the marker's timestamp. */
-export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp'>;
+export declare function stdoutShipMarkers(app: string, dataset: string): Pick<ShipperOptions, 'onAttemptFailure' | 'onGiveUp' | 'onPartialFailure'>;
 export declare function createLogger(opts: LoggerOptions): Logger;

package/dist/log.js

@@ -4,10 +4,11 @@
 //
 // Conventions (new PW OPERATIONS.md): ERROR for anything that could reflect
 // a real problem; log entry points, outbound calls, and DB writes with ids;
-// rich meta is encouraged, but keep meta keys disciplined — every distinct
-// flattened key becomes an Axiom field (dataset cap 1024).
+// rich meta is encouraged, but keys are columns — every distinct flattened
+// key becomes an Axiom field (dataset cap 1025), so identity goes in values.
+// log-core's key-shape guard rewrites offenders and WARNs once per site.
 import { writeSync } from 'node:fs';
-import { LogShipper, makeEvent, } from './log-core.js';
+import { LogShipper, makeEvents, truncate, } from './log-core.js';
 export { LogShipper, serializeError, truncate } from './log-core.js';
 // Registry of every shipper created here (plus any registered explicitly),
 // used by flushAllLoggers and graceful shutdown. Shipper-level on purpose:
@@ -73,6 +74,26 @@ export function stdoutShipMarkers(app, dataset) {
             message: `axiom ship dropped a batch (${reason}): ${String(error)} ` +
                 `(${batch.events.length} events, ${batch.bytes} bytes; stdout mirror has them)`,
         }),
+        // Same marker kind, sized to what was actually lost, so the daily
+        // ship-failure report's dropped-event total stays exact.
+        onPartialFailure: (batch, failed, failures) => {
+            const detail = truncate(JSON.stringify(failures), 300);
+            write({
+                severity: 'ERROR',
+                app,
+                dataset,
+                kind: 'log-ship-drop',
+                batch_id: batch.id,
+                events: failed,
+                bytes: batch.bytes,
+                first_time: batch.firstTime,
+                last_time: batch.lastTime,
+                reason: 'partial',
+                error: detail,
+                message: `axiom accepted a batch but rejected ${failed} of ${batch.events.length} events: ${detail} ` +
+                    `(stdout mirror has them)`,
+            });
+        },
     };
 }
 export function createLogger(opts) {
@@ -97,10 +118,11 @@ export function createLogger(opts) {
     function build(context) {
         const factory = { app: opts.app, source: 'server', context };
         const emit = (level, message, meta) => {
-            const event = makeEvent(factory, level, message, meta);
-            if (stdout)
-                process.stdout.write(JSON.stringify(event) + '\n');
-            shipper?.enqueue(event);
+            for (const event of makeEvents(factory, level, message, meta)) {
+                if (stdout)
+                    process.stdout.write(JSON.stringify(event) + '\n');
+                shipper?.enqueue(event);
+            }
         };
         return {
             debug: (m, meta) => emit('debug', m, meta),

package/dist/s2s.d.ts

@@ -61,7 +61,7 @@ export declare function resolveS2sCaller(opts: S2sOptions & {
  *  sessions) — e.g. scheduler pokes, Pub/Sub push, admin APIs. */
 export declare function s2sAuth(opts: S2sOptions): import("hono").MiddlewareHandler<any, string, {}, Response | (Response & import("hono").TypedResponse<{
     error: string;
-}, 401 | 403 | 429, "json">)>;
+}, 429 | 401 | 403, "json">)>;
 /** fetch() with an OIDC identity token for the target's canonical audience.
  *  In test mode, sends the synthetic X-Test-S2S-Caller header instead
  *  (identity from TEST_S2S_IDENTITY, default 'test-service@test'). */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",