@code-pushup/utils 0.109.0 → 0.111.0

package/src/lib/profiler/wal-json-trace.js

@@ -0,0 +1,77 @@
+import { defaultClock } from '../clock-epoch.js';
+import { decodeTraceEvent, encodeTraceEvent, getCompleteEvent, getInstantEventTracingStartedInBrowser, getTraceFile, } from './trace-file-utils.js';
+/** Name for the trace start margin event */
+const TRACE_START_MARGIN_NAME = '[trace padding start]';
+/** Name for the trace end margin event */
+const TRACE_END_MARGIN_NAME = '[trace padding end]';
+/** Microseconds of padding to add before/after trace events (1000ms = 1,000,000μs) */
+const TRACE_MARGIN_US = 1_000_000;
+/** Duration in microseconds for margin events (20ms = 20,000μs) */
+const TRACE_MARGIN_DURATION_US = 20_000;
+/**
+ * Generates a complete Chrome DevTools trace file content as JSON string.
+ * Adds margin events around the trace events and includes metadata.
+ * @param events - Array of user timing trace events to include
+ * @param metadata - Optional custom metadata to include in the trace file
+ * @returns JSON string representation of the complete trace file
+ */
+export function generateTraceContent(events, metadata) {
+    const traceContainer = getTraceFile({
+        traceEvents: events,
+        startTime: new Date().toISOString(),
+        metadata: {
+            ...metadata,
+            generatedAt: new Date().toISOString(),
+        },
+    });
+    const marginUs = TRACE_MARGIN_US;
+    const marginDurUs = TRACE_MARGIN_DURATION_US;
+    const sortedEvents = [...events].sort((a, b) => a.ts - b.ts);
+    const fallbackTs = defaultClock.epochNowUs();
+    const firstTs = sortedEvents.at(0)?.ts ?? fallbackTs;
+    const lastTs = sortedEvents.at(-1)?.ts ?? fallbackTs;
+    const startTs = firstTs - marginUs;
+    const endTs = lastTs + marginUs;
+    const traceEvents = [
+        getInstantEventTracingStartedInBrowser({
+            ts: startTs,
+            url: events.length === 0 ? 'empty-trace' : 'generated-trace',
+        }),
+        getCompleteEvent({
+            name: TRACE_START_MARGIN_NAME,
+            ts: startTs,
+            dur: marginDurUs,
+        }),
+        ...sortedEvents,
+        getCompleteEvent({
+            name: TRACE_END_MARGIN_NAME,
+            ts: endTs,
+            dur: marginDurUs,
+        }),
+    ];
+    return JSON.stringify({ ...traceContainer, traceEvents });
+}
+/**
+ * Creates a WAL (Write-Ahead Logging) format configuration for Chrome DevTools trace files.
+ * Automatically finalizes shards into complete trace files with proper metadata and margin events.
+ * @returns WalFormat configuration object with baseName, codec, extensions, and finalizer
+ */
+export function traceEventWalFormat() {
+    const baseName = 'trace';
+    const walExtension = '.jsonl';
+    const finalExtension = '.json';
+    return {
+        baseName,
+        walExtension,
+        finalExtension,
+        codec: {
+            encode: (event) => JSON.stringify(encodeTraceEvent(event)),
+            decode: (json) => decodeTraceEvent(JSON.parse(json)),
+        },
+        finalizer: (records, metadata) => {
+            const validRecords = records.filter((r) => !(typeof r === 'object' && r != null && '__invalid' in r));
+            return generateTraceContent(validRecords, metadata);
+        },
+    };
+}
+//# sourceMappingURL=wal-json-trace.js.map

package/src/lib/profiler/wal-json-trace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wal-json-trace.js","sourceRoot":"","sources":["../../../../src/lib/profiler/wal-json-trace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,sCAAsC,EACtC,YAAY,GACb,MAAM,uBAAuB,CAAC;AAG/B,4CAA4C;AAC5C,MAAM,uBAAuB,GAAG,uBAAuB,CAAC;AACxD,0CAA0C;AAC1C,MAAM,qBAAqB,GAAG,qBAAqB,CAAC;AACpD,sFAAsF;AACtF,MAAM,eAAe,GAAG,SAAS,CAAC;AAClC,mEAAmE;AACnE,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAExC;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAA8B,EAC9B,QAAkC;IAElC,MAAM,cAAc,GAAG,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM;QACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE;YACR,GAAG,QAAQ;YACX,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC;KACF,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,eAAe,CAAC;IACjC,MAAM,WAAW,GAAG,wBAAwB,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAW,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,UAAU,CAAC;IAC7D,MAAM,MAAM,GAAW,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,UAAU,CAAC;IAE7D,MAAM,OAAO,GAAG,OAAO,GAAG,QAAQ,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IAEhC,MAAM,WAAW,GAAiB;QAChC,sCAAsC,CAAC;YACrC,EAAE,EAAE,OAAO;YACX,GAAG,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,iBAAiB;SAC7D,CAAC;QACF,gBAAgB,CAAC;YACf,IAAI,EAAE,uBAAuB;YAC7B,EAAE,EAAE,OAAO;YACX,GAAG,EAAE,WAAW;SACjB,CAAC;QACF,GAAG,YAAY;QACf,gBAAgB,CAAC;YACf,IAAI,EAAE,qBAAqB;YAC3B,EAAE,EAAE,KAAK;YACT,GAAG,EAAE,WAAW;SACjB,CAAC;KACH,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IACjC,MAAM,QAAQ,GAAG,OAAO,CAAC;IACzB,MAAM,YAAY,GAAG,QAAQ,CAAC;IAC9B,MAAM,cAAc,GAAG,OAAO,CAAC;IAC/B,OAAO;QACL,QAAQ;QACR,YAAY;QACZ,cAAc;QACd,KAAK,EAAE;YACL,MAAM,EAAE,CAAC,KAA2B,EAAE,EAAE,CACtC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CACvB,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAyB;SAC7D;QACD,SAAS,EAAE,CACT,OAAwD,EACxD,QAAkC,EAClC,EAAE;YACF,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CACjC,CAAC,CAAC,EAA6B,EAAE,CAC/B,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,WAAW,IAAI,CAAC,CAAC,CAC5D,CAAC;YACF,OAAO,oBAAoB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACtD,CAAC;KACwC,CAAC;AAC9C,CAAC"}

package/src/lib/wal.d.ts

@@ -0,0 +1,248 @@
+/**
+ * Codec for encoding/decoding values to/from strings for WAL storage.
+ * Used to serialize/deserialize records written to and read from WAL files.
+ */
+export type Codec<I, O = string> = {
+    /** Encode a value to a string for storage */
+    encode: (v: I) => O;
+    /** Decode a string back to the original value type */
+    decode: (data: O) => I;
+};
+export type InvalidEntry<O = string> = {
+    __invalid: true;
+    raw: O;
+};
+/**
+ * Interface for sinks that can append items.
+ * Allows for different types of appendable storage (WAL, in-memory, etc.)
+ */
+export type AppendableSink<T> = Recoverable & {
+    append: (item: T) => void;
+    isClosed: () => boolean;
+    open?: () => void;
+    close?: () => void;
+};
+/**
+ * Interface for sinks that support recovery operations.
+ * Represents the recoverable subset of AppendableSink functionality.
+ */
+export type Recoverable = {
+    recover: () => RecoverResult<unknown>;
+    repack: (out?: string) => void;
+    finalize?: (opt?: Record<string, unknown>) => void;
+};
+/**
+ * Result of recovering records from a WAL file.
+ * Contains successfully recovered records and any errors encountered during parsing.
+ */
+export type RecoverResult<T> = {
+    /** Successfully recovered records */
+    records: T[];
+    /** Errors encountered during recovery with line numbers and context */
+    errors: {
+        lineNo: number;
+        line: string;
+        error: Error;
+    }[];
+    /** Last incomplete line if file was truncated (null if clean) */
+    partialTail: string | null;
+};
+/**
+ * Statistics about the WAL file state and last recovery operation.
+ */
+export type WalStats<T> = {
+    /** File path for this WAL */
+    filePath: string;
+    /** Whether the WAL file is currently closed */
+    isClosed: boolean;
+    /** Whether the WAL file exists on disk */
+    fileExists: boolean;
+    /** File size in bytes (0 if file doesn't exist) */
+    fileSize: number;
+    /** Last recovery state from the most recent {@link recover} or {@link repack} operation */
+    lastRecovery: RecoverResult<T | InvalidEntry<string>> | null;
+};
+export declare const createTolerantCodec: <I, O = string>(codec: {
+    encode: (v: I) => O;
+    decode: (d: O) => I;
+}) => Codec<I | InvalidEntry<O>, O>;
+export declare function filterValidRecords<T>(records: (T | InvalidEntry<unknown>)[]): T[];
+/**
+ * Pure helper function to recover records from WAL file content.
+ * @param content - Raw file content as string
+ * @param decode - function for decoding records
+ * @returns Recovery result with records, errors, and partial tail
+ */
+export declare function recoverFromContent<T>(content: string, decode: Codec<T>['decode']): RecoverResult<T>;
+/**
+ * Write-Ahead Log implementation for crash-safe append-only logging.
+ * Provides atomic operations for writing, recovering, and repacking log entries.
+ */
+export declare class WriteAheadLogFile<T> implements AppendableSink<T> {
+    #private;
+    /**
+     * Create a new WAL file instance.
+     * @param options - Configuration options
+     */
+    constructor(options: {
+        file: string;
+        codec: Codec<T>;
+    });
+    /** Get the file path for this WAL */
+    getPath: () => string;
+    /** Open the WAL file for writing (creates directories if needed) */
+    open: () => void;
+    /**
+     * Append a record to the WAL.
+     * @param v - Record to append
+     * @throws Error if WAL cannot be opened
+     */
+    append: (v: T) => void;
+    /** Close the WAL file */
+    close: () => void;
+    isClosed: () => boolean;
+    /**
+     * Recover all records from the WAL file.
+     * Handles partial writes and decode errors gracefully.
+     * Updates the recovery state (accessible via {@link getStats}).
+     * @returns Recovery result with records, errors, and partial tail
+     */
+    recover(): RecoverResult<T | InvalidEntry<string>>;
+    /**
+     * Repack the WAL by recovering all valid records and rewriting cleanly.
+     * Removes corrupted entries and ensures clean formatting.
+     * Updates the recovery state (accessible via {@link getStats}).
+     * @param out - Output path (defaults to current file)
+     */
+    repack(out?: string): void;
+    /**
+     * Get comprehensive statistics about the WAL file state.
+     * Includes file information, open/close status, and last recovery state.
+     * @returns Statistics object with file info and last recovery state
+     */
+    getStats(): WalStats<T>;
+}
+/**
+ * Format descriptor that binds codec and file extension together.
+ * Prevents misconfiguration by keeping related concerns in one object.
+ */
+export type WalFormat<T extends object | string> = {
+    /** Base name for the WAL (e.g., "trace") */
+    baseName: string;
+    /** Shard file extension (e.g., ".jsonl") */
+    walExtension: string;
+    /** Final file extension (e.g., ".json", ".trace.json") falls back to walExtension if not provided */
+    finalExtension: string;
+    /** Codec for encoding/decoding records */
+    codec: Codec<T, string>;
+    /** Finalizer for converting records to a string */
+    finalizer: (records: (T | InvalidEntry<string>)[], opt?: Record<string, unknown>) => string;
+};
+export declare const stringCodec: <T extends string | object = string>() => Codec<T>;
+/**
+ * Parses a partial WalFormat configuration and returns a complete WalFormat object.
+ * All fallback values are targeting string types.
+ *  - baseName defaults to 'wal'
+ *  - walExtension defaults to '.log'
+ *  - finalExtension defaults to '.log'
+ *  - codec defaults to stringCodec<T>()
+ *  - finalizer defaults to encoding each record using codec.encode() and joining with newlines.
+ *    For object types, this properly JSON-stringifies them (not [object Object]).
+ *    InvalidEntry records use their raw string value directly.
+ * @param format - Partial WalFormat configuration
+ * @returns Parsed WalFormat with defaults filled in
+ */
+export declare function parseWalFormat<T extends object | string = object>(format: Partial<WalFormat<T>>): WalFormat<T>;
+/**
+ * Determines if this process is the leader WAL process using the origin PID heuristic.
+ *
+ * The leader is the process that first enabled profiling (the one that set CP_PROFILER_ORIGIN_PID).
+ * All descendant processes inherit the environment but have different PIDs.
+ *
+ * @returns true if this is the leader WAL process, false otherwise
+ */
+export declare function isLeaderWal(envVarName: string, profilerID: string): boolean;
+/**
+ * Initialize the origin PID environment variable if not already set.
+ * This must be done as early as possible before any user code runs.
+ * Sets envVarName to the current process ID if not already defined.
+ */
+export declare function setLeaderWal(envVarName: string, profilerID: string): void;
+/**
+ * Generates a human-readable shard ID.
+ * This ID is unique per process/thread/shard combination and used in the file name.
+ * Format: readable-timestamp.pid.threadId.shardCount
+ * Example: "20240101-120000-000.12345.1.1"
+ * Becomes file: trace.20240101-120000-000.12345.1.1.log
+ */
+export declare function getShardId(): string;
+/**
+ * Generates a human-readable sharded group ID.
+ * This ID is a globally unique, sortable, human-readable date string per run.
+ * Used directly as the folder name to group shards.
+ * Format: yyyymmdd-hhmmss-ms
+ * Example: "20240101-120000-000"
+ */
+export declare function getShardedGroupId(): string;
+/**
+ * Regex patterns for validating WAL ID formats
+ */
+export declare const WAL_ID_PATTERNS: {
+    /** Readable date format: yyyymmdd-hhmmss-ms */
+    readonly READABLE_DATE: RegExp;
+    /** Group ID format: yyyymmdd-hhmmss-ms */
+    readonly GROUP_ID: RegExp;
+    /** Shard ID format: readable-date.pid.threadId.count */
+    readonly SHARD_ID: RegExp;
+};
+export declare function sortableReadableDateString(timestampMs: string): string;
+/**
+ * Generates a path to a shard file using human-readable IDs.
+ * Both groupId and shardId are already in readable date format.
+ *
+ * Example with groupId "20240101-120000-000" and shardId "20240101-120000-000.12345.1.1":
+ * Full path: /base/20240101-120000-000/trace.20240101-120000-000.12345.1.1.log
+ *
+ * @param opt.dir - The directory to store the shard file
+ * @param opt.format - The WalFormat to use for the shard file
+ * @param opt.groupId - The human-readable group ID (yyyymmdd-hhmmss-ms format)
+ * @param opt.shardId - The human-readable shard ID (readable-timestamp.pid.threadId.count format)
+ * @returns The path to the shard file
+ */
+export declare function getShardedPath<T extends object | string = object>(opt: {
+    dir?: string;
+    format: WalFormat<T>;
+    groupId: string;
+    shardId: string;
+}): string;
+export declare function getShardedFinalPath<T extends object | string = object>(opt: {
+    dir?: string;
+    format: WalFormat<T>;
+    groupId: string;
+}): string;
+/**
+ * Sharded Write-Ahead Log manager for coordinating multiple WAL shards.
+ * Handles distributed logging across multiple processes/files with atomic finalization.
+ */
+export declare class ShardedWal<T extends object | string = object> {
+    #private;
+    readonly groupId: string;
+    /**
+     * Create a sharded WAL manager.
+     */
+    constructor(opt: {
+        dir?: string;
+        format: Partial<WalFormat<T>>;
+        groupId?: string;
+    });
+    shard(shardId?: string): WriteAheadLogFile<T>;
+    /** Get all shard file paths matching this WAL's base name */
+    private shardFiles;
+    /**
+     * Finalize all shards by merging them into a single output file.
+     * Recovers all records from all shards, validates no errors, and writes merged result.
+     * @throws Error if any shard contains decode errors
+     */
+    finalize(opt?: Record<string, unknown>): void;
+    cleanup(): void;
+}