@smoothbricks/lmao 0.1.0

package/dist/lib/flushScheduler.js

@@ -0,0 +1,388 @@
+/**
+ * Background flush scheduler with adaptive flushing
+ *
+ * Per specs/lmao/01b2_buffer_self_tuning.md:
+ * - Adaptive flush intervals based on capacity, time, memory pressure
+ * - Idle detection for opportunistic flushing
+ * - Background processing (cold path)
+ */
+import { Column, tableFromColumns } from '@uwdata/flechette';
+import { convertSpanTreeToArrowTable } from './convertToArrow.js';
+/**
+ * Number of flushes before logging capacity stats.
+ * Capacity stats are logged periodically to avoid overhead on every flush.
+ */
+const CAPACITY_STATS_FLUSH_INTERVAL = 100;
+function isSpanBufferConstructor(value) {
+    return typeof value === 'function' && typeof Reflect.get(value, 'stats') === 'object';
+}
+function isNodeTimer(value) {
+    return typeof value === 'object' && value !== null && typeof Reflect.get(value, 'hasRef') === 'function';
+}
+function mergeTables(first, rest) {
+    if (rest.length === 0)
+        return first;
+    const names = first.names;
+    const merged = names.map((name, index) => {
+        const batches = [...(first.getChildAt(index)?.data ?? [])];
+        for (const table of rest) {
+            const col = table.getChild(name);
+            if (!col)
+                continue;
+            batches.push(...col.data);
+        }
+        return [name, new Column(batches)];
+    });
+    return tableFromColumns(merged);
+}
+/**
+ * Background flush scheduler
+ * Manages automatic flushing of span buffers to Arrow tables
+ */
+export class FlushScheduler {
+    config;
+    handler;
+    buffers = new Set();
+    lastFlushTime = Date.now();
+    lastActivityTime = Date.now();
+    flushTimer;
+    idleTimer;
+    memoryTimer;
+    isRunning = false;
+    /** Flush counter for periodic capacity stats logging */
+    _flushCount = 0;
+    /**
+     * Track unique (SpanBufferConstructor, OpMetadata) pairs that have been flushed since last capacity stats log.
+     * We use SpanBufferConstructor as the key (for dedup) and store the OpMetadata for building PreEncodedEntry at conversion time.
+     * The constructor has static stats property with the capacity stats we need.
+     */
+    _bufferClassesSinceLastStatsLog = new Map();
+    constructor(handler, config = {}) {
+        this.handler = handler;
+        // Apply defaults
+        this.config = {
+            maxFlushInterval: config.maxFlushInterval ?? 10000,
+            minFlushInterval: config.minFlushInterval ?? 1000,
+            capacityThreshold: config.capacityThreshold ?? 0.8,
+            idleDetection: config.idleDetection ?? true,
+            idleTimeout: config.idleTimeout ?? 5000,
+            memoryPressureDetection: config.memoryPressureDetection ?? true,
+            memoryPressureThreshold: config.memoryPressureThreshold ?? 100 * 1024 * 1024, // 100MB
+        };
+    }
+    /**
+     * Register a buffer for automatic flushing
+     */
+    register(buffer) {
+        this.buffers.add(buffer);
+        this.lastActivityTime = Date.now();
+        // Start scheduler if not running
+        if (!this.isRunning) {
+            this.start();
+        }
+    }
+    /**
+     * Start the flush scheduler
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        this.lastFlushTime = Date.now();
+        this.lastActivityTime = Date.now();
+        // Set up periodic flush timer
+        this.flushTimer = setInterval(() => {
+            this.checkFlushConditions();
+        }, 1000); // Check every second
+        // Set up idle detection timer
+        if (this.config.idleDetection) {
+            this.idleTimer = setInterval(() => {
+                this.checkIdleFlush();
+            }, this.config.idleTimeout);
+        }
+        // Set up memory pressure timer (Node.js only)
+        if (this.config.memoryPressureDetection && typeof process !== 'undefined') {
+            this.memoryTimer = setInterval(() => {
+                this.checkMemoryPressure();
+            }, 2000); // Check every 2 seconds
+        }
+    }
+    /**
+     * Stop the flush scheduler
+     */
+    stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.flushTimer) {
+            clearInterval(this.flushTimer);
+            this.flushTimer = undefined;
+        }
+        if (this.idleTimer) {
+            clearInterval(this.idleTimer);
+            this.idleTimer = undefined;
+        }
+        if (this.memoryTimer) {
+            clearInterval(this.memoryTimer);
+            this.memoryTimer = undefined;
+        }
+    }
+    /**
+     * Manually trigger a flush
+     */
+    async flush() {
+        await this.doFlush('manual');
+    }
+    /**
+     * Check flush conditions and flush if needed
+     */
+    checkFlushConditions() {
+        const now = Date.now();
+        const timeSinceLastFlush = now - this.lastFlushTime;
+        // Check max flush interval
+        if (timeSinceLastFlush >= this.config.maxFlushInterval) {
+            this.doFlush('time').catch(console.error);
+            return;
+        }
+        // Check capacity threshold
+        if (this.shouldFlushByCapacity()) {
+            this.doFlush('capacity').catch(console.error);
+            return;
+        }
+    }
+    /**
+     * Check if should flush based on capacity
+     */
+    shouldFlushByCapacity() {
+        for (const buffer of this.buffers) {
+            const utilizationRatio = buffer._writeIndex / buffer._capacity;
+            if (utilizationRatio >= this.config.capacityThreshold) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Check idle state and flush if idle
+     */
+    checkIdleFlush() {
+        const now = Date.now();
+        const timeSinceActivity = now - this.lastActivityTime;
+        if (timeSinceActivity >= this.config.idleTimeout) {
+            // Only flush if there's data and min interval has passed
+            const timeSinceLastFlush = now - this.lastFlushTime;
+            if (this.buffers.size > 0 && timeSinceLastFlush >= this.config.minFlushInterval) {
+                this.doFlush('idle').catch(console.error);
+            }
+        }
+    }
+    /**
+     * Check memory pressure and flush if needed
+     * Node.js only
+     */
+    checkMemoryPressure() {
+        if (typeof process === 'undefined')
+            return;
+        try {
+            const memUsage = process.memoryUsage();
+            const availableMemory = memUsage.heapTotal - memUsage.heapUsed;
+            if (availableMemory < this.config.memoryPressureThreshold) {
+                this.doFlush('memory').catch(console.error);
+            }
+        }
+        catch {
+            // Ignore errors in memory detection
+        }
+    }
+    /**
+     * Perform the actual flush operation
+     */
+    async doFlush(reason) {
+        if (this.buffers.size === 0)
+            return;
+        const now = Date.now();
+        const timeSinceLastFlush = now - this.lastFlushTime;
+        // Respect minimum flush interval
+        if (reason !== 'manual' && timeSinceLastFlush < this.config.minFlushInterval) {
+            return;
+        }
+        // Collect all buffers to flush
+        const buffersToFlush = Array.from(this.buffers);
+        // Collect unique (SpanBufferConstructor, OpMetadata) pairs from buffers being flushed
+        // Use SpanBufferConstructor as the key to dedup - multiple buffers may share the same class
+        // We pick the first OpMetadata we see for each class (they should be consistent)
+        const bufferClassesInThisFlush = new Map();
+        for (const buffer of buffersToFlush) {
+            let currentBuffer = buffer;
+            while (currentBuffer) {
+                const bufferClass = currentBuffer.constructor;
+                if (!isSpanBufferConstructor(bufferClass)) {
+                    throw new TypeError('Expected span buffer constructor with shared stats metadata');
+                }
+                // Only add if we haven't seen this buffer class before
+                if (!bufferClassesInThisFlush.has(bufferClass)) {
+                    bufferClassesInThisFlush.set(bufferClass, currentBuffer._opMetadata);
+                }
+                currentBuffer = currentBuffer._overflow;
+            }
+        }
+        // Add to tracking map (for periodic capacity stats logging)
+        for (const [bufferClass, metadata] of bufferClassesInThisFlush) {
+            // Only set if not already present (preserve first metadata seen)
+            if (!this._bufferClassesSinceLastStatsLog.has(bufferClass)) {
+                this._bufferClassesSinceLastStatsLog.set(bufferClass, metadata);
+            }
+        }
+        // Increment flush counter
+        this._flushCount++;
+        // Check if we should log capacity stats
+        const shouldLogCapacityStats = this._flushCount >= CAPACITY_STATS_FLUSH_INTERVAL;
+        // Convert Map to CapacityStatsEntry[] for the conversion function
+        const modulesToLogStatsForConversion = shouldLogCapacityStats
+            ? Array.from(this._bufferClassesSinceLastStatsLog.entries()).map(([bufferClass, metadata]) => ({
+                bufferClass,
+                metadata,
+            }))
+            : undefined;
+        // Count total rows and buffers
+        let totalRows = 0;
+        let totalBuffers = 0;
+        for (const buffer of buffersToFlush) {
+            // Count rows in buffer chain
+            let currentBuffer = buffer;
+            while (currentBuffer) {
+                totalRows += currentBuffer._writeIndex;
+                totalBuffers++;
+                currentBuffer = currentBuffer._overflow;
+            }
+        }
+        if (totalRows === 0 && !shouldLogCapacityStats)
+            return;
+        // Convert all buffers to Arrow tables and concatenate
+        const tables = [];
+        for (const buffer of buffersToFlush) {
+            try {
+                const table = convertSpanTreeToArrowTable(buffer, undefined, modulesToLogStatsForConversion);
+                if (table.numRows > 0) {
+                    tables.push(table);
+                }
+            }
+            catch (error) {
+                console.error('Error converting buffer to Arrow table:', error);
+            }
+        }
+        // Concatenate all tables
+        if (tables.length === 0) {
+            // No tables but might have capacity stats - create empty table
+            // Capacity stats will be in the first table if any buffer had data
+            // If no buffers had data, we still need to handle capacity stats
+            // For now, return early - capacity stats will be logged on next flush with data
+            if (!shouldLogCapacityStats)
+                return;
+            // If we should log stats but have no tables, we need to create a table with just capacity stats
+            // This is handled by convertSpanTreeToArrowTable returning a table with capacity stats batch
+            // But since we have no buffers, we can't call it. So we skip logging stats this time.
+            // Stats will be logged when there's actual data to flush.
+            return;
+        }
+        const [firstTable, ...otherTables] = tables;
+        const combinedTable = mergeTables(firstTable, otherTables);
+        // Call handler
+        const metadata = {
+            totalRows,
+            totalBuffers,
+            flushReason: reason,
+            timestamp: now,
+        };
+        try {
+            await this.handler(combinedTable, metadata);
+            this.lastFlushTime = now;
+            // Reset flush counter and module tracking if we logged capacity stats
+            if (shouldLogCapacityStats) {
+                this._flushCount = 0;
+                this._bufferClassesSinceLastStatsLog.clear();
+            }
+            // Reset buffers after successful flush to avoid duplicate re-processing
+            for (const buffer of buffersToFlush) {
+                // Reset writeIndex to 0 for the root buffer
+                buffer._writeIndex = 0;
+                // Clear any linked/chained buffers
+                let nextBuffer = buffer._overflow;
+                while (nextBuffer) {
+                    nextBuffer._writeIndex = 0;
+                    const temp = nextBuffer._overflow;
+                    nextBuffer._overflow = undefined; // Unlink chained buffer
+                    nextBuffer = temp;
+                }
+                buffer._overflow = undefined; // Clear link from root buffer
+                // A flushed root buffer is no longer pending; callers must re-register on future writes.
+                this.buffers.delete(buffer);
+            }
+        }
+        catch (error) {
+            console.error('Error in flush handler:', error);
+            // Do NOT reset buffers on failure to avoid data loss
+        }
+    }
+    /**
+     * Record activity (called when buffers are written to)
+     */
+    recordActivity() {
+        this.lastActivityTime = Date.now();
+    }
+}
+/**
+ * Global flush scheduler instance (optional)
+ * Applications can use this or create their own
+ */
+let globalScheduler = null;
+/**
+ * Get or create global flush scheduler
+ */
+export function getGlobalFlushScheduler(handler, config) {
+    if (!globalScheduler) {
+        globalScheduler = new FlushScheduler(handler, config);
+    }
+    return globalScheduler;
+}
+/**
+ * Reset global flush scheduler (for testing)
+ */
+export function resetGlobalFlushScheduler() {
+    if (globalScheduler) {
+        globalScheduler.stop();
+        globalScheduler = null;
+    }
+}
+/**
+ * Test utilities for accessing FlushScheduler internals
+ * @internal
+ */
+export const FlushSchedulerTestUtils = {
+    /**
+     * Get the flush timer for a scheduler instance (for testing)
+     */
+    getFlushTimer(scheduler) {
+        const timer = Reflect.get(scheduler, 'flushTimer');
+        return isNodeTimer(timer) ? timer : undefined;
+    },
+    /**
+     * Set the flush timer for a scheduler instance (for testing)
+     */
+    setFlushTimer(scheduler, timer) {
+        Reflect.set(scheduler, 'flushTimer', timer);
+    },
+    /**
+     * Get the last activity time for a scheduler instance (for testing)
+     */
+    getLastActivityTime(scheduler) {
+        const time = Reflect.get(scheduler, 'lastActivityTime');
+        return typeof time === 'number' ? time : undefined;
+    },
+    /**
+     * Set the last activity time for a scheduler instance (for testing)
+     */
+    setLastActivityTime(scheduler, time) {
+        Reflect.set(scheduler, 'lastActivityTime', time);
+    },
+};

package/dist/lib/library.d.ts

@@ -0,0 +1,149 @@
+/**
+ * Library Prefix/Remapping Utilities
+ *
+ * WHY: Libraries need to write clean, domain-focused code (ctx.tag.status(200))
+ * while avoiding naming conflicts when composed with other libraries.
+ *
+ * HOW: At composition time (cold path), we generate a remapped SpanLogger class
+ * that exposes clean method names but writes to prefixed buffer columns.
+ *
+ * Per specs/lmao/01e_library_integration_pattern.md:
+ * - Libraries define clean schemas without prefixes (status, method, url)
+ * - Prefixing happens at composition time (http_status, http_method, http_url)
+ * - Zero hot path overhead - all remapping done via code generation at module creation
+ *
+ * WHAT: This module provides:
+ * - prefixSchema() - Rename schema fields with prefix
+ * - createPrefixMapping() - Build clean→prefixed name mapping
+ * - generateRemappedBufferViewClass() - Generate view for Arrow conversion
+ * - generateRemappedSpanLoggerClass() - Generate class with clean methods, prefixed writes
+ * - createRemappedSpanLoggerClass() - Compile and cache SpanLogger classes
+ */
+import type { SpanLoggerImpl } from './codegen/spanLoggerGenerator.js';
+import { LogSchema } from './schema/types.js';
+import type { AnySpanBuffer } from './types.js';
+type RemappedBufferViewConstructor = new (buffer: AnySpanBuffer) => AnySpanBuffer;
+type RemappedSpanLoggerConstructor<T extends LogSchema = LogSchema> = new (buffer: AnySpanBuffer, createOverflowBuffer: (buffer: AnySpanBuffer) => AnySpanBuffer, initialScopedAttributes?: Record<string, unknown>) => SpanLoggerImpl<T>;
+/**
+ * Prefix a tag attribute schema
+ * Renames all fields with a prefix to avoid conflicts
+ *
+ * Example:
+ * - Input: { status: S.number(), method: S.enum(['GET', 'POST']) }
+ * - Prefix: 'http'
+ * - Output: { http_status: S.number(), http_method: S.enum(['GET', 'POST']) }
+ */
+export declare function prefixSchema(schema: LogSchema, prefix: string): LogSchema & Record<string, unknown>;
+/**
+ * Prefix mapping: maps clean names to prefixed column names
+ *
+ * WHY: Library authors write ctx.tag.status() but buffer column is http_status
+ * HOW: At module creation time, we build this mapping once (cold path)
+ *
+ * @example
+ * // Input: schema = { status: S.number(), method: S.enum([...]) }, prefix = 'http'
+ * // Output: { status: 'http_status', method: 'http_method' }
+ */
+export interface PrefixMapping {
+    [cleanName: string]: string;
+}
+/**
+ * Create a mapping from clean field names to prefixed column names
+ *
+ * WHY: This mapping is used by generateRemappedSpanLoggerClass to create methods
+ * that write to prefixed columns but expose clean API names.
+ *
+ * @param schema - Clean schema (LogSchema instance)
+ * @param prefix - Prefix to apply (e.g., 'http')
+ * @returns Mapping from clean name to prefixed name
+ *
+ * @example
+ * const schema = defineLogSchema({ status: S.number() });
+ * const mapping = createPrefixMapping(schema, 'http');
+ * // Returns: { status: 'http_status' }
+ */
+export declare function createPrefixMapping<T extends LogSchema>(schema: T, prefix: string): PrefixMapping;
+/**
+ * Generate a RemappedBufferView class for library prefix support.
+ *
+ * WHY: Libraries write to unprefixed columns (status, method) but Arrow conversion
+ * iterates using prefixed names from the root schema (http_status, http_method).
+ * RemappedBufferView bridges this gap for tree traversal during cold-path Arrow conversion.
+ *
+ * HOW: Uses new Function() to generate a class at library composition time (cold path).
+ * The generated class wraps a SpanBuffer and remaps getColumnIfAllocated/getNullsIfAllocated
+ * calls from prefixed names to unprefixed names.
+ *
+ * WHAT: Returns a class constructor that:
+ * - Passes through tree traversal properties (children, next)
+ * - Passes through system columns (timestamps, operations, message, etc.)
+ * - Passes through identity properties (traceId, spanId, etc.)
+ * - Remaps getColumnIfAllocated() and getNullsIfAllocated() from prefixed→unprefixed
+ *
+ * Per specs/lmao/01e_library_integration_pattern.md:
+ * - Hot path: Library writes directly to unprefixed columns (zero overhead)
+ * - Cold path: Arrow conversion uses RemappedBufferView to access via prefixed names
+ *
+ * @param prefixToUnprefixedMapping - Maps prefixed names to unprefixed names
+ *   e.g., { 'http_status': 'status', 'http_method': 'method' }
+ *   NOTE: The mapping is prefixed→unprefixed (reverse of createPrefixMapping output)
+ *
+ * @returns Constructor for RemappedBufferView class
+ *
+ * @example
+ * ```typescript
+ * // Create mapping (prefixed → unprefixed)
+ * const mapping = { 'http_status': 'status', 'http_method': 'method' };
+ * const ViewClass = generateRemappedBufferViewClass(mapping);
+ *
+ * // Wrap library buffer
+ * const view = new ViewClass(libraryBuffer);
+ *
+ * // Access via prefixed name returns unprefixed column
+ * view.getColumnIfAllocated('http_status'); // Returns buffer.getColumnIfAllocated('status')
+ * ```
+ */
+export declare function generateRemappedBufferViewClass(prefixToUnprefixedMapping: Record<string, string>): RemappedBufferViewConstructor;
+/**
+ * Generate a remapped SpanLogger class for library prefix support
+ *
+ * WHY: Library authors write clean code (ctx.tag.status(200)) but buffers use prefixed
+ * columns (http_status). This function generates a class at module creation time
+ * (cold path) that exposes clean method names but writes to prefixed columns.
+ *
+ * HOW: Uses new Function() to generate optimized JavaScript code. The generated class
+ * has methods named after clean schema fields but writes to prefixed buffer columns.
+ *
+ * WHAT: Returns executable JavaScript code string for a SpanLogger class with:
+ * - Clean method names (status, method, url)
+ * - Writes to prefixed columns (http_status, http_method, http_url)
+ * - Full support for enum/category/text type handling
+ * - Method chaining support
+ *
+ * @param cleanSchema - Original clean schema (without prefix)
+ * @param prefixMapping - Mapping from clean names to prefixed names
+ * @param className - Name for the generated class
+ * @returns Executable JavaScript code string
+ *
+ * @example
+ * const code = generateRemappedSpanLoggerClass(
+ *   { status: S.number(), method: S.enum(['GET', 'POST']) },
+ *   { status: 'http_status', method: 'http_method' },
+ *   'HttpSpanLogger'
+ * );
+ * // Generated class has .status() method that writes to http_status
+ */
+export declare function generateRemappedSpanLoggerClass<T extends LogSchema>(cleanSchema: T, prefixMapping: PrefixMapping, className?: string): string;
+/**
+ * Create a remapped SpanLogger class constructor from clean schema and prefix mapping
+ *
+ * WHY: This is the cold-path function called at module creation time. It compiles
+ * the generated class code once and caches the constructor.
+ *
+ * @param cleanSchema - Clean schema (without prefix)
+ * @param prefixMapping - Mapping from clean names to prefixed names
+ * @returns Constructor for the remapped SpanLogger class
+ */
+export declare function createRemappedSpanLoggerClass<T extends LogSchema>(cleanSchema: T, prefixMapping: PrefixMapping): RemappedSpanLoggerConstructor<T>;
+export {};
+//# sourceMappingURL=library.d.ts.map

package/dist/lib/library.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"library.d.ts","sourceRoot":"","sources":["../../src/lib/library.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AASvE,OAAO,EAAE,SAAS,EAAqB,MAAM,mBAAmB,CAAC;AACjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,KAAK,6BAA6B,GAAG,KAAK,MAAM,EAAE,aAAa,KAAK,aAAa,CAAC;AAClF,KAAK,6BAA6B,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS,IAAI,KACpE,MAAM,EAAE,aAAa,EACrB,oBAAoB,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,aAAa,EAC9D,uBAAuB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9C,cAAc,CAAC,CAAC,CAAC,CAAC;AAiCvB;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAenG;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC5B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC7B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,aAAa,CASjG;AAsCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAgB,+BAA+B,CAC7C,yBAAyB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChD,6BAA6B,CA0F/B;AAkHD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,+BAA+B,CAAC,CAAC,SAAS,SAAS,EACjE,WAAW,EAAE,CAAC,EACd,aAAa,EAAE,aAAa,EAC5B,SAAS,SAAuB,GAC/B,MAAM,CAkMR;AAED;;;;;;;;;GASG;AACH,wBAAgB,6BAA6B,CAAC,CAAC,SAAS,SAAS,EAC/D,WAAW,EAAE,CAAC,EACd,aAAa,EAAE,aAAa,GAC3B,6BAA6B,CAAC,CAAC,CAAC,CAkBlC"}