@nathanvale/chatline 0.0.1

package/dist/utils/incremental-state.js

@@ -0,0 +1,237 @@
+/**
+ * Incremental State Tracking Module (INCREMENTAL--T01)
+ *
+ * Implements resumable enrichment by tracking:
+ * - AC01: State file with last run metadata
+ * - AC02: Last enriched date, total messages, config hash
+ * - AC03: GUID delta detection for new messages
+ * - AC04: Atomic writes with temp + rename pattern
+ * - AC05: --incremental flag integration
+ *
+ * Architecture:
+ * - IncrementalState: Complete state schema with version, metadata, GUID tracking
+ * - createIncrementalState: Factory for new state
+ * - loadIncrementalState: Safe loading from disk with corruption handling
+ * - saveIncrementalState: Atomic writes with temp file + rename
+ * - detectNewMessages: O(n) GUID comparison using Set intersection
+ * - updateStateWithEnrichedGuids: Add new enrichments and update metadata
+ * - verifyConfigHash: Detect config changes between runs
+ */
+import crypto from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { humanWarn } from '#utils/human';
+import { createLogger } from '#utils/logger';
+/**
+ * AC01 + AC02: Create new incremental state with current metadata
+ *
+ * @param options - Optional initial values
+ * @returns New IncrementalState with current timestamp and config hash
+ */
+export function createIncrementalState(options = {}) {
+    return {
+        version: '1.0',
+        lastEnrichedAt: new Date().toISOString(),
+        totalMessages: options.totalMessages ?? 0,
+        enrichedGuids: options.enrichedGuids ?? [],
+        pipelineConfig: {
+            configHash: generateConfigHash(),
+        },
+        enrichmentStats: options.enrichmentStats ?? null,
+    };
+}
+/**
+ * AC02: Generate hash of pipeline configuration
+ * Used to detect when settings change (API keys, flags, etc.)
+ *
+ * @returns SHA-256 hex digest of current config
+ */
+function generateConfigHash() {
+    // Hash config including API key presence (not the actual key)
+    const hasGeminiKey = process.env.GEMINI_API_KEY !== undefined;
+    const hasFirecrawlKey = process.env.FIRECRAWL_API_KEY !== undefined;
+    const config = JSON.stringify({
+        version: '1.0',
+        // Add API key presence (not the actual key)
+        hasGeminiKey,
+        hasFirecrawlKey,
+    });
+    return crypto.createHash('sha256').update(config).digest('hex');
+}
+// ============================================================================
+// AC03: GUID Delta Detection
+// ============================================================================
+/**
+ * AC03: Detect new messages by comparing GUIDs with state
+ *
+ * Performance: O(n) where n = number of current messages
+ * Using Set for fast O(1) lookup of previously enriched GUIDs
+ *
+ * @param currentGuids - Set of message GUIDs from normalized output
+ * @param state - Previous state with enriched GUIDs
+ * @returns Array of new GUID strings not in enriched set
+ */
+export function detectNewMessages(currentGuids, state) {
+    const enrichedSet = new Set(state.enrichedGuids);
+    const newGuids = [];
+    for (const guid of currentGuids) {
+        if (!enrichedSet.has(guid)) {
+            newGuids.push(guid);
+        }
+    }
+    return newGuids;
+}
+// ============================================================================
+// AC04: Atomic Writes (Temp + Rename Pattern)
+// ============================================================================
+/**
+ * AC04: Save state atomically to disk
+ *
+ * Pattern:
+ * 1. Write to temp file with .tmp suffix
+ * 2. Atomic rename (temp → final)
+ * 3. Prevents corruption from power loss or crashes
+ *
+ * @param state - IncrementalState to persist
+ * @param filePath - Target .imessage-state.json path
+ * @throws Error if write fails (permission, disk full, etc.)
+ */
+export async function saveIncrementalState(state, filePath) {
+    const dir = path.dirname(filePath);
+    const tempFile = `${filePath}.${Date.now()}.tmp`;
+    try {
+        // Ensure parent directory exists
+        await fs.mkdir(dir, { recursive: true });
+        // Write to temp file
+        const content = JSON.stringify(state, null, 2);
+        await fs.writeFile(tempFile, content, 'utf-8');
+        // Atomic rename
+        await fs.rename(tempFile, filePath);
+    }
+    catch (error) {
+        // Clean up temp file if rename failed
+        try {
+            await fs.unlink(tempFile);
+        }
+        catch {
+            // ignore cleanup errors
+        }
+        throw error;
+    }
+}
+/**
+ * AC05: Load state from disk safely
+ *
+ * - Returns null if file doesn't exist (treat as first run)
+ * - Returns null if JSON is corrupted (ignore stale state)
+ * - Validates schema version for future compatibility
+ *
+ * @param filePath - Path to .imessage-state.json
+ * @returns IncrementalState if valid, null if missing or corrupted
+ */
+export async function loadIncrementalState(filePath) {
+    try {
+        const content = await fs.readFile(filePath, 'utf-8');
+        const parsed = JSON.parse(content);
+        // Validate schema version
+        if (parsed.version !== '1.0') {
+            const logger = createLogger('utils:incremental-state');
+            logger.warn('Unknown state version. Ignoring.', {
+                version: parsed.version,
+            });
+            return null;
+        }
+        return parsed;
+    }
+    catch (error) {
+        if (error instanceof Error && error.message.includes('ENOENT')) {
+            // File doesn't exist - first run
+            return null;
+        }
+        // Corrupted JSON or other read error - ignore
+        return null;
+    }
+}
+// ============================================================================
+// Config Hash Verification
+// ============================================================================
+/**
+ * AC02: Verify config hash to detect changes
+ *
+ * @param currentHash - Hash from current state
+ * @param expectedHash - Hash to verify against (default: newly generated)
+ * @returns true if hashes match
+ */
+export function verifyConfigHash(currentHash, expectedHash = generateConfigHash()) {
+    return currentHash === expectedHash;
+}
+/**
+ * AC02: Check if state is stale (old enrichment run)
+ *
+ * Useful for warning about old state that may be out of sync
+ *
+ * @param state - IncrementalState to check
+ * @param daysThreshold - Days before state is considered stale (default: 7)
+ * @returns true if state is older than threshold
+ */
+export function isStateOutdated(state, daysThreshold = 7) {
+    const lastEnrichedTime = new Date(state.lastEnrichedAt).getTime();
+    const now = Date.now();
+    const ageMs = now - lastEnrichedTime;
+    const ageDays = ageMs / (1000 * 60 * 60 * 24);
+    if (ageDays > daysThreshold) {
+        const logger = createLogger('utils:incremental-state');
+        logger.warn('State file is old. Consider full re-enrichment.', {
+            ageDays: Math.floor(ageDays),
+            daysThreshold,
+        });
+        // Human-visible warning for tests & interactive CLI (console.warn spied in tests)
+        humanWarn(`State file is old (ageDays=${Math.floor(ageDays)}, threshold=${daysThreshold}). Consider full re-enrichment.`);
+        return true;
+    }
+    return false;
+}
+// ============================================================================
+// Update State with Enrichment Results
+// ============================================================================
+/**
+ * Update state with newly enriched GUIDs
+ *
+ * Called after successful enrichment to:
+ * - Add new enriched GUIDs (avoid duplicates)
+ * - Update lastEnrichedAt timestamp
+ * - Record enrichment statistics
+ *
+ * @param state - State to update (mutated in place)
+ * @param newGuids - GUIDs that were just enriched
+ * @param enrichmentStats - Optional enrichment stats
+ */
+export function updateStateWithEnrichedGuids(state, newGuids, enrichmentStats) {
+    // Add new GUIDs, avoiding duplicates
+    const existingSet = new Set(state.enrichedGuids);
+    for (const guid of newGuids) {
+        if (!existingSet.has(guid)) {
+            state.enrichedGuids.push(guid);
+        }
+    }
+    // Update timestamp
+    state.lastEnrichedAt = new Date().toISOString();
+    // Update stats if provided
+    if (enrichmentStats) {
+        state.enrichmentStats = enrichmentStats;
+    }
+}
+// ============================================================================
+// Helper: Reset state (for testing or --force-refresh)
+// ============================================================================
+/**
+ * Create fresh state, discarding incremental tracking
+ * Used with --force-refresh flag to re-enrich everything
+ *
+ * @param totalMessages - Total messages to initialize with
+ * @returns Fresh IncrementalState with no enriched GUIDs
+ */
+export function resetIncrementalState(totalMessages = 0) {
+    return createIncrementalState({ totalMessages });
+}
+//# sourceMappingURL=incremental-state.js.map

package/dist/utils/incremental-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-state.js","sourceRoot":"","sources":["../../src/utils/incremental-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAA;AACxC,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAoD5C;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACrC,UAA8B,EAAE;IAEhC,OAAO;QACN,OAAO,EAAE,KAAK;QACd,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxC,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC;QACzC,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,EAAE;QAC1C,cAAc,EAAE;YACf,UAAU,EAAE,kBAAkB,EAAE;SAChC;QACD,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB;IAC1B,8DAA8D;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAA;IAC7D,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,SAAS,CAAA;IAEnE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,OAAO,EAAE,KAAK;QACd,4CAA4C;QAC5C,YAAY;QACZ,eAAe;KACf,CAAC,CAAA;IACF,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAChE,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAChC,YAAyB,EACzB,KAAuB;IAEvB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,+EAA+E;AAC/E,8CAA8C;AAC9C,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,KAAuB,EACvB,QAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAClC,MAAM,QAAQ,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAA;IAEhD,IAAI,CAAC;QACJ,iCAAiC;QACjC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAExC,qBAAqB;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QAC9C,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;QAE9C,gBAAgB;QAChB,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,sCAAsC;QACtC,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAC1B,CAAC;QAAC,MAAM,CAAC;YACR,wBAAwB;QACzB,CAAC;QACD,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,QAAgB;IAEhB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAqB,CAAA;QAEtD,0BAA0B;QAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,YAAY,CAAC,yBAAyB,CAAC,CAAA;YACtD,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC/C,OAAO,EAAE,MAAM,CAAC,OAAO;aACvB,CAAC,CAAA;YACF,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChE,iCAAiC;YACjC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,8CAA8C;QAC9C,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC/B,WAAmB,EACnB,eAAuB,kBAAkB,EAAE;IAE3C,OAAO,WAAW,KAAK,YAAY,CAAA;AACpC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC9B,KAAuB,EACvB,aAAa,GAAG,CAAC;IAEjB,MAAM,gBAAgB,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAA;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,KAAK,GAAG,GAAG,GAAG,gBAAgB,CAAA;IACpC,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IAE7C,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,yBAAyB,CAAC,CAAA;QACtD,MAAM,CAAC,IAAI,CAAC,iDAAiD,EAAE;YAC9D,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;YAC5B,aAAa;SACb,CAAC,CAAA;QACF,kFAAkF;QAClF,SAAS,CACR,8BAA8B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,aAAa,iCAAiC,CAC9G,CAAA;QACD,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,+EAA+E;AAC/E,uCAAuC;AACvC,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,4BAA4B,CAC3C,KAAuB,EACvB,QAAkB,EAClB,eAAiC;IAEjC,qCAAqC;IACrC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAChD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,mBAAmB;IACnB,KAAK,CAAC,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAE/C,2BAA2B;IAC3B,IAAI,eAAe,EAAE,CAAC;QACrB,KAAK,CAAC,eAAe,GAAG,eAAe,CAAA;IACxC,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,uDAAuD;AACvD,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAa,GAAG,CAAC;IACtD,OAAO,sBAAsB,CAAC,EAAE,aAAa,EAAE,CAAC,CAAA;AACjD,CAAC"}

package/dist/utils/logger.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Structured JSON Logger (Pino + JSONL)
+ *
+ * - Uses Pino for fast JSON logging to stdout
+ * - Writes JSON Lines to ./logs/YYYY-MM-DD.jsonl when enabled
+ * - Stable keys for Pinot ingestion (ts, level, component, msg, context, pid, ver, seq)
+ *
+ * Environment variables:
+ *  LOG_LEVEL=debug|info|warn|error  minimum level (default info)
+ *  LOG_FORMAT=json|pretty           pretty prints to stdout when set to pretty
+ *  LOG_TO_FILE=true|false           write JSONL to ./logs (default true except during tests)
+ */
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+export type LogEntry = {
+    ts: string;
+    level: LogLevel;
+    component: string;
+    msg: string;
+    context?: Record<string, unknown>;
+    pid: number;
+    ver?: string;
+    seq: number;
+    correlationId?: string;
+};
+export declare function log(component: string, level: LogLevel, msg: string, context?: Record<string, unknown>): void;
+export type ComponentLogger = {
+    debug: (msg: string, context?: Record<string, unknown>) => void;
+    info: (msg: string, context?: Record<string, unknown>) => void;
+    warn: (msg: string, context?: Record<string, unknown>) => void;
+    error: (msg: string, context?: Record<string, unknown>) => void;
+};
+export declare function createLogger(component: string): ComponentLogger;
+export type LogSink = (entry: LogEntry) => void;
+export declare function registerSink(sink: LogSink): void;
+export declare function clearSinks(): void;
+export declare function setCorrelationId(id: string | undefined): void;
+export declare function getCorrelationId(): string | undefined;
+export declare function withCorrelationId<T>(id: string, fn: () => Promise<T> | T): Promise<T>;
+export declare function setLogLevel(level: LogLevel): void;
+//# sourceMappingURL=logger.d.ts.map

package/dist/utils/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;AAE1D,MAAM,MAAM,QAAQ,GAAG;IACtB,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,QAAQ,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAyED,wBAAgB,GAAG,CAClB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,QAAQ,EACf,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,IAAI,CA2CN;AAED,MAAM,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC/D,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC9D,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC9D,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;CAC/D,CAAA;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe,CAO/D;AAKD,MAAM,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAA;AAE/C,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAEhD;AACD,wBAAgB,UAAU,IAAI,IAAI,CAEjC;AAGD,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAE7D;AAED,wBAAgB,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAErD;AAED,wBAAsB,iBAAiB,CAAC,CAAC,EACxC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GACtB,OAAO,CAAC,CAAC,CAAC,CAUZ;AAID,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,IAAI,CAQjD"}

package/dist/utils/logger.js

@@ -0,0 +1,176 @@
+/**
+ * Structured JSON Logger (Pino + JSONL)
+ *
+ * - Uses Pino for fast JSON logging to stdout
+ * - Writes JSON Lines to ./logs/YYYY-MM-DD.jsonl when enabled
+ * - Stable keys for Pinot ingestion (ts, level, component, msg, context, pid, ver, seq)
+ *
+ * Environment variables:
+ *  LOG_LEVEL=debug|info|warn|error  minimum level (default info)
+ *  LOG_FORMAT=json|pretty           pretty prints to stdout when set to pretty
+ *  LOG_TO_FILE=true|false           write JSONL to ./logs (default true except during tests)
+ */
+const LEVEL_ORDER = {
+    debug: 10,
+    info: 20,
+    warn: 30,
+    error: 40,
+};
+import { AsyncLocalStorage } from 'node:async_hooks';
+import fs from 'node:fs';
+import path from 'node:path';
+import pino, {} from 'pino';
+import pkgJson from '../../package.json' with { type: 'json' };
+let sequenceCounter = 0;
+let cachedVersion;
+let currentCorrelationId;
+const correlationStore = new AsyncLocalStorage();
+function loadVersion() {
+    if (cachedVersion)
+        return cachedVersion;
+    const version = pkgJson.version || '0.0.0';
+    cachedVersion = version;
+    return version;
+}
+const isTestEnv = process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
+const shouldWriteFile = (process.env.LOG_TO_FILE ?? (isTestEnv ? 'false' : 'true')) === 'true';
+// stdout logger via Pino
+const baseLevel = (process.env.LOG_LEVEL || 'info');
+const pinoTransport = process.env.LOG_FORMAT === 'pretty'
+    ? pino.transport({
+        target: 'pino-pretty',
+        options: { colorize: true, singleLine: true },
+    })
+    : undefined;
+const pinoStdout = pino({ level: baseLevel, base: null }, pinoTransport);
+// JSONL file stream (lazy)
+let fileStream;
+let fileDate;
+function ensureFileStream() {
+    if (!shouldWriteFile)
+        return;
+    const nowDate = new Date().toISOString().slice(0, 10);
+    if (fileStream && fileDate === nowDate)
+        return;
+    try {
+        const logsDir = path.resolve(process.cwd(), 'logs');
+        fs.mkdirSync(logsDir, { recursive: true });
+        const filePath = path.join(logsDir, `${nowDate}.jsonl`);
+        if (fileStream)
+            fileStream.end();
+        fileStream = fs.createWriteStream(filePath, { flags: 'a' });
+        fileDate = nowDate;
+    }
+    catch {
+        // swallow file sink errors to avoid impacting app
+        fileStream = undefined;
+        fileDate = undefined;
+    }
+}
+// will be overridden with dynamic version later in file
+// initial implementation kept for reference; dynamicShouldLog is used instead
+// kept for backward compatibility in case other modules import it later
+function _format(entry) {
+    return JSON.stringify(entry);
+}
+export function log(component, level, msg, context) {
+    if (!dynamicShouldLog(level))
+        return;
+    const effectiveCorrelationId = correlationStore.getStore() ?? currentCorrelationId;
+    const entryBase = {
+        ts: new Date().toISOString(),
+        level,
+        component,
+        msg,
+        pid: process.pid,
+        ver: loadVersion(),
+        seq: ++sequenceCounter,
+    };
+    const withCorr = effectiveCorrelationId ? { correlationId: effectiveCorrelationId } : {};
+    const entryNoCtx = { ...entryBase, ...withCorr };
+    const entry = context ? { ...entryNoCtx, context } : entryNoCtx;
+    // 1) stdout via Pino (keeps human-friendly output and fast JSON)
+    const bindings = {
+        component,
+        ver: entry.ver,
+        seq: entry.seq,
+    };
+    if (entry.correlationId)
+        bindings.correlationId = entry.correlationId;
+    const logger = pinoStdout.child(bindings);
+    // Pino expects object then msg for structured logs
+    logger[level](context ?? {}, msg);
+    // 2) JSONL file sink (Pinot-friendly schema)
+    if (shouldWriteFile) {
+        ensureFileStream();
+        if (fileStream)
+            fileStream.write(`${JSON.stringify(entry)}\n`);
+    }
+    // 3) Custom sinks (if any registered)
+    if (sinks.length > 0) {
+        try {
+            for (const s of sinks)
+                s(entry);
+        }
+        catch {
+            // ignore sink errors
+        }
+    }
+}
+export function createLogger(component) {
+    return {
+        debug: (msg, context) => log(component, 'debug', msg, context),
+        info: (msg, context) => log(component, 'info', msg, context),
+        warn: (msg, context) => log(component, 'warn', msg, context),
+        error: (msg, context) => log(component, 'error', msg, context),
+    };
+}
+let sinks = [];
+export function registerSink(sink) {
+    sinks.push(sink);
+}
+export function clearSinks() {
+    sinks = [];
+}
+// Correlation ID management
+export function setCorrelationId(id) {
+    currentCorrelationId = id;
+}
+export function getCorrelationId() {
+    return correlationStore.getStore() ?? currentCorrelationId;
+}
+export async function withCorrelationId(id, fn) {
+    const prev = currentCorrelationId;
+    currentCorrelationId = id;
+    return await correlationStore.run(id, async () => {
+        try {
+            return await fn();
+        }
+        finally {
+            currentCorrelationId = prev;
+        }
+    });
+}
+// Dynamic log level control
+let dynamicLevel;
+export function setLogLevel(level) {
+    dynamicLevel = level;
+    try {
+        // biome-ignore lint/suspicious/noExplicitAny: pino internal API
+        ;
+        pinoStdout.level = level;
+    }
+    catch {
+        // ignore if pinoStdout not ready
+    }
+}
+function getEffectiveLevel() {
+    return dynamicLevel || 'info';
+}
+// Override original shouldLog: replace earlier implementation by assigning new function
+// Use a const to avoid duplicate declarations
+const dynamicShouldLog = (level) => {
+    const envLevel = getEffectiveLevel();
+    return LEVEL_ORDER[level] >= LEVEL_ORDER[envLevel];
+};
+//# sourceMappingURL=logger.js.map

package/dist/utils/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAgBH,MAAM,WAAW,GAA6B;IAC7C,KAAK,EAAE,EAAE;IACT,IAAI,EAAE,EAAE;IACR,IAAI,EAAE,EAAE;IACR,KAAK,EAAE,EAAE;CACT,CAAA;AAED,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,IAAI,EAAE,EAA6B,MAAM,MAAM,CAAA;AAEtD,OAAO,OAAO,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAA;AAE9D,IAAI,eAAe,GAAG,CAAC,CAAA;AACvB,IAAI,aAAiC,CAAA;AACrC,IAAI,oBAAwC,CAAA;AAC5C,MAAM,gBAAgB,GAAG,IAAI,iBAAiB,EAAU,CAAA;AACxD,SAAS,WAAW;IACnB,IAAI,aAAa;QAAE,OAAO,aAAc,CAAA;IACxC,MAAM,OAAO,GAAI,OAAgC,CAAC,OAAO,IAAI,OAAO,CAAA;IACpE,aAAa,GAAG,OAAO,CAAA;IACvB,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,SAAS,GACd,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAA;AACjE,MAAM,eAAe,GACpB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAA;AAEvE,yBAAyB;AACzB,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAa,CAAA;AAC/D,MAAM,aAAa,GAClB,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,QAAQ;IAClC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACf,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;KAC7C,CAAC;IACH,CAAC,CAAC,SAAS,CAAA;AACb,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;AAExE,2BAA2B;AAC3B,IAAI,UAAsC,CAAA;AAC1C,IAAI,QAA4B,CAAA;AAChC,SAAS,gBAAgB;IACxB,IAAI,CAAC,eAAe;QAAE,OAAM;IAC5B,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACrD,IAAI,UAAU,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAM;IAC9C,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;QACnD,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,QAAQ,CAAC,CAAA;QACvD,IAAI,UAAU;YAAE,UAAU,CAAC,GAAG,EAAE,CAAA;QAChC,UAAU,GAAG,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QAC3D,QAAQ,GAAG,OAAO,CAAA;IACnB,CAAC;IAAC,MAAM,CAAC;QACR,kDAAkD;QAClD,UAAU,GAAG,SAAS,CAAA;QACtB,QAAQ,GAAG,SAAS,CAAA;IACrB,CAAC;AACF,CAAC;AAED,wDAAwD;AACxD,8EAA8E;AAE9E,wEAAwE;AACxE,SAAS,OAAO,CAAC,KAAe;IAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,GAAG,CAClB,SAAiB,EACjB,KAAe,EACf,GAAW,EACX,OAAiC;IAEjC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAM;IACpC,MAAM,sBAAsB,GAC3B,gBAAgB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAA;IACpD,MAAM,SAAS,GAAgD;QAC9D,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAC5B,KAAK;QACL,SAAS;QACT,GAAG;QACH,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,EAAE,WAAW,EAAE;QAClB,GAAG,EAAE,EAAE,eAAe;KACtB,CAAA;IACD,MAAM,QAAQ,GACb,sBAAsB,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IACxE,MAAM,UAAU,GAAa,EAAE,GAAI,SAAsB,EAAE,GAAG,QAAQ,EAAE,CAAA;IACxE,MAAM,KAAK,GAAa,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAA;IAEzE,iEAAiE;IACjE,MAAM,QAAQ,GAA4B;QACzC,SAAS;QACT,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,GAAG,EAAE,KAAK,CAAC,GAAG;KACd,CAAA;IACD,IAAI,KAAK,CAAC,aAAa;QAAE,QAAQ,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAA;IACrE,MAAM,MAAM,GAAe,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IACrD,mDAAmD;IACnD,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,CAAA;IAEjC,6CAA6C;IAC7C,IAAI,eAAe,EAAE,CAAC;QACrB,gBAAgB,EAAE,CAAA;QAClB,IAAI,UAAU;YAAE,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC;IAED,sCAAsC;IACtC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC;YACJ,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,CAAC,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC;QAAC,MAAM,CAAC;YACR,qBAAqB;QACtB,CAAC;IACF,CAAC;AACF,CAAC;AASD,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC7C,OAAO;QACN,KAAK,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC;QAC9D,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC;QAC5D,IAAI,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC;QAC5D,KAAK,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC;KAC9D,CAAA;AACF,CAAC;AAMD,IAAI,KAAK,GAAc,EAAE,CAAA;AACzB,MAAM,UAAU,YAAY,CAAC,IAAa;IACzC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACjB,CAAC;AACD,MAAM,UAAU,UAAU;IACzB,KAAK,GAAG,EAAE,CAAA;AACX,CAAC;AAED,4BAA4B;AAC5B,MAAM,UAAU,gBAAgB,CAAC,EAAsB;IACtD,oBAAoB,GAAG,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO,gBAAgB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAA;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,EAAU,EACV,EAAwB;IAExB,MAAM,IAAI,GAAG,oBAAoB,CAAA;IACjC,oBAAoB,GAAG,EAAE,CAAA;IACzB,OAAO,MAAM,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAChD,IAAI,CAAC;YACJ,OAAO,MAAM,EAAE,EAAE,CAAA;QAClB,CAAC;gBAAS,CAAC;YACV,oBAAoB,GAAG,IAAI,CAAA;QAC5B,CAAC;IACF,CAAC,CAAC,CAAA;AACH,CAAC;AAED,4BAA4B;AAC5B,IAAI,YAAkC,CAAA;AACtC,MAAM,UAAU,WAAW,CAAC,KAAe;IAC1C,YAAY,GAAG,KAAK,CAAA;IACpB,IAAI,CAAC;QACJ,gEAAgE;QAChE,CAAC;QAAC,UAAkB,CAAC,KAAK,GAAG,KAAK,CAAA;IACnC,CAAC;IAAC,MAAM,CAAC;QACR,iCAAiC;IAClC,CAAC;AACF,CAAC;AAED,SAAS,iBAAiB;IACzB,OAAO,YAAY,IAAI,MAAM,CAAA;AAC9B,CAAC;AAED,wFAAwF;AACxF,8CAA8C;AAC9C,MAAM,gBAAgB,GAAG,CAAC,KAAe,EAAW,EAAE;IACrD,MAAM,QAAQ,GAAG,iBAAiB,EAAE,CAAA;IACpC,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAA;AACnD,CAAC,CAAA"}

package/package.json

@@ -0,0 +1,165 @@
+{
+	"author": {
+		"name": "Nathan Vale",
+		"url": "https://github.com/nathanvale"
+	},
+	"bin": {
+		"chatline": "./dist/bin/index.js"
+	},
+	"bugs": {
+		"url": "https://github.com/nathanvale/chatline/issues"
+	},
+	"dependencies": {
+		"@google/generative-ai": "^0.21.0",
+		"@mendable/firecrawl-js": "^4.3.7",
+		"cli-progress": "^3.12.0",
+		"commander": "^14.0.1",
+		"csv-parse": "^5.6.0",
+		"csv-stringify": "^6.5.2",
+		"js-yaml": "^4.1.0",
+		"pino": "^10.0.0",
+		"sharp": "^0.34.4",
+		"zod": "^3.23.8"
+	},
+	"description": "Extract, transform, and analyze iMessage conversations with AI-powered enrichment and timeline rendering.",
+	"devDependencies": {
+		"@arethetypeswrong/cli": "^0.18.2",
+		"@biomejs/biome": "^2.3.7",
+		"@changesets/changelog-github": "^0.5.1",
+		"@changesets/cli": "^2.29.7",
+		"@commitlint/cli": "^20.1.0",
+		"@commitlint/config-conventional": "^20.0.0",
+		"@testing-library/jest-dom": "^6.9.1",
+		"@testing-library/react": "^16.3.0",
+		"@types/cli-progress": "^3.11.6",
+		"@types/js-yaml": "^4.0.9",
+		"@types/node": "^24.8.1",
+		"@vitest/coverage-v8": "^4.0.14",
+		"better-sqlite3": "^12.4.1",
+		"bun-types": "^1.3.3",
+		"bunup": "^0.16.10",
+		"changesets": "^1.0.2",
+		"husky": "^9.1.7",
+		"jsdom": "^25.0.1",
+		"lint-staged": "^15.2.10",
+		"pino-pretty": "^11.2.2",
+		"publint": "^0.3.15",
+		"rimraf": "^6.0.1",
+		"tsx": "^4.20.6",
+		"typescript": "^5.9.3",
+		"vitest": "^4.0.14"
+	},
+	"engines": {
+		"node": ">=22.20"
+	},
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js"
+		},
+		"./package.json": "./package.json"
+	},
+	"files": [
+		"dist/**",
+		"README.md",
+		"LICENSE",
+		"CHANGELOG.md"
+	],
+	"funding": [
+		{
+			"type": "github",
+			"url": "https://github.com/sponsors/nathanvale"
+		}
+	],
+	"homepage": "https://github.com/nathanvale/chatline#readme",
+	"imports": {
+		"#enrich/*": "./src/enrich/*",
+		"#ingest/*": "./src/ingest/*",
+		"#normalize/*": "./src/normalize/*",
+		"#render/*": "./src/render/*",
+		"#schema/*": "./src/schema/*",
+		"#utils/*": "./src/utils/*"
+	},
+	"keywords": [
+		"imessage",
+		"typescript",
+		"cli",
+		"data-pipeline",
+		"message-analysis",
+		"timeline",
+		"ai-enrichment"
+	],
+	"overrides": {
+		"express": "4.22.0"
+	},
+	"license": "MIT",
+	"lint-staged": {
+		"*.{ts,tsx,js,jsx,mts,cts,json}": [
+			"biome check --write"
+		]
+	},
+	"main": "./dist/index.js",
+	"name": "@nathanvale/chatline",
+	"publishConfig": {
+		"access": "public",
+		"provenance": true
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/nathanvale/chatline.git"
+	},
+	"scripts": {
+		"research:actionlint": "bun scripts/firecrawl/actionlint-best-practices.ts",
+		"build": "bunx bunup",
+		"check:publint": "publint",
+		"check:types": "attw --pack",
+		"check:updates": "bun pm ls || true",
+		"clean": "rimraf dist 2>/dev/null || true",
+		"cli": "node ./dist/bin/index.js",
+		"coverage": "bun test --coverage",
+		"dev": "bun src/cli.ts",
+		"docs:build": "cd website && bun run build",
+		"docs:dev": "cd website && bun run start",
+		"docs:serve": "cd website && bun run serve",
+		"check": "biome check --write .",
+		"format": "biome format --write .",
+		"format:check": "biome format .",
+		"hygiene": "bun run check:publint && bun run check:types",
+		"lint": "biome lint .",
+		"lint:fix": "biome lint --write .",
+		"lint:scripts": "shellcheck .github/scripts/*.sh",
+		"lint:workflows": "actionlint -color -verbose",
+		"pack:dry": "mkdir -p .pack && bun pm pack --destination .pack --ignore-scripts && ls -lah .pack && tar -tf .pack/*.tgz | sort | sed 's/^/ - /'",
+		"pre:enter:beta": "changeset pre enter beta",
+		"pre:enter:next": "changeset pre enter next",
+		"pre:enter:rc": "changeset pre enter rc",
+		"quality-check:ci": "biome check . && bun run typecheck",
+		"pre:exit": "changeset pre exit",
+		"prepare": "husky",
+		"publish:pre": "changeset publish --provenance",
+		"release": "changeset publish --provenance",
+		"release:snapshot:canary": "changeset version --snapshot canary && changeset publish --tag canary",
+		"security:audit": "npm audit",
+		"start": "node ./dist/bin/index.js",
+		"test": "bun test --recursive",
+		"test:ci": "TF_BUILD=true bun test --recursive",
+		"test:coverage": "bun test --coverage",
+		"test:scripts": "bash tests/smoke-test-scripts.sh",
+		"test:watch": "bun test --watch",
+		"typecheck": "tsc -p tsconfig.eslint.json --noEmit",
+		"upgrade:minor": "bun update",
+		"validate": "bun run lint && bun run typecheck && bun run build && TF_BUILD=true bun run test",
+		"validate:json": "bun scripts/validate-json.ts",
+		"version:gen": "changeset",
+		"version:pre": "changeset version",
+		"watch:types": "tsc -p tsconfig.eslint.json --noEmit --watch"
+	},
+	"sideEffects": false,
+	"type": "module",
+	"types": "./dist/index.d.ts",
+	"version": "0.0.1",
+	"workspaces": [
+		".",
+		"website"
+	]
+}