@path58/n8n-mcp 0.1.0

package/dist/services/CheckpointManager.js

@@ -0,0 +1,281 @@
+/**
+ * Checkpoint Manager Service
+ *
+ * Enhanced checkpoint/resume capability for long-running pipeline operations.
+ * Saves full state to database for reliable resume after interruptions.
+ *
+ * **Extends shared-utilities patterns:**
+ * - Follows shared-utilities CheckpointManager concepts
+ * - Enhanced for pipeline operations with full state tracking
+ * - Supports arrays of processed/failed items (not just last ID)
+ * - Detailed progress metrics (not just counts)
+ * - Full state restoration for complex operations
+ *
+ * **Integration:**
+ * - Integrates with banner service for progress display
+ * - Works with script runner for automatic checkpoint management
+ * - Reports checkpoint progress to banner service when provided
+ *
+ * Features:
+ * - Save checkpoint state to database
+ * - Load checkpoint state
+ * - Update checkpoint progress
+ * - Clear checkpoints
+ * - List checkpoints
+ * - Automatic state persistence
+ * - Banner service integration (optional)
+ *
+ * @example
+ * ```typescript
+ * const checkpointManager = new CheckpointManager();
+ *
+ * // Save checkpoint
+ * await checkpointManager.saveCheckpoint({
+ *   key: 'node-embeddings-v1',
+ *   pipelineType: 'embedding',
+ *   state: { processed: 100, success: 95, failed: 5 },
+ *   processedItems: ['node1', 'node2'],
+ *   failedItems: ['node3'],
+ *   progress: { total: 1000, processed: 100 }
+ * });
+ *
+ * // Load checkpoint
+ * const checkpoint = await checkpointManager.loadCheckpoint('node-embeddings-v1');
+ * if (checkpoint) {
+ *   console.log(`Resuming from checkpoint: ${checkpoint.progress.processed} items processed`);
+ * }
+ * ```
+ */
+import { validatorQuery } from '../db/validatorPostgresClient.js';
+export class CheckpointManager {
+    bannerService;
+    /**
+     * Create a new CheckpointManager
+     *
+     * @param bannerService Optional banner service for progress display
+     */
+    constructor(bannerService) {
+        this.bannerService = bannerService;
+    }
+    /**
+     * Set banner service for progress reporting
+     *
+     * @param bannerService Banner service instance
+     */
+    setBannerService(bannerService) {
+        this.bannerService = bannerService;
+    }
+    /**
+     * Save a checkpoint to the database
+     *
+     * Creates a new checkpoint or updates existing one with same key.
+     * Reports progress to banner service if provided.
+     *
+     * @param data Checkpoint data to save
+     * @returns Saved checkpoint
+     */
+    async saveCheckpoint(data) {
+        const { key, pipelineType, state, processedItems = [], failedItems = [], progress = {} } = data;
+        const result = await validatorQuery(`
+      INSERT INTO validator.pipeline_checkpoints (
+        checkpoint_key,
+        pipeline_type,
+        state,
+        processed_items,
+        failed_items,
+        progress
+      )
+      VALUES ($1, $2, $3::jsonb, $4, $5, $6::jsonb)
+      ON CONFLICT (checkpoint_key) DO UPDATE SET
+        pipeline_type = EXCLUDED.pipeline_type,
+        state = EXCLUDED.state,
+        processed_items = EXCLUDED.processed_items,
+        failed_items = EXCLUDED.failed_items,
+        progress = EXCLUDED.progress,
+        updated_at = NOW()
+      RETURNING
+        id::text,
+        checkpoint_key as key,
+        pipeline_type as "pipelineType",
+        state,
+        processed_items as "processedItems",
+        failed_items as "failedItems",
+        progress,
+        created_at as "createdAt",
+        updated_at as "updatedAt"
+      `, [key, pipelineType, JSON.stringify(state), processedItems, failedItems, JSON.stringify(progress)]);
+        const row = Array.isArray(result) ? result[0] : result.rows[0];
+        const saved = {
+            id: row.id,
+            key: row.key,
+            pipelineType: row.pipelineType,
+            state: row.state,
+            processedItems: row.processedItems || [],
+            failedItems: row.failedItems || [],
+            progress: row.progress || {},
+            createdAt: row.createdAt,
+            updatedAt: row.updatedAt,
+        };
+        // Report checkpoint progress to banner service if available
+        if (this.bannerService && saved.progress) {
+            const processed = saved.progress.processed || (saved.processedItems || []).length;
+            this.bannerService.setCheckpoint(processed);
+        }
+        return saved;
+    }
+    /**
+     * Load a checkpoint from the database
+     *
+     * @param key Checkpoint key
+     * @returns Checkpoint if found, null otherwise
+     */
+    async loadCheckpoint(key) {
+        const result = await validatorQuery(`
+      SELECT
+        id::text,
+        checkpoint_key as key,
+        pipeline_type as "pipelineType",
+        state,
+        processed_items as "processedItems",
+        failed_items as "failedItems",
+        progress,
+        created_at as "createdAt",
+        updated_at as "updatedAt"
+      FROM validator.pipeline_checkpoints
+      WHERE checkpoint_key = $1
+      `, [key]);
+        const rows = Array.isArray(result) ? result : result.rows;
+        if (rows.length === 0) {
+            return null;
+        }
+        const row = rows[0];
+        return {
+            id: row.id,
+            key: row.key,
+            pipelineType: row.pipelineType,
+            state: row.state,
+            processedItems: row.processedItems || [],
+            failedItems: row.failedItems || [],
+            progress: row.progress || {},
+            createdAt: row.createdAt,
+            updatedAt: row.updatedAt,
+        };
+    }
+    /**
+     * Update checkpoint progress without changing state
+     *
+     * Useful for updating progress metrics during processing.
+     * Reports progress to banner service if provided.
+     *
+     * @param key Checkpoint key
+     * @param progress Progress metrics to update
+     */
+    async updateProgress(key, progress) {
+        await validatorQuery(`
+      UPDATE validator.pipeline_checkpoints
+      SET progress = COALESCE(progress, '{}'::jsonb) || $1::jsonb,
+          updated_at = NOW()
+      WHERE checkpoint_key = $2
+      `, [JSON.stringify(progress), key]);
+        // Report checkpoint progress to banner service if available
+        if (this.bannerService && progress.processed !== undefined) {
+            this.bannerService.setCheckpoint(progress.processed);
+        }
+    }
+    /**
+     * Add processed items to checkpoint
+     *
+     * @param key Checkpoint key
+     * @param itemIds Item IDs to add
+     */
+    async addProcessedItems(key, itemIds) {
+        await validatorQuery(`
+      UPDATE validator.pipeline_checkpoints
+      SET processed_items = array_cat(COALESCE(processed_items, '{}'), $1),
+          updated_at = NOW()
+      WHERE checkpoint_key = $2
+      `, [itemIds, key]);
+    }
+    /**
+     * Add failed items to checkpoint
+     *
+     * @param key Checkpoint key
+     * @param itemIds Item IDs to add
+     */
+    async addFailedItems(key, itemIds) {
+        await validatorQuery(`
+      UPDATE validator.pipeline_checkpoints
+      SET failed_items = array_cat(COALESCE(failed_items, '{}'), $1),
+          updated_at = NOW()
+      WHERE checkpoint_key = $2
+      `, [itemIds, key]);
+    }
+    /**
+     * Clear a checkpoint
+     *
+     * @param key Checkpoint key
+     */
+    async clearCheckpoint(key) {
+        await validatorQuery(`
+      DELETE FROM validator.pipeline_checkpoints
+      WHERE checkpoint_key = $1
+      `, [key]);
+    }
+    /**
+     * List all checkpoints
+     *
+     * @param pipelineType Optional filter by pipeline type
+     * @returns Array of checkpoints
+     */
+    async listCheckpoints(pipelineType) {
+        let query = `
+      SELECT
+        id::text,
+        checkpoint_key as key,
+        pipeline_type as "pipelineType",
+        state,
+        processed_items as "processedItems",
+        failed_items as "failedItems",
+        progress,
+        created_at as "createdAt",
+        updated_at as "updatedAt"
+      FROM validator.pipeline_checkpoints
+    `;
+        const params = [];
+        if (pipelineType) {
+            query += ` WHERE pipeline_type = $1`;
+            params.push(pipelineType);
+        }
+        query += ` ORDER BY updated_at DESC`;
+        const result = await validatorQuery(query, params);
+        const rows = Array.isArray(result) ? result : result.rows;
+        return rows.map((row) => ({
+            id: row.id,
+            key: row.key,
+            pipelineType: row.pipelineType,
+            state: row.state,
+            processedItems: row.processedItems || [],
+            failedItems: row.failedItems || [],
+            progress: row.progress || {},
+            createdAt: row.createdAt,
+            updatedAt: row.updatedAt,
+        }));
+    }
+    /**
+     * Check if a checkpoint exists
+     *
+     * @param key Checkpoint key
+     * @returns True if checkpoint exists
+     */
+    async checkpointExists(key) {
+        const result = await validatorQuery(`
+      SELECT EXISTS(
+        SELECT 1 FROM validator.pipeline_checkpoints
+        WHERE checkpoint_key = $1
+      ) as exists
+      `, [key]);
+        const row = Array.isArray(result) ? result[0] : result.rows[0];
+        return row.exists;
+    }
+}
+//# sourceMappingURL=CheckpointManager.js.map

package/dist/services/CheckpointManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CheckpointManager.js","sourceRoot":"","sources":["../../src/services/CheckpointManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAiD/D,MAAM,OAAO,iBAAiB;IACpB,aAAa,CAAiB;IAEtC;;;;OAIG;IACH,YAAY,aAA6B;QACvC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,gBAAgB,CAAC,aAA4B;QAC3C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAAC,IAAoB;QACvC,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,GAAG,EAAE,EAAE,WAAW,GAAG,EAAE,EAAE,QAAQ,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC;QAEhG,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BC,EACD,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAClG,CAAC;QAEF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAoB;YAC7B,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,YAAY,EAAE,GAAG,CAAC,YAA4B;YAC9C,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,EAAE;YACxC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;QAEF,4DAA4D;QAC5D,IAAI,IAAI,CAAC,aAAa,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAClF,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,GAAW;QAC9B,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC;;;;;;;;;;;;;OAaC,EACD,CAAC,GAAG,CAAC,CACN,CAAC;QAEF,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,YAAY,EAAE,GAAG,CAAC,YAA4B;YAC9C,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,EAAE;YACxC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,QAAqC;QACrE,MAAM,cAAc,CAClB;;;;;OAKC,EACD,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAChC,CAAC;QAEF,4DAA4D;QAC5D,IAAI,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC3D,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,GAAW,EAAE,OAAiB;QACpD,MAAM,cAAc,CAClB;;;;;OAKC,EACD,CAAC,OAAO,EAAE,GAAG,CAAC,CACf,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,OAAiB;QACjD,MAAM,cAAc,CAClB;;;;;OAKC,EACD,CAAC,OAAO,EAAE,GAAG,CAAC,CACf,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,eAAe,CAAC,GAAW;QAC/B,MAAM,cAAc,CAClB;;;OAGC,EACD,CAAC,GAAG,CAAC,CACN,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,YAA2B;QAC/C,IAAI,KAAK,GAAG;;;;;;;;;;;;KAYX,CAAC;QACF,MAAM,MAAM,GAAU,EAAE,CAAC;QAEzB,IAAI,YAAY,EAAE,CAAC;YACjB,KAAK,IAAI,2BAA2B,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5B,CAAC;QAED,KAAK,IAAI,2BAA2B,CAAC;QAErC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAkB,KAAK,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QAE1D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,YAAY,EAAE,GAAG,CAAC,YAA4B;YAC9C,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,EAAE;YACxC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,gBAAgB,CAAC,GAAW;QAChC,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC;;;;;OAKC,EACD,CAAC,GAAG,CAAC,CACN,CAAC;QAEF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;CACF"}

package/dist/services/CostCalculator.js

@@ -0,0 +1,211 @@
+/**
+ * Cost Calculator Service
+ *
+ * Unified cost calculation for embedding generation scripts.
+ * Replaces duplicate cost tracking code across multiple files.
+ *
+ * Features:
+ * - Accurate token estimation
+ * - Cost calculation for OpenAI API
+ * - Support for different models/pricing
+ * - Formatted cost output
+ *
+ * @example
+ * ```typescript
+ * const calculator = new CostCalculator();
+ *
+ * // Estimate tokens before API call
+ * const estimatedTokens = calculator.estimateTokens(text);
+ * logger.info('Estimated cost', { cost: calculator.estimateCost(estimatedTokens) });
+ *
+ * // Track actual usage after API call
+ * calculator.addTokens(response.usage.total_tokens);
+ *
+ * // Get totals
+ * logger.info('Cost summary', {
+ *   total_tokens: calculator.getTotalTokens(),
+ *   total_cost: calculator.getTotalCost(),
+ *   formatted: calculator.getFormattedCost(),
+ * });
+ * ```
+ */
+import { logger } from '@tsvika58/shared-utilities/logging';
+/**
+ * OpenAI API Pricing (as of 2024)
+ */
+export const OPENAI_PRICING = {
+    // Embedding models
+    'text-embedding-3-small': 0.00002, // $0.02 per 1M tokens
+    'text-embedding-3-large': 0.00013, // $0.13 per 1M tokens
+    'text-embedding-ada-002': 0.00010, // $0.10 per 1M tokens
+    // GPT models (for reference)
+    'gpt-4-turbo': 0.01, // $10 per 1M input tokens
+    'gpt-4': 0.03, // $30 per 1M input tokens
+    'gpt-3.5-turbo': 0.0005, // $0.50 per 1M input tokens
+};
+export class CostCalculator {
+    totalTokens = 0;
+    itemsProcessed = 0;
+    costPer1K;
+    model;
+    constructor(config = {}) {
+        this.model = config.model || 'text-embedding-3-small';
+        this.costPer1K = config.costPer1K || OPENAI_PRICING[this.model] || 0.00002;
+    }
+    /**
+     * Add tokens to the total count
+     *
+     * @param tokenCount Number of tokens used
+     */
+    addTokens(tokenCount) {
+        this.totalTokens += tokenCount;
+        this.itemsProcessed++;
+    }
+    /**
+     * Estimate tokens for a text string
+     *
+     * Uses a simple heuristic: ~4 characters per token for English text
+     * This is a rough estimate - actual tokenization may vary
+     *
+     * @param text Text to estimate
+     * @returns Estimated token count
+     */
+    estimateTokens(text) {
+        // Average: ~4 characters per token for English
+        // Add 10% buffer for safety
+        const estimate = Math.ceil((text.length / 4) * 1.1);
+        return Math.max(estimate, 1); // Minimum 1 token
+    }
+    /**
+     * Estimate cost for a given token count
+     *
+     * @param tokenCount Number of tokens
+     * @returns Estimated cost in dollars
+     */
+    estimateCost(tokenCount) {
+        return (tokenCount / 1000) * this.costPer1K;
+    }
+    /**
+     * Estimate cost for a text string
+     *
+     * @param text Text to estimate
+     * @returns Estimated cost in dollars
+     */
+    estimateTextCost(text) {
+        const tokens = this.estimateTokens(text);
+        return this.estimateCost(tokens);
+    }
+    /**
+     * Get total tokens used
+     *
+     * @returns Total token count
+     */
+    getTotalTokens() {
+        return this.totalTokens;
+    }
+    /**
+     * Get total cost
+     *
+     * @returns Total cost in dollars
+     */
+    getTotalCost() {
+        return (this.totalTokens / 1000) * this.costPer1K;
+    }
+    /**
+     * Get formatted cost string
+     *
+     * @param decimals Number of decimal places (default: 4)
+     * @returns Formatted cost (e.g., "$0.0025")
+     */
+    getFormattedCost(decimals = 4) {
+        return `$${this.getTotalCost().toFixed(decimals)}`;
+    }
+    /**
+     * Get formatted token count with thousands separators
+     *
+     * @returns Formatted token count (e.g., "125,000")
+     */
+    getFormattedTokens() {
+        return this.totalTokens.toLocaleString();
+    }
+    /**
+     * Get average tokens per item
+     *
+     * @returns Average tokens
+     */
+    getAvgTokensPerItem() {
+        if (this.itemsProcessed === 0)
+            return 0;
+        return this.totalTokens / this.itemsProcessed;
+    }
+    /**
+     * Get average cost per item
+     *
+     * @returns Average cost in dollars
+     */
+    getAvgCostPerItem() {
+        if (this.itemsProcessed === 0)
+            return 0;
+        return this.getTotalCost() / this.itemsProcessed;
+    }
+    /**
+     * Get number of items processed
+     *
+     * @returns Item count
+     */
+    getItemsProcessed() {
+        return this.itemsProcessed;
+    }
+    /**
+     * Get complete cost summary
+     *
+     * @returns Cost summary object
+     */
+    getSummary() {
+        return {
+            totalTokens: this.totalTokens,
+            totalCost: this.getTotalCost(),
+            avgTokensPerItem: this.getAvgTokensPerItem(),
+            avgCostPerItem: this.getAvgCostPerItem(),
+            itemsProcessed: this.itemsProcessed,
+        };
+    }
+    /**
+     * Reset all counters
+     */
+    reset() {
+        this.totalTokens = 0;
+        this.itemsProcessed = 0;
+    }
+    /**
+     * Get model name
+     *
+     * @returns Model name
+     */
+    getModel() {
+        return this.model;
+    }
+    /**
+     * Get cost per 1K tokens
+     *
+     * @returns Cost rate
+     */
+    getCostPer1K() {
+        return this.costPer1K;
+    }
+    /**
+     * Print cost summary to console
+     */
+    printSummary() {
+        const summary = this.getSummary();
+        logger.info('Cost Summary', {
+            model: this.model,
+            total_tokens: this.getFormattedTokens(),
+            total_cost: this.getFormattedCost(),
+            items_processed: summary.itemsProcessed > 0 ? summary.itemsProcessed.toLocaleString() : undefined,
+            avg_tokens_per_item: summary.itemsProcessed > 0 ? Math.round(summary.avgTokensPerItem).toLocaleString() : undefined,
+            avg_cost_per_item: summary.itemsProcessed > 0 ? `$${summary.avgCostPerItem.toFixed(6)}` : undefined,
+        });
+    }
+}
+//# sourceMappingURL=CostCalculator.js.map

package/dist/services/CostCalculator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CostCalculator.js","sourceRoot":"","sources":["../../src/services/CostCalculator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAyC5D;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,mBAAmB;IACnB,wBAAwB,EAAE,OAAO,EAAE,sBAAsB;IACzD,wBAAwB,EAAE,OAAO,EAAE,sBAAsB;IACzD,wBAAwB,EAAE,OAAO,EAAE,sBAAsB;IAEzD,6BAA6B;IAC7B,aAAa,EAAE,IAAI,EAAE,0BAA0B;IAC/C,OAAO,EAAE,IAAI,EAAE,0BAA0B;IACzC,eAAe,EAAE,MAAM,EAAE,4BAA4B;CAC7C,CAAC;AAEX,MAAM,OAAO,cAAc;IACjB,WAAW,GAAW,CAAC,CAAC;IACxB,cAAc,GAAW,CAAC,CAAC;IAClB,SAAS,CAAS;IAClB,KAAK,CAAS;IAE/B,YAAY,SAAqB,EAAE;QACjC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QACtD,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,KAAoC,CAAC,IAAI,OAAO,CAAC;IAC5G,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,UAAkB;QAC1B,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC;QAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,IAAY;QACzB,+CAA+C;QAC/C,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,kBAAkB;IAClD,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,UAAkB;QAC7B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;IAC9C,CAAC;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,WAAmB,CAAC;QACnC,OAAO,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,kBAAkB;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,mBAAmB;QACjB,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,iBAAiB;QACf,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC;IACnD,CAAC;IAED;;;;OAIG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACH,UAAU;QACR,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;YAC9B,gBAAgB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC5C,cAAc,EAAE,IAAI,CAAC,iBAAiB,EAAE;YACxC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAElC,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE;YAC1B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,IAAI,CAAC,kBAAkB,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,gBAAgB,EAAE;YACnC,eAAe,EAAE,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;YACjG,mBAAmB,EAAE,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;YACnH,iBAAiB,EAAE,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;SACpG,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/services/EmbeddingCache.js

@@ -0,0 +1,68 @@
+/**
+ * Embedding Cache Service
+ *
+ * Caches query embeddings to avoid regenerating them for repeated queries.
+ * Reduces API calls and improves latency for common queries.
+ */
+import NodeCache from 'node-cache';
+export class EmbeddingCache {
+    cache;
+    constructor() {
+        const ttl = process.env.EMBEDDING_CACHE_TTL
+            ? Number(process.env.EMBEDDING_CACHE_TTL)
+            : 604800; // Default: 7 days (query embeddings are deterministic, database is static)
+        const maxKeys = process.env.EMBEDDING_CACHE_MAX_SIZE
+            ? Number(process.env.EMBEDDING_CACHE_MAX_SIZE)
+            : 1000; // Default: 1000 queries
+        this.cache = new NodeCache({
+            stdTTL: ttl,
+            checkperiod: 600, // Check for expired keys every 10 minutes
+            useClones: false, // Don't clone arrays (faster, but be careful with mutations)
+            maxKeys: maxKeys,
+        });
+    }
+    /**
+     * Get cached embedding for a query
+     */
+    get(query) {
+        const key = this.getKey(query);
+        return this.cache.get(key);
+    }
+    /**
+     * Cache an embedding for a query
+     */
+    set(query, embedding) {
+        const key = this.getKey(query);
+        this.cache.set(key, embedding);
+    }
+    /**
+     * Check if a query is cached
+     */
+    has(query) {
+        const key = this.getKey(query);
+        return this.cache.has(key);
+    }
+    /**
+     * Get cache statistics
+     */
+    stats() {
+        return this.cache.getStats();
+    }
+    /**
+     * Clear all cached embeddings
+     */
+    flush() {
+        this.cache.flushAll();
+    }
+    /**
+     * Generate cache key from query
+     */
+    getKey(query) {
+        // Normalize query: lowercase, trim, remove extra whitespace
+        const normalized = query.toLowerCase().trim().replace(/\s+/g, ' ');
+        return `embedding:${normalized}`;
+    }
+}
+// Singleton instance
+export const embeddingCache = new EmbeddingCache();
+//# sourceMappingURL=EmbeddingCache.js.map

package/dist/services/EmbeddingCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EmbeddingCache.js","sourceRoot":"","sources":["../../src/services/EmbeddingCache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,SAAS,MAAM,YAAY,CAAC;AAEnC,MAAM,OAAO,cAAc;IACjB,KAAK,CAAY;IAEzB;QACE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;YACzC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;YACzC,CAAC,CAAC,MAAM,CAAC,CAAC,2EAA2E;QAEvF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB;YAClD,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC9C,CAAC,CAAC,IAAI,CAAC,CAAC,wBAAwB;QAElC,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC;YACzB,MAAM,EAAE,GAAG;YACX,WAAW,EAAE,GAAG,EAAE,0CAA0C;YAC5D,SAAS,EAAE,KAAK,EAAE,6DAA6D;YAC/E,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,KAAa;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAW,GAAG,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,KAAa,EAAE,SAAmB;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,KAAa;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,KAAa;QAC1B,4DAA4D;QAC5D,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnE,OAAO,aAAa,UAAU,EAAE,CAAC;IACnC,CAAC;CACF;AAED,qBAAqB;AACrB,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}