@meller/tokentalos 1.0.0

package/api/index.js

@@ -0,0 +1,111 @@
+import express from 'express';
+import cors from 'cors';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import chalk from 'chalk';
+import { initDb } from '../lib/engine/db.js';
+import usageRouter, { setConfig as setUsageConfig } from './api/v1/usage.js';
+import analyticsRouter from './api/v1/analytics.js';
+import opvRouter, { setConfig as setOpvConfig } from './api/v1/opv.js';
+import net from 'net';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+async function isPortInUse(port) {
+  return new Promise((resolve) => {
+    const server = net.createServer()
+      .once('error', (err) => {
+        if (err.code === 'EADDRINUSE') resolve(true);
+        else resolve(false);
+      })
+      .once('listening', () => {
+        server.close();
+        resolve(false);
+      })
+      .listen(port, '0.0.0.0');
+  });
+}
+
+export async function startServer(config) {
+  await initDb(config);
+  setUsageConfig(config);
+  setOpvConfig(config);
+  const app = express();
+  return createApp(app, config);
+}
+
+export function createApp(app, config) {
+  const isCollectorOnly = config._service === 'collector';
+  const isDashboardOnly = config._service === 'dashboard';
+  
+  const PORT = isDashboardOnly ? (config.dashboardPort || 8060) : (config.gatewayPort || 8060);
+  const publicPath = path.join(__dirname, 'public');
+
+  // Store config for middleware access
+  app.set('tokentalosConfig', config);
+
+  app.use(cors());
+  app.use(express.json());
+
+  // --- API Router ---
+  // The Dashboard ALWAYS needs the Analytics and Recent Usage APIs to function.
+  if (config.enableCollector !== false || config.enableDashboard !== false) {
+    const apiRouter = express.Router();
+
+    // Shared / Analytics Routes (Needed by Dashboard)
+    apiRouter.use('/v1/analytics', analyticsRouter);
+    
+    // Usage Router has both Read (recent) and Write (ingest/execute).
+    // We mount it for both, as the dashboard needs the 'recent' logs.
+    apiRouter.use('/v1/usage', usageRouter);
+    apiRouter.use('/v1/opv', opvRouter);
+
+    apiRouter.get('/', (req, res) => {
+      res.json({
+        name: isDashboardOnly ? 'TokenTalos Dashboard API' : 'TokenTalos Collector API',
+        version: '0.1.0',
+        services: {
+          collector: !isDashboardOnly && (config.enableCollector !== false),
+          dashboard: !isCollectorOnly && (config.enableDashboard !== false)
+        }
+      });
+    });
+
+    apiRouter.get('/health', (req, res) => {
+      res.json({ status: 'ok', service: isDashboardOnly ? 'dashboard' : 'collector' });
+    });
+
+    app.use('/api', apiRouter);
+  }
+
+  // --- Dashboard Static Assets ---
+  if (config.enableDashboard !== false && !isCollectorOnly) {
+    app.use(express.static(publicPath));
+
+    // Catch-all for SPA (excluding /api)
+    app.use((req, res, next) => {
+      if (req.path.startsWith('/api')) return next();
+      res.sendFile(path.join(publicPath, 'index.html'), (err) => {
+        if (err) res.status(404).send('TokenTalos Dashboard build not found.');
+      });
+    });
+  }
+
+  if (process.env.NODE_ENV !== 'test') {
+    (async () => {
+      if (await isPortInUse(PORT)) {
+        console.error(chalk.red.bold(`\n❌ Error: Port ${PORT} is already in use.`));
+        process.exit(1);
+      }
+
+      app.listen(PORT, '0.0.0.0', () => {
+        const serviceName = isCollectorOnly ? 'Collector' : (isDashboardOnly ? 'Dashboard' : 'TokenTalos');
+        console.log(chalk.cyan.bold(`\n🚀 ${serviceName} running at http://localhost:${PORT}`));
+        if (!isDashboardOnly && (config.enableCollector !== false)) console.log(chalk.gray(`   API Endpoint: http://localhost:${PORT}/api/v1`));
+      });
+    })();
+  }
+
+  return app;
+}

package/api/middleware/auth.js

@@ -0,0 +1,45 @@
+import { TokenTalosEngine } from '../../lib/engine/index.js';
+
+/**
+ * Authentication Middleware
+ * 
+ * Secure ingestion and execution endpoints.
+ * In Managed Mode: Requires valid X-TokenTalos-Key.
+ * In Local Mode: Optional key, defaults to default_org.
+ */
+export async function authMiddleware(req, res, next) {
+  // Config is passed to the app during startServer
+  const config = req.app.get('tokentalosConfig');
+  const managedMode = config.managedMode || false;
+  const apiKey = req.headers['x-tokentalos-key'];
+
+  try {
+    const engine = new TokenTalosEngine(config);
+    await engine.init();
+
+    if (apiKey) {
+      const orgId = await engine.validateApiKey(apiKey);
+      if (orgId) {
+        req.orgId = orgId;
+        return next();
+      }
+      
+      // If key provided but invalid
+      if (managedMode) {
+        return res.status(401).json({ error: 'Invalid API Key' });
+      }
+    }
+
+    // No key provided
+    if (managedMode) {
+      return res.status(401).json({ error: 'API Key required (X-TokenTalos-Key)' });
+    }
+
+    // Fallback for Local Mode
+    req.orgId = config.orgId || 'default_org';
+    next();
+  } catch (err) {
+    console.error('[TokenTalos] Auth Error:', err);
+    res.status(500).json({ error: 'Authentication internal error' });
+  }
+}

package/api/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "tokentalos-api",
+  "version": "0.1.0",
+  "description": "",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "start": "node index.js",
+    "dev": "nodemon index.js",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "jest": "^29.0.0",
+    "nodemon": "^3.0.0",
+    "supertest": "^7.2.2"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.76.0",
+    "@google/generative-ai": "^0.24.1",
+    "anthropic": "^0.0.0",
+    "axios": "^1.13.5",
+    "cli-table3": "^0.6.5",
+    "cors": "^2.8.6",
+    "dotenv": "^17.3.1",
+    "express": "^5.2.1",
+    "fs-extra": "^11.3.3",
+    "js-tiktoken": "^1.0.21",
+    "openai": "^6.22.0",
+    "path": "^0.12.7",
+    "pg": "^8.18.0",
+    "sqlite": "^5.1.1",
+    "sqlite3": "^5.1.7",
+    "uuid": "^13.0.0"
+  }
+}

package/bin/tokentalos.js

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+
+import { loadConfig, runSetup } from '../api/setup.js';
+import { startServer } from '../api/index.js';
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import axios from 'axios';
+import { Command } from 'commander';
+import { exec, spawn } from 'child_process';
+import fs from 'fs-extra';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const program = new Command();
+
+const getPidFile = (service) => path.join(process.cwd(), `.tokentalos-${service || 'full'}.pid`);
+
+program
+  .name('tokentalos')
+  .description('Standalone LLM Token Usage Analyzer and Proxy')
+  .version('0.1.0');
+
+program
+  .command('setup')
+  .description('Run the interactive configuration wizard')
+  .action(async () => {
+    await runSetup();
+  });
+
+program
+  .command('start [service]')
+  .description('Start Token Talos services (all, collector, or dashboard)')
+  .option('-d, --daemon', 'Run in background', false)
+  .action(async (service, options) => {
+    const validServices = ['collector', 'dashboard'];
+    if (service && !validServices.includes(service)) {
+      console.log(chalk.red(`Invalid service: ${service}. Use 'collector', 'dashboard', or leave empty for both.`));
+      process.exit(1);
+    }
+
+    let config = await loadConfig();
+    if (!config) {
+      console.log(chalk.yellow('No configuration found. Starting setup...'));
+      config = await runSetup();
+    }
+
+    // Inject service context into config
+    if (service) config._service = service;
+
+    const pidFile = getPidFile(service);
+
+    if (options.daemon) {
+      if (fs.existsSync(pidFile)) {
+        console.log(chalk.red(`Token Talos ${service || ''} is already running (PID file exists).`));
+        process.exit(1);
+      }
+
+      const logFile = `./tokentalos-${service || 'all'}.log`;
+      const out = fs.openSync(logFile, 'a');
+      const err = fs.openSync(logFile, 'a');
+
+      const args = ['start'];
+      if (service) args.push(service);
+
+      const child = spawn('node', [path.join(__dirname, 'tokentalos.js'), ...args], {
+        detached: true,
+        stdio: ['ignore', out, err]
+      });
+
+      fs.writeFileSync(pidFile, child.pid.toString());
+      child.unref();
+      console.log(chalk.green(`Token Talos ${service || 'services'} started in background (PID: ${child.pid})`));
+      console.log(chalk.gray(`Logs: ${logFile}`));
+      process.exit(0);
+    } else {
+      await startServer(config);
+    }
+  });
+
+program
+  .command('stop [service]')
+  .description('Stop Token Talos services (all, collector, or dashboard)')
+  .action(async (service) => {
+    const config = await loadConfig();
+    const pidFile = getPidFile(service);
+    
+    // Determine port based on service
+    let port = config?.gatewayPort || 8060;
+    if (service === 'dashboard') port = config?.dashboardPort || 8060;
+
+    let stopped = false;
+
+    if (fs.existsSync(pidFile)) {
+      const pid = fs.readFileSync(pidFile, 'utf8');
+      try {
+        process.kill(parseInt(pid), 'SIGTERM');
+        fs.removeSync(pidFile);
+        console.log(chalk.green(`Token Talos ${service || ''} (PID: ${pid}) stopped.`));
+        stopped = true;
+      } catch (err) {
+        console.log(chalk.red(`Failed to stop Token Talos ${service || ''} via PID: ${err.message}`));
+        fs.removeSync(pidFile);
+      }
+    }
+
+    // Secondary check: kill by port if PID file didn't work
+    if (!stopped) {
+      console.log(chalk.gray(`Checking for processes on port ${port}...`));
+      try {
+        const { stdout } = await new Promise((resolve) => {
+          exec(`lsof -t -i:${port}`, (err, stdout) => resolve({ stdout }));
+        });
+
+        if (stdout && stdout.trim()) {
+          const pids = stdout.trim().split('\n');
+          for (const pidToKill of pids) {
+            process.kill(parseInt(pidToKill), 'SIGKILL');
+            console.log(chalk.green(`Process on port ${port} (PID: ${pidToKill}) terminated.`));
+          }
+          stopped = true;
+        }
+      } catch (err) {
+        // Silently ignore if lsof fails (likely no process)
+      }
+    }
+
+    if (!stopped) {
+      console.log(chalk.yellow(`No running Token Talos ${service || 'service'} found.`));
+    }
+  });
+
+program
+  .command('dashboard')
+  .description('Launch the Token Talos Dashboard (Reader Mode)')
+  .action(async () => {
+    let config = await loadConfig();
+    if (!config) {
+      console.log(chalk.yellow('No configuration found. Starting setup...'));
+      config = await runSetup();
+    }
+    // Start server in foreground for dashboard access
+    console.log(chalk.blue('Launching Dashboard...'));
+    await startServer(config);
+  });
+
+program
+  .command('stats')
+  .description('Show aggregate token and cost statistics')
+  .action(async () => {
+    const config = await loadConfig();
+    const port = config?.gatewayPort || 8060;
+    const API_URL = process.env.TOKENTALOS_API_URL || `http://localhost:${port}/api/v1`;
+    try {
+      const { data } = await axios.get(`${API_URL}/usage/stats`);
+      const table = new Table({ head: [chalk.blue('Metric'), chalk.blue('Value')] });
+      table.push(['Total Tokens', data.total_tokens.toLocaleString()]);
+      table.push(['Total Cost', `$${data.total_cost.toFixed(4)}`]);
+      table.push(['Total Requests', data.total_requests]);
+      console.log(table.toString());
+    } catch (err) {
+      console.error(chalk.red('Error: Could not connect to API. Is Token Talos running?'));
+    }
+  });
+
+program
+  .command('list')
+  .description('List recent prompt logs')
+  .action(async () => {
+    const config = await loadConfig();
+    const port = config?.gatewayPort || 8060;
+    const API_URL = process.env.TOKENTALOS_API_URL || `http://localhost:${port}/api/v1`;
+    try {
+      const { data } = await axios.get(`${API_URL}/usage/recent`);
+      const table = new Table({ head: [chalk.blue('ID'), chalk.blue('Model'), chalk.blue('Tokens'), chalk.blue('Cost')] });
+      data.forEach(r => table.push([r.id.substring(0, 8), r.model, r.total_tokens, `$${r.total_cost.toFixed(4)}`]));
+      console.log(table.toString());
+    } catch (err) {
+      console.error(chalk.red('Error: Could not connect to API.'));
+    }
+  });
+
+program
+  .command('export')
+  .description('Export usage logs to external formats')
+  .option('-f, --format <type>', 'Export format (jsonl, langsmith)', 'jsonl')
+  .option('-o, --output <path>', 'Output file path', './tokentalos_export.json')
+  .action(async (options) => {
+    const config = await loadConfig();
+    const port = config?.gatewayPort || 8060;
+    const API_URL = process.env.TOKENTALOS_API_URL || `http://localhost:${port}/api/v1`;
+    try {
+      const { data } = await axios.get(`${API_URL}/usage/recent?limit=1000`);
+      
+      let outputData = '';
+      if (options.format === 'jsonl') {
+        outputData = data.map(r => JSON.stringify(r)).join('\n');
+      } else if (options.format === 'langsmith') {
+        // Simple mapping to LangSmith schema
+        outputData = JSON.stringify(data.map(r => ({
+          name: r.endpoint || 'tokentalos_gateway',
+          inputs: r.variables ? Object.fromEntries(r.variables.map(v => [v.name, v.content])) : {},
+          outputs: { content: '...' }, // Responses not stored in passive ingest
+          usage: { prompt_tokens: r.input_tokens, completion_tokens: r.output_tokens },
+          extra: { model: r.model, provider: r.provider }
+        })), null, 2);
+      }
+
+      fs.writeFileSync(options.output, outputData);
+      console.log(chalk.green(`Successfully exported ${data.length} records to ${options.output} (${options.format})`));
+    } catch (err) {
+      console.error(chalk.red(`Export failed: ${err.message}`));
+    }
+  });
+
+// Handle default case
+if (process.argv.length === 2) {
+  program.help();
+}
+
+program.parse(process.argv);

package/index.js

@@ -0,0 +1,151 @@
+import { TokenTalosEngine } from './lib/engine/index.js';
+import { TokenTalosPrompt } from './lib/engine/parameterizer.js';
+import axios from 'axios';
+
+/**
+ * TokenTalos Client SDK
+ * 
+ * Supports both Standalone (direct DB) and Proxy (HTTP) modes.
+ */
+export class TokenTalos {
+  /**
+   * @param {Object} options
+   * @param {string} [options.mode='standalone'] - 'standalone' or 'proxy'
+   * @param {string} [options.apiUrl] - Base URL for proxy mode
+   * @param {Object} [options.config] - TokenTalos configuration for standalone mode
+   */
+  constructor(options = {}) {
+    this.mode = options.mode || 'standalone';
+    this.apiUrl = options.apiUrl || 'http://localhost:8060/api/v1';
+    this.reportUrl = options.reportUrl || options.apiUrl || null; // URL to report usage to if in standalone
+    this.projectId = options.projectId || 'default';
+    this.apiKey = options.apiKey || null;
+    
+    if (this.mode === 'standalone') {
+      this.engine = new TokenTalosEngine({ 
+        ...options.config, 
+        projectId: this.projectId 
+      });
+    }
+  }
+
+  /**
+   * Internal helper for reporting to collector
+   */
+  async _report(usageId, result, params) {
+    if (!this.reportUrl || this.mode === 'proxy') return;
+
+    try {
+      // Re-map engine result to ingestion format
+      const reportData = {
+        id: usageId,
+        projectId: this.projectId,
+        provider: params.provider,
+        model: params.model,
+        full_prompt: result.full_prompt_string || result.full_prompt || null,
+        response_content: result.content || null,
+        input_tokens: result.usage?.input_tokens || result.input_tokens || 0,
+        output_tokens: result.usage?.output_tokens || result.output_tokens || 0,
+        latency_ms: result.metadata?.latency_ms || 0,
+        endpoint: params.endpoint || 'sdk_standalone',
+        variables: result.variables || [],
+        actions_taken: result.metadata?.actions_taken || [],
+        timestamp: new Date().toISOString()
+      };
+
+      await axios.post(`${this.reportUrl}/usage/ingest`, reportData, { 
+        headers: this._getHeaders() 
+      });
+    } catch (err) {
+      console.warn('[TokenTalos] Failed to report usage to collector:', err.message);
+    }
+  }
+
+  async init() {
+    if (this.mode === 'standalone') {
+      await this.engine.init();
+    }
+  }
+
+  /**
+   * Construct and process a prompt without executing
+   */
+  async construct(params) {
+    if (this.mode === 'standalone') {
+      const { processedParts, metadata } = await this.engine.process(params.parts);
+      const prompt = this.engine.createPrompt(params.provider, params.model);
+      
+      for (const key in processedParts) {
+        if (key === 'system') prompt.addSystem(processedParts[key], params.parts[key]);
+        else if (key === 'context') prompt.addContext(processedParts[key], params.parts[key]);
+        else if (key === 'history') prompt.addHistory(processedParts[key], params.parts[key]);
+        else if (key === 'user_query') prompt.addUserQuery(processedParts[key], params.parts[key]);
+        else prompt.add(key, processedParts[key], params.parts[key]);
+      }
+
+      const trackingData = prompt.getTrackingData();
+      const result = {
+        id: trackingData.id,
+        messages: prompt.toMessages(),
+        full_prompt_string: prompt.toString(),
+        metadata,
+        variables: trackingData.variables
+      };
+
+      // Construction is "passive" ingestion - we report the intent
+      await this._report(trackingData.id, { ...result, input_tokens: trackingData.total_tokens }, params);
+
+      return result;
+    } else {
+      const { data } = await axios.post(`${this.apiUrl}/usage/prompt/construct`, {
+        projectId: this.projectId,
+        ...params
+      }, { headers: this._getHeaders() });
+      return data;
+    }
+  }
+
+  /**
+   * Execute a prompt through the Gateway
+   */
+  async execute(params) {
+    if (this.mode === 'standalone') {
+      const result = await this.engine.execute({
+        projectId: this.projectId,
+        ...params
+      });
+
+      // Report execution result
+      await this._report(result.id, result, params);
+
+      return result;
+    } else {
+      const { data } = await axios.post(`${this.apiUrl}/usage/prompt/execute`, {
+        projectId: this.projectId,
+        ...params
+      }, { headers: this._getHeaders() });
+      return data;
+    }
+  }
+
+  /**
+   * Verify reasoning chunks (OPV)
+   */
+  async verifyReasoning(params) {
+    if (this.mode === 'standalone') {
+      return await this.engine.verifyReasoning({
+        projectId: this.projectId,
+        ...params
+      });
+    } else {
+      const { data } = await axios.post(`${this.apiUrl}/opv/heartbeat`, {
+        projectId: this.projectId,
+        ...params
+      }, { headers: this._getHeaders() });
+      return data;
+    }
+  }
+}
+
+export { TokenTalosPrompt };
+export default TokenTalos;

package/lib/engine/ai_analyzer.js

@@ -0,0 +1,66 @@
+import { GoogleGenerativeAI } from '@google/generative-ai';
+import { getCostCalculator } from './pricing.js';
+
+export async function runAIAnalysis(config, usageRecord, variables) {
+  if (config.llmProvider !== 'gemini') return null;
+
+  try {
+    const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY);
+    const model = genAI.getGenerativeModel({ model: 'gemini-1.5-flash' });
+
+    const variableInfo = variables.map(v => `${v.name} (${v.token_count} tokens): "${v.content.substring(0, 100)}..."`).join('\n');
+    
+    const prompt = `
+    Analyze this LLM prompt structure and suggest optimizations to reduce costs while maintaining performance.
+    
+    Usage Context:
+    - Provider: ${usageRecord.provider}
+    - Model: ${usageRecord.model}
+    - Total Tokens: ${usageRecord.total_tokens}
+    - Total Cost: $${usageRecord.total_cost}
+    
+    Variables:
+    ${variableInfo}
+    
+    Return a JSON object with:
+    - detected_issues (array of strings)
+    - optimization_suggestions (array of strings)
+    - estimated_savings_pct (number)
+    `;
+
+    const result = await model.generateContent(prompt);
+    const response = await result.response;
+    const text = response.text();
+    
+    // Attempt to parse JSON from response
+    try {
+      const jsonStart = text.indexOf('{');
+      const jsonEnd = text.lastIndexOf('}') + 1;
+      const analysis = JSON.parse(text.substring(jsonStart, jsonEnd));
+      
+      const calculator = getCostCalculator();
+      const mceResult = calculator.getBestAlternative(
+        usageRecord.provider, 
+        usageRecord.model, 
+        usageRecord.input_tokens, 
+        usageRecord.output_tokens,
+        config.comparisonProviders
+      );
+      
+      if (mceResult) {
+        analysis.mce_best_alternative_model = mceResult.model;
+        analysis.mce_best_alternative_provider = mceResult.provider;
+        analysis.mce_best_alternative_cost = mceResult.cost;
+        analysis.mce_savings_pct = ((usageRecord.total_cost - mceResult.cost) / usageRecord.total_cost) * 100;
+      }
+      
+      return analysis;
+    } catch (err) {
+      console.warn('AI analysis JSON parsing failed:', err);
+      return null;
+    }
+  } catch (err) {
+    console.error('AI analysis error:', err);
+    return null;
+  }
+}