@oas-tools/oas-telemetry 0.7.1 → 0.8.0-alpha.0

package/dist/esm/telemetry/telemetryConfigurator.js

@@ -1,37 +1,60 @@
-import { BatchSpanProcessor, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node';
-import { dynamicMultiLogProcessor, dynamicMultiSpanProcessor, inMemoryDbLogExporter, inMemoryDbMetricExporter, inMemoryDbSpanExporter, multiLogExporter, multiSpanExporter, oasTelemetryResource } from './telemetryRegistry.js';
+import { BatchLogRecordProcessor as LogBatchLogRecordProcessor, SimpleLogRecordProcessor as LogSimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+import { BatchSpanProcessor as TraceBatchSpanProcessor, SimpleSpanProcessor as TraceSimpleSpanProcessor } from '@opentelemetry/sdk-trace-node';
 import logger from '../utils/logger.js';
-import { BatchLogRecordProcessor, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
-import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
-import { HostMetrics } from '@opentelemetry/host-metrics';
+import { inMemoryDbLogExporter, inMemoryDbMetricExporter, inMemoryDbSpanExporter, instrumentations, multiLogExporter, multiSpanExporter, oasTelemetryResource } from './telemetryRegistry.js';
+import { NodeSDK } from '@opentelemetry/sdk-node';
 import { bootEnvVariables } from '../config/bootConfig.js';
 import { pluginService } from '../tlm-plugin/pluginService.js';
-export const configureTelemetry = (oasTlmConfig) => {
+export function configureTelemetry(oasTlmConfig) {
+    logger.info("🚀 Configuring Telemetry...");
+    if (oasTlmConfig.instrumentations) {
+        instrumentations.push(...oasTlmConfig.instrumentations);
+    }
     configurePlugins(oasTlmConfig);
-    configureTraces(oasTlmConfig);
-    configureMetrics(oasTlmConfig);
-    configureLogs(oasTlmConfig);
-    logger.info("✅ Telemetry configured successfully. All exporters are ready");
-};
+    const mainTraceProcessor = configureTraces(oasTlmConfig);
+    const mainMetricReader = configureMetrics(oasTlmConfig);
+    const mainLogProcessor = configureLogs(oasTlmConfig);
+    const sdk = new NodeSDK({
+        instrumentations: instrumentations,
+        resource: oasTelemetryResource,
+        traceExporter: inMemoryDbSpanExporter,
+        spanProcessors: [mainTraceProcessor, ...oasTlmConfig.traces.extraProcessors || []],
+        metricReaders: [mainMetricReader, ...(oasTlmConfig.metrics.extraReaders || [])],
+        logRecordProcessors: [mainLogProcessor, ...oasTlmConfig.logs.extraProcessors || []],
+    });
+    sdk.start();
+    logger.info("✅ Node SDK started with telemetry configuration");
+    return true;
+}
 function configurePlugins(oasTlmConfig) {
     pluginService.enabled = oasTlmConfig.plugins.enabled;
 }
 function configureTraces(oasTlmConfig) {
     // TRACES CONFIGURATION
     // [OT]Provider -> [OT]SpanProcessor(multiSpan) -> n Processors(eg mainProcessor, extra) -> 1 SpanExporter
-    inMemoryDbSpanExporter.baseUrl = oasTlmConfig.general.baseUrl; // TODO this will be done with filters
     inMemoryDbSpanExporter.retentionTimeInSeconds = oasTlmConfig.traces.memoryExporter.retentionTimeSeconds;
     inMemoryDbSpanExporter.setEnabledValue(oasTlmConfig.traces.memoryExporter.enabled);
     const mainExporter = multiSpanExporter;
-    let mainProcessor = new BatchSpanProcessor(mainExporter);
+    let mainProcessor = new TraceBatchSpanProcessor(mainExporter);
     if (bootEnvVariables.OASTLM_BOOT_ENV !== 'production') {
         logger.info('Not in production, using SimpleSpanProcessor for traces');
-        mainProcessor = new SimpleSpanProcessor(mainExporter);
+        mainProcessor = new TraceSimpleSpanProcessor(mainExporter);
     }
     mainExporter.addExporters(inMemoryDbSpanExporter); // Main exporter have at least the in-memory exporter used by the traces controller
     mainExporter.addExporters(oasTlmConfig.traces.extraExporters);
-    dynamicMultiSpanProcessor.addProcessors(mainProcessor);
-    dynamicMultiSpanProcessor.addProcessors(oasTlmConfig.traces.extraProcessors);
+    return mainProcessor;
+}
+function configureMetrics(oasTlmConfig) {
+    // METRICS CONFIGURATION
+    inMemoryDbMetricExporter.setEnabledValue(oasTlmConfig.metrics.memoryExporter.enabled);
+    inMemoryDbMetricExporter.retentionTimeInSeconds = oasTlmConfig.metrics.memoryExporter.retentionTimeSeconds;
+    const mainReader = new PeriodicExportingMetricReader({
+        exporter: inMemoryDbMetricExporter,
+        exportIntervalMillis: oasTlmConfig.metrics.mainMetricReaderOptions.exportIntervalMillis,
+        metricProducers: oasTlmConfig.metrics.mainMetricReaderOptions.metricProducers
+    });
+    return mainReader;
 }
 function configureLogs(oasTlmConfig) {
     // LOGS CONFIGURATION
@@ -39,33 +62,12 @@ function configureLogs(oasTlmConfig) {
     inMemoryDbLogExporter.setEnabledValue(oasTlmConfig.logs.memoryExporter.enabled);
     inMemoryDbLogExporter.retentionTimeInSeconds = oasTlmConfig.logs.memoryExporter.retentionTimeSeconds;
     const mainExporter = multiLogExporter;
-    let mainProcessor = new BatchLogRecordProcessor(mainExporter);
+    let mainProcessor = new LogBatchLogRecordProcessor(mainExporter);
     if (bootEnvVariables.OASTLM_BOOT_ENV !== 'production') {
         logger.info('Not in production, using SimpleLogRecordProcessor for logs');
-        mainProcessor = new SimpleLogRecordProcessor(mainExporter);
+        mainProcessor = new LogSimpleLogRecordProcessor(mainExporter);
     }
     mainExporter.addExporters(inMemoryDbLogExporter); // Main exporter have at least the in-memory exporter used by the logs controller
     mainExporter.addExporters(oasTlmConfig.logs.extraExporters);
-    dynamicMultiLogProcessor.addProcessors(mainProcessor);
-    dynamicMultiLogProcessor.addProcessors(oasTlmConfig.logs.extraProcessors);
-}
-function configureMetrics(oasTlmConfig) {
-    // METRICS CONFIGURATION
-    // [CUSTOM]MeterProvider -> n [OTel]MetricReader -> (0-1) MetricExporter (only if push-based reader)
-    inMemoryDbMetricExporter.setEnabledValue(oasTlmConfig.metrics.memoryExporter.enabled);
-    inMemoryDbMetricExporter.retentionTimeInSeconds = oasTlmConfig.metrics.memoryExporter.retentionTimeSeconds;
-    const mainReader = new PeriodicExportingMetricReader({
-        exporter: inMemoryDbMetricExporter,
-        exportIntervalMillis: oasTlmConfig.metrics.mainMetricReaderOptions.exportIntervalMillis,
-        metricProducers: oasTlmConfig.metrics.mainMetricReaderOptions.metricProducers
-    });
-    const meterProvider = new MeterProvider({
-        resource: oasTelemetryResource,
-        readers: [mainReader, ...oasTlmConfig.metrics.extraReaders],
-        views: oasTlmConfig.metrics.extraViews || []
-    });
-    // TODO maybe hostMetrics are too much, consider using only a subset of them.
-    const hostMetrics = new HostMetrics({ meterProvider: meterProvider });
-    // AFTER adding all readers, start the instrumentations
-    hostMetrics.start();
+    return mainProcessor;
 }

package/dist/esm/telemetry/telemetryRegistry.js

@@ -8,6 +8,15 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 import { resourceFromAttributes } from "@opentelemetry/resources";
 import { bootEnvVariables } from "../config/bootConfig.js";
 // GLOBAL REGISTRY of telemetry components, used by SDKs and controllers.
+let _bootInitialized = false;
+let _telemetryConfigured = false;
+let _router = undefined;
+export function isBootInitialized() { return _bootInitialized; }
+export function setBootInitialized(v) { _bootInitialized = v; }
+export function isTelemetryConfigured() { return _telemetryConfigured; }
+export function setTelemetryConfigured(v) { _telemetryConfigured = v; }
+export function setTelemetryRouter(router) { _router = router; }
+export function getTelemetryRouter() { return _router; }
 export const oasTelemetryResource = resourceFromAttributes({
     [ATTR_SERVICE_NAME]: bootEnvVariables.OASTLM_BOOT_SERVICE_NAME
 });
@@ -22,7 +31,7 @@ export const dynamicMultiSpanProcessor = new DynamicMultiSpanProcessor();
 export const inMemoryDbLogExporter = new InMemoryDbLogExporter();
 export const multiLogExporter = new EnablerMultiLogExporter();
 export const dynamicMultiLogProcessor = new DynamicMultiLogRecordProcessor();
-// Override console methods to emit logs via OpenTelemetry, like an instrumentation
+// For custom log instrumentation, and lib logging
 export const originalConsoleMethods = {
     log: console.log,
     warn: console.warn,
@@ -35,3 +44,5 @@ export const originalConsoleMethods = {
 export const inMemoryDbMetricExporter = new InMemoryDbMetricExporter();
 // Readers and their exporters cannot be grouped together in a MultiReader or similar construct
 // due to differences in aggregation temporality and aggregation selection.
+// INSTRUMENTATIONS -----------------------------------------------------------------------
+export const instrumentations = [];

package/dist/esm/tlm-ai/agent.js

@@ -1,6 +1,6 @@
 import { tools, availableTools } from './tools.js';
 import logger from '../utils/logger.js';
-export async function agent(openai, messages, model = "gpt-3.5-turbo", extraPrompts = []) {
+export async function agent(openai, messages, model = "gpt-3.5-turbo") {
     for (let i = 0; i < 5; i++) {
         const modelResponse = await openai.chat.completions.create({
             model,
@@ -12,11 +12,13 @@ export async function agent(openai, messages, model = "gpt-3.5-turbo", extraProm
             logger.debug("Tool calls detected:", message.tool_calls);
             const results = [];
             for (const toolCall of message.tool_calls) {
+                // Type guard for tool_calls
+                if (toolCall.type !== 'function' || !toolCall.function)
+                    continue;
                 const functionName = toolCall.function.name;
                 const functionToCall = availableTools[functionName];
                 const functionArgs = JSON.parse(toolCall.function.arguments);
                 const functionArgsArr = Object.values(functionArgs);
-                // @ts-expect-error yes
                 // eslint-disable-next-line prefer-spread
                 const functionResponse = await functionToCall.apply(null, functionArgsArr);
                 results.push({
@@ -26,7 +28,7 @@ export async function agent(openai, messages, model = "gpt-3.5-turbo", extraProm
             }
             const resultMessage = results.map(({ name, response }, idx) => {
                 const toolCall = message.tool_calls?.[idx];
-                const params = toolCall ? JSON.parse(toolCall.function.arguments) : {};
+                const params = toolCall && toolCall.type === 'function' && toolCall.function ? JSON.parse(toolCall.function.arguments) : {};
                 return `Tool "${name}" called with parameters:\n${JSON.stringify(params, null, 2)}\nResult:\n${JSON.stringify(response, null, 2)}`;
             }).join("\n\n");
             messages.push({

package/dist/esm/tlm-ai/aiController.js

@@ -20,7 +20,7 @@ export async function listConversations(req, res) {
 }
 export async function getConversationHistory(req, res) {
     try {
-        const { conversationId } = req.params;
+        const conversationId = req.params.conversationId;
         const conversation = getAiService().getConversation(conversationId);
         if (!conversation) {
             res.status(404).json({ error: 'Not found' });
@@ -34,7 +34,7 @@ export async function getConversationHistory(req, res) {
 }
 export async function deleteConversation(req, res) {
     try {
-        const { conversationId } = req.params;
+        const conversationId = req.params.conversationId;
         const deleted = getAiService().deleteConversation(conversationId);
         if (!deleted) {
             res.status(404).json({ error: 'Not found' });
@@ -48,7 +48,7 @@ export async function deleteConversation(req, res) {
 }
 export async function sendMessage(req, res) {
     try {
-        const { conversationId } = req.params;
+        const conversationId = req.params.conversationId;
         const { content } = req.body;
         if (!content) {
             res.status(400).json({ error: 'Missing content' });

package/dist/esm/tlm-ai/aiService.js

@@ -1,9 +1,13 @@
 import OpenAI from 'openai';
 import { agent } from './agent.js';
 class AIService {
+    config;
+    conversations = new Map();
+    openai;
+    model;
+    extraPrompts;
     constructor(config) {
         this.config = config;
-        this.conversations = new Map();
         this.openai = new OpenAI({
             apiKey: this.config.apiKey,
         });
@@ -47,7 +51,7 @@ class AIService {
         // Use the agent function with tools
         // Internaly pushes new messages to conversation.messages
         const currentMessagesCount = conversation.messages.length;
-        await agent(this.openai, conversation.messages, model || this.model, this.extraPrompts);
+        await agent(this.openai, conversation.messages, model || this.model);
         // Return the last assistant message (the one just added by agent)
         const generatedMessages = conversation.messages.slice(currentMessagesCount);
         return generatedMessages;

package/dist/esm/tlm-ai/tools.js

@@ -3,14 +3,10 @@ import { inMemoryDbLogExporter, inMemoryDbMetricExporter, inMemoryDbSpanExporter
 const getTraces = async (searchInput) => {
     logger.debug("getTraces called with searchInput:", searchInput);
     try {
-        const search = searchInput || {};
-        const traces = await new Promise((resolve, reject) => {
-            inMemoryDbSpanExporter.find(search, (err, docs) => {
-                if (err)
-                    reject(err);
-                else
-                    resolve(docs);
-            });
+        const search = searchInput ? JSON.parse(searchInput) : {};
+        const traces = await inMemoryDbSpanExporter.find({
+            query: search,
+            limit: 1000
         });
         const simplifiedTraces = getSimplifiedTraces(traces);
         logger.debug(`Searching for traces with searchInput: ${JSON.stringify(search)}`);
@@ -38,7 +34,7 @@ const getLogs = async (startDate, endDate) => {
         const logs = (await inMemoryDbLogExporter.find({
             query: nedbQuery,
             messageSearch: null,
-            limit: 50 // or any appropriate limit
+            limit: 1000 // or any appropriate limit
         })) || [];
         logger.debug(`Found ${logs.length} logs in the specified range.`);
         const simplifiedLogs = getSimplifiedLogs(logs);

package/dist/esm/tlm-auth/authController.js

@@ -1,5 +1,6 @@
 import jwt from 'jsonwebtoken';
 import logger from '../utils/logger.js';
+import { bootEnvVariables } from '../config/bootConfig.js';
 function generateAccessToken(secret, expiresIn) {
     return jwt.sign({ type: "access" }, secret, { expiresIn: Math.floor(expiresIn / 1000) });
 }
@@ -28,7 +29,7 @@ export const getLogin = (oasTlmConfig) => (req, res) => {
                 httpOnly: true,
                 secure: process.env.NODE_ENV === "production",
                 sameSite: "lax",
-                path: oasTlmConfig.general.baseUrl + "/auth/refresh"
+                path: bootEnvVariables.OASTLM_BOOT_BASE_URL + "/auth/refresh"
             });
             res.status(200).json({ valid: true, message: "Login successful" });
             return;
@@ -46,7 +47,7 @@ export const getLogout = (oasTlmConfig) => (req, res) => {
         return;
     }
     res.clearCookie('oas-tlm-access-token', { path: '/' });
-    res.clearCookie('oas-tlm-refresh-token', { path: oasTlmConfig.general.baseUrl + '/auth/refresh' });
+    res.clearCookie('oas-tlm-refresh-token', { path: bootEnvVariables.OASTLM_BOOT_BASE_URL + '/auth/refresh' });
     res.status(200).json({ valid: true, message: "Logged out" });
 };
 export const getRefresh = (oasTlmConfig) => (req, res) => {

package/dist/esm/tlm-log/logController.js

@@ -1,6 +1,27 @@
 import { inMemoryDbLogExporter } from '../telemetry/telemetryRegistry.js';
 import logger from '../utils/logger.js';
 import { convertRegexRecursively } from '../utils/regexUtils.js';
+/**
+ * Parse NDJSON import data
+ * Each line must be a valid JSON object
+ * @param body - Request body (raw string)
+ * @returns Array of log objects
+ */
+function parseImportData(body) {
+    if (typeof body !== 'string') {
+        throw new Error('Import must be NDJSON format (plain text with one JSON object per line)');
+    }
+    const lines = body.split('\n').filter((line) => line.trim());
+    return lines.map((line, index) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch (e) {
+            logger.error(`Failed to parse NDJSON line ${index + 1}: ${line}`);
+            throw new Error(`Invalid JSON on line ${index + 1}`);
+        }
+    });
+}
 export const findLogs = async (req, res) => {
     const body = req.body || {};
     const messageSearch = body.textSearch || null;
@@ -27,7 +48,8 @@ export const findLogs = async (req, res) => {
         };
         const docs = await inMemoryDbLogExporter.find(findConfig);
         res.send({
-            items: docs,
+            logsCount: docs.length,
+            logs: docs,
         });
     }
     catch (err) {
@@ -40,14 +62,13 @@ export const resetLogs = (req, res) => {
     res.send('Logs reset');
 };
 export const insertLogsToDb = async (req, res) => {
-    const jsonContent = req.body.logs;
+    const jsonContent = (req.body || {}).logs;
     const resetData = req.query.reset === 'true';
     if (!Array.isArray(jsonContent)) {
         res.status(400).send({ error: 'Invalid data format.' });
         return;
     }
     const cleanedLogs = jsonContent.map((log) => {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const { _id, ...rest } = log; // Remove _id if it exists
         return rest; // Return the cleaned log object
     });
@@ -74,6 +95,41 @@ export const insertLogsToDb = async (req, res) => {
         res.status(500).send({ error: 'Failed to reset and insert data', details: err.message });
     }
 };
+export const importLogs = async (req, res) => {
+    const resetData = req.query.reset === 'true';
+    try {
+        // Parse NDJSON format
+        const logs = parseImportData(req.body);
+        if (logs.length === 0) {
+            res.status(400).send({ error: 'No valid logs found in import data' });
+            return;
+        }
+        const cleanedLogs = logs.map((log) => {
+            const { _id, ...rest } = log; // Remove _id if it exists
+            return rest;
+        });
+        let message = '';
+        if (resetData) {
+            inMemoryDbLogExporter.reset();
+            message += 'Logs Database reset. ';
+        }
+        await new Promise((resolve, reject) => {
+            inMemoryDbLogExporter.insert(cleanedLogs, (err, newDocs) => {
+                if (err) {
+                    logger.error('Error importing logs:', err);
+                    return reject(err);
+                }
+                resolve(newDocs);
+            });
+        });
+        message += `Imported ${cleanedLogs.length} logs.`;
+        res.send({ message, ImportedLogsCount: cleanedLogs.length });
+    }
+    catch (err) {
+        logger.error('Import failed:', err);
+        res.status(400).send({ error: 'Failed to import logs', details: err.message });
+    }
+};
 export const startLogs = (req, res) => {
     inMemoryDbLogExporter.enable();
     res.send('Log collection started');
@@ -87,7 +143,7 @@ export const statusLogs = (req, res) => {
     res.send({ active: isRunning });
 };
 export const setLogRetentionTime = (req, res) => {
-    const retentionTimeInSeconds = req.body.retentionTimeInSeconds;
+    const retentionTimeInSeconds = (req.body || {}).retentionTimeInSeconds;
     if (typeof retentionTimeInSeconds !== 'number' || retentionTimeInSeconds <= 0) {
         res.status(400).send({ error: 'Invalid retention time. Must be a positive number.' });
         return;
@@ -99,3 +155,27 @@ export const getLogRetentionTime = (req, res) => {
     const retentionTimeInSeconds = inMemoryDbLogExporter.retentionTimeInSeconds || 0;
     res.send({ retentionTimeInSeconds: retentionTimeInSeconds });
 };
+export const exportLogs = async (req, res) => {
+    try {
+        // Get ALL logs without practical limit
+        const findConfig = {
+            query: {},
+            messageSearch: null,
+            sortOrder: { timestamp: -1 }
+        };
+        const docs = await inMemoryDbLogExporter.find(findConfig);
+        const timestamp = new Date().toISOString().slice(0, 19).replace(/[-T:]/g, '');
+        res.setHeader('Content-Type', 'application/x-ndjson');
+        res.setHeader('Content-Disposition', `attachment; filename="logs-${timestamp}.ndjson"`);
+        res.setHeader('Transfer-Encoding', 'chunked');
+        // Stream as NDJSON (one JSON object per line)
+        docs.forEach(doc => {
+            res.write(JSON.stringify(doc) + '\n');
+        });
+        res.end();
+    }
+    catch (err) {
+        logger.error('Failed to export logs:', err);
+        res.status(500).send({ error: 'Failed to export logs', details: err.message });
+    }
+};

package/dist/esm/tlm-log/logRoutes.js

@@ -1,5 +1,5 @@
-import { Router } from 'express';
-import { startLogs, stopLogs, statusLogs, resetLogs, findLogs, insertLogsToDb, setLogRetentionTime, getLogRetentionTime } from './logController.js';
+import { Router, text } from 'express';
+import { startLogs, stopLogs, statusLogs, resetLogs, findLogs, insertLogsToDb, importLogs, setLogRetentionTime, getLogRetentionTime, exportLogs } from './logController.js';
 export const getLogRoutes = () => {
     const router = Router();
     // Logs Control
@@ -9,6 +9,9 @@ export const getLogRoutes = () => {
     router.post('/reset', resetLogs);
     router.post('/retention-time', setLogRetentionTime);
     router.get('/retention-time', getLogRetentionTime);
+    router.get('/export', exportLogs);
+    // Use text middleware for import to handle NDJSON format
+    router.post('/import', text({ type: 'application/x-ndjson', limit: '500mb' }), importLogs);
     router.get('/', findLogs);
     router.post('/', insertLogsToDb);
     router.post('/find', findLogs);