@portel/photon 1.21.0 → 1.22.0

package/dist/loader.js

@@ -11,6 +11,8 @@ import * as path from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
 import * as crypto from 'crypto';
 import { startToolSpan } from './telemetry/otel.js';
+import { recordToolCall, recordCircuitStateChange, recordRateLimitRejection, recordBulkheadRejection, } from './telemetry/metrics.js';
+import { runWithRequestContext } from './telemetry/context.js';
 import { spawn } from 'child_process';
 import { SchemaExtractor, DependencyManager, 
 // Generator utilities (ask/emit pattern from 1.2.0)
@@ -235,6 +237,19 @@ export class PhotonLoader {
     middlewareStates = new Map();
     /** Per-photon custom middleware definitions discovered from module exports */
     photonMiddleware = new Map();
+    /** Shadow registry of circuit breaker states, keyed by `${photon}:${instance}:${tool}` */
+    circuitHealthTracker = new Map();
+    /**
+     * Returns all tracked circuit breaker states across all photons.
+     * Each entry uses the key format `${photon}:${instance}:${tool}`.
+     */
+    getCircuitHealth() {
+        const result = {};
+        for (const [key, entry] of this.circuitHealthTracker) {
+            result[key] = { ...entry };
+        }
+        return result;
+    }
     /** Base directory for state/config/cache (defaults to ~/.photon) */
     baseDir;
     /**
@@ -698,9 +713,31 @@ export class PhotonLoader {
             if (typeof instance._callHandler === 'undefined' || instance._callHandler === undefined) {
                 const callBaseDir = this.baseDir;
                 instance._callHandler = async (photonName, method, params, targetInstance) => {
+                    // Propagate trace context to nested photon-to-photon calls so the
+                    // downstream span chains as a child of the current span. Uses
+                    // in-band `_meta.traceparent` which executeTool peeks at before
+                    // schema validation — works through worker threads transparently.
+                    const ctxMod = await import('./telemetry/context.js');
+                    const ctx = ctxMod.getRequestContext();
+                    let forwardedParams = params;
+                    if (ctx) {
+                        const traceparent = ctx.parentTraceparent ||
+                            (ctx.traceId
+                                ? `00-${ctx.traceId}-${crypto.randomBytes(8).toString('hex')}-01`
+                                : undefined);
+                        if (traceparent) {
+                            const existingMeta = params?._meta;
+                            forwardedParams = {
+                                ...(params || {}),
+                                _meta: existingMeta && typeof existingMeta === 'object'
+                                    ? { traceparent, ...existingMeta }
+                                    : { traceparent },
+                            };
+                        }
+                    }
                     // Dynamic import to avoid circular dependency
                     const { sendCommand } = await import('./daemon/client.js');
-                    return sendCommand(photonName, method, params, {
+                    return sendCommand(photonName, method, forwardedParams, {
                         workingDir: callBaseDir,
                         targetInstance,
                     });
@@ -1494,7 +1531,7 @@ export class PhotonLoader {
         if (!description)
             return '';
         if (description.includes('@emits') && process.env.PHOTON_DEBUG_EXTRACT) {
-            console.log(`[stripJSDocTags] Input: "${description}"`);
+            this.logger.debug(`[stripJSDocTags] Input: "${description}"`);
         }
         // Remove lines that start with @ (full line removal)
         let cleaned = description
@@ -1505,7 +1542,7 @@ export class PhotonLoader {
         // Also remove inline @ tags (e.g., "text @emits ... " at end of line)
         cleaned = cleaned.replace(/\s*@\w+.*$/gm, '').trim();
         if (description.includes('@emits') && process.env.PHOTON_DEBUG_EXTRACT) {
-            console.log(`[stripJSDocTags] Output: "${cleaned}"`);
+            this.logger.debug(`[stripJSDocTags] Output: "${cleaned}"`);
         }
         return cleaned;
     }
@@ -1588,7 +1625,7 @@ export class PhotonLoader {
                     if (process.env.PHOTON_DEBUG_EXTRACT) {
                         tools.forEach((t) => {
                             if (t.description?.includes('@')) {
-                                console.log(`[EXTRACTOR] Before clean: ${t.name}: "${t.description}"`);
+                                this.logger.debug(`[EXTRACTOR] Before clean: ${t.name}: "${t.description}"`);
                             }
                         });
                     }
@@ -1601,7 +1638,7 @@ export class PhotonLoader {
                     if (process.env.PHOTON_DEBUG_EXTRACT) {
                         tools.forEach((t) => {
                             if (t.name === 'clear') {
-                                console.log(`[EXTRACTOR] After clean: ${t.name}: "${t.description}"`);
+                                this.logger.debug(`[EXTRACTOR] After clean: ${t.name}: "${t.description}"`);
                             }
                         });
                     }
@@ -2593,14 +2630,17 @@ Run: photon mcp ${mcpName} --config
      * Tags become composable wrappers applied in the correct order via phase-sorted middleware.
      * All middleware (built-in and custom) follows the same code path — no if-chain.
      */
-    applyMiddleware(execute, toolMeta, photonName, toolName, parameters, instanceName) {
+    applyMiddleware(execute, toolMeta, photonName, toolName, parameters, instanceName, outputHandler) {
         // Build middleware context
+        const store = executionContext.getStore();
         const ctx = {
             photon: photonName,
             tool: toolName,
             instance: instanceName || 'default',
             params: parameters,
-        };
+            caller: store?.caller,
+            outputHandler,
+        }; // MiddlewareContext type is in photon-core (read-only); caller/outputHandler added via runtime extension
         // Get declarations from the new middleware[] field
         const declarations = toolMeta.middleware || [];
         if (declarations.length === 0) {
@@ -2652,6 +2692,94 @@ Run: photon mcp ${mcpName} --config
                 return withLockHelper(lockName, next);
             };
         });
+        // Handler override for circuitBreaker — mirror state into circuitHealthTracker for introspection
+        handlerOverrides.set('circuitBreaker', (config, _state) => {
+            const tracker = this.circuitHealthTracker;
+            return async (ctx, next) => {
+                const key = `${ctx.photon}:${ctx.instance}:${ctx.tool}`;
+                let circuit = tracker.get(key);
+                if (!circuit) {
+                    circuit = { failures: 0, state: 'closed', openedAt: 0 };
+                    tracker.set(key, circuit);
+                }
+                const openError = () => {
+                    const error = new Error(`Circuit open: ${ctx.photon}.${ctx.tool} has failed ${config.threshold} consecutive times. Resets in ${Math.ceil((config.resetAfterMs - (Date.now() - circuit.openedAt)) / 1000)}s`);
+                    error.name = 'PhotonCircuitOpenError';
+                    return error;
+                };
+                const emitTransition = (from, to) => {
+                    if (from === to)
+                        return;
+                    recordCircuitStateChange({
+                        photon: ctx.photon,
+                        tool: ctx.tool,
+                        instance: ctx.instance,
+                        from,
+                        to,
+                    });
+                    // Broadcast state-change as a domain event so AG-UI clients
+                    // (via createAGUIOutputHandler) receive a STATE_DELTA in real
+                    // time instead of polling /api/health/circuits.
+                    const handler = ctx.outputHandler;
+                    if (handler) {
+                        try {
+                            handler({
+                                channel: `${ctx.photon}:circuits`,
+                                event: 'state-change',
+                                data: {
+                                    key: `${ctx.photon}:${ctx.instance}:${ctx.tool}`,
+                                    from,
+                                    to,
+                                    timestamp: Date.now(),
+                                },
+                            });
+                        }
+                        catch {
+                            /* best-effort — don't fail the middleware chain */
+                        }
+                    }
+                };
+                if (circuit.state === 'open') {
+                    if (Date.now() - circuit.openedAt >= config.resetAfterMs) {
+                        // Transition to half-open and allow exactly one probe through.
+                        emitTransition('open', 'half-open');
+                        circuit.state = 'half-open';
+                        circuit.probeInFlight = true;
+                    }
+                    else {
+                        throw openError();
+                    }
+                }
+                else if (circuit.state === 'half-open') {
+                    // Probe already in flight — reject concurrent requests to avoid stampede.
+                    if (circuit.probeInFlight) {
+                        throw openError();
+                    }
+                    circuit.probeInFlight = true;
+                }
+                const stateBefore = circuit.state;
+                try {
+                    const result = await next();
+                    circuit.failures = 0;
+                    if (stateBefore !== 'closed')
+                        emitTransition(stateBefore, 'closed');
+                    circuit.state = 'closed';
+                    circuit.probeInFlight = false;
+                    return result;
+                }
+                catch (error) {
+                    circuit.failures++;
+                    circuit.probeInFlight = false;
+                    // A failed probe in half-open immediately reopens the circuit.
+                    if (stateBefore === 'half-open' || circuit.failures >= config.threshold) {
+                        emitTransition(circuit.state, 'open');
+                        circuit.state = 'open';
+                        circuit.openedAt = Date.now();
+                    }
+                    throw error;
+                }
+            };
+        });
         return buildMiddlewareChain(execute, declarations, combinedRegistry, this.middlewareStates, ctx, handlerOverrides);
     }
     /**
@@ -2668,11 +2796,54 @@ Run: photon mcp ${mcpName} --config
      * @returns Tool result, or wrapped result with runId for stateful workflows
      */
     async executeTool(mcp, toolName, parameters, options) {
+        // Resolve parentTraceparent from options or in-band `_meta.traceparent` so
+        // the ALS context reflects the true parent for any nested `this.call()`.
+        let resolvedParentTraceparent = options?.parentTraceparent;
+        if (!resolvedParentTraceparent &&
+            parameters &&
+            typeof parameters === 'object' &&
+            '_meta' in parameters) {
+            const metaPeek = parameters._meta;
+            if (metaPeek && typeof metaPeek.traceparent === 'string') {
+                resolvedParentTraceparent = metaPeek.traceparent;
+            }
+        }
+        return runWithRequestContext({
+            photon: mcp.name,
+            tool: toolName,
+            traceId: options?.traceId,
+            parentTraceparent: resolvedParentTraceparent,
+            caller: options?.caller,
+            startedAt: Date.now(),
+        }, () => this._executeToolInner(mcp, toolName, parameters, {
+            ...options,
+            parentTraceparent: resolvedParentTraceparent,
+        }));
+    }
+    async _executeToolInner(mcp, toolName, parameters, options) {
         // Start audit trail recording
         const audit = getAuditTrail();
         const { finish: auditFinish } = audit.start(mcp.name, toolName, parameters || {});
+        // Start wall-clock timer for tool-call duration histogram.
+        const toolStartedAt = Date.now();
+        let metricsStatus = 'ok';
+        let metricsErrorType;
+        // Peek at _meta.traceparent before span creation so distributed traces chain.
+        // Explicit option wins over in-band _meta (server may have extracted from headers).
+        let parentTraceparent = options?.parentTraceparent;
+        if (!parentTraceparent &&
+            parameters &&
+            typeof parameters === 'object' &&
+            '_meta' in parameters) {
+            const metaPeek = parameters._meta;
+            if (metaPeek && typeof metaPeek === 'object' && typeof metaPeek.traceparent === 'string') {
+                parentTraceparent = metaPeek.traceparent;
+            }
+        }
         // Start OTel span for tool execution (no-op if SDK not installed)
-        const span = startToolSpan(mcp.name, toolName, parameters);
+        const toolMetaForSpan = mcp?.meta?.tools?.[toolName];
+        const isStateful = Boolean(toolMetaForSpan?.stateful ?? mcp?.meta?.stateful);
+        const span = startToolSpan(mcp.name, toolName, parameters, options?.traceId, isStateful, parentTraceparent);
         if (mcp.instance?.instanceName) {
             span.setAttribute('photon.instance', mcp.instance.instanceName);
         }
@@ -2787,7 +2958,7 @@ Run: photon mcp ${mcpName} --config
                 };
                 // Apply functional tag middleware if any tags present
                 if (hasFunctionalTags) {
-                    executeBase = this.applyMiddleware(executeBase, toolMeta, mcp.name, toolName, parameters, mcp.instance.instanceName);
+                    executeBase = this.applyMiddleware(executeBase, toolMeta, mcp.name, toolName, parameters, mcp.instance.instanceName, options?.outputHandler);
                 }
                 const result = await executeBase();
                 this.progressRenderer.done();
@@ -2815,30 +2986,36 @@ Run: photon mcp ${mcpName} --config
                                 configurable: true,
                             });
                         }
-                        // Construct event data with full context for transmission
-                        // CRITICAL: emit() expects { channel, event, data } structure for daemon pub/sub routing
-                        const eventPayload = {
+                        // Construct CloudEvents 1.0 envelope for @stateful event emissions.
+                        // The `data` field carries the photon-specific payload; `channel` and `event`
+                        // are transport metadata used by the daemon pub/sub router.
+                        const cloudEventData = {
                             method: toolName,
                             params: parameters,
                             result,
-                            timestamp: new Date().toISOString(),
+                            instance: mcp.instance.instanceName ?? null,
                         };
-                        // Add instance name if available
-                        if (mcp.instance.instanceName) {
-                            eventPayload.instance = mcp.instance.instanceName;
-                        }
                         // Add index/pagination info if result is from items array
                         if (result && typeof result === 'object' && Array.isArray(mcp.instance.items)) {
                             const index = mcp.instance.items.findIndex((item) => item === result);
                             if (index !== -1) {
-                                eventPayload.index = index;
-                                eventPayload.totalCount = mcp.instance.items.length;
-                                eventPayload.affectedRange = {
+                                cloudEventData.index = index;
+                                cloudEventData.totalCount = mcp.instance.items.length;
+                                cloudEventData.affectedRange = {
                                     start: index,
                                     end: index + 1,
                                 };
                             }
                         }
+                        // CloudEvents 1.0 envelope (https://cloudevents.io)
+                        const eventPayload = {
+                            specversion: '1.0',
+                            id: crypto.randomUUID(),
+                            source: `photon/${photonName}`,
+                            type: `photon.${photonName}.${toolName}.executed`,
+                            time: new Date().toISOString(),
+                            data: cloudEventData,
+                        };
                         // Wrap in daemon pub/sub format: { channel, event, data }
                         const eventData = {
                             channel: `${photonName}:${toolName}`, // For daemon pub/sub routing
@@ -2850,18 +3027,24 @@ Run: photon mcp ${mcpName} --config
                         if (options?.outputHandler) {
                             try {
                                 if (process.env.PHOTON_DEBUG_EMIT === '1') {
-                                    console.error(`[EMIT-DEBUG] Sending event: method=${eventPayload.method}, channel=${eventData.channel}, hasMeta=${!!result?.__meta}`);
+                                    this.logger.debug('[emit] sending event', {
+                                        method: eventPayload.data.method,
+                                        channel: eventData.channel,
+                                        hasMeta: !!result?.__meta,
+                                    });
                                 }
                                 // Cast to DaemonEventEnvelope - outputHandler is flexible and routes any object with channel property
                                 void Promise.resolve(options.outputHandler(eventData)).catch((e) => {
                                     this.logger.debug('Output handler failed for event', { error: e?.message || e });
                                 });
                                 if (process.env.PHOTON_DEBUG_EMIT === '1') {
-                                    console.error(`[EMIT-DEBUG] Event transmitted to outputHandler`);
+                                    this.logger.debug('[emit] event transmitted to outputHandler');
                                 }
                             }
                             catch (e) {
-                                console.error(`[EMIT-ERROR] Failed to send event through outputHandler: ${e instanceof Error ? e.message : String(e)}`);
+                                this.logger.error('Failed to send event through outputHandler', {
+                                    error: e instanceof Error ? e.message : String(e),
+                                });
                             }
                         }
                         else if (mcp.instance && typeof mcp.instance.emit === 'function') {
@@ -2869,19 +3052,22 @@ Run: photon mcp ${mcpName} --config
                             // (this path won't route to daemon pub/sub, but at least calls emit)
                             try {
                                 if (process.env.PHOTON_DEBUG_EMIT === '1') {
-                                    console.error(`[EMIT-DEBUG] No outputHandler, falling back to instance.emit`);
+                                    this.logger.debug('[emit] no outputHandler, falling back to instance.emit');
                                 }
                                 mcp.instance.emit(eventData);
                             }
                             catch (e) {
-                                console.error(`[EMIT-ERROR] Failed to emit: ${e instanceof Error ? e.message : String(e)}`);
+                                this.logger.error('Failed to emit event via instance.emit', {
+                                    error: e instanceof Error ? e.message : String(e),
+                                });
                             }
                         }
                     }
                     catch (e) {
                         // Log emit errors but don't break tool execution
-                        console.error(`[EMIT-ERROR] Failed to emit @stateful event: ${e instanceof Error ? e.message : String(e)}`);
-                        this.logger.debug(`Failed to emit @stateful event: ${e instanceof Error ? e.message : String(e)}`);
+                        this.logger.error('Failed to emit @stateful event', {
+                            error: e instanceof Error ? e.message : String(e),
+                        });
                     }
                 }
                 // Apply _meta post-processing (format transformation, field selection)
@@ -2938,7 +3124,7 @@ Run: photon mcp ${mcpName} --config
                 });
             // Apply functional tag middleware if any tags present
             if (hasFunctionalTags) {
-                generatorFn = this.applyMiddleware(generatorFn, toolMeta, mcp.name, toolName, parameters, mcp.instance.instanceName);
+                generatorFn = this.applyMiddleware(generatorFn, toolMeta, mcp.name, toolName, parameters, mcp.instance.instanceName, options?.outputHandler);
             }
             // Use maybeStatefulExecute for all executions
             // It handles both regular async and generators, detecting checkpoint yields
@@ -2961,6 +3147,7 @@ Run: photon mcp ${mcpName} --config
                 if (execResult.runId) {
                     error.runId = execResult.runId;
                 }
+                span.recordException(error);
                 span.setStatus('ERROR', error.message);
                 auditFinish(null, error);
                 throw error;
@@ -2993,13 +3180,41 @@ Run: photon mcp ${mcpName} --config
         catch (error) {
             // Clear progress on error too
             this.progressRenderer.done();
+            span.recordException(error);
             span.setStatus('ERROR', error instanceof Error ? error.message : String(error));
             auditFinish(null, error);
+            metricsStatus = 'error';
+            metricsErrorType = error instanceof Error ? error.name || 'Error' : 'unknown';
+            // Emit a dedicated counter for rate-limit rejections so dashboards can
+            // alert on sustained throttling without scanning the generic error
+            // stream.
+            if (metricsErrorType === 'PhotonRateLimitError') {
+                recordRateLimitRejection({
+                    photon: mcp.name,
+                    tool: toolName,
+                    instance: mcp.instance?.instanceName,
+                });
+            }
+            else if (metricsErrorType === 'PhotonBulkheadFullError') {
+                recordBulkheadRejection({
+                    photon: mcp.name,
+                    tool: toolName,
+                    instance: mcp.instance?.instanceName,
+                });
+            }
             this.logger.error(`Tool execution failed: ${toolName} - ${getErrorMessage(error)}`);
             throw error;
         }
         finally {
             span.end();
+            recordToolCall({
+                photon: mcp.name,
+                tool: toolName,
+                durationMs: Date.now() - toolStartedAt,
+                status: metricsStatus,
+                errorType: metricsErrorType,
+                stateful: isStateful,
+            });
         }
     }
     normalizeNestedParamsTool(toolMeta, parameters) {
@@ -3320,6 +3535,10 @@ Run: photon mcp ${mcpName} --config
                         });
                     }
                 };
+                // Generators must pass through unwrapped - .then() would kill the iterator protocol
+                if (result && typeof result[Symbol.asyncIterator] === 'function') {
+                    return result;
+                }
                 // For async methods, attach __meta to the resolved value, not the Promise
                 if (result && typeof result.then === 'function') {
                     return result.then((resolved) => {