@portel/photon 1.22.0 → 1.23.0

package/dist/loader.js

@@ -4,7 +4,7 @@
  * Loads a single .photon.ts file and extracts tools
  */
 import * as fs from 'fs/promises';
-import { realpathSync, existsSync, mkdirSync, symlinkSync, readFileSync } from 'fs';
+import { realpathSync, existsSync, mkdirSync, symlinkSync, readFileSync, statSync, } from 'fs';
 import { readText, readJSON, writeText, writeJSON } from './shared/io.js';
 import { createRequire } from 'module';
 import * as path from 'path';
@@ -13,7 +13,7 @@ import * as crypto from 'crypto';
 import { startToolSpan } from './telemetry/otel.js';
 import { recordToolCall, recordCircuitStateChange, recordRateLimitRejection, recordBulkheadRejection, } from './telemetry/metrics.js';
 import { runWithRequestContext } from './telemetry/context.js';
-import { spawn } from 'child_process';
+import { spawn, execSync } from 'child_process';
 import { SchemaExtractor, DependencyManager, 
 // Generator utilities (ask/emit pattern from 1.2.0)
 isAsyncGenerator, executeGenerator, 
@@ -38,9 +38,9 @@ executionContext,
 // Lock helper
 withLock as withLockHelper, 
 // Middleware system
-builtinRegistry, MiddlewareRegistry, buildMiddlewareChain, detectNamespace, getCacheDir, } from '@portel/photon-core';
+builtinRegistry, MiddlewareRegistry, buildMiddlewareChain, getCacheDir, } from '@portel/photon-core';
 import { getDefaultContext } from './context.js';
-import * as os from 'os';
+import { getInstanceStatePath } from './context-store.js';
 import { MarketplaceManager } from './marketplace-manager.js';
 import { PHOTON_VERSION, getResolvedPhotonCoreVersion } from './version.js';
 // Timeout for external fetch requests (marketplace, GitHub)
@@ -686,10 +686,75 @@ export class PhotonLoader {
             // Set photon name and namespace for event source identification and data paths
             instance._photonName = name;
             instance._photonNamespace = this.resolveNamespace(absolutePath);
+            // Pin baseDir so this.memory (and any other .data/-rooted state)
+            // resolves to the same root no matter which process reads back.
+            // Without this, MemoryProvider falls through to getDefaultContext()
+            // and drifts between cwd-derived locations across daemon restarts.
+            instance._baseDir = this.baseDir;
             // Inject instance name for named instances (runtime concept, not code)
             instance.instanceName = options?.instanceName ?? '';
             // Inject file path for storage()/assets() resolution
             instance._photonFilePath = absolutePath;
+            // Inject this.shell — execSync wrapper with cwd defaulted to the
+            // photon's own folder. Shelling out from a photon method should run
+            // from where the photon lives, not the daemon's cwd, so that
+            // `photon cli peer method` inherits the same marketplace context the
+            // user would see when running the command from that folder directly.
+            // User-defined shell() on the class wins (no clobber).
+            if (typeof instance.shell !== 'function') {
+                const photonDir = path.dirname(absolutePath);
+                const loaderLogger = this.logger;
+                instance.shell = (cmd, timeoutMs = 30000) => {
+                    try {
+                        return execSync(cmd, {
+                            cwd: photonDir,
+                            timeout: timeoutMs,
+                            encoding: 'utf-8',
+                            stdio: ['ignore', 'pipe', 'pipe'],
+                        });
+                    }
+                    catch (err) {
+                        // execSync throws on non-zero exit or timeout. Preserve any
+                        // partial stdout for callers that parse incremental output;
+                        // surface stderr through the loader logger so the failure is
+                        // not silent.
+                        const e = err;
+                        const stderr = typeof e.stderr === 'string' ? e.stderr : e.stderr?.toString('utf-8');
+                        if (stderr) {
+                            loaderLogger.warn(`this.shell error in ${name}`, {
+                                cmd,
+                                stderr: stderr.slice(0, 500),
+                            });
+                        }
+                        const stdout = typeof e.stdout === 'string' ? e.stdout : e.stdout?.toString('utf-8');
+                        return stdout ?? '';
+                    }
+                };
+            }
+            // Stat-gate baseline for CLI-direct dispatch. Daemon-routed calls
+            // have their own equivalent at src/daemon/server.ts; this ensures
+            // `photon cli foo bar` run immediately after a `sed -i` sees the
+            // new code on the first call.
+            try {
+                const s = statSync(absolutePath);
+                instance._photonSourceStat = { mtimeMs: s.mtimeMs, size: s.size, ino: s.ino };
+            }
+            catch {
+                // No stat — the gate stays a no-op for this instance.
+            }
+            instance._photonReloader = async () => {
+                try {
+                    await this.reloadFile(absolutePath);
+                    const s = statSync(absolutePath);
+                    instance._photonSourceStat = { mtimeMs: s.mtimeMs, size: s.size, ino: s.ino };
+                }
+                catch (err) {
+                    this.logger.debug('Stat-gate reload failed', {
+                        path: absolutePath,
+                        error: getErrorMessage(err),
+                    });
+                }
+            };
             // Inject dynamic photon resolver for this.photon.use()
             instance._photonResolver = (photonName, instanceName) => {
                 return this.resolveAndLoadPhoton(photonName, absolutePath, instanceName);
@@ -754,9 +819,12 @@ export class PhotonLoader {
                     this.log(`🔍 Detected capabilities for ${name}: ${[...caps].join(', ')}`);
                 }
                 this.injectPathHelpers(instance, tsContent);
-                if (caps.has('emit')) {
-                    // Inject emit() that reads from executionContext (set during executeTool)
-                    // Falls back to last known outputHandler for timer-based emits after method returns
+                // Always-inject emit + its helpers (render/toast/log/status/progress/
+                // thinking). The injection is a pure closure — zero cost if unused.
+                // Dropping the capability-detection gate means typed-access patterns
+                // like `(this as any).emit(...)` work even if the photon-core regex
+                // misses them. User-defined emit on the class wins (no clobber).
+                if (!instance.emit) {
                     instance.emit = (data) => {
                         const store = executionContext.getStore();
                         const emitData = instance._photonName && typeof data === 'object' && data !== null
@@ -811,8 +879,12 @@ export class PhotonLoader {
                         configurable: true,
                     });
                 }
-                if (caps.has('call') && !instance.call) {
-                    // Inject call() for cross-photon communication
+                // Inject call() for cross-photon communication. Always-inject (no
+                // capability-detection gate) because the underlying _callHandler
+                // is ALWAYS wired above. Gating on a regex that matches literal
+                // `this.call(` misses typed-access patterns like
+                // `(this as any).call(...)` and fails silently at runtime.
+                if (!instance.call) {
                     instance.call = async (target, params = {}, options) => {
                         const dotIndex = target.indexOf('.');
                         if (dotIndex === -1) {
@@ -826,7 +898,8 @@ export class PhotonLoader {
                         return instance._callHandler(photonName, methodName, params, options?.instance);
                     };
                 }
-                if (caps.has('mcp') && !instance.mcp) {
+                // Always-inject mcp(). Pure closure. User-defined mcp wins.
+                if (!instance.mcp) {
                     // Inject mcp() accessor for external MCP server access
                     const mcpClients = new Map();
                     instance.mcp = (mcpName) => {
@@ -842,9 +915,10 @@ export class PhotonLoader {
                         return client;
                     };
                 }
-                if (caps.has('caller') &&
-                    !Object.getOwnPropertyDescriptor(Object.getPrototypeOf(instance), 'caller')) {
-                    // Inject caller getter that reads from executionContext
+                // Always-inject caller getter. The prototype check preserves any
+                // user-defined getter on the class (e.g. Photon base class has its
+                // own) so we don't clobber.
+                if (!Object.getOwnPropertyDescriptor(Object.getPrototypeOf(instance), 'caller')) {
                     Object.defineProperty(instance, 'caller', {
                         get() {
                             const store = executionContext.getStore();
@@ -853,8 +927,8 @@ export class PhotonLoader {
                         configurable: true,
                     });
                 }
-                if (caps.has('lock') && !instance.withLock) {
-                    // Inject withLock() helper
+                // Always-inject withLock. Pure closure. User-defined wins.
+                if (!instance.withLock) {
                     instance.withLock = async (lockName, fn, timeout) => {
                         return withLockHelper(lockName, fn, timeout);
                     };
@@ -884,8 +958,7 @@ export class PhotonLoader {
                 if (caps.has('instanceMeta')) {
                     // Inject instance metadata (file-stat-based timestamps)
                     const instanceName = options?.instanceName || 'default';
-                    const stateDir = path.join(this.baseDir, 'state', name);
-                    const stateFile = path.join(stateDir, `${instanceName}.json`);
+                    const stateFile = getInstanceStatePath(name, instanceName, this.baseDir);
                     try {
                         const stat = await fs.stat(stateFile);
                         instance.instanceMeta = {
@@ -903,37 +976,43 @@ export class PhotonLoader {
                         };
                     }
                 }
-                if (caps.has('allInstances')) {
-                    // Inject cross-instance iterator
+                // Always-inject allInstances. The async generator is lazy — directory
+                // reads only happen when the caller iterates. User-defined wins.
+                if (!instance.allInstances) {
+                    // Inject cross-instance iterator. Walks the Option B layout —
+                    // `.data/{photon}/state/{instance}/state.json` — one subdirectory
+                    // per instance. getInstanceStatePath keeps the path source truth
+                    // in one place.
                     const photonName = name;
                     const loaderBaseDir = this.baseDir;
                     instance.allInstances = async function* () {
-                        const stateDir = path.join(loaderBaseDir, 'state', photonName);
+                        // Derive the parent state dir from a known instance path.
+                        const stateDir = path.dirname(path.dirname(getInstanceStatePath(photonName, 'default', loaderBaseDir)));
+                        let subdirs;
                         try {
-                            const files = await fs.readdir(stateDir);
-                            for (const file of files.filter((f) => f.endsWith('.json'))) {
-                                const instName = file.replace('.json', '');
-                                const statePath = path.join(stateDir, file);
-                                try {
-                                    const snapshot = await readJSON(statePath);
-                                    const stat = await fs.stat(statePath);
-                                    yield {
-                                        name: instName,
-                                        meta: {
-                                            name: instName,
-                                            createdAt: stat.birthtime.toISOString(),
-                                            updatedAt: stat.mtime.toISOString(),
-                                        },
-                                        state: snapshot,
-                                    };
-                                }
-                                catch {
-                                    // Skip corrupted state files
-                                }
-                            }
+                            subdirs = await fs.readdir(stateDir);
                         }
                         catch {
-                            // State dir doesn't exist yet — no instances
+                            return; // state dir doesn't exist yet — no instances
+                        }
+                        for (const instName of subdirs) {
+                            const statePath = getInstanceStatePath(photonName, instName, loaderBaseDir);
+                            try {
+                                const snapshot = await readJSON(statePath);
+                                const stat = await fs.stat(statePath);
+                                yield {
+                                    name: instName,
+                                    meta: {
+                                        name: instName,
+                                        createdAt: stat.birthtime.toISOString(),
+                                        updatedAt: stat.mtime.toISOString(),
+                                    },
+                                    state: snapshot,
+                                };
+                            }
+                            catch {
+                                // Not a state subdir, or a corrupted / legacy file — skip.
+                            }
                         }
                     };
                 }
@@ -1042,26 +1121,11 @@ export class PhotonLoader {
             if (tsContent) {
                 await this.checkCLIDependencies(tsContent, name);
             }
-            // Call lifecycle hook if present with error handling
-            // skipInitialize is used during hot-reload: state is transferred from the old
-            // instance after loadFile returns, then onInitialize is called manually.
-            const onInitialize = instance.onInitialize;
-            if (typeof onInitialize === 'function' && !options?.skipInitialize) {
-                this.onProgress?.('running onInitialize');
-                try {
-                    await onInitialize.call(instance);
-                }
-                catch (error) {
-                    const initError = new Error(`Initialization failed for ${name}: ${getErrorMessage(error)}\n` +
-                        `\nThe onInitialize() lifecycle hook threw an error.\n` +
-                        `Check your constructor configuration and initialization logic.`);
-                    initError.name = 'PhotonInitializationError';
-                    if (error instanceof Error && error.stack) {
-                        initError.stack = error.stack;
-                    }
-                    throw initError;
-                }
-            }
+            // Call lifecycle hook if present with error handling.
+            // skipInitialize is used during hot-reload: state is transferred from
+            // the old instance after loadFile returns, then onInitialize is called
+            // manually.
+            await this.invokeInitialize(instance, name, options);
             // Auto-wrap public methods in @stateful classes to emit events
             // Must happen AFTER capability injection so that emit() is available
             if (tsContent) {
@@ -1180,13 +1244,46 @@ export class PhotonLoader {
         }
         instance._photonName = name;
         instance._photonNamespace = this.resolveNamespace(absolutePath);
+        // Same pin as the file-load path above — see that comment for why
+        // this.memory drifts without it.
+        instance._baseDir = this.baseDir;
         instance.instanceName = options?.instanceName ?? '';
         // Inject file path for storage()/assets() resolution.
-        // For preloaded modules (compiled binaries), remap to ~/.photon/ so storage()
-        // resolves under the deployment directory instead of the build machine's source tree.
+        // For preloaded modules (compiled binaries), remap to the resolved
+        // PHOTON_DIR (this.baseDir) so storage() resolves under the deployment
+        // directory instead of the build machine's source tree. Respects Option B:
+        // storage follows wherever the photon is deployed, not always ~/.photon.
         // Assets still use absolutePath (resolved from embedded source paths).
-        const storagePath = path.join(os.homedir(), '.photon', path.basename(absolutePath));
+        const storagePath = path.join(this.baseDir, path.basename(absolutePath));
         instance._photonFilePath = storagePath;
+        // Inject this.shell — see the primary load path for rationale.
+        // User-defined shell() wins; we don't clobber.
+        if (typeof instance.shell !== 'function') {
+            const photonDir = path.dirname(absolutePath);
+            const loaderLogger = this.logger;
+            instance.shell = (cmd, timeoutMs = 30000) => {
+                try {
+                    return execSync(cmd, {
+                        cwd: photonDir,
+                        timeout: timeoutMs,
+                        encoding: 'utf-8',
+                        stdio: ['ignore', 'pipe', 'pipe'],
+                    });
+                }
+                catch (err) {
+                    const e = err;
+                    const stderr = typeof e.stderr === 'string' ? e.stderr : e.stderr?.toString('utf-8');
+                    if (stderr) {
+                        loaderLogger.warn(`this.shell error in ${name}`, {
+                            cmd,
+                            stderr: stderr.slice(0, 500),
+                        });
+                    }
+                    const stdout = typeof e.stdout === 'string' ? e.stdout : e.stdout?.toString('utf-8');
+                    return stdout ?? '';
+                }
+            };
+        }
         // Inject dynamic photon resolver for this.photon.use()
         instance._photonResolver = (photonName, instanceName) => {
             return this.resolveAndLoadPhoton(photonName, absolutePath, instanceName);
@@ -1225,7 +1322,8 @@ export class PhotonLoader {
             if (detectEmitHelperUsage(tsContent))
                 caps.add('emit');
             this.injectPathHelpers(instance, tsContent);
-            if (caps.has('emit')) {
+            // Always-inject emit (see primary path). User-defined wins.
+            if (!instance.emit) {
                 instance.emit = (data) => {
                     const store = executionContext.getStore();
                     const emitData = instance._photonName && typeof data === 'object' && data !== null
@@ -1273,7 +1371,8 @@ export class PhotonLoader {
                     configurable: true,
                 });
             }
-            if (caps.has('call') && !instance.call) {
+            // Always-inject call() (see the primary load path for rationale).
+            if (!instance.call) {
                 instance.call = async (target, params = {}, opts) => {
                     const dotIndex = target.indexOf('.');
                     if (dotIndex === -1) {
@@ -1348,18 +1447,7 @@ export class PhotonLoader {
             await this.checkCLIDependencies(tsContent, name);
         }
         // Call lifecycle hook
-        const onInitialize = instance.onInitialize;
-        if (typeof onInitialize === 'function' && !options?.skipInitialize) {
-            this.onProgress?.('running onInitialize');
-            try {
-                await onInitialize.call(instance);
-            }
-            catch (error) {
-                const initError = new Error(`Initialization failed for ${name}: ${getErrorMessage(error)}`);
-                initError.name = 'PhotonInitializationError';
-                throw initError;
-            }
-        }
+        await this.invokeInitialize(instance, name, options);
         // Extract tools and metadata from embedded source (no disk I/O)
         const { tools, templates, statics, settingsSchema, auth: extractedAuth, } = await this.extractTools(MCPClass, absolutePath, tsContent);
         // Settings injection
@@ -1419,7 +1507,7 @@ export class PhotonLoader {
     /**
      * Reload a Photon MCP file (for hot reload)
      */
-    async reloadFile(filePath) {
+    async reloadFile(filePath, options) {
         // Invalidate the cache for this file
         const absolutePath = path.resolve(filePath);
         if (absolutePath.endsWith('.ts')) {
@@ -1445,7 +1533,7 @@ export class PhotonLoader {
                 }
             }
         }
-        return this.loadFile(filePath);
+        return this.loadFile(filePath, options);
     }
     /**
      * Compile TypeScript file to JavaScript and cache it
@@ -1677,10 +1765,16 @@ export class PhotonLoader {
     // SETTINGS — property-driven configuration with auto-persistence
     // ════════════════════════════════════════════════════════════════════════════
     /**
-     * Get the settings persistence path for a photon instance
+     * Get the settings persistence path for a photon instance.
+     * Co-located with state under Option B: the settings JSON lives next to
+     * state.json inside the per-instance directory.
      */
     getSettingsPath(photonName, instanceName) {
-        return path.join(this.baseDir, 'state', photonName, `${instanceName}-settings.json`);
+        // getInstanceStatePath returns `.../state/{instance}/state.json`; swap
+        // the filename so settings land at `.../state/{instance}/settings.json`.
+        // Keeps the source of truth for the layout in context-store.
+        const statePath = getInstanceStatePath(photonName, instanceName, this.baseDir);
+        return path.join(path.dirname(statePath), 'settings.json');
     }
     /**
      * Load persisted settings from disk
@@ -3668,23 +3762,49 @@ Run: photon mcp ${mcpName} --config
         });
     }
     /**
-     * Resolve the namespace for a photon based on its file path.
-     * Extracts the directory name between baseDir and the photon file.
+     * Resolve the namespace for a photon based purely on its directory
+     * position relative to baseDir. See docs/internals/PHOTON-DIR-AND-NAMESPACE.md.
+     *
+     *   {baseDir}/foo.photon.ts            → '' (flat at root)
+     *   {baseDir}/alice/foo.photon.ts      → 'alice'
+     *   {baseDir}/org/team/foo.photon.ts   → 'org/team'
      *
-     * Examples:
-     *   ~/.photon/portel-dev/todo.photon.ts → 'portel-dev'
-     *   ~/.photon/acme/todo.photon.ts       → 'acme'
-     *   ~/.photon/todo.photon.ts            → detected from git or 'local'
+     * The runtime never consults git state. PHOTON_DIR is the outer boundary;
+     * the file's position within it is the only namespace signal.
      */
     resolveNamespace(absolutePath) {
         const rel = path.relative(this.baseDir, absolutePath);
         const parts = rel.split(path.sep);
-        // If file is in a subdirectory (namespace/photon.ts), use that as namespace
-        if (parts.length >= 2) {
-            return parts[0];
+        if (parts.length < 2)
+            return '';
+        return parts.slice(0, -1).join(path.sep);
+    }
+    /**
+     * Invoke the `onInitialize` lifecycle hook on a loaded photon instance,
+     * honoring the `skipInitialize` option and wrapping any thrown error in
+     * a PhotonInitializationError so the caller can surface hook failures
+     * distinctly from runtime errors.
+     */
+    async invokeInitialize(instance, name, options, ctx) {
+        const hook = instance?.onInitialize;
+        if (typeof hook !== 'function' || options?.skipInitialize)
+            return;
+        this.onProgress?.('running onInitialize');
+        try {
+            await hook.call(instance, ctx);
+        }
+        catch (error) {
+            const message = ctx
+                ? `Initialization failed for ${name}: ${getErrorMessage(error)}`
+                : `Initialization failed for ${name}: ${getErrorMessage(error)}\n` +
+                    `\nThe onInitialize() lifecycle hook threw an error.\n` +
+                    `Check your constructor configuration and initialization logic.`;
+            const initError = new Error(message);
+            initError.name = 'PhotonInitializationError';
+            if (error instanceof Error && error.stack)
+                initError.stack = error.stack;
+            throw initError;
         }
-        // Flat file at root — detect from git remote or default to 'local'
-        return detectNamespace(this.baseDir);
     }
     /**
      * Discover and extract assets from a Photon file