@samanhappy/mcphub 1.0.45 → 1.1.0

package/dist/services/mcpService.js

@@ -1,3 +1,5 @@
+import { preserveListFreshness, combineListFreshness, remainingListTtl, } from '../utils/listFreshness.js';
+import { expandServerConfig, getServerEnvironment } from './serverConfigEnvironment.js';
 import { getMcpRequestGroup } from '../utils/mcpRequestGroup.js';
 import { LegacyMcpClient } from '../clients/legacyMcpClient.js';
 import { LEGACY_PROTOCOL_VERSIONS } from '../utils/mcpProtocol.js';
@@ -29,7 +31,7 @@ import { removeServerToolEmbeddings, removeToolEmbeddings, saveToolsAsVectorEmbe
 import { OpenAPIClient } from '../clients/openapi.js';
 import { RequestContextService } from './requestContextService.js';
 import { getDataService } from './services.js';
-import { getServerDao, getGroupDao, getSystemConfigDao, getBuiltinPromptDao, getBuiltinResourceDao, } from '../dao/index.js';
+import { getBearerKeyDao, getServerDao, getGroupDao, getSystemConfigDao, getBuiltinPromptDao, getBuiltinResourceDao, } from '../dao/index.js';
 import { initializeAllOAuthClients } from './oauthService.js';
 import { createOAuthProvider } from './mcpOAuthProvider.js';
 import { initSmartRoutingService, getSmartRoutingTools, handleSearchToolsRequest, handleDescribeToolRequest, isSmartRoutingGroup, } from './smartRoutingService.js';
@@ -781,7 +783,7 @@ export const setServerInfosForTest = (infos) => {
     serverInfos = infos;
 };
 export const updateServerToolsCache = (serverInfo, tools, options) => {
-    serverInfo.tools = tools.map((tool) => normalizeToolForCache(serverInfo.name, tool));
+    serverInfo.tools = preserveListFreshness(tools, tools.map((tool) => normalizeToolForCache(serverInfo.name, tool)));
     syncToolsAsVectorEmbeddings(serverInfo.name, serverInfo.tools, {
         reportProgress: options?.reportEmbeddingProgress === true,
     }).catch(() => {
@@ -789,10 +791,10 @@ export const updateServerToolsCache = (serverInfo, tools, options) => {
     });
 };
 const updateServerPromptsCache = (serverInfo, prompts) => {
-    serverInfo.prompts = prompts.map((prompt) => normalizePromptForCache(serverInfo.name, prompt));
+    serverInfo.prompts = preserveListFreshness(prompts, prompts.map((prompt) => normalizePromptForCache(serverInfo.name, prompt)));
 };
 const updateServerResourcesCache = (serverInfo, resources) => {
-    serverInfo.resources = resources.map(normalizeResourceForCache);
+    serverInfo.resources = preserveListFreshness(resources, resources.map(normalizeResourceForCache));
 };
 const logListChangedRefreshError = (listType) => {
     logger.warn(`Failed to refresh ${listType} list after upstream notification`);
@@ -806,6 +808,7 @@ const createUpstreamMcpClient = (name, getServerInfo) => {
         jsonSchemaValidator: new ResilientJsonSchemaValidator(),
         listChanged: {
             tools: {
+                debounceMs: 0,
                 onChanged: (error, tools) => {
                     const serverInfo = getServerInfo();
                     if (error) {
@@ -820,6 +823,7 @@ const createUpstreamMcpClient = (name, getServerInfo) => {
                 },
             },
             prompts: {
+                debounceMs: 0,
                 onChanged: (error, prompts) => {
                     const serverInfo = getServerInfo();
                     if (error) {
@@ -834,6 +838,7 @@ const createUpstreamMcpClient = (name, getServerInfo) => {
                 },
             },
             resources: {
+                debounceMs: 0,
                 onChanged: (error, resources) => {
                     const serverInfo = getServerInfo();
                     if (error) {
@@ -1171,10 +1176,7 @@ const stripAuthorizationHeader = (headers) => {
 };
 export const createTransportFromConfig = async (name, conf) => {
     let transport;
-    const env = {
-        ...process.env,
-        ...(hasCredentialTemplate(conf) ? conf.env : replaceEnvVars(conf.env || {})),
-    };
+    const env = await getServerEnvironment(conf);
     // The encryption key belongs only to the hub, never to a child process.
     delete env.MCPHUB_CREDENTIAL_ENCRYPTION_KEY;
     // SSRF guard: block URL/streamable-http transports from reaching
@@ -1275,7 +1277,7 @@ export const createTransportFromConfig = async (name, conf) => {
             env['npm_config_registry'] = systemConfig.install.npmRegistry;
         }
         // Apply proxychains4 wrapper if proxy is configured (Linux/macOS only)
-        let resolvedArgs = replaceEnvVars(conf.args ?? []);
+        let resolvedArgs = replaceEnvVars(conf.args ?? [], env);
         // If this server is pending a reinstall, inject cache-busting flags (uvx only).
         // For npx, the cache directory was already cleared before reconnect.
         if (pendingReinstalls.has(name)) {
@@ -1344,7 +1346,7 @@ const callToolWithReconnect = async (serverInfo, toolParams, options, maxRetries
                         });
                     }
                     // Match initial connection expansion before URL validation and transport creation.
-                    const server = replaceEnvVars(rawConfig);
+                    const server = await expandServerConfig(rawConfig);
                     const newTransport = await createTransportFromConfig(serverInfo.name, server);
                     const newClient = createUpstreamMcpClient(serverInfo.name, () => serverInfo);
                     // Reconnect with new transport
@@ -1476,7 +1478,7 @@ export const initializeClientsFromSettings = async (isInit, serverName, options)
                 }
             }
             // Expand environment variables in all configuration values
-            const expandedConf = replaceEnvVars(conf);
+            const expandedConf = await expandServerConfig(conf);
             // Skip disabled servers
             if (expandedConf.enabled === false) {
                 logger.log(`Skipping disabled server: ${name}`);
@@ -2542,7 +2544,7 @@ const ensureServerReady = async (serverInfo) => {
         if (rawConfig.enabled === false) {
             throw new Error(`Cannot start disabled on-demand server: ${rawConfig.name}`);
         }
-        const expandedConf = replaceEnvVars(rawConfig);
+        const expandedConf = await expandServerConfig(rawConfig);
         // startOnDemand is a stdio-only optimisation; OpenAPI/HTTP servers have no
         // deferred process to spawn, so there is nothing to wake.
         if (expandedConf.type === 'openapi') {
@@ -3766,19 +3768,24 @@ const handleListResourceTemplatesRequestImpl = async (_, extra) => {
     const { filteredServerInfos, serverConfigsByName } = await getFilteredServerInfosForGroup(lookupGroup, {
         requireClient: true,
     });
+    const snapshots = [];
     const results = await Promise.allSettled(filteredServerInfos.map(async (serverInfo) => {
         if (!serverInfo.client?.listResourceTemplates) {
             return [];
         }
         const templates = await serverInfo.client.listResourceTemplates({}, serverInfo.options || {});
+        snapshots.push(templates.resourceTemplates);
         const filteredTemplates = await filterResourceTemplatesByGroup(lookupGroup, serverInfo.name, templates.resourceTemplates || [], serverConfigsByName.get(serverInfo.name));
         return appsRouteContext.enabled
             ? filteredTemplates
             : filteredTemplates.map((template) => stripMcpAppsMetadata(template));
     }));
-    return {
-        resourceTemplates: results.flatMap((result) => result.status === 'fulfilled' ? result.value : []),
-    };
+    const resourceTemplates = results.flatMap((result) => result.status === 'fulfilled' ? result.value : []);
+    if (results.every((result) => result.status === 'fulfilled') &&
+        snapshots.length === filteredServerInfos.length) {
+        combineListFreshness(snapshots, resourceTemplates);
+    }
+    return { resourceTemplates };
 };
 const handleReadResourceRequestImpl = async (request, extra) => {
     try {
@@ -3818,8 +3825,15 @@ const handleReadResourceRequestImpl = async (request, extra) => {
             }
         }
         let result;
+        // A cached SDK body carries its original TTL, not the remaining lifetime.
+        // Fetch modern reads afresh so forwarding the TTL cannot renew stale content.
+        const readOptions = RequestContextService.getInstance().getRequestContext()?.stateless
+            ? { cacheMode: 'bypass' }
+            : undefined;
         if (server?.client) {
-            result = await server.client.readResource({ uri });
+            result = readOptions
+                ? await server.client.readResource({ uri }, readOptions)
+                : await server.client.readResource({ uri });
             if (!result || !Array.isArray(result.contents)) {
                 throw new Error(`Failed to read resource: ${uri}`);
             }
@@ -3839,7 +3853,9 @@ const handleReadResourceRequestImpl = async (request, extra) => {
                     continue;
                 }
                 try {
-                    const candidateResult = await candidate.client.readResource({ uri });
+                    const candidateResult = readOptions
+                        ? await candidate.client.readResource({ uri }, readOptions)
+                        : await candidate.client.readResource({ uri });
                     if (candidateResult && Array.isArray(candidateResult.contents)) {
                         result = candidateResult;
                         break;
@@ -3947,12 +3963,18 @@ const createPrincipalRuntime = async (name, resolvedConfig) => {
             info.client = protectClient(client);
             await client.connect(transport, resolvedConfig.options || {});
             const capabilities = client.getServerCapabilities();
-            if (capabilities?.tools)
-                info.tools = (await client.listTools({}, resolvedConfig.options)).tools.map((tool) => normalizeToolForCache(name, tool));
-            if (capabilities?.prompts)
-                info.prompts = (await client.listPrompts({}, resolvedConfig.options)).prompts.map((prompt) => normalizePromptForCache(name, prompt));
-            if (capabilities?.resources)
-                info.resources = (await client.listResources({}, resolvedConfig.options)).resources.map(normalizeResourceForCache);
+            if (capabilities?.tools) {
+                const { tools } = await client.listTools({}, resolvedConfig.options);
+                info.tools = preserveListFreshness(tools, tools.map((tool) => normalizeToolForCache(name, tool)));
+            }
+            if (capabilities?.prompts) {
+                const { prompts } = await client.listPrompts({}, resolvedConfig.options);
+                info.prompts = preserveListFreshness(prompts, prompts.map((prompt) => normalizePromptForCache(name, prompt)));
+            }
+            if (capabilities?.resources) {
+                const { resources } = await client.listResources({}, resolvedConfig.options);
+                info.resources = preserveListFreshness(resources, resources.map(normalizeResourceForCache));
+            }
             info.version = client.getServerVersion()?.version;
             // Record the resolved npx/uvx package version backing this server.
             applyResolvedPackageVersion(info, resolvedConfig);
@@ -4104,13 +4126,100 @@ const withPrincipalServers = (handler, operation) => async (request, extra) => {
             release();
     }
 };
-export const handleListToolsRequest = withPrincipalServers(handleListToolsRequestImpl, 'list');
+// Re-read authoritative inputs across awaits; no projected response is cached.
+const listProjectionInputs = async (group, field) => {
+    const user = UserContextService.getInstance().getCurrentUser();
+    const keyId = RequestContextService.getInstance().getBearerKeyContext().keyId;
+    const liveUser = user?.username ? await getUserDao().findByUsername(user.username) : undefined;
+    const key = keyId ? await getBearerKeyDao().findById(keyId) : undefined;
+    if ((user?.username && (!liveUser || Boolean(user.isAdmin) !== Boolean(liveUser.isAdmin))) ||
+        (keyId && !key?.enabled))
+        return undefined;
+    const lookupGroup = getGroupLookupName(group);
+    const groupDao = getGroupDao();
+    let routeGroup = lookupGroup ? await groupDao.findByName(lookupGroup) : undefined;
+    if (!routeGroup &&
+        lookupGroup &&
+        /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(lookupGroup)) {
+        routeGroup = await groupDao.findById(lookupGroup);
+    }
+    return JSON.stringify([
+        await getServerDao().findAll(),
+        routeGroup,
+        await getSystemConfigDao().get(),
+        field === 'prompts' ? await getBuiltinPromptDao().findEnabled() : undefined,
+        field === 'resources' ? await getBuiltinResourceDao().findEnabled() : undefined,
+        liveUser,
+        key,
+        RequestContextService.getInstance().getRequestContext()?.clientCapabilities,
+        getVisibleServerInfos().map((info) => ({
+            name: info.name,
+            status: info.status,
+            enabled: info.enabled,
+            owner: info.owner,
+            visibility: info.visibility,
+            sharedWithUsers: info.sharedWithUsers,
+            config: info.config,
+        })),
+    ]);
+};
+// Lists combine cached discovery with live gateway configuration and authorization.
+// Upstream freshness alone cannot guarantee freshness of the resulting projection.
+const withCacheHints = (handler, passthroughTtl = false, listField) => async (request, extra) => {
+    if (!RequestContextService.getInstance().getRequestContext()?.stateless) {
+        return handler(request, extra);
+    }
+    const startedAt = performance.now();
+    const group = getMcpRequestGroup(extra);
+    const inputs = listField ? await listProjectionInputs(group, listField) : undefined;
+    const before = listField
+        ? await getFilteredServerInfosForGroup(getGroupLookupName(group))
+        : undefined;
+    const snapshots = before && listField && listField !== 'resourceTemplates'
+        ? before.filteredServerInfos.map((info) => info[listField])
+        : [];
+    const clients = before?.filteredServerInfos.map((info) => info.client);
+    const result = await handler(request, extra);
+    if (listField) {
+        const after = await getFilteredServerInfosForGroup(getGroupLookupName(group));
+        const stable = typeof inputs === 'string' &&
+            inputs === (await listProjectionInputs(group, listField)) &&
+            before?.filteredServerInfos.length === after.filteredServerInfos.length &&
+            before.filteredServerInfos.every((info, index) => info === after.filteredServerInfos[index] &&
+                info.client === clients?.[index] &&
+                info.status === 'connected' &&
+                (listField === 'resourceTemplates' || info[listField] === snapshots[index]));
+        const generated = (listField === 'tools' && isSmartRoutingGroup(group)) ||
+            (listField === 'prompts' && (await getBuiltinPromptDao().findEnabled()).length > 0) ||
+            (listField === 'resources' && (await getBuiltinResourceDao().findEnabled()).length > 0);
+        const unknownPermissions = RequestContextService.getInstance().getHostedAuthContext();
+        const dependencies = listField === 'resourceTemplates' ? [result.resourceTemplates] : snapshots;
+        return {
+            ...result,
+            ttlMs: stable && !generated && !unknownPermissions
+                ? remainingListTtl(dependencies, startedAt, performance.now())
+                : 0,
+            cacheScope: 'private',
+        };
+    }
+    const upstreamTtl = result.ttlMs;
+    const ttlMs = passthroughTtl &&
+        typeof upstreamTtl === 'number' &&
+        Number.isSafeInteger(upstreamTtl) &&
+        upstreamTtl >= 0
+        ? Math.max(0, upstreamTtl - Math.ceil(performance.now() - startedAt))
+        : 0;
+    // Even public upstream content is projected through caller-specific permissions
+    // and capabilities. Shared caches must not reuse the gateway result across users.
+    return { ...result, ttlMs, cacheScope: 'private' };
+};
+export const handleListToolsRequest = withPrincipalServers(withCacheHints(handleListToolsRequestImpl, false, 'tools'), 'list');
 export const handleCallToolRequest = withPrincipalServers(handleCallToolRequestImpl, 'tool');
 export const handleGetPromptRequest = withPrincipalServers(handleGetPromptRequestImpl, 'prompt');
-export const handleListPromptsRequest = withPrincipalServers(handleListPromptsRequestImpl, 'list');
-export const handleListResourcesRequest = withPrincipalServers(handleListResourcesRequestImpl, 'list');
-export const handleListResourceTemplatesRequest = withPrincipalServers(handleListResourceTemplatesRequestImpl, 'list');
-export const handleReadResourceRequest = withPrincipalServers(handleReadResourceRequestImpl, 'resource');
+export const handleListPromptsRequest = withPrincipalServers(withCacheHints(handleListPromptsRequestImpl, false, 'prompts'), 'list');
+export const handleListResourcesRequest = withPrincipalServers(withCacheHints(handleListResourcesRequestImpl, false, 'resources'), 'list');
+export const handleListResourceTemplatesRequest = withPrincipalServers(withCacheHints(handleListResourceTemplatesRequestImpl, false, 'resourceTemplates'), 'list');
+export const handleReadResourceRequest = withPrincipalServers(withCacheHints(handleReadResourceRequestImpl, true), 'resource');
 export const createMcpServer = (name, version, options) => {
     const normalizedOptions = typeof options === 'string' ? { group: options } : (options ?? {});
     // Determine server name based on routing type