coding-tool-x 3.8.7 → 3.8.9

package/src/platforms/drivers/dsh/native-logs.js

@@ -0,0 +1,116 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { resolvePaths } = require('./common');
+const { decodeArtifact } = require('./sessions');
+const {
+  normalizeCost,
+  normalizeUsage,
+  walkFiles,
+  readJsonLines,
+  createScannedFileCursor
+} = require('../native-log-utils');
+const { calculateUsageCost } = require('../../../server/services/usage-log-utils');
+
+const SESSION_FILE_PATTERN = /^session(?:\.v\d+)?\.jsonl(?:\.zstd)?$/;
+
+function firstObject(...candidates) {
+  return candidates.find(value => value && typeof value === 'object' && !Array.isArray(value)) || null;
+}
+
+function extractUsage(record = {}) {
+  const data = record.data || {};
+  const message = data.message || record.message || {};
+  const source = message.source || data.source || {};
+  return firstObject(
+    record.usage,
+    record.tokens,
+    data.usage,
+    data.tokens,
+    message.usage,
+    message.tokens,
+    source.usage,
+    source.tokens,
+    data.response?.usage,
+    data.result?.usage
+  );
+}
+
+function normalizeTimestamp(value) {
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    if (value > 1e11) return value;
+    if (value > 1e9) return value * 1000;
+  }
+  if (typeof value === 'string' && value.trim()) {
+    const parsed = Date.parse(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return Date.now();
+}
+
+function normalizeDshEvent(record = {}, filePath = '', index = 0) {
+  const usage = extractUsage(record);
+  if (!usage) return null;
+
+  const data = record.data || {};
+  const message = data.message || record.message || {};
+  const source = message.source || data.source || {};
+  const model = record.model || data.model || message.model || source.model || '';
+  const provider = record.provider || data.provider || message.provider || source.provider || '';
+  const tokens = normalizeUsage(usage);
+  const id = record.id
+    || data.id
+    || message.id
+    || `${path.basename(filePath)}:${record.seq ?? index}`;
+  const timestamp = normalizeTimestamp(record.timestamp || record.time || data.timestamp || message.timestamp);
+  const cost = normalizeCost(record.cost ?? data.cost ?? message.cost ?? usage.cost)
+    || calculateUsageCost('dsh', model, tokens);
+
+  return {
+    id: `dsh:${path.basename(filePath)}:${id}`,
+    source: 'dsh',
+    sessionId: path.basename(path.dirname(filePath)),
+    timestamp,
+    provider,
+    model,
+    tokens,
+    cost,
+    channelId: record.channelId || data.channelId || message.channelId || source.channelId,
+    channel: record.channel || data.channel || message.channel || provider
+  };
+}
+
+function resolveSessionRoot({ nativeRoot, pathContext, paths } = {}) {
+  if (paths?.sessions) return paths.sessions;
+  if (pathContext?.customized && pathContext.native?.sessions) return pathContext.native.sessions;
+  if (nativeRoot) return nativeRoot;
+  return resolvePaths({ pathContext }).sessions;
+}
+
+function createDriver({ nativeRoot, pathContext, paths, fsImpl = fs } = {}) {
+  const resolvedNativeRoot = resolveSessionRoot({ nativeRoot, pathContext, paths });
+
+  return {
+    platform: 'dsh',
+    capability: 'nativeLogs',
+    createNativeLogCursor({ fs: cursorFs = fsImpl, skipInitialParse = false } = {}) {
+      return createScannedFileCursor({
+        scanFiles: () => walkFiles(
+          resolvedNativeRoot,
+          name => SESSION_FILE_PATTERN.test(name),
+          cursorFs
+        ),
+        parseFile: filePath => filePath.endsWith('.zstd')
+          ? decodeArtifact(filePath).events
+          : readJsonLines(filePath, cursorFs),
+        normalizeEvent: normalizeDshEvent,
+        fsImpl: cursorFs,
+        skipInitialParse
+      });
+    },
+    createCursor(options) { return this.createNativeLogCursor(options); }
+  };
+}
+
+module.exports = { createDriver, extractUsage, normalizeDshEvent };

package/src/platforms/drivers/dsh/plugin-manager.js

@@ -0,0 +1,117 @@
+'use strict';
+
+const { execFile } = require('child_process');
+const path = require('path');
+const { promisify } = require('util');
+const { resolvePaths, listProfilePlugins, safeProfileName } = require('./common');
+
+const execFileAsync = promisify(execFile);
+
+function redactCommandOutput(value) {
+  return String(value || '')
+    .replace(/((?:api[-_]?key|access[-_]?token|refresh[-_]?token|token|secret|password|authorization)\s*[=:]\s*)([^\s,;]+)/gi, '$1[REDACTED]')
+    .replace(/(https?:\/\/[^\s/@]+:)[^@\s]+@/gi, '$1[REDACTED]@');
+}
+
+function normalizeSpecs(value) {
+  const values = Array.isArray(value) ? value : [value];
+  const specs = values
+    .map(spec => String(spec || '').trim())
+    .filter(Boolean);
+  if (!specs.length) throw new Error('至少需要一个 DSH plugin package spec');
+  if (specs.some(spec => spec.includes('\0') || spec.startsWith('-'))) {
+    throw new Error('DSH plugin package spec 不能是选项或包含 NUL 字符');
+  }
+  return specs;
+}
+
+function resolveWorkingDirectory(paths, value) {
+  if (value === undefined || value === null || value === '') return paths.dir;
+  const cwd = path.resolve(String(value));
+  try {
+    if (!require('fs').statSync(cwd).isDirectory()) throw new Error('not a directory');
+  } catch (_) {
+    throw new Error(`DSH plugin working directory does not exist: ${cwd}`);
+  }
+  return cwd;
+}
+
+function commandFor(context = {}) {
+  return context.dshCommand
+    || context.dependencies?.dshCommand
+    || process.env.DSH_COMMAND
+    || 'dsh';
+}
+
+function commandEnvironment(paths, context = {}) {
+  const env = { ...process.env, ...(context.dshEnv || {}) };
+  env.DSH_HOME = paths.dir;
+  if (paths.installAnchor) env.DSH_INSTALL_ANCHOR = paths.installAnchor;
+  if (paths.installRoot) env.DSH_INSTALL_ROOT = paths.installRoot;
+  return env;
+}
+
+async function runDshPlugin(context = {}, profileName, args, options = {}) {
+  const paths = resolvePaths(context);
+  const profile = safeProfileName(profileName);
+  const command = commandFor(context);
+  const cwd = resolveWorkingDirectory(paths, options.cwd);
+  const commandArgs = ['plugin', '--profile', profile, ...args];
+  let result;
+  try {
+    result = await execFileAsync(command, commandArgs, {
+      cwd,
+      env: commandEnvironment(paths, context),
+      encoding: 'utf8',
+      maxBuffer: 4 * 1024 * 1024,
+      timeout: options.timeoutMs || 15 * 60 * 1000,
+      windowsHide: true
+    });
+  } catch (error) {
+    const status = error && typeof error.code === 'number' ? error.code : null;
+    const wrapped = new Error(redactCommandOutput(error?.stderr?.trim() || error?.message || 'DSH plugin command failed'));
+    wrapped.code = error?.code === 'ENOENT' ? 'DSH_COMMAND_NOT_FOUND' : 'DSH_PLUGIN_COMMAND_FAILED';
+    wrapped.statusCode = error?.code === 'ENOENT' ? 503 : 422;
+    wrapped.exitCode = status;
+    wrapped.stdout = redactCommandOutput(error?.stdout);
+    wrapped.stderr = redactCommandOutput(error?.stderr);
+    throw wrapped;
+  }
+  return {
+    command,
+    args: commandArgs,
+    cwd,
+    profile,
+    exitCode: 0,
+    stdout: redactCommandOutput(result.stdout),
+    stderr: redactCommandOutput(result.stderr),
+    plugins: listProfilePlugins(context, profile)
+  };
+}
+
+async function installPlugin(context = {}, request = {}) {
+  const body = request.body || {};
+  return runDshPlugin(context, request.params?.profileName, ['add', ...normalizeSpecs(body.specs || body.spec || body.package)], {
+    cwd: body.cwd
+  });
+}
+
+async function uninstallPlugin(context = {}, request = {}) {
+  const body = request.body || {};
+  const specs = normalizeSpecs(body.specs || body.spec || request.params?.pluginName);
+  return runDshPlugin(context, request.params?.profileName, ['remove', ...specs], { cwd: body.cwd });
+}
+
+async function updatePlugin(context = {}, request = {}) {
+  const body = request.body || {};
+  const specs = body.specs || body.spec || body.package;
+  const args = ['update', ...(specs === undefined ? [] : normalizeSpecs(specs))];
+  return runDshPlugin(context, request.params?.profileName, args, { cwd: body.cwd });
+}
+
+module.exports = {
+  runDshPlugin,
+  installPlugin,
+  uninstallPlugin,
+  updatePlugin
+};

package/src/platforms/drivers/dsh/projects.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const { createDriver: createSessionsDriver } = require('./sessions');
+
+function createDriver(context = {}) {
+  const sessions = createSessionsDriver(context);
+  const unsupported = operation => () => ({
+    status: 'unsupported',
+    platform: 'dsh',
+    capability: 'projects',
+    operation
+  });
+  return {
+    platform: 'dsh',
+    capability: 'projects',
+    listProjects: sessions.listProjects,
+    listProjectsPage: sessions.listProjectsPage,
+    saveProjectOrder: unsupported('saveProjectOrder'),
+    deleteProject: unsupported('deleteProject')
+  };
+}
+
+module.exports = { createDriver };

package/src/platforms/drivers/dsh/proxy-implementation.js

@@ -0,0 +1,260 @@
+'use strict';
+
+const express = require('express');
+const httpProxy = require('http-proxy');
+const http = require('http');
+const chalk = require('chalk');
+const { broadcastSchedulerState } = require('../../../server/websocket-server');
+const { allocateChannel, releaseChannel, getSchedulerState } = require('../../../server/services/channel-scheduler');
+const { recordSuccess, recordFailure } = require('../../../server/services/channel-health');
+const { loadConfig } = require('../../../config/loader');
+const {
+  getEnabledChannels,
+  getEffectiveApiKey,
+  setDshProxyConfig,
+  restoreDshNativeConfig,
+  clearActiveMarker,
+  isProxyModeEnabled,
+  readActiveMarker
+} = require('./channels-implementation');
+const { persistProxyRequestSnapshot } = require('../../../server/services/request-logger');
+const { redirectModel, resolveTargetUrl, isChatCompletionsPath, ensureOpenAiStreamUsage } = require('../../../shared/proxy-utils');
+const { saveProxyStartTime, clearProxyStartTime, getProxyStartTime, getProxyRuntime } = require('../../../server/services/proxy-runtime');
+const { attachServerShutdownHandling, expediteServerShutdown } = require('../../../server/services/server-shutdown');
+
+let proxyServer = null;
+let currentPort = null;
+const requestMetadata = new Map();
+
+const jsonBodyParser = express.json({
+  limit: '100mb',
+  verify: (req, _res, buffer) => {
+    req.rawBody = Buffer.from(buffer);
+  }
+});
+
+function shouldParseJson(req) {
+  return req.method === 'POST' && String(req.headers['content-type'] || '').includes('application/json');
+}
+
+function openAiChannels() {
+  return getEnabledChannels().filter(channel => channel.providerApi === 'openai-completions');
+}
+
+function releaseOnce(channel) {
+  let released = false;
+  return () => {
+    if (released) return;
+    released = true;
+    if (channel?.id) {
+      releaseChannel(channel.id, 'dsh');
+      broadcastSchedulerState('dsh', getSchedulerState('dsh'));
+    }
+  };
+}
+
+async function startDshProxyServer(options = {}) {
+  if (proxyServer) {
+    const activeChannelId = readActiveMarker()?.activeChannelId;
+    const activeChannel = activeChannelId
+      ? getEnabledChannels().find(channel => channel.id === activeChannelId) || null
+      : null;
+    return { success: true, port: currentPort, provider: 'ctx-dsh-proxy', activeChannel };
+  }
+
+  if (openAiChannels().length === 0) {
+    throw new Error('DSH 动态代理当前只支持 openai-completions；请先添加并启用至少一个兼容渠道');
+  }
+
+  const preserveStartTime = options.preserveStartTime === true;
+  const config = loadConfig();
+  const requestedPort = Number(options.port ?? config.ports?.dshProxy ?? 20093);
+  const app = express();
+  const proxy = httpProxy.createProxyServer({});
+
+  app.use((req, res, next) => shouldParseJson(req) ? jsonBodyParser(req, res, next) : next());
+  app.use(async (req, res) => {
+    let channel;
+    let release;
+    try {
+      channel = await allocateChannel({
+        source: 'dsh',
+        providerApi: 'openai-completions',
+        enableSessionBinding: false
+      });
+      req.selectedChannel = channel;
+      release = releaseOnce(channel);
+      res.on('close', release);
+      res.on('error', release);
+      broadcastSchedulerState('dsh', getSchedulerState('dsh'));
+
+      const apiKey = getEffectiveApiKey(channel);
+      if (channel.authMode !== 'none' && !apiKey) {
+        release();
+        return res.status(401).json({
+          error: { message: 'DSH channel API key is not configured', type: 'authentication_error' }
+        });
+      }
+      req.effectiveApiKey = apiKey;
+
+      if (req.body?.model) {
+        const originalModel = req.body.model;
+        const redirectedModel = redirectModel(originalModel, channel, { useTierFallback: false });
+        if (redirectedModel !== originalModel) req.body.model = redirectedModel;
+      }
+      if (shouldParseJson(req) && isChatCompletionsPath(req.url)) {
+        ensureOpenAiStreamUsage(req.body);
+      }
+      if (shouldParseJson(req) && req.body) {
+        req.rawBody = Buffer.from(JSON.stringify(req.body));
+      }
+
+      persistProxyRequestSnapshot('dsh', {
+        timestamp: Date.now(),
+        source: 'dsh',
+        channel: channel.name,
+        request: { method: req.method, url: req.url, headers: req.headers, body: req.body || null }
+      });
+
+      proxy.web(req, res, {
+        target: resolveTargetUrl(channel.baseUrl, req.url),
+        changeOrigin: true,
+        proxyTimeout: 120000,
+        timeout: 120000
+      }, error => {
+        release?.();
+        if (!error) return;
+        recordFailure(channel.id, 'dsh', error);
+        if (!res.headersSent) {
+          res.status(502).json({ error: { message: `Proxy error: ${error.message}`, type: 'proxy_error' } });
+        }
+      });
+    } catch (error) {
+      release?.();
+      if (!res.headersSent) {
+        res.status(503).json({ error: { message: error.message || 'No DSH channel available', type: 'channel_pool_exhausted' } });
+      }
+    }
+  });
+
+  proxy.on('proxyReq', (proxyReq, req) => {
+    const channel = req.selectedChannel;
+    if (!channel) return;
+    const metadata = {
+      channelId: channel.id,
+      channel: channel.name,
+      startTime: Date.now(),
+      requestModel: req.body?.model || ''
+    };
+    requestMetadata.set(req, metadata);
+    proxyReq.removeHeader('authorization');
+    if (req.effectiveApiKey) proxyReq.setHeader('authorization', `Bearer ${req.effectiveApiKey}`);
+    proxyReq.setHeader('content-type', 'application/json');
+    if (shouldParseJson(req) && (req.rawBody || req.body)) {
+      const body = req.rawBody
+        ? (Buffer.isBuffer(req.rawBody) ? req.rawBody : Buffer.from(req.rawBody))
+        : Buffer.from(JSON.stringify(req.body));
+      proxyReq.setHeader('Content-Length', body.length);
+      proxyReq.write(body);
+      proxyReq.end();
+    }
+  });
+
+  proxy.on('proxyRes', (proxyRes, req, res) => {
+    const metadata = requestMetadata.get(req);
+    if (!metadata) return;
+    if (Number(proxyRes.statusCode) >= 400) {
+      recordFailure(metadata.channelId, 'dsh', new Error(`DSH upstream HTTP ${proxyRes.statusCode}`));
+    } else {
+      recordSuccess(metadata.channelId, 'dsh');
+    }
+    requestMetadata.delete(req);
+    res?.on('close', () => requestMetadata.delete(req));
+  });
+
+  proxy.on('error', (error, req, res) => {
+    if (req?.selectedChannel) {
+      recordFailure(req.selectedChannel.id, 'dsh', error);
+      releaseChannel(req.selectedChannel.id, 'dsh');
+      broadcastSchedulerState('dsh', getSchedulerState('dsh'));
+    }
+    if (res && !res.headersSent) res.status(502).json({ error: { message: `Proxy error: ${error.message}`, type: 'proxy_error' } });
+  });
+
+  const server = http.createServer(app);
+  attachServerShutdownHandling(server);
+  proxyServer = server;
+  currentPort = requestedPort;
+
+  try {
+    await new Promise((resolve, reject) => {
+      server.once('error', reject);
+      server.listen(requestedPort, '127.0.0.1', resolve);
+    });
+    const port = server.address()?.port || requestedPort;
+    currentPort = port;
+    const active = openAiChannels()[0];
+    setDshProxyConfig(port, { activeChannelId: active?.id });
+    saveProxyStartTime('dsh', preserveStartTime);
+    console.log(`DSH proxy server started on http://127.0.0.1:${port}`);
+    return { success: true, port, provider: 'ctx-dsh-proxy', activeChannel: active };
+  } catch (error) {
+    proxyServer = null;
+    currentPort = null;
+    clearActiveMarker();
+    try { await new Promise(resolve => server.close(() => resolve())); } catch (_) {}
+    if (error.code === 'EADDRINUSE') {
+      console.error(chalk.red(`DSH proxy port ${requestedPort} is already in use`));
+    }
+    throw error;
+  }
+}
+
+async function stopDshProxyServer(options = {}) {
+  const clearStartTime = options.clearStartTime !== false;
+  const stoppedPort = currentPort;
+  requestMetadata.clear();
+
+  if (proxyServer) {
+    const shutdownTimer = expediteServerShutdown(proxyServer);
+    await new Promise(resolve => proxyServer.close(resolve));
+    if (shutdownTimer) clearTimeout(shutdownTimer);
+    proxyServer = null;
+    currentPort = null;
+  }
+
+  if (!isProxyModeEnabled()) {
+    if (clearStartTime) clearProxyStartTime('dsh');
+    return { success: true, port: stoppedPort, restored: null };
+  }
+
+  const restored = restoreDshNativeConfig();
+  if (clearStartTime) clearProxyStartTime('dsh');
+  return { success: true, port: stoppedPort, restored: restored ? restored.id || restored.name : null };
+}
+
+function getDshProxyStatus() {
+  const config = loadConfig();
+  const running = Boolean(proxyServer);
+  const activeChannelId = readActiveMarker()?.activeChannelId;
+  const activeChannel = activeChannelId
+    ? getEnabledChannels().find(channel => channel.id === activeChannelId) || null
+    : null;
+  return {
+    running,
+    port: currentPort,
+    defaultPort: config.ports?.dshProxy || 20093,
+    startTime: getProxyStartTime('dsh', { allowRecovery: running }),
+    runtime: getProxyRuntime('dsh', { allowRecovery: running }),
+    mode: 'managed-provider',
+    activeChannel,
+    enabledChannelsCount: getEnabledChannels().length,
+    supportedChannelsCount: openAiChannels().length
+  };
+}
+
+module.exports = {
+  startDshProxyServer,
+  stopDshProxyServer,
+  getDshProxyStatus
+};

package/src/platforms/drivers/dsh/proxy.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const { createProxyDriver } = require('../../../shared/proxy-driver');
+
+function createDriver(context = {}) {
+  return createProxyDriver({
+    ...context,
+    platform: 'dsh',
+    servicePath: './dsh/proxy-implementation',
+    localServicePath: '../platforms/drivers/dsh/proxy-implementation',
+    exports: {
+      status: 'getDshProxyStatus',
+      start: 'startDshProxyServer',
+      stop: 'stopDshProxyServer'
+    },
+    cliMetadata: {
+      managedProviderConfig: true,
+      defaultPort: 20093
+    }
+  });
+}
+
+module.exports = { createDriver };