@supen-ai/cli 1.4.4 → 1.4.5

package/daemon/scripts/bench-message-history.ts

@@ -0,0 +1,584 @@
+import http from 'node:http';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { pathToFileURL } from 'node:url';
+
+type SampleSummary = {
+  runs: number;
+  samplesMs: number[];
+  minMs: number;
+  p50Ms: number;
+  p95Ms: number;
+  maxMs: number;
+  avgMs: number;
+};
+
+type ThreadHistoryPayload = {
+  messages?: unknown[];
+  events?: unknown[];
+};
+
+type HttpHistorySample = {
+  totalMs: number;
+  status: number;
+  bytes: number;
+  jsonParseMs: number;
+  replayMs: number;
+  messages: number;
+  events: number;
+  uiMessages: number;
+};
+
+type SseSample = {
+  responseMs: number;
+  firstEventMs: number;
+  drainMs: number;
+  status: number;
+  events: number;
+  bytes: number;
+};
+
+type BenchMessageHistoryOptions = {
+  baseUrl?: string;
+  threadId?: string;
+  computerId: string;
+  apiKey?: string;
+  authorization?: string;
+  runs: number;
+  limit: number;
+  streamSampleMs: number;
+};
+
+export function summarize(samples: number[]): SampleSummary {
+  const sorted = [...samples].sort((a, b) => a - b);
+  const percentile = (value: number) => {
+    if (sorted.length === 0) return 0;
+    return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * value) - 1)] ?? 0;
+  };
+  const avg = samples.reduce((sum, sample) => sum + sample, 0) / Math.max(1, samples.length);
+  return {
+    runs: samples.length,
+    samplesMs: samples.map((sample) => Number(sample.toFixed(3))),
+    minMs: Number((sorted[0] ?? 0).toFixed(3)),
+    p50Ms: Number(percentile(0.5).toFixed(3)),
+    p95Ms: Number(percentile(0.95).toFixed(3)),
+    maxMs: Number((sorted.at(-1) ?? 0).toFixed(3)),
+    avgMs: Number(avg.toFixed(3)),
+  };
+}
+
+function readPositiveInteger(value: string | undefined, fallback: number): number {
+  const parsed = Number.parseInt(value || '', 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export function parseBenchMessageHistoryArgs(argv: string[]): BenchMessageHistoryOptions {
+  const options: BenchMessageHistoryOptions = {
+    baseUrl: process.env.BENCH_MESSAGE_HISTORY_REMOTE_BASE_URL?.trim(),
+    threadId: process.env.BENCH_MESSAGE_HISTORY_REMOTE_THREAD_ID?.trim(),
+    computerId: process.env.BENCH_MESSAGE_HISTORY_REMOTE_COMPUTER_ID?.trim() || 'local',
+    apiKey: process.env.BENCH_MESSAGE_HISTORY_REMOTE_API_KEY?.trim(),
+    authorization: (
+      process.env.BENCH_MESSAGE_HISTORY_REMOTE_AUTHORIZATION ||
+      (process.env.BENCH_MESSAGE_HISTORY_REMOTE_BEARER
+        ? `Bearer ${process.env.BENCH_MESSAGE_HISTORY_REMOTE_BEARER}`
+        : '')
+    ).trim() || undefined,
+    runs: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_RUNS, 12),
+    limit: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_LIMIT, 400),
+    streamSampleMs: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_STREAM_SAMPLE_MS, 5_000),
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    const next = argv[index + 1];
+    if (arg === '--base-url') {
+      options.baseUrl = next?.trim();
+      index += 1;
+    } else if (arg === '--thread-id') {
+      options.threadId = next?.trim();
+      index += 1;
+    } else if (arg === '--computer-id') {
+      options.computerId = next?.trim() || options.computerId;
+      index += 1;
+    } else if (arg === '--api-key') {
+      options.apiKey = next?.trim();
+      index += 1;
+    } else if (arg === '--authorization') {
+      options.authorization = next?.trim();
+      index += 1;
+    } else if (arg === '--runs') {
+      options.runs = readPositiveInteger(next, options.runs);
+      index += 1;
+    } else if (arg === '--limit') {
+      options.limit = readPositiveInteger(next, options.limit);
+      index += 1;
+    } else if (arg === '--stream-sample-ms') {
+      options.streamSampleMs = readPositiveInteger(next, options.streamSampleMs);
+      index += 1;
+    }
+  }
+
+  return options;
+}
+
+export function buildCodexThreadEndpoint(
+  baseUrl: string,
+  threadId: string,
+  endpoint: 'messages' | 'stream',
+  params: Record<string, string> = {},
+  computerId = 'local',
+): string {
+  const normalizedBaseUrl = baseUrl.replace(/\/+$/, '');
+  const url = new URL(
+    `${normalizedBaseUrl}/api/computers/${encodeURIComponent(computerId)}/agents/codex/threads/${encodeURIComponent(threadId)}/${endpoint}`,
+  );
+  for (const [key, value] of Object.entries(params)) {
+    url.searchParams.set(key, value);
+  }
+  return url.toString();
+}
+
+function summarizeField<T>(samples: T[], selector: (sample: T) => number): SampleSummary {
+  return summarize(samples.map(selector));
+}
+
+function denseTimestamp(index: number, suffixMs = 0): string {
+  return `2026-06-04T12:${String(Math.floor(index / 60)).padStart(2, '0')}:${String(index % 60).padStart(2, '0')}.${String(suffixMs).padStart(3, '0')}Z`;
+}
+
+function measure<T>(runs: number, fn: () => T): { summary: SampleSummary; last: T } {
+  const samples: number[] = [];
+  let last: T;
+  for (let run = 0; run < runs; run += 1) {
+    const startedAt = performance.now();
+    last = fn();
+    samples.push(performance.now() - startedAt);
+  }
+  return { summary: summarize(samples), last: last! };
+}
+
+function warmup(runs: number, fn: () => void): void {
+  for (let run = 0; run < runs; run += 1) {
+    fn();
+  }
+}
+
+async function measureAsync<T>(runs: number, fn: () => Promise<T>): Promise<{ samples: T[]; total: SampleSummary }> {
+  const samples: T[] = [];
+  const totals: number[] = [];
+  for (let run = 0; run < runs; run += 1) {
+    const startedAt = performance.now();
+    samples.push(await fn());
+    totals.push(performance.now() - startedAt);
+  }
+  return { samples, total: summarize(totals) };
+}
+
+async function warmupAsync(runs: number, fn: () => Promise<void>): Promise<void> {
+  for (let run = 0; run < runs; run += 1) {
+    await fn();
+  }
+}
+
+function createFixture(agentHome: string, supenHome: string) {
+  const threadId = '019e8b00-0000-7000-9000-000000000099';
+  const threadsDir = path.join(agentHome, 'threads', '2026', '06', '04');
+  fs.mkdirSync(threadsDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(agentHome, 'thread_index.jsonl'),
+    `${JSON.stringify({ id: threadId, thread_name: 'Benchmark thread', updated_at: '2026-06-04T12:00:00.000Z' })}\n`,
+    'utf-8',
+  );
+
+  const threadPath = path.join(threadsDir, `rollout-2026-06-04T12-00-00-${threadId}.jsonl`);
+  const largeAssistantText = 'assistant-detail '.repeat(1024);
+  const lines: string[] = [];
+  for (let index = 0; index < 250; index += 1) {
+    lines.push(JSON.stringify({
+      timestamp: denseTimestamp(index, 0),
+      type: 'response_item',
+      payload: {
+        type: 'message',
+        role: 'user',
+        content: [{ type: 'input_text', text: `request ${index}` }],
+      },
+    }));
+    lines.push(JSON.stringify({
+      timestamp: denseTimestamp(index, 100),
+      type: 'response_item',
+      payload: {
+        type: 'function_call',
+        name: 'exec_command',
+        call_id: `call-${index}`,
+        arguments: JSON.stringify({ cmd: `echo ${index}` }),
+      },
+    }));
+    lines.push(JSON.stringify({
+      timestamp: denseTimestamp(index, 200),
+      type: 'response_item',
+      payload: {
+        type: 'message',
+        role: 'assistant',
+        content: [{ type: 'output_text', text: `answer ${index}\n${largeAssistantText}` }],
+      },
+    }));
+  }
+  fs.writeFileSync(threadPath, `${lines.join('\n')}\n`, 'utf-8');
+
+  const eventLogDir = path.join(supenHome, 'threads', threadId, 'event-log');
+  fs.mkdirSync(eventLogDir, { recursive: true });
+  const rawEvents: string[] = [];
+  for (let index = 0; index < 120; index += 1) {
+    rawEvents.push(JSON.stringify({
+      event_id: `bench-event-${index}`,
+      runtime_event_id: `bench-runtime-event-${index}`,
+      thread_id: threadId,
+      runtime_thread_id: threadId,
+      sequence: index + 1,
+      runtime_sequence: index + 1,
+      source: 'codex-app-server',
+      event_type: 'item/agentMessage/delta',
+      raw_payload: {
+        type: 'data-codex-event',
+        data: {
+          eventType: 'item/agentMessage/delta',
+          raw: {
+            method: 'item/agentMessage/delta',
+            params: {
+              threadId,
+              turnId: 'turn-bench',
+              itemId: 'assistant-bench',
+              delta: `stream-delta-${index} `,
+            },
+          },
+        },
+      },
+      received_at: denseTimestamp(index, 300),
+      payload_hash: `bench-${index}`,
+    }));
+  }
+  fs.writeFileSync(path.join(eventLogDir, 'raw-events.jsonl'), `${rawEvents.join('\n')}\n`, 'utf-8');
+  fs.writeFileSync(
+    path.join(eventLogDir, 'head.json'),
+    `${JSON.stringify({ last_sequence: rawEvents.length, updated_at: denseTimestamp(rawEvents.length, 0) })}\n`,
+    'utf-8',
+  );
+
+  return {
+    threadId,
+    threadPath,
+    jsonlLines: lines.length,
+    seededStreamEvents: rawEvents.length,
+  };
+}
+
+async function startLoopbackServer() {
+  const { dispatchRequest } = await import('../src/http/router.js');
+  const { setCorsHeaders } = await import('../src/http/context.js');
+  const server = http.createServer(async (req, res) => {
+    if (req.method === 'OPTIONS') {
+      setCorsHeaders(req, res);
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+    setCorsHeaders(req, res);
+    await dispatchRequest(req, res, () => {});
+  });
+  await new Promise<void>((resolve, reject) => {
+    server.listen(0, '127.0.0.1', resolve);
+    server.on('error', reject);
+  });
+  const address = server.address();
+  if (!address || typeof address === 'string') {
+    throw new Error('Expected loopback TCP server address.');
+  }
+  return {
+    server,
+    baseUrl: `http://127.0.0.1:${address.port}`,
+  };
+}
+
+async function measureHttpHistory(
+  url: string,
+  buildThreadUIMessages: (payload: ThreadHistoryPayload, options: { includeSilentEvents: boolean }) => unknown[],
+  headers: HeadersInit = {},
+): Promise<HttpHistorySample> {
+  const startedAt = performance.now();
+  const response = await fetch(url, { headers });
+  const text = await response.text();
+  const totalMs = performance.now() - startedAt;
+  if (!response.ok) {
+    throw new Error(`History request failed (${response.status}) for ${url}: ${text.slice(0, 300)}`);
+  }
+  const parseStartedAt = performance.now();
+  const history = JSON.parse(text) as ThreadHistoryPayload;
+  const jsonParseMs = performance.now() - parseStartedAt;
+  const replayStartedAt = performance.now();
+  const uiMessages = buildThreadUIMessages(history, { includeSilentEvents: false });
+  const replayMs = performance.now() - replayStartedAt;
+  return {
+    totalMs,
+    status: response.status,
+    bytes: Buffer.byteLength(text),
+    jsonParseMs,
+    replayMs,
+    messages: history.messages?.length ?? 0,
+    events: history.events?.length ?? 0,
+    uiMessages: uiMessages.length,
+  };
+}
+
+async function measureSse(
+  url: string,
+  headers: HeadersInit = {},
+  options: { targetEvents?: number; sampleMs?: number } = {},
+): Promise<SseSample> {
+  const startedAt = performance.now();
+  const targetEvents = Math.max(1, options.targetEvents ?? 120);
+  const sampleMs = Math.max(1, options.sampleMs ?? 5_000);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), sampleMs);
+  let response: Response;
+  try {
+    response = await fetch(url, { headers, signal: controller.signal });
+  } catch (error) {
+    clearTimeout(timeout);
+    if ((error as Error).name === 'AbortError') {
+      const elapsedMs = performance.now() - startedAt;
+      return {
+        responseMs: elapsedMs,
+        firstEventMs: 0,
+        drainMs: elapsedMs,
+        status: 0,
+        events: 0,
+        bytes: 0,
+      };
+    }
+    throw error;
+  }
+  const responseMs = performance.now() - startedAt;
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    clearTimeout(timeout);
+    throw new Error(`SSE request failed (${response.status}) for ${url}: ${body.slice(0, 300)}`);
+  }
+  if (!response.body) {
+    clearTimeout(timeout);
+    throw new Error(`SSE response body missing (${response.status})`);
+  }
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = '';
+  let bytes = 0;
+  let events = 0;
+  let firstEventMs = 0;
+  const processLine = (line: string) => {
+    if (!line.startsWith('data:')) return;
+    const data = line.slice(5).trim();
+    if (!data) return;
+    events += 1;
+    if (firstEventMs === 0) firstEventMs = performance.now() - startedAt;
+  };
+
+  try {
+    while (events < targetEvents) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split(/\r?\n/);
+      buffer = lines.pop() ?? '';
+      for (const line of lines) processLine(line);
+    }
+  } catch (error) {
+    if ((error as Error).name !== 'AbortError') throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+  await reader.cancel().catch(() => {});
+  return {
+    responseMs,
+    firstEventMs,
+    drainMs: performance.now() - startedAt,
+    status: response.status,
+    events,
+    bytes,
+  };
+}
+
+function sampleShape<T extends Record<string, unknown>>(sample: T | undefined): T | undefined {
+  if (!sample) return undefined;
+  return Object.fromEntries(
+    Object.entries(sample).map(([key, value]) => [
+      key,
+      typeof value === 'number' ? Number(value.toFixed(3)) : value,
+    ]),
+  ) as T;
+}
+
+async function main() {
+  const options = parseBenchMessageHistoryArgs(process.argv.slice(2));
+  const agentHome = fs.mkdtempSync(path.join(os.tmpdir(), 'supen-message-history-bench-'));
+  const supenHome = path.join(agentHome, '.supen');
+  process.env.CODEX_HOME = agentHome;
+  process.env.HOME = agentHome;
+  process.env.SUPEN_HOME = supenHome;
+  process.env.HTTP_API_KEY = '';
+  process.env.NODE_ENV = 'test';
+
+  let server: http.Server | null = null;
+  try {
+    const fixture = createFixture(agentHome, supenHome);
+    const { readCodexThreadHistory } = await import('../src/http/routes/system.js');
+    const { directHttpAccessToken } = await import('../src/core/direct-http-access.js');
+    const { buildThreadUIMessages } = await import('../../app/app/lib/chat-ui-message-replay.ts');
+
+    const requestedHistoryLimit = options.limit;
+    warmup(2, () => {
+      readCodexThreadHistory(fixture.threadId, requestedHistoryLimit);
+    });
+    const historyBench = measure(options.runs, () => readCodexThreadHistory(fixture.threadId, requestedHistoryLimit));
+    const history = historyBench.last ?? { messages: [], events: [] };
+    warmup(5, () => {
+      buildThreadUIMessages(history, { includeSilentEvents: false });
+    });
+    const replayBench = measure(80, () =>
+      buildThreadUIMessages(history, { includeSilentEvents: false }),
+    );
+
+    const loopback = await startLoopbackServer();
+    server = loopback.server;
+    const directHeaders = {
+      Origin: 'https://hub.supen.ai',
+      'X-API-Key': directHttpAccessToken(),
+    };
+    const historyUrl = buildCodexThreadEndpoint(loopback.baseUrl, fixture.threadId, 'messages', {
+      limit: String(requestedHistoryLimit),
+      eventLimit: '200',
+    });
+    const streamUrl = buildCodexThreadEndpoint(loopback.baseUrl, fixture.threadId, 'stream', {
+      after: '0',
+    });
+
+    await warmupAsync(2, async () => {
+      await measureHttpHistory(historyUrl, buildThreadUIMessages, directHeaders);
+    });
+    const httpHistoryBench = await measureAsync(options.runs, () =>
+      measureHttpHistory(historyUrl, buildThreadUIMessages, directHeaders),
+    );
+    const sseBench = await measureAsync(Math.max(1, Math.min(options.runs, 8)), () =>
+      measureSse(streamUrl, directHeaders, {
+        targetEvents: fixture.seededStreamEvents,
+        sampleMs: options.streamSampleMs,
+      }),
+    );
+    const remoteBaseUrl = (options.baseUrl || '').trim().replace(/\/+$/, '');
+    const remoteThreadId = (options.threadId || '').trim();
+    const remoteComputerId = options.computerId.trim() || 'local';
+    const remoteHeaders: Record<string, string> = {};
+    if (options.apiKey) remoteHeaders['X-API-Key'] = options.apiKey;
+    if (options.authorization) remoteHeaders.Authorization = options.authorization;
+    const shouldRunRemote = Boolean(remoteBaseUrl && remoteThreadId);
+    const remoteHistoryUrl = shouldRunRemote
+      ? buildCodexThreadEndpoint(
+          remoteBaseUrl,
+          remoteThreadId,
+          'messages',
+          { limit: String(requestedHistoryLimit), eventLimit: '200' },
+          remoteComputerId,
+        )
+      : '';
+    const remoteStreamUrl = shouldRunRemote
+      ? buildCodexThreadEndpoint(remoteBaseUrl, remoteThreadId, 'stream', { after: '0' }, remoteComputerId)
+      : '';
+    const remoteRuns = Math.max(1, Math.min(options.runs, 6));
+    const remoteHistoryBench = shouldRunRemote
+      ? await measureAsync(remoteRuns, () => measureHttpHistory(remoteHistoryUrl, buildThreadUIMessages, remoteHeaders))
+      : null;
+    const shouldRunRemoteStream = shouldRunRemote && process.env.BENCH_MESSAGE_HISTORY_REMOTE_STREAM !== '0';
+    const remoteSseBench = shouldRunRemoteStream
+      ? await measureAsync(Math.max(1, Math.min(options.runs, 3)), () =>
+          measureSse(remoteStreamUrl, remoteHeaders, {
+            targetEvents: Number.parseInt(process.env.BENCH_MESSAGE_HISTORY_REMOTE_STREAM_EVENTS || '1', 10) || 1,
+            sampleMs: options.streamSampleMs,
+          }),
+        )
+      : null;
+
+    console.log(JSON.stringify({
+      benchmark: {
+        generatedAt: new Date().toISOString(),
+        options,
+      },
+      fixture: {
+        threadJsonlBytes: fs.statSync(fixture.threadPath).size,
+        jsonlLines: fixture.jsonlLines,
+        seededStreamEvents: fixture.seededStreamEvents,
+        requestedHistoryLimit,
+        loadedMessages: history.messages.length,
+        loadedEvents: history.events.length,
+        replayMessages: replayBench.last.length,
+      },
+      inProcess: {
+        daemonHistory: historyBench.summary,
+        frontendReplay: replayBench.summary,
+      },
+      loopbackHttp: {
+        historyTotal: summarizeField(httpHistoryBench.samples, (sample) => sample.totalMs),
+        historyJsonParse: summarizeField(httpHistoryBench.samples, (sample) => sample.jsonParseMs),
+        historyReplay: summarizeField(httpHistoryBench.samples, (sample) => sample.replayMs),
+        last: sampleShape(httpHistoryBench.samples.at(-1)),
+      },
+      loopbackSse: {
+        response: summarizeField(sseBench.samples, (sample) => sample.responseMs),
+        firstEvent: summarizeField(sseBench.samples, (sample) => sample.firstEventMs),
+        drain: summarizeField(sseBench.samples, (sample) => sample.drainMs),
+        last: sampleShape(sseBench.samples.at(-1)),
+      },
+      compareRemote: {
+        ...(remoteHistoryBench
+          ? {
+              baseUrl: remoteBaseUrl,
+              computerId: remoteComputerId,
+              threadId: remoteThreadId,
+              historyTotal: summarizeField(remoteHistoryBench.samples, (sample) => sample.totalMs),
+              historyJsonParse: summarizeField(remoteHistoryBench.samples, (sample) => sample.jsonParseMs),
+              historyReplay: summarizeField(remoteHistoryBench.samples, (sample) => sample.replayMs),
+              lastHistory: sampleShape(remoteHistoryBench.samples.at(-1)),
+              ...(remoteSseBench
+                ? {
+                    sseResponse: summarizeField(remoteSseBench.samples, (sample) => sample.responseMs),
+                    sseFirstEvent: summarizeField(remoteSseBench.samples, (sample) => sample.firstEventMs),
+                    sseDrain: summarizeField(remoteSseBench.samples, (sample) => sample.drainMs),
+                    lastSse: sampleShape(remoteSseBench.samples.at(-1)),
+                  }
+                : {
+                    sseSkipped:
+                      'Remote SSE timing is enabled by default when a remote base URL and thread ID are provided. BENCH_MESSAGE_HISTORY_REMOTE_STREAM=0 skips it.',
+                  }),
+            }
+          : {
+              skipped: true,
+              note:
+                'Set --base-url plus --thread-id or BENCH_MESSAGE_HISTORY_REMOTE_BASE_URL plus BENCH_MESSAGE_HISTORY_REMOTE_THREAD_ID. Optional: --computer-id, --api-key, --authorization, BENCH_MESSAGE_HISTORY_REMOTE_COMPUTER_ID/API_KEY/AUTHORIZATION/BEARER.',
+            }),
+      },
+    }, null, 2));
+  } finally {
+    if (server) {
+      await new Promise<void>((resolve) => server!.close(() => resolve()));
+    }
+    fs.rmSync(agentHome, { recursive: true, force: true });
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });
+}

package/dist/computer.js

@@ -11,7 +11,7 @@ import { hostOsIdentityEnv } from './os-identity.js';
 import { createTransport, resolveTransportConfig } from './transport/index.js';
 const DEFAULT_LOCAL_GATEWAY_URL = 'http://127.0.0.1:2755';
 const DEFAULT_PRODUCTION_HUB_URL = 'https://hub.supen.ai';
-const HOST_DAEMON_PACKAGE_SPEC = '@supen-ai/cli@1.4.4';
+const HOST_DAEMON_PACKAGE_SPEC = '@supen-ai/cli@1.4.5';
 const DEFAULT_NPM_REGISTRY = 'https://registry.npmjs.org';
 function normalizeGatewayUrl(raw) {
     const trimmed = (raw || '').trim();

package/dist/index.js

@@ -78,7 +78,7 @@ const program = new Command();
 program
     .name('supen')
     .description('Supen CLI — bootstrap, manage skills, agents, and control the service')
-    .version('1.4.4');
+    .version('1.4.5');
 program
     .command('bootstrap')
     .description('First-time setup: choose gateway mode or local Docker daemon mode')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@supen-ai/cli",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "description": "Supen CLI — command-line tool to manage local Supen daemon, agents, and skills",
   "type": "module",
   "main": "dist/index.js",