@cursor/july 0.1.109 → 0.1.112

package/src/extensions/slack-communication/evals/harness.ts

@@ -0,0 +1,728 @@
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { begin, end } from "../../../channels/slack/agentic-posted.js";
+import { buildDefaultEvents } from "../../../channels/slack/defaults.js";
+import {
+  dispatchInboundMessage,
+  emptySlackChannelState,
+  type SlackWakeState,
+} from "../../../channels/slack/dispatch.js";
+import { resetExternalSenderCache } from "../../../channels/slack/external-policy.js";
+import { resetPlaceholderSlots } from "../../../channels/slack/placeholder.js";
+import {
+  createRecordingSlackSink,
+  type RecordingSlackSink,
+  type SlackSinkCall,
+} from "../../../channels/slack/recording-sink.js";
+import type {
+  SlackChannelState,
+  SlackReplyFooter,
+} from "../../../channels/slack/types.js";
+import { createDeferredChannelSession } from "../../../internal/deferred-channel-session.js";
+import { createHostFilesApi } from "../../../internal/host-files.js";
+import { createHostOtelApi } from "../../../internal/host-otel.js";
+import { isSlackImChannel } from "../../../internal/slack-thread-key.js";
+import type {
+  ActionCall,
+  ArtifactsApi,
+  ChannelEventContext,
+  ChannelEventHandlers,
+  ChannelSession,
+  HostContext,
+  SendMessageFn,
+  SessionContext,
+  SessionEvent,
+  ToolContext,
+} from "../../../types.js";
+import postUpdate from "../tools/post-update.js";
+import react from "../tools/react.js";
+import setStatus from "../tools/set-status.js";
+
+/** Jenny's configured ack. Slack renders `paw_prints` as :feet:. */
+export const JENNY_ACK: string = "paw_prints";
+
+const TOOL_STALL_MS = 100;
+
+export type PlayStep =
+  | { type: "turn.started" }
+  | { type: "reasoning.appended"; text: string; parentCallId?: string }
+  | {
+      type: "actions.requested";
+      calls: Array<{
+        tool: string;
+        args?: unknown;
+        id?: string;
+        parentCallId?: string;
+      }>;
+      parentCallId?: string;
+    }
+  | {
+      type: "action.result";
+      tool: string;
+      output?: unknown;
+      isError?: boolean;
+      stubbed?: true;
+      id?: string;
+      parentCallId?: string;
+    }
+  | { type: "turn.completed"; result?: string }
+  | {
+      type: "message.completed";
+      text: string;
+      finishReason?: "stop" | "tool_call";
+    }
+  | { type: "turn.failed"; message?: string };
+
+let eventIndex = 0;
+
+function evt<T extends SessionEvent["type"]>(
+  partial: Omit<Extract<SessionEvent, { type: T }>, "index" | "at"> &
+    Partial<Pick<SessionEvent, "index" | "at">>
+): Extract<SessionEvent, { type: T }> {
+  return {
+    index: eventIndex++,
+    at: "2026-01-01T00:00:00.000Z",
+    ...partial,
+  } as Extract<SessionEvent, { type: T }>;
+}
+
+function sessionOf(dryRun: boolean | undefined): SessionContext {
+  return {
+    session: {
+      id: "s1",
+      channelId: "slack",
+      mode: "chat",
+      auth: null,
+      dryRun,
+    },
+    host: {} as HostContext,
+    artifacts: {} as never,
+  };
+}
+
+function channelOf(
+  state: SlackChannelState
+): ChannelEventContext<SlackChannelState> {
+  return {
+    channelId: "slack",
+    state,
+    continuationToken: null,
+    setContinuationToken: async () => {},
+  };
+}
+
+async function emit(
+  handlers: ChannelEventHandlers<SlackChannelState>,
+  channel: ChannelEventContext<SlackChannelState>,
+  event: SessionEvent,
+  ctx: SessionContext
+): Promise<void> {
+  const handler = handlers[event.type];
+  if (handler === undefined) {
+    return;
+  }
+  await handler(event as never, channel, ctx);
+}
+
+export function signal(
+  tool: string,
+  output: unknown,
+  options: { id?: string; args?: unknown } = {}
+): PlayStep[] {
+  const id = options.id ?? "s0";
+  return [
+    {
+      type: "actions.requested",
+      calls: [{ tool, args: options.args ?? output, id }],
+    },
+    {
+      type: "action.result",
+      tool,
+      id,
+      output,
+    },
+  ];
+}
+
+export function publish(
+  message: string,
+  options: { id?: string; tool?: string; ts?: string } = {}
+): PlayStep[] {
+  const id = options.id ?? "u0";
+  const tool = options.tool ?? "post-update";
+  const output: { message: string; ts?: string } = { message };
+  output.ts = options.ts;
+  return [
+    {
+      type: "actions.requested",
+      calls: [{ tool, args: { message }, id }],
+    },
+    {
+      type: "action.result",
+      tool,
+      id,
+      output,
+    },
+  ];
+}
+
+export async function playAgentic(
+  steps: readonly PlayStep[],
+  options: {
+    dryRun?: boolean;
+    ack?: string;
+    channelId?: string;
+    startPainted?: boolean;
+    placeholder?: string;
+    sink?: RecordingSlackSink;
+    footer?: SlackReplyFooter;
+  } = {}
+): Promise<RecordingSlackSink> {
+  eventIndex = 0;
+  const sink = options.sink ?? createRecordingSlackSink();
+  if (options.sink === undefined) {
+    resetPlaceholderSlots();
+  }
+  const handlers = buildDefaultEvents({
+    reply: {
+      mode: "agentic",
+      ack: options.ack,
+      placeholder: options.placeholder,
+      footer: options.footer,
+    },
+    credentials: { botToken: sink.transport },
+  });
+  const ctx = sessionOf(options.dryRun);
+  const channel = channelOf(
+    emptySlackChannelState({
+      channelId: options.channelId ?? "C1",
+      threadTs: "1.0",
+      teamId: "T1",
+      triggeringUserId: "U1",
+      startPainted: options.startPainted === true,
+    })
+  );
+  for (const step of steps) {
+    await playStep(handlers, channel, step, ctx);
+  }
+  return sink;
+}
+
+/** Channel Slack I/O is held. The next tool batch must still finish. */
+export async function playHeldWrites(): Promise<{
+  stalled: boolean;
+  sink: RecordingSlackSink;
+}> {
+  eventIndex = 0;
+  const sink = createRecordingSlackSink({ hold: true });
+  const handlers = buildDefaultEvents({
+    reply: { mode: "agentic" },
+    credentials: { botToken: sink.transport },
+  });
+  const ctx = sessionOf(undefined);
+  const channel = channelOf(
+    emptySlackChannelState({
+      channelId: "C1",
+      threadTs: "1.0",
+      teamId: "T1",
+      triggeringUserId: "U1",
+    })
+  );
+  const writes = Promise.all([
+    playStep(
+      handlers,
+      channel,
+      {
+        type: "action.result",
+        tool: "react",
+        id: "r0",
+        output: { emoji: "white_check_mark" },
+      },
+      ctx
+    ),
+    playStep(
+      handlers,
+      channel,
+      {
+        type: "action.result",
+        tool: "post-update",
+        id: "u0",
+        output: { message: "Looking." },
+      },
+      ctx
+    ),
+  ]);
+  const tools = await runSlackTools("s-next");
+  sink.release();
+  await writes;
+  return { stalled: tools.stalled, sink };
+}
+
+async function playStep(
+  handlers: ChannelEventHandlers<SlackChannelState>,
+  channel: ChannelEventContext<SlackChannelState>,
+  step: PlayStep,
+  ctx: SessionContext
+): Promise<void> {
+  if (step.type === "turn.started") {
+    await emit(
+      handlers,
+      channel,
+      evt({ type: "turn.started", sessionId: "s1", data: {} }),
+      ctx
+    );
+    return;
+  }
+  if (step.type === "reasoning.appended") {
+    await emit(
+      handlers,
+      channel,
+      evt({
+        type: "reasoning.appended",
+        sessionId: "s1",
+        data: {
+          delta: "",
+          text: step.text,
+          parentCallId: step.parentCallId,
+        },
+      }),
+      ctx
+    );
+    return;
+  }
+  if (step.type === "actions.requested") {
+    const calls: ActionCall[] = step.calls.map((call, i) => ({
+      callId: call.id ?? `c${i}`,
+      toolName: call.tool,
+      args: call.args,
+      parentCallId: call.parentCallId,
+    }));
+    await emit(
+      handlers,
+      channel,
+      evt({
+        type: "actions.requested",
+        sessionId: "s1",
+        data: {
+          calls,
+          parentCallId: step.parentCallId,
+        },
+      }),
+      ctx
+    );
+    return;
+  }
+  if (step.type === "action.result") {
+    await emit(
+      handlers,
+      channel,
+      evt({
+        type: "action.result",
+        sessionId: "s1",
+        data: {
+          callId: step.id ?? "c0",
+          toolName: step.tool,
+          output: step.output,
+          isError: step.isError === true,
+          stubbed: step.stubbed,
+          parentCallId: step.parentCallId,
+        },
+      }),
+      ctx
+    );
+    return;
+  }
+  if (step.type === "turn.completed") {
+    await emit(
+      handlers,
+      channel,
+      evt({
+        type: "turn.completed",
+        sessionId: "s1",
+        data: { result: step.result ?? "" },
+      }),
+      ctx
+    );
+    return;
+  }
+  if (step.type === "message.completed") {
+    await emit(
+      handlers,
+      channel,
+      evt({
+        type: "message.completed",
+        sessionId: "s1",
+        data: {
+          text: step.text,
+          finishReason: step.finishReason ?? "stop",
+        },
+      }),
+      ctx
+    );
+    return;
+  }
+  await emit(
+    handlers,
+    channel,
+    evt({
+      type: "turn.failed",
+      sessionId: "s1",
+      data: { message: step.message ?? "failed" },
+    }),
+    ctx
+  );
+}
+
+export function uniqueTexts(sink: RecordingSlackSink): string[] {
+  return [...new Set(sink.texts())];
+}
+
+export function chipStatuses(sink: RecordingSlackSink): string[] {
+  return sink.calls
+    .filter(call => call.method === "assistant.threads.setStatus")
+    .map(call =>
+      typeof call.args.status === "string" ? call.args.status : ""
+    );
+}
+
+export function countMethod(sink: RecordingSlackSink, method: string): number {
+  return sink.methods().filter(name => name === method).length;
+}
+
+export function firstCall(
+  sink: RecordingSlackSink,
+  method: string
+): SlackSinkCall | undefined {
+  return sink.calls.find(call => call.method === method);
+}
+
+/** Surviving Slack messages: posts plus streams minus deletes. */
+export function netPosted(sink: RecordingSlackSink): number {
+  return (
+    countMethod(sink, "chat.postMessage") +
+    countMethod(sink, "chat.startStream") -
+    countMethod(sink, "chat.delete")
+  );
+}
+
+/** Slack methods recorded before the first ack or thinking write. */
+export function methodsBeforePaint(sink: RecordingSlackSink): string[] {
+  const i = sink.calls.findIndex(
+    call =>
+      call.method === "reactions.add" ||
+      call.method === "assistant.threads.setStatus"
+  );
+  if (i === -1) {
+    return sink.methods();
+  }
+  return sink.calls.slice(0, i).map(call => call.method);
+}
+
+export function outputField(output: unknown, key: string): unknown {
+  if (typeof output !== "object" || output === null) {
+    return undefined;
+  }
+  return (output as Record<string, unknown>)[key];
+}
+
+export type SlackToolOutputs = {
+  react: unknown;
+  status: unknown;
+  post: unknown;
+};
+
+function toolCtx(sessionId: string): ToolContext {
+  return { session: { id: sessionId } } as ToolContext;
+}
+
+export async function runSlackTools(
+  sessionId = "s1"
+): Promise<{ outputs: SlackToolOutputs; stalled: boolean }> {
+  begin(sessionId);
+  const ctx = toolCtx(sessionId);
+  try {
+    const finished = Promise.all([
+      Promise.resolve(
+        react.execute!({ emoji: "white_check_mark" } as never, ctx)
+      ),
+      Promise.resolve(
+        setStatus.execute!({ status: "Reading a.ts" } as never, ctx)
+      ),
+      Promise.resolve(
+        postUpdate.execute!({ message: "Looking." } as never, ctx)
+      ),
+    ]).then(([reactOut, statusOut, postOut]) => ({
+      react: reactOut,
+      status: statusOut,
+      post: postOut,
+    }));
+    return await Promise.race([
+      finished.then(outputs => ({ outputs, stalled: false as const })),
+      new Promise<{ outputs: SlackToolOutputs; stalled: true }>(resolve => {
+        setTimeout(() => {
+          resolve({
+            outputs: { react: undefined, status: undefined, post: undefined },
+            stalled: true,
+          });
+        }, TOOL_STALL_MS);
+      }),
+    ]);
+  } finally {
+    end(sessionId);
+  }
+}
+
+function evalHost(): HostContext {
+  return {
+    mcp: {
+      names: () => [],
+      listTools: async () => [],
+      callTool: async () => ({ content: [] }),
+    },
+    github: {
+      getOctokit: async () => {
+        throw new Error("github client not configured in eval");
+      },
+    },
+    slack: {
+      getClient: async () => {
+        throw new Error("slack client not configured in eval");
+      },
+    },
+    kv: {
+      get: async () => undefined,
+      put: async () => {},
+      delete: async () => {},
+    },
+    files: createHostFilesApi({
+      agentName: "slack-eval",
+      stateRoot: join(tmpdir(), "slack-eval-files"),
+      env: {},
+    }),
+    otel: createHostOtelApi({ agentName: "slack-eval" }),
+  };
+}
+
+function evalArtifacts(): ArtifactsApi {
+  return {
+    tag: async () => {
+      throw new Error("artifacts not configured in eval");
+    },
+    list: async () => [],
+  };
+}
+
+function evalSession(coalesced?: boolean): ChannelSession {
+  return {
+    id: "ses_eval",
+    continuationToken: null,
+    isNew: true,
+    coalesced,
+    info: {
+      id: "ses_eval",
+      channelId: "slack",
+      mode: "chat",
+      auth: null,
+    },
+    getEventStream: async () => new ReadableStream(),
+    waitForCompletion: async () => ({ status: "finished" }),
+    setContinuationToken: async () => undefined,
+  };
+}
+
+export type InboundPlay = {
+  sink: RecordingSlackSink;
+  methodsBeforeSend: string[];
+  state: SlackChannelState | undefined;
+  refreshState: SlackWakeState | undefined;
+};
+
+export type InboundPlayArgs = {
+  ack: string;
+  channelId?: string;
+  ts?: string;
+  threadTs?: string;
+  blockExternals?: boolean;
+  hold?: boolean | readonly string[];
+  hasContinuationSession?: () => Promise<boolean>;
+  getContinuationLastBotMessageTs?: () => Promise<string | undefined>;
+  placeholder?: string;
+  text?: string;
+  /** Hosted admission captured `send()`. */
+  deferred?: boolean;
+  /** Busy follow-up; the running turn keeps going. */
+  coalesced?: boolean;
+  sink?: RecordingSlackSink;
+};
+
+export function startInbound(args: InboundPlayArgs): {
+  sink: RecordingSlackSink;
+  sent: () => boolean;
+  whenSent: Promise<void>;
+  done: Promise<InboundPlay>;
+} {
+  if (args.blockExternals === true) {
+    resetExternalSenderCache();
+  }
+  if (args.sink === undefined) {
+    resetPlaceholderSlots();
+  }
+  const sink = args.sink ?? createRecordingSlackSink({ hold: args.hold });
+  let methodsBeforeSend: string[] = [];
+  let capturedState: SlackChannelState | undefined;
+  let capturedRefresh: SlackWakeState | undefined;
+  let sent = false;
+  let resolveSent: () => void = () => undefined;
+  const whenSent = new Promise<void>(resolve => {
+    resolveSent = resolve;
+  });
+  const send: SendMessageFn = async (_message, options) => {
+    sent = true;
+    methodsBeforeSend = sink.methods();
+    capturedState = options?.state as SlackChannelState | undefined;
+    capturedRefresh = options?.refreshState as SlackWakeState | undefined;
+    resolveSent();
+    if (args.deferred === true) {
+      return createDeferredChannelSession();
+    }
+    return evalSession(args.coalesced);
+  };
+  const channelId = args.channelId ?? "D1";
+  const ts = args.ts ?? "1.0";
+  const text = args.text ?? "hi";
+  const done = dispatchInboundMessage({
+    kind: isSlackImChannel(channelId) ? "direct_message" : "app_mention",
+    message: {
+      text,
+      body: text,
+      markdown: text,
+      ts,
+      threadTs: args.threadTs ?? ts,
+      channelId,
+      teamId: "T1",
+      author: { userId: "U1", isBot: false },
+    },
+    handler: async () => ({
+      auth: {
+        authenticator: "eval",
+        principalId: "U1",
+        principalType: "user",
+      },
+    }),
+    send,
+    credentials: { botToken: sink.transport },
+    artifacts: evalArtifacts(),
+    host: evalHost(),
+    ack: args.ack,
+    placeholder: args.placeholder,
+    propagateHandlerErrors: args.deferred === true,
+    blockExternals: args.blockExternals === true,
+    hasContinuationSession: args.hasContinuationSession ?? (async () => false),
+    getContinuationLastBotMessageTs: args.getContinuationLastBotMessageTs,
+  }).then(() => ({
+    sink,
+    methodsBeforeSend,
+    state: capturedState,
+    refreshState: capturedRefresh,
+  }));
+  return { sink, sent: () => sent, whenSent, done };
+}
+
+export async function playInbound(args: InboundPlayArgs): Promise<InboundPlay> {
+  return startInbound(args).done;
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise(resolve => {
+    setTimeout(resolve, ms);
+  });
+}
+
+export type PlaceholderPlay = {
+  sink: RecordingSlackSink;
+  startedAt: number;
+  first: ReturnType<RecordingSlackSink["firstReadable"]>;
+  last: ReturnType<RecordingSlackSink["lastReadable"]>;
+};
+
+export async function playPlaceholder(args: {
+  placeholder?: string;
+  agentDelayMs?: number;
+  waitForPlaceholder?: boolean;
+  text?: string;
+  steps: readonly PlayStep[];
+}): Promise<PlaceholderPlay> {
+  resetPlaceholderSlots();
+  const startedAt = Date.now();
+  const inbound = startInbound({
+    ack: JENNY_ACK,
+    channelId: "D1",
+    placeholder: args.placeholder,
+    text: args.text,
+  });
+  await inbound.whenSent;
+  if (args.waitForPlaceholder === true) {
+    await inbound.done;
+  }
+  if ((args.agentDelayMs ?? 0) > 0) {
+    await delay(args.agentDelayMs ?? 0);
+  }
+  await playAgentic(args.steps, {
+    sink: inbound.sink,
+    channelId: "D1",
+    startPainted: true,
+    placeholder: args.placeholder,
+  });
+  if (args.waitForPlaceholder !== true) {
+    await inbound.done;
+  }
+  return {
+    sink: inbound.sink,
+    startedAt,
+    first: inbound.sink.firstReadable(),
+    last: inbound.sink.lastReadable(),
+  };
+}
+
+/** Hosted admission skips the post. The execution turn posts, then overlays. */
+export async function playHostedPlaceholder(args: {
+  placeholder?: string;
+  text?: string;
+  steps: readonly PlayStep[];
+}): Promise<
+  PlaceholderPlay & {
+    admittedPosts: number;
+  }
+> {
+  resetPlaceholderSlots();
+  const startedAt = Date.now();
+  const inbound = startInbound({
+    ack: JENNY_ACK,
+    channelId: "D1",
+    placeholder: args.placeholder,
+    text: args.text,
+    deferred: true,
+  });
+  const admitted = await inbound.done;
+  const admittedPosts =
+    countMethod(inbound.sink, "chat.postMessage") +
+    countMethod(inbound.sink, "chat.startStream");
+  resetPlaceholderSlots();
+  const steps =
+    args.steps[0]?.type === "turn.started"
+      ? args.steps
+      : [{ type: "turn.started" as const }, ...args.steps];
+  await playAgentic(steps, {
+    sink: inbound.sink,
+    channelId: "D1",
+    startPainted: true,
+    placeholder: args.placeholder,
+  });
+  return {
+    sink: inbound.sink,
+    startedAt,
+    first: inbound.sink.firstReadable(),
+    last: inbound.sink.lastReadable(),
+    admittedPosts,
+  };
+}