@cursor/bdk 0.2.2 → 0.2.4

package/src/extensions/slack-communication/evals/harness.ts

@@ -1,705 +0,0 @@
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { begin, end } from "../../../channels/slack/agentic-posted.js";
-import { buildDefaultEvents } from "../../../channels/slack/defaults.js";
-import {
-  dispatchInboundMessage,
-  emptySlackChannelState,
-  type SlackWakeState,
-} from "../../../channels/slack/dispatch.js";
-import { resetExternalSenderCache } from "../../../channels/slack/external-policy.js";
-import { resetPlaceholderSlots } from "../../../channels/slack/placeholder.js";
-import {
-  createRecordingSlackSink,
-  type RecordingSlackSink,
-  type SlackSinkCall,
-} from "../../../channels/slack/recording-sink.js";
-import type { SlackChannelState, SlackReplyFooter } from "../../../channels/slack/types.js";
-import { createDeferredChannelSession } from "../../../internal/deferred-channel-session.js";
-import { createHostFilesApi } from "../../../internal/host-files.js";
-import { createHostOtelApi } from "../../../internal/host-otel.js";
-import { isSlackImChannel } from "../../../internal/slack-thread-key.js";
-import type {
-  ActionCall,
-  ArtifactsApi,
-  ChannelEventContext,
-  ChannelEventHandlers,
-  ChannelSession,
-  HostContext,
-  SendMessageFn,
-  SessionContext,
-  SessionEvent,
-  ToolContext,
-} from "../../../types.js";
-import postUpdate from "../tools/post-update.js";
-import react from "../tools/react.js";
-import setStatus from "../tools/set-status.js";
-
-/** Jenny's configured ack. Slack renders `paw_prints` as :feet:. */
-export const JENNY_ACK: string = "paw_prints";
-
-const TOOL_STALL_MS = 100;
-
-export type PlayStep =
-  | { type: "turn.started" }
-  | { type: "reasoning.appended"; text: string; parentCallId?: string }
-  | {
-      type: "actions.requested";
-      calls: Array<{
-        tool: string;
-        args?: unknown;
-        id?: string;
-        parentCallId?: string;
-      }>;
-      parentCallId?: string;
-    }
-  | {
-      type: "action.result";
-      tool: string;
-      output?: unknown;
-      isError?: boolean;
-      stubbed?: true;
-      id?: string;
-      parentCallId?: string;
-    }
-  | { type: "turn.completed"; result?: string }
-  | {
-      type: "message.completed";
-      text: string;
-      finishReason?: "stop" | "tool_call";
-    }
-  | { type: "turn.failed"; message?: string };
-
-let eventIndex = 0;
-
-function evt<T extends SessionEvent["type"]>(
-  partial: Omit<Extract<SessionEvent, { type: T }>, "index" | "at"> &
-    Partial<Pick<SessionEvent, "index" | "at">>,
-): Extract<SessionEvent, { type: T }> {
-  return {
-    index: eventIndex++,
-    at: "2026-01-01T00:00:00.000Z",
-    ...partial,
-  } as Extract<SessionEvent, { type: T }>;
-}
-
-function sessionOf(dryRun: boolean | undefined): SessionContext {
-  return {
-    session: {
-      id: "s1",
-      channelId: "slack",
-      mode: "chat",
-      auth: null,
-      dryRun,
-    },
-    host: {} as HostContext,
-    artifacts: {} as never,
-  };
-}
-
-function channelOf(state: SlackChannelState): ChannelEventContext<SlackChannelState> {
-  return {
-    channelId: "slack",
-    state,
-    continuationToken: null,
-    setContinuationToken: async () => {},
-  };
-}
-
-async function emit(
-  handlers: ChannelEventHandlers<SlackChannelState>,
-  channel: ChannelEventContext<SlackChannelState>,
-  event: SessionEvent,
-  ctx: SessionContext,
-): Promise<void> {
-  const handler = handlers[event.type];
-  if (handler === undefined) {
-    return;
-  }
-  await handler(event as never, channel, ctx);
-}
-
-export function signal(
-  tool: string,
-  output: unknown,
-  options: { id?: string; args?: unknown } = {},
-): PlayStep[] {
-  const id = options.id ?? "s0";
-  return [
-    {
-      type: "actions.requested",
-      calls: [{ tool, args: options.args ?? output, id }],
-    },
-    {
-      type: "action.result",
-      tool,
-      id,
-      output,
-    },
-  ];
-}
-
-export function publish(
-  message: string,
-  options: { id?: string; tool?: string; ts?: string } = {},
-): PlayStep[] {
-  const id = options.id ?? "u0";
-  const tool = options.tool ?? "post-update";
-  const output: { message: string; ts?: string } = { message };
-  output.ts = options.ts;
-  return [
-    {
-      type: "actions.requested",
-      calls: [{ tool, args: { message }, id }],
-    },
-    {
-      type: "action.result",
-      tool,
-      id,
-      output,
-    },
-  ];
-}
-
-export async function playAgentic(
-  steps: readonly PlayStep[],
-  options: {
-    dryRun?: boolean;
-    ack?: string;
-    channelId?: string;
-    startPainted?: boolean;
-    placeholder?: string;
-    sink?: RecordingSlackSink;
-    footer?: SlackReplyFooter;
-  } = {},
-): Promise<RecordingSlackSink> {
-  eventIndex = 0;
-  const sink = options.sink ?? createRecordingSlackSink();
-  if (options.sink === undefined) {
-    resetPlaceholderSlots();
-  }
-  const handlers = buildDefaultEvents({
-    reply: {
-      mode: "agentic",
-      ack: options.ack,
-      placeholder: options.placeholder,
-      footer: options.footer,
-    },
-    credentials: { botToken: sink.transport },
-  });
-  const ctx = sessionOf(options.dryRun);
-  const channel = channelOf(
-    emptySlackChannelState({
-      channelId: options.channelId ?? "C1",
-      threadTs: "1.0",
-      teamId: "T1",
-      triggeringUserId: "U1",
-      startPainted: options.startPainted === true,
-    }),
-  );
-  for (const step of steps) {
-    await playStep(handlers, channel, step, ctx);
-  }
-  return sink;
-}
-
-/** Channel Slack I/O is held. The next tool batch must still finish. */
-export async function playHeldWrites(): Promise<{
-  stalled: boolean;
-  sink: RecordingSlackSink;
-}> {
-  eventIndex = 0;
-  const sink = createRecordingSlackSink({ hold: true });
-  const handlers = buildDefaultEvents({
-    reply: { mode: "agentic" },
-    credentials: { botToken: sink.transport },
-  });
-  const ctx = sessionOf(undefined);
-  const channel = channelOf(
-    emptySlackChannelState({
-      channelId: "C1",
-      threadTs: "1.0",
-      teamId: "T1",
-      triggeringUserId: "U1",
-    }),
-  );
-  const writes = Promise.all([
-    playStep(
-      handlers,
-      channel,
-      {
-        type: "action.result",
-        tool: "react",
-        id: "r0",
-        output: { emoji: "white_check_mark" },
-      },
-      ctx,
-    ),
-    playStep(
-      handlers,
-      channel,
-      {
-        type: "action.result",
-        tool: "post-update",
-        id: "u0",
-        output: { message: "Looking." },
-      },
-      ctx,
-    ),
-  ]);
-  const tools = await runSlackTools("s-next");
-  sink.release();
-  await writes;
-  return { stalled: tools.stalled, sink };
-}
-
-async function playStep(
-  handlers: ChannelEventHandlers<SlackChannelState>,
-  channel: ChannelEventContext<SlackChannelState>,
-  step: PlayStep,
-  ctx: SessionContext,
-): Promise<void> {
-  if (step.type === "turn.started") {
-    await emit(handlers, channel, evt({ type: "turn.started", sessionId: "s1", data: {} }), ctx);
-    return;
-  }
-  if (step.type === "reasoning.appended") {
-    await emit(
-      handlers,
-      channel,
-      evt({
-        type: "reasoning.appended",
-        sessionId: "s1",
-        data: {
-          delta: "",
-          text: step.text,
-          parentCallId: step.parentCallId,
-        },
-      }),
-      ctx,
-    );
-    return;
-  }
-  if (step.type === "actions.requested") {
-    const calls: ActionCall[] = step.calls.map((call, i) => ({
-      callId: call.id ?? `c${i}`,
-      toolName: call.tool,
-      args: call.args,
-      parentCallId: call.parentCallId,
-    }));
-    await emit(
-      handlers,
-      channel,
-      evt({
-        type: "actions.requested",
-        sessionId: "s1",
-        data: {
-          calls,
-          parentCallId: step.parentCallId,
-        },
-      }),
-      ctx,
-    );
-    return;
-  }
-  if (step.type === "action.result") {
-    await emit(
-      handlers,
-      channel,
-      evt({
-        type: "action.result",
-        sessionId: "s1",
-        data: {
-          callId: step.id ?? "c0",
-          toolName: step.tool,
-          output: step.output,
-          isError: step.isError === true,
-          stubbed: step.stubbed,
-          parentCallId: step.parentCallId,
-        },
-      }),
-      ctx,
-    );
-    return;
-  }
-  if (step.type === "turn.completed") {
-    await emit(
-      handlers,
-      channel,
-      evt({
-        type: "turn.completed",
-        sessionId: "s1",
-        data: { result: step.result ?? "" },
-      }),
-      ctx,
-    );
-    return;
-  }
-  if (step.type === "message.completed") {
-    await emit(
-      handlers,
-      channel,
-      evt({
-        type: "message.completed",
-        sessionId: "s1",
-        data: {
-          text: step.text,
-          finishReason: step.finishReason ?? "stop",
-        },
-      }),
-      ctx,
-    );
-    return;
-  }
-  await emit(
-    handlers,
-    channel,
-    evt({
-      type: "turn.failed",
-      sessionId: "s1",
-      data: { message: step.message ?? "failed" },
-    }),
-    ctx,
-  );
-}
-
-export function uniqueTexts(sink: RecordingSlackSink): string[] {
-  return [...new Set(sink.texts())];
-}
-
-export function chipStatuses(sink: RecordingSlackSink): string[] {
-  return sink.calls
-    .filter(call => call.method === "assistant.threads.setStatus")
-    .map(call => (typeof call.args.status === "string" ? call.args.status : ""));
-}
-
-export function countMethod(sink: RecordingSlackSink, method: string): number {
-  return sink.methods().filter(name => name === method).length;
-}
-
-export function firstCall(sink: RecordingSlackSink, method: string): SlackSinkCall | undefined {
-  return sink.calls.find(call => call.method === method);
-}
-
-/** Surviving Slack messages: posts plus streams minus deletes. */
-export function netPosted(sink: RecordingSlackSink): number {
-  return (
-    countMethod(sink, "chat.postMessage") +
-    countMethod(sink, "chat.startStream") -
-    countMethod(sink, "chat.delete")
-  );
-}
-
-/** Slack methods recorded before the first ack or thinking write. */
-export function methodsBeforePaint(sink: RecordingSlackSink): string[] {
-  const i = sink.calls.findIndex(
-    call => call.method === "reactions.add" || call.method === "assistant.threads.setStatus",
-  );
-  if (i === -1) {
-    return sink.methods();
-  }
-  return sink.calls.slice(0, i).map(call => call.method);
-}
-
-export function outputField(output: unknown, key: string): unknown {
-  if (typeof output !== "object" || output === null) {
-    return undefined;
-  }
-  return (output as Record<string, unknown>)[key];
-}
-
-export type SlackToolOutputs = {
-  react: unknown;
-  status: unknown;
-  post: unknown;
-};
-
-function toolCtx(sessionId: string): ToolContext {
-  return { session: { id: sessionId } } as ToolContext;
-}
-
-export async function runSlackTools(
-  sessionId = "s1",
-): Promise<{ outputs: SlackToolOutputs; stalled: boolean }> {
-  begin(sessionId);
-  const ctx = toolCtx(sessionId);
-  try {
-    const finished = Promise.all([
-      Promise.resolve(react.execute!({ emoji: "white_check_mark" } as never, ctx)),
-      Promise.resolve(setStatus.execute!({ status: "Reading a.ts" } as never, ctx)),
-      Promise.resolve(postUpdate.execute!({ message: "Looking." } as never, ctx)),
-    ]).then(([reactOut, statusOut, postOut]) => ({
-      react: reactOut,
-      status: statusOut,
-      post: postOut,
-    }));
-    return await Promise.race([
-      finished.then(outputs => ({ outputs, stalled: false as const })),
-      new Promise<{ outputs: SlackToolOutputs; stalled: true }>(resolve => {
-        setTimeout(() => {
-          resolve({
-            outputs: { react: undefined, status: undefined, post: undefined },
-            stalled: true,
-          });
-        }, TOOL_STALL_MS);
-      }),
-    ]);
-  } finally {
-    end(sessionId);
-  }
-}
-
-function evalHost(): HostContext {
-  return {
-    mcp: {
-      names: () => [],
-      listTools: async () => [],
-      callTool: async () => ({ content: [] }),
-    },
-    github: {
-      getOctokit: async () => {
-        throw new Error("github client not configured in eval");
-      },
-    },
-    slack: {
-      getClient: async () => {
-        throw new Error("slack client not configured in eval");
-      },
-    },
-    kv: {
-      get: async () => undefined,
-      put: async () => {},
-      delete: async () => {},
-    },
-    files: createHostFilesApi({
-      agentName: "slack-eval",
-      stateRoot: join(tmpdir(), "slack-eval-files"),
-      env: {},
-    }),
-    otel: createHostOtelApi({ agentName: "slack-eval" }),
-  };
-}
-
-function evalArtifacts(): ArtifactsApi {
-  return {
-    tag: async () => {
-      throw new Error("artifacts not configured in eval");
-    },
-    list: async () => [],
-    page: async () => ({ records: [] }),
-  };
-}
-
-function evalSession(coalesced?: boolean): ChannelSession {
-  return {
-    id: "ses_eval",
-    continuationToken: null,
-    isNew: true,
-    coalesced,
-    info: {
-      id: "ses_eval",
-      channelId: "slack",
-      mode: "chat",
-      auth: null,
-    },
-    getEventStream: async () => new ReadableStream(),
-    waitForCompletion: async () => ({ status: "finished" }),
-    setContinuationToken: async () => undefined,
-  };
-}
-
-export type InboundPlay = {
-  sink: RecordingSlackSink;
-  methodsBeforeSend: string[];
-  state: SlackChannelState | undefined;
-  refreshState: SlackWakeState | undefined;
-};
-
-export type InboundPlayArgs = {
-  ack: string;
-  channelId?: string;
-  ts?: string;
-  threadTs?: string;
-  blockExternals?: boolean;
-  hold?: boolean | readonly string[];
-  hasContinuationSession?: () => Promise<boolean>;
-  getContinuationLastBotMessageTs?: () => Promise<string | undefined>;
-  placeholder?: string;
-  text?: string;
-  /** Hosted admission captured `send()`. */
-  deferred?: boolean;
-  /** Busy follow-up; the running turn keeps going. */
-  coalesced?: boolean;
-  sink?: RecordingSlackSink;
-};
-
-export function startInbound(args: InboundPlayArgs): {
-  sink: RecordingSlackSink;
-  sent: () => boolean;
-  whenSent: Promise<void>;
-  done: Promise<InboundPlay>;
-} {
-  if (args.blockExternals === true) {
-    resetExternalSenderCache();
-  }
-  if (args.sink === undefined) {
-    resetPlaceholderSlots();
-  }
-  const sink = args.sink ?? createRecordingSlackSink({ hold: args.hold });
-  let methodsBeforeSend: string[] = [];
-  let capturedState: SlackChannelState | undefined;
-  let capturedRefresh: SlackWakeState | undefined;
-  let sent = false;
-  let resolveSent: () => void = () => undefined;
-  const whenSent = new Promise<void>(resolve => {
-    resolveSent = resolve;
-  });
-  const send: SendMessageFn = async (_message, options) => {
-    sent = true;
-    methodsBeforeSend = sink.methods();
-    capturedState = options?.state as SlackChannelState | undefined;
-    capturedRefresh = options?.refreshState as SlackWakeState | undefined;
-    resolveSent();
-    if (args.deferred === true) {
-      return createDeferredChannelSession();
-    }
-    return evalSession(args.coalesced);
-  };
-  const channelId = args.channelId ?? "D1";
-  const ts = args.ts ?? "1.0";
-  const text = args.text ?? "hi";
-  const done = dispatchInboundMessage({
-    kind: isSlackImChannel(channelId) ? "direct_message" : "app_mention",
-    message: {
-      text,
-      body: text,
-      markdown: text,
-      ts,
-      threadTs: args.threadTs ?? ts,
-      channelId,
-      teamId: "T1",
-      author: { userId: "U1", isBot: false },
-    },
-    handler: async () => ({
-      auth: {
-        authenticator: "eval",
-        principalId: "U1",
-        principalType: "user",
-      },
-    }),
-    send,
-    credentials: { botToken: sink.transport },
-    artifacts: evalArtifacts(),
-    host: evalHost(),
-    ack: args.ack,
-    placeholder: args.placeholder,
-    propagateHandlerErrors: args.deferred === true,
-    blockExternals: args.blockExternals === true,
-    hasContinuationSession: args.hasContinuationSession ?? (async () => false),
-    getContinuationLastBotMessageTs: args.getContinuationLastBotMessageTs,
-  }).then(() => ({
-    sink,
-    methodsBeforeSend,
-    state: capturedState,
-    refreshState: capturedRefresh,
-  }));
-  return { sink, sent: () => sent, whenSent, done };
-}
-
-export async function playInbound(args: InboundPlayArgs): Promise<InboundPlay> {
-  return startInbound(args).done;
-}
-
-function delay(ms: number): Promise<void> {
-  return new Promise(resolve => {
-    setTimeout(resolve, ms);
-  });
-}
-
-export type PlaceholderPlay = {
-  sink: RecordingSlackSink;
-  startedAt: number;
-  first: ReturnType<RecordingSlackSink["firstReadable"]>;
-  last: ReturnType<RecordingSlackSink["lastReadable"]>;
-};
-
-export async function playPlaceholder(args: {
-  placeholder?: string;
-  agentDelayMs?: number;
-  waitForPlaceholder?: boolean;
-  text?: string;
-  steps: readonly PlayStep[];
-}): Promise<PlaceholderPlay> {
-  resetPlaceholderSlots();
-  const startedAt = Date.now();
-  const inbound = startInbound({
-    ack: JENNY_ACK,
-    channelId: "D1",
-    placeholder: args.placeholder,
-    text: args.text,
-  });
-  await inbound.whenSent;
-  if (args.waitForPlaceholder === true) {
-    await inbound.done;
-  }
-  if ((args.agentDelayMs ?? 0) > 0) {
-    await delay(args.agentDelayMs ?? 0);
-  }
-  await playAgentic(args.steps, {
-    sink: inbound.sink,
-    channelId: "D1",
-    startPainted: true,
-    placeholder: args.placeholder,
-  });
-  if (args.waitForPlaceholder !== true) {
-    await inbound.done;
-  }
-  return {
-    sink: inbound.sink,
-    startedAt,
-    first: inbound.sink.firstReadable(),
-    last: inbound.sink.lastReadable(),
-  };
-}
-
-/** Hosted admission skips the post. The execution turn posts, then overlays. */
-export async function playHostedPlaceholder(args: {
-  placeholder?: string;
-  text?: string;
-  steps: readonly PlayStep[];
-}): Promise<
-  PlaceholderPlay & {
-    admittedPosts: number;
-  }
-> {
-  resetPlaceholderSlots();
-  const startedAt = Date.now();
-  const inbound = startInbound({
-    ack: JENNY_ACK,
-    channelId: "D1",
-    placeholder: args.placeholder,
-    text: args.text,
-    deferred: true,
-  });
-  const admitted = await inbound.done;
-  const admittedPosts =
-    countMethod(inbound.sink, "chat.postMessage") + countMethod(inbound.sink, "chat.startStream");
-  resetPlaceholderSlots();
-  const steps =
-    args.steps[0]?.type === "turn.started"
-      ? args.steps
-      : [{ type: "turn.started" as const }, ...args.steps];
-  await playAgentic(steps, {
-    sink: inbound.sink,
-    channelId: "D1",
-    startPainted: true,
-    placeholder: args.placeholder,
-  });
-  return {
-    sink: inbound.sink,
-    startedAt,
-    first: inbound.sink.firstReadable(),
-    last: inbound.sink.lastReadable(),
-    admittedPosts,
-  };
-}