@dxos/plugin-assistant 0.13.0 → 0.14.0

package/src/review/turn-review.ts

@@ -0,0 +1,213 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { AiSession } from '@dxos/assistant';
+import { type Message } from '@dxos/types';
+import { trim } from '@dxos/util';
+
+/** PostHog event emitted when the reviewer finds a turn the agent struggled through. */
+export const STRUGGLE_EVENT = 'agent_struggle';
+
+/** Small enough to be cheap per turn, strong enough to judge an agentic transcript. */
+export const REVIEW_MODEL = 'com.anthropic.model.claude-sonnet-5.default';
+
+/** Bounds the transcript the reviewer reads; the uploaded trajectory is not cut. */
+const MAX_TRANSCRIPT_CHARS = 120_000;
+const MAX_BLOCK_CHARS = 4_000;
+
+/**
+ * Why the agent struggled. Only the first four are reported: they are the ones fixable on our side
+ * (prompting or tooling); the model's own reasoning and an unclear user request are not.
+ */
+export const StruggleCause = Schema.Literals([
+  'system_prompt_missing',
+  'system_prompt_wrong',
+  'tool_missing',
+  'tool_faulty',
+  'model',
+  'user',
+  'none',
+]);
+export type StruggleCause = Schema.Schema.Type<typeof StruggleCause>;
+
+export const TurnVerdict = Schema.Struct({
+  struggled: Schema.Boolean.annotate({
+    description: 'Whether the agent visibly struggled in the latest turn.',
+  }),
+  cause: StruggleCause.annotate({
+    description: 'The root cause of the struggle, or "none".',
+  }),
+  severity: Schema.Literals(['low', 'medium', 'high']).annotate({
+    description: 'low: recovered quickly; medium: wasted several steps; high: failed the request.',
+  }),
+  summary: Schema.String.annotate({
+    description: 'One sentence naming what went wrong and what instruction or tool change would fix it.',
+  }),
+  tools: Schema.Array(Schema.String).annotate({
+    description: 'Names of the tools implicated, if any.',
+  }),
+});
+export type TurnVerdict = Schema.Schema.Type<typeof TurnVerdict>;
+
+export type CauseGroup = 'system_prompt' | 'tooling';
+
+/** The dashboard's top-level split; `undefined` for a cause that is not reported. */
+export const causeGroup = (cause: StruggleCause): CauseGroup | undefined => {
+  switch (cause) {
+    case 'system_prompt_missing':
+    case 'system_prompt_wrong':
+      return 'system_prompt';
+    case 'tool_missing':
+    case 'tool_faulty':
+      return 'tooling';
+    default:
+      return undefined;
+  }
+};
+
+export const isReportable = (verdict: TurnVerdict): boolean =>
+  verdict.struggled && causeGroup(verdict.cause) !== undefined;
+
+export const REVIEW_SYSTEM_PROMPT = trim`
+  You review transcripts of an AI agent working inside Composer, a collaborative workspace app.
+  Decide whether the agent struggled in the LATEST TURN (messages inside the <latest_turn> tags).
+  Earlier messages are context only.
+
+  Signs of struggling: repeated or looping tool calls, tool errors, calling tools with invalid
+  arguments, guessing at tool names or object ids, apologizing and retrying, giving up, asking the
+  user for information it should have been able to obtain, or giving an incomplete answer.
+
+  Classify the root cause:
+  - system_prompt_missing: the agent lacked instructions or context it needed (e.g. how a feature
+    works, which tool to use for a task, conventions of the app).
+  - system_prompt_wrong: instructions were present but misleading, contradictory, or outdated.
+  - tool_missing: no tool existed for an action the request clearly required.
+  - tool_faulty: a tool errored, returned unusable output, had a confusing schema or description.
+  - model: the tools and instructions were adequate; the model reasoned poorly.
+  - user: the request was ambiguous or impossible.
+  - none: the agent did not struggle.
+
+  Be conservative: a single quickly-recovered tool error is not a struggle. When unsure, answer
+  struggled=false with cause=none.
+`;
+
+const clip = (text: string, max: number): string =>
+  text.length <= max ? text : `${text.slice(0, max)}… [${text.length - max} chars truncated]`;
+
+const stringify = (value: unknown): string => (typeof value === 'string' ? value : JSON.stringify(value));
+
+/** One line per block, so the reviewer sees the shape of the turn without a tool result drowning it. */
+const formatMessage = (message: Message.Message): string => {
+  const lines = message.blocks.map((block) => {
+    switch (block._tag) {
+      case 'text':
+        return clip(block.text, MAX_BLOCK_CHARS);
+      case 'toolCall':
+        return `[tool call ${block.name}] ${clip(stringify(block.input), MAX_BLOCK_CHARS)}`;
+      case 'toolResult':
+        return block.error
+          ? `[tool error ${block.name}] ${clip(stringify(block.error), MAX_BLOCK_CHARS)}`
+          : `[tool result ${block.name}] ${clip(stringify(block.result), MAX_BLOCK_CHARS)}`;
+      default:
+        return `[${block._tag}]`;
+    }
+  });
+  return `${(message.sender.role ?? 'unknown').toUpperCase()}:\n${lines.join('\n')}`;
+};
+
+/**
+ * Splits the history at the turn that just ended: it starts at the last user message, which is what
+ * the agent was answering. Hooks fire per request, so everything after it is this turn's work.
+ */
+export const splitTurn = (history: readonly Message.Message[]) => {
+  const index = history.findLastIndex((message) => message.sender.role === 'user');
+  return index === -1
+    ? { context: [] as readonly Message.Message[], turn: history }
+    : { context: history.slice(0, index), turn: history.slice(index) };
+};
+
+/** The reviewer's prompt; context is trimmed from the front when the transcript runs long. */
+export const formatReviewPrompt = (history: readonly Message.Message[]): string => {
+  const { context, turn } = splitTurn(history);
+  const turnText = turn.map(formatMessage).join('\n\n');
+  const budget = Math.max(0, MAX_TRANSCRIPT_CHARS - turnText.length);
+  const contextText = context.map(formatMessage).join('\n\n');
+  const clippedContext =
+    contextText.length > budget ? `…${contextText.slice(contextText.length - budget)}` : contextText;
+  return [clippedContext, '<latest_turn>', clip(turnText, MAX_TRANSCRIPT_CHARS), '</latest_turn>']
+    .filter((part) => part.length > 0)
+    .join('\n\n');
+};
+
+export const countToolCalls = (messages: readonly Message.Message[]) => {
+  let calls = 0;
+  let errors = 0;
+  for (const message of messages) {
+    for (const block of message.blocks) {
+      if (block._tag === 'toolCall') {
+        calls++;
+      } else if (block._tag === 'toolResult' && block.error) {
+        errors++;
+      }
+    }
+  }
+  return { calls, errors };
+};
+
+export type TrajectoryHeader = {
+  sessionId: string;
+  model?: string;
+  codeMode: boolean;
+  skills: readonly string[];
+  verdict: TurnVerdict;
+};
+
+/** A header line then one line per message, in the span serialization the LLM analytics traces use. */
+export const toTrajectoryNdjson = (header: TrajectoryHeader, history: readonly Message.Message[]): string =>
+  [
+    JSON.stringify({ type: 'header', ...header }),
+    ...history.map((message) =>
+      JSON.stringify({ type: 'message', created: message.created, ...(AiSession.serializeMessage(message) as object) }),
+    ),
+  ].join('\n');
+
+export type StruggleEventInput = TrajectoryHeader & {
+  history: readonly Message.Message[];
+  trajectoryKey?: string;
+};
+
+/**
+ * Flat properties so every one is a PostHog breakdown; lists are comma-joined for the same reason.
+ * `$ai_session_id` joins the event to the conversation's LLM analytics traces.
+ */
+export const toStruggleEventProperties = ({
+  sessionId,
+  model,
+  codeMode,
+  skills,
+  verdict,
+  history,
+  trajectoryKey,
+}: StruggleEventInput): Record<string, unknown> => {
+  const { turn } = splitTurn(history);
+  const { calls, errors } = countToolCalls(turn);
+  return {
+    $ai_session_id: sessionId,
+    model,
+    code_mode: codeMode,
+    skills: skills.join(','),
+    cause: verdict.cause,
+    cause_group: causeGroup(verdict.cause),
+    severity: verdict.severity,
+    summary: verdict.summary,
+    tools: verdict.tools.join(','),
+    turn_message_count: turn.length,
+    turn_tool_call_count: calls,
+    turn_tool_error_count: errors,
+    review_model: REVIEW_MODEL,
+    trajectory_key: trajectoryKey,
+  };
+};

package/src/skills/index.ts

@@ -4,3 +4,4 @@
 
 export * from './assistant/index.ts';
 export * from './plugin-manager/index.ts';
+export * from './turn-review/index.ts';

package/src/skills/turn-review/TurnReviewSkill.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+// @import-as-namespace
+
+import * as Schema from 'effect/Schema';
+
+import { AiService } from '@dxos/ai';
+import * as Capability from '@dxos/app-framework/Capability';
+import * as Harness from '@dxos/assistant/Harness';
+import * as Operation from '@dxos/compute/Operation';
+import * as Skill from '@dxos/compute/Skill';
+import { Database, DXN, Ref } from '@dxos/echo';
+
+/**
+ * The skill's end-request hook: asks a small model whether the agent struggled in the turn that just
+ * ended because of its prompting or tooling, and if so reports the trajectory. Defined beside the
+ * skill rather than in `#types` because the harness service it needs carries the session runtime,
+ * which the UI must not load.
+ */
+export const AnalyzeTurn = Operation.make({
+  meta: {
+    key: DXN.make('org.dxos.operation.assistant.analyzeTurn'),
+    name: 'Analyze Agent Turn',
+    icon: 'ph--magnifying-glass--regular',
+  },
+  services: [Harness.HarnessService, Database.Service, AiService.AiService, Capability.Service],
+  input: Schema.Struct({}),
+  output: Schema.Void,
+});
+
+export const key = 'org.dxos.skill.assistant.turnReview';
+
+/**
+ * Bound to a chat by the chat processor while the user has opted into struggle reports. It adds no
+ * tools or instructions; it exists only for its hook, which runs in the background so the user never
+ * waits on the review.
+ */
+export const make = (): Skill.Skill =>
+  Skill.make({
+    key,
+    name: 'Turn Review',
+    description: 'Reports agent turns that struggled because of their instructions or tools (opt-in telemetry).',
+    hooks: [{ spec: { _tag: 'end-request' }, async: true, function: Ref.make(Operation.serialize(AnalyzeTurn)) }],
+  });

package/src/skills/turn-review/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * as TurnReviewSkill from './TurnReviewSkill.ts';

package/src/translations.ts

@@ -192,6 +192,8 @@ export const translations: Theme.Resource[] = [
         'options.mcp.title': 'MCP',
         'options.chat-model.title': 'Models',
         'options.environment.title': 'Environment',
+        'options.agent.title': 'Agent',
+        'agent-locked.message': 'The agent is chosen before the first message. Start a new chat to use another.',
         'remove-object.label': 'Remove object',
 
         'chat-view.title': 'View',

package/src/types/AssistantCapabilities.ts

@@ -9,7 +9,7 @@ import type * as Atom from 'effect/reactivity/Atom';
 import * as Schema from 'effect/Schema';
 import * as Struct from 'effect/Struct';
 
-import type { MakeTurnProducer } from '@dxos/agent-runtime';
+import type { AgentProcessDefinition, MakeTurnProducer } from '@dxos/agent-runtime';
 import * as Capability from '@dxos/app-framework/Capability';
 import type { AiContext } from '@dxos/assistant';
 import type * as Chat from '@dxos/assistant/Chat';
@@ -107,6 +107,13 @@ export type Agent = {
 
 export const Agent = Capability.make<Agent>()(`${meta.profile.key}.capability.agent`);
 
+/**
+ * A durable process a chat can run on in place of the assistant's own agent process, named by its
+ * key in `chat.session.process`. The agent service spawns it for such a chat; a chat that names
+ * none runs on the assistant's.
+ */
+export const AgentProcess = Capability.make<AgentProcessDefinition>()(`${meta.profile.key}.capability.agentProcess`);
+
 /** Context a chat receives when it runs against a subject object. */
 export type SubjectBindings = AiContext.BindingProps & {
   /** Applied to `chat.instructions` only when the chat has none; reaches the model via the system prompt. */

package/src/types/Settings.ts

@@ -81,6 +81,13 @@ export const Settings = Schema.Struct({
         'Agent that one-click "Assign to agent" hands tasks to, by id: composer, or another installed agent such as claude-code. Falls back to Composer where that agent is unavailable.',
     }),
   ),
+  reportStruggles: Schema.optional(
+    Schema.Boolean.annotate({
+      title: 'Report agent struggles',
+      description:
+        'After each agent turn, a small model reviews the conversation. If the agent struggled because of its instructions or tools, the conversation (including content from your space) is sent to the DXOS team to improve Composer. Requires telemetry to be enabled.',
+    }),
+  ),
   tracePanelDebug: Schema.optional(
     Schema.Boolean.annotate({
       title: 'Trace panel debug',

package/src/util/agent-options.test.ts

@@ -0,0 +1,54 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { SessionConfig } from '@dxos/ai';
+
+import { NOT_INSTALLED, agentOptions } from './agent-options.ts';
+
+const composer = { id: SessionConfig.COMPOSER_HARNESS, label: 'Composer', icon: 'ph--sparkle--regular' };
+const local = { id: 'claude-code', label: 'Claude Code', icon: 'px--anthropic--regular' };
+const cloud = { id: 'claude-code-edge', label: 'Claude Code (cloud)', icon: 'px--anthropic--regular' };
+
+describe('agentOptions', () => {
+  test('lists Composer first, then the other agents in registration order', ({ expect }) => {
+    const options = agentOptions({
+      agents: [local, composer, cloud],
+      availability: [{ available: true }, { available: true }, { available: true }],
+      current: SessionConfig.COMPOSER_HARNESS,
+    });
+    expect(options.map(({ value }) => value)).toEqual([composer.id, local.id, cloud.id]);
+    expect(options.every(({ disabled }) => !disabled)).toBe(true);
+  });
+
+  test('disables an unavailable agent with its reason', ({ expect }) => {
+    const options = agentOptions({
+      agents: [composer, local, cloud],
+      availability: [
+        { available: true },
+        { available: false, reason: 'needs the Composer desktop app' },
+        { available: true },
+      ],
+      current: SessionConfig.COMPOSER_HARNESS,
+    });
+    expect(options[1]).toEqual({
+      value: local.id,
+      label: local.label,
+      icon: local.icon,
+      disabled: true,
+      description: 'needs the Composer desktop app',
+    });
+    expect(options[2].disabled).toBe(false);
+  });
+
+  test('keeps an agent the chat names that this device does not register', ({ expect }) => {
+    const options = agentOptions({
+      agents: [composer],
+      availability: [{ available: true }],
+      current: 'claude-code-edge',
+    });
+    expect(options.at(-1)).toMatchObject({ value: 'claude-code-edge', disabled: true, description: NOT_INSTALLED });
+  });
+});

package/src/util/agent-options.ts

@@ -0,0 +1,60 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { SessionConfig } from '@dxos/ai';
+
+import type * as AssistantCapabilities from '../types/AssistantCapabilities.ts';
+
+/** One row of the chat's agent picker. */
+export type AgentOption = {
+  value: string;
+  label: string;
+  icon: string;
+  disabled: boolean;
+  /** Why the row cannot be picked, shown under its name. */
+  description?: string;
+};
+
+export type AgentOptionsProps = {
+  agents: readonly Pick<AssistantCapabilities.Agent, 'id' | 'label' | 'icon'>[];
+  /** Availability of each agent, by index into `agents`. */
+  availability: readonly AssistantCapabilities.AgentAvailability[];
+  /** The agent the chat names (`chat.session.harness`). */
+  current: string;
+};
+
+/** Reason shown for an agent a chat names that no plugin registers on this device. */
+export const NOT_INSTALLED = 'not installed here';
+
+/**
+ * The rows of the agent picker: Composer's own loop first, as the default, then the rest in
+ * registration order, with an unavailable agent disabled and its reason as the description. An
+ * agent the chat already names but this device does not register is kept, so the picker still
+ * shows what the chat runs on.
+ */
+export const agentOptions = ({ agents, availability, current }: AgentOptionsProps): AgentOption[] => {
+  const options = agents.map((agent, index): AgentOption => {
+    const status = availability[index];
+    return status && !status.available
+      ? { value: agent.id, label: agent.label, icon: agent.icon, disabled: true, description: status.reason }
+      : { value: agent.id, label: agent.label, icon: agent.icon, disabled: false };
+  });
+
+  const sorted = [
+    ...options.filter(({ value }) => value === SessionConfig.COMPOSER_HARNESS),
+    ...options.filter(({ value }) => value !== SessionConfig.COMPOSER_HARNESS),
+  ];
+
+  if (!sorted.some(({ value }) => value === current)) {
+    sorted.push({
+      value: current,
+      label: current,
+      icon: 'ph--robot--regular',
+      disabled: true,
+      description: NOT_INSTALLED,
+    });
+  }
+
+  return sorted;
+};

package/src/util/default-skills.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type AiContext } from '@dxos/assistant';
+import * as AlarmSkill from '@dxos/assistant-toolkit/AlarmSkill';
+import * as ChatContextSkill from '@dxos/assistant-toolkit/ChatContextSkill';
+import * as PlanningSkill from '@dxos/assistant-toolkit/PlanningSkill';
+import * as SkillManagerSkill from '@dxos/assistant-toolkit/SkillManagerSkill';
+import type * as Chat from '@dxos/assistant/Chat';
+import * as Skill from '@dxos/compute/Skill';
+import { Ref } from '@dxos/echo';
+import * as DatabaseSkill from '@dxos/plugin-space/DatabaseSkill';
+
+import { AssistantSkill, PluginManagerSkill } from '#skills';
+
+/** Only an extensible host contributes the plugin-manager skill; elsewhere its tools cannot run. */
+export const contributesPluginManager = (contributed: readonly Pick<Skill.Definition, 'key'>[]): boolean =>
+  contributed.some(({ key }) => key === PluginManagerSkill.key);
+
+const defaultChatSkills = (pluginManager: boolean): Ref.Ref<Skill.Skill>[] =>
+  [
+    AssistantSkill,
+    DatabaseSkill,
+    ChatContextSkill,
+    SkillManagerSkill,
+    AlarmSkill,
+    PlanningSkill,
+    ...(pluginManager ? [PluginManagerSkill] : []),
+  ].map(({ key }) => Ref.fromURI(Skill.registryURI(key)));
+
+export const bindChatDefaults = (
+  binder: AiContext.Binder,
+  { chat, pluginManager }: { chat: Chat.Chat; pluginManager: boolean },
+): Promise<void> => binder.bind({ skills: defaultChatSkills(pluginManager), objects: [Ref.make(chat)] });