@vellumai/assistant 0.3.4 → 0.3.6

package/src/daemon/ipc-contract/computer-use.ts

@@ -0,0 +1,151 @@
+// Computer use, task routing, ride shotgun, and watch observation types.
+
+import type { UserMessageAttachment, IpcBlobRef } from './shared.js';
+
+// === Client → Server ===
+
+export interface CuSessionCreate {
+  type: 'cu_session_create';
+  sessionId: string;
+  task: string;
+  screenWidth: number;
+  screenHeight: number;
+  attachments?: UserMessageAttachment[];
+  interactionType?: 'computer_use' | 'text_qa';
+}
+
+export interface CuSessionAbort {
+  type: 'cu_session_abort';
+  sessionId: string;
+}
+
+export interface CuObservation {
+  type: 'cu_observation';
+  sessionId: string;
+  axTree?: string;
+  axDiff?: string;
+  secondaryWindows?: string;
+  screenshot?: string;
+  /** Screenshot image width in pixels (`Px`). */
+  screenshotWidthPx?: number;
+  /** Screenshot image height in pixels (`Px`). */
+  screenshotHeightPx?: number;
+  /** Screen width in macOS points (`Pt`) used by native execution. */
+  screenWidthPt?: number;
+  /** Screen height in macOS points (`Pt`) used by native execution. */
+  screenHeightPt?: number;
+  /** Coordinate origin convention used by the observation payload. */
+  coordinateOrigin?: 'top_left';
+  /** Display ID used by screenshot capture for this observation. */
+  captureDisplayId?: number;
+  executionResult?: string;
+  executionError?: string;
+  axTreeBlob?: IpcBlobRef;
+  screenshotBlob?: IpcBlobRef;
+}
+
+export interface TaskSubmit {
+  type: 'task_submit';
+  task: string;
+  screenWidth: number;
+  screenHeight: number;
+  attachments?: UserMessageAttachment[];
+  source?: 'voice' | 'text';
+}
+
+export interface RideShotgunStart {
+  type: 'ride_shotgun_start';
+  durationSeconds: number;
+  intervalSeconds: number;
+  mode?: 'observe' | 'learn';
+  targetDomain?: string;
+  /** Domain to auto-navigate (may differ from targetDomain, e.g. open.spotify.com vs spotify.com). */
+  navigateDomain?: string;
+  autoNavigate?: boolean;
+}
+
+export interface RideShotgunStop {
+  type: 'ride_shotgun_stop';
+  watchId: string;
+}
+
+export interface WatchObservation {
+  type: 'watch_observation';
+  watchId: string;
+  sessionId: string;
+  ocrText: string;
+  appName?: string;
+  windowTitle?: string;
+  bundleIdentifier?: string;
+  timestamp: number;
+  captureIndex: number;
+  totalExpected: number;
+}
+
+// === Server → Client ===
+
+export interface CuAction {
+  type: 'cu_action';
+  sessionId: string;
+  toolName: string;
+  input: Record<string, unknown>;
+  reasoning?: string;
+  stepNumber: number;
+}
+
+export interface CuComplete {
+  type: 'cu_complete';
+  sessionId: string;
+  summary: string;
+  stepCount: number;
+  isResponse?: boolean;
+}
+
+export interface CuError {
+  type: 'cu_error';
+  sessionId: string;
+  message: string;
+}
+
+export interface TaskRouted {
+  type: 'task_routed';
+  sessionId: string;
+  interactionType: 'computer_use' | 'text_qa';
+  /** The task text passed to the escalated session. */
+  task?: string;
+  /** Set when a text_qa session escalates to computer_use via computer_use_request_control. */
+  escalatedFrom?: string;
+}
+
+export interface RideShotgunProgress {
+  type: 'ride_shotgun_progress';
+  watchId: string;
+  message: string;
+  networkEntryCount?: number;
+  statusMessage?: string;
+  idleHint?: boolean;
+}
+
+export interface RideShotgunResult {
+  type: 'ride_shotgun_result';
+  sessionId: string;
+  watchId: string;
+  summary: string;
+  observationCount: number;
+  recordingId?: string;
+  recordingPath?: string;
+}
+
+export interface WatchStarted {
+  type: 'watch_started';
+  sessionId: string;
+  watchId: string;
+  durationSeconds: number;
+  intervalSeconds: number;
+}
+
+export interface WatchCompleteRequest {
+  type: 'watch_complete_request';
+  sessionId: string;
+  watchId: string;
+}

package/src/daemon/ipc-contract/diagnostics.ts

@@ -0,0 +1,56 @@
+// Diagnostics, environment, blob probe, and dictation types.
+
+import type { DictationContext } from './shared.js';
+
+// === Client → Server ===
+
+export interface DiagnosticsExportRequest {
+  type: 'diagnostics_export_request';
+  conversationId: string;
+  anchorMessageId?: string;  // if omitted, use latest assistant message
+}
+
+export interface EnvVarsRequest {
+  type: 'env_vars_request';
+}
+
+export interface IpcBlobProbe {
+  type: 'ipc_blob_probe';
+  probeId: string;
+  nonceSha256: string;
+}
+
+export interface DictationRequest {
+  type: 'dictation_request';
+  transcription: string;
+  context: DictationContext;
+}
+
+// === Server → Client ===
+
+export interface DiagnosticsExportResponse {
+  type: 'diagnostics_export_response';
+  success: boolean;
+  filePath?: string;   // path to the zip file on success
+  error?: string;      // error message on failure
+}
+
+export interface EnvVarsResponse {
+  type: 'env_vars_response';
+  vars: Record<string, string>;
+}
+
+export interface IpcBlobProbeResult {
+  type: 'ipc_blob_probe_result';
+  probeId: string;
+  ok: boolean;
+  observedNonceSha256?: string;
+  reason?: string;
+}
+
+export interface DictationResponse {
+  type: 'dictation_response';
+  text: string;
+  mode: 'dictation' | 'command' | 'action';
+  actionPlan?: string;
+}

package/src/daemon/ipc-contract/documents.ts

@@ -0,0 +1,74 @@
+// Document editor and document persistence types.
+
+// === Server → Client ===
+
+export interface DocumentEditorShow {
+  type: 'document_editor_show';
+  sessionId: string;
+  surfaceId: string;
+  title: string;
+  initialContent: string;
+}
+
+export interface DocumentEditorUpdate {
+  type: 'document_editor_update';
+  sessionId: string;
+  surfaceId: string;
+  markdown: string;
+  mode: string;
+}
+
+// === Client → Server ===
+
+export interface DocumentSaveRequest {
+  type: 'document_save';
+  surfaceId: string;
+  conversationId: string;
+  title: string;
+  content: string;
+  wordCount: number;
+}
+
+export interface DocumentLoadRequest {
+  type: 'document_load';
+  surfaceId: string;
+}
+
+export interface DocumentListRequest {
+  type: 'document_list';
+  conversationId?: string;
+}
+
+// === Server → Client ===
+
+export interface DocumentSaveResponse {
+  type: 'document_save_response';
+  surfaceId: string;
+  success: boolean;
+  error?: string;
+}
+
+export interface DocumentLoadResponse {
+  type: 'document_load_response';
+  surfaceId: string;
+  conversationId: string;
+  title: string;
+  content: string;
+  wordCount: number;
+  createdAt: number;
+  updatedAt: number;
+  success: boolean;
+  error?: string;
+}
+
+export interface DocumentListResponse {
+  type: 'document_list_response';
+  documents: Array<{
+    surfaceId: string;
+    conversationId: string;
+    title: string;
+    wordCount: number;
+    createdAt: number;
+    updatedAt: number;
+  }>;
+}

package/src/daemon/ipc-contract/inbox.ts

@@ -0,0 +1,209 @@
+// Assistant inbox: invite management, member management, inbox threads, escalations, and replies.
+
+// === Client → Server ===
+
+export interface IngressInviteRequest {
+  type: 'ingress_invite';
+  action: 'create' | 'list' | 'revoke' | 'redeem';
+  /** Source channel for the invite (required for create and redeem). */
+  sourceChannel?: string;
+  /** Optional note describing the invite (create only). */
+  note?: string;
+  /** Maximum number of times the invite can be redeemed (create only). */
+  maxUses?: number;
+  /** Expiration time in milliseconds from now (create only). */
+  expiresInMs?: number;
+  /** Invite ID to revoke (revoke only). */
+  inviteId?: string;
+  /** Invite token to redeem (redeem only). */
+  token?: string;
+  /** External user ID of the redeemer (redeem only). */
+  externalUserId?: string;
+  /** External chat ID of the redeemer (redeem only). */
+  externalChatId?: string;
+  /** Filter by status (list only). */
+  status?: string;
+}
+
+export interface IngressMemberRequest {
+  type: 'ingress_member';
+  action: 'list' | 'upsert' | 'revoke' | 'block';
+  /** Source channel (required for upsert, optional filter for list). */
+  sourceChannel?: string;
+  /** External user ID (upsert only). */
+  externalUserId?: string;
+  /** External chat ID (upsert only). */
+  externalChatId?: string;
+  /** Display name (upsert only). */
+  displayName?: string;
+  /** Username (upsert only). */
+  username?: string;
+  /** Access policy (upsert only). */
+  policy?: 'allow' | 'deny' | 'escalate';
+  /** Member status (upsert only for setting, list only for filtering). */
+  status?: 'pending' | 'active';
+  /** Member ID (revoke and block only). */
+  memberId?: string;
+  /** Reason for revoke or block (revoke and block only). */
+  reason?: string;
+}
+
+export interface AssistantInboxRequest {
+  type: 'assistant_inbox';
+  action: 'list_threads' | 'get_thread_messages';
+  /** Filter by assistant ID (list_threads only). */
+  assistantId?: string;
+  /** Maximum number of results to return (list_threads and get_thread_messages). */
+  limit?: number;
+  /** Offset for pagination (list_threads only). */
+  offset?: number;
+  /** Conversation ID (required for get_thread_messages). */
+  conversationId?: string;
+  /** Cursor for message pagination — return messages before this ID (get_thread_messages only). */
+  beforeMessageId?: string;
+}
+
+export interface AssistantInboxEscalationRequest {
+  type: 'assistant_inbox_escalation';
+  action: 'list' | 'decide';
+  /** Filter by assistant ID (list only). */
+  assistantId?: string;
+  /** Filter by status (list only). */
+  status?: string;
+  /** Approval request ID (required for decide). */
+  approvalRequestId?: string;
+  /** Decision (required for decide). */
+  decision?: 'approve' | 'deny';
+  /** Reason for the decision (decide only). */
+  reason?: string;
+}
+
+export interface AssistantInboxReplyRequest {
+  type: 'assistant_inbox_reply';
+  conversationId: string;
+  content: string;
+}
+
+// === Server → Client ===
+
+export interface IngressInviteResponse {
+  type: 'ingress_invite_response';
+  success: boolean;
+  error?: string;
+  /** Single invite (returned on create/revoke). Token field is only present on create. */
+  invite?: {
+    id: string;
+    sourceChannel: string;
+    token?: string;
+    tokenHash: string;
+    maxUses: number;
+    useCount: number;
+    expiresAt: number | null;
+    status: string;
+    note?: string;
+    createdAt: number;
+  };
+  /** List of invites (returned on list). */
+  invites?: Array<{
+    id: string;
+    sourceChannel: string;
+    tokenHash: string;
+    maxUses: number;
+    useCount: number;
+    expiresAt: number | null;
+    status: string;
+    note?: string;
+    createdAt: number;
+  }>;
+}
+
+export interface IngressMemberResponse {
+  type: 'ingress_member_response';
+  success: boolean;
+  error?: string;
+  /** Single member (returned on upsert/revoke/block). */
+  member?: {
+    id: string;
+    sourceChannel: string;
+    externalUserId?: string;
+    externalChatId?: string;
+    displayName?: string;
+    username?: string;
+    status: string;
+    policy: string;
+    lastSeenAt?: number;
+    createdAt: number;
+  };
+  /** List of members (returned on list). */
+  members?: Array<{
+    id: string;
+    sourceChannel: string;
+    externalUserId?: string;
+    externalChatId?: string;
+    displayName?: string;
+    username?: string;
+    status: string;
+    policy: string;
+    lastSeenAt?: number;
+    createdAt: number;
+  }>;
+}
+
+export interface AssistantInboxResponse {
+  type: 'assistant_inbox_response';
+  success: boolean;
+  error?: string;
+  /** List of inbox threads (returned on list_threads). */
+  threads?: Array<{
+    conversationId: string;
+    sourceChannel: string;
+    externalChatId: string;
+    externalUserId?: string;
+    displayName?: string;
+    username?: string;
+    lastMessageAt?: number;
+    unreadCount: number;
+    hasPendingEscalation: boolean;
+    lastInboundAt?: number;
+    lastOutboundAt?: number;
+  }>;
+  /** List of messages (returned on get_thread_messages). */
+  messages?: Array<{
+    id: string;
+    role: string;
+    content: string;
+    createdAt: number;
+    metadata?: Record<string, unknown>;
+  }>;
+}
+
+export interface AssistantInboxEscalationResponse {
+  type: 'assistant_inbox_escalation_response';
+  success: boolean;
+  error?: string;
+  /** List of escalations (returned on list). */
+  escalations?: Array<{
+    id: string;
+    runId: string;
+    conversationId: string;
+    channel: string;
+    requesterExternalUserId: string;
+    requesterChatId: string;
+    status: string;
+    requestSummary?: string;
+    createdAt: number;
+  }>;
+  /** Decision result (returned on decide). */
+  decision?: {
+    id: string;
+    status: string;
+    decidedAt: number;
+  };
+}
+
+export interface AssistantInboxReplyResponse {
+  type: 'assistant_inbox_reply_response';
+  success: boolean;
+  error?: string;
+  messageId?: string;
+}