@moxxy/sdk 0.1.3 → 0.3.0

package/src/provider.ts

@@ -10,7 +10,16 @@ export type ContentBlock =
   | { readonly type: 'tool_use'; readonly id: string; readonly name: string; readonly input: unknown }
   | { readonly type: 'tool_result'; readonly toolUseId: string; readonly content: string; readonly isError?: boolean }
   | { readonly type: 'image'; readonly mediaType: string; readonly data: string }
-  | { readonly type: 'audio'; readonly mediaType: string; readonly data: string };
+  | { readonly type: 'audio'; readonly mediaType: string; readonly data: string }
+  /**
+   * A document the model reads natively (e.g. a PDF). `data` is base64-encoded
+   * bytes and `mediaType` the document MIME (e.g. `application/pdf`). Providers
+   * that support documents translate it to their native shape (Anthropic
+   * `document`, OpenAI `file`, Responses `input_file`); those that don't degrade
+   * to a text placeholder. Text/Office files are inlined as `text` instead — only
+   * formats a model ingests as bytes become `document` blocks.
+   */
+  | { readonly type: 'document'; readonly mediaType: string; readonly data: string; readonly name?: string };
 
 /**
  * Provider-neutral instruction for where a prompt-cache breakpoint should be
@@ -68,6 +77,13 @@ export interface ModelDescriptor {
    * refuses or warns instead of silently dropping the bytes.
    */
   readonly supportsImages?: boolean;
+  /**
+   * Whether this model accepts `document` ContentBlocks (e.g. native PDF) in
+   * user messages. When false, the desktop reader extracts text instead of
+   * shipping the raw bytes, so the attachment still reaches the model — just
+   * without full-document fidelity (figures/layout).
+   */
+  readonly supportsDocuments?: boolean;
   /**
    * Whether this model accepts `audio` ContentBlocks in user messages
    * (GPT-4o, Gemini-Live-class models). When false, channels with audio

package/src/requirements.ts

@@ -3,6 +3,7 @@ export type RequirementKind =
   | 'provider'
   | 'tool'
   | 'transcriber'
+  | 'synthesizer'
   | 'mode'
   | 'compactor'
   | 'channel'

package/src/schemas.ts

@@ -15,6 +15,7 @@ const pluginKindSchema = z.enum([
   'agent',
   'command',
   'transcriber',
+  'synthesizer',
 ]);
 
 export const requirementSchema = z.object({
@@ -23,6 +24,7 @@ export const requirementSchema = z.object({
     'provider',
     'tool',
     'transcriber',
+    'synthesizer',
     'mode',
     'compactor',
     'channel',

package/src/session-like.ts

@@ -1,7 +1,7 @@
 import type { MoxxyEvent, UserPromptAttachment } from './events.js';
 import type { SessionId, TurnId } from './ids.js';
 import type { EventLogReader } from './log.js';
-import type { ApprovalResolver } from './mode.js';
+import type { ApprovalResolver, ModeBadge } from './mode.js';
 import type { PermissionResolver } from './permission.js';
 import type { ModelDescriptor } from './provider.js';
 import type { ToolCompactPresentation } from './tool.js';
@@ -96,6 +96,13 @@ export interface SessionInfo {
   readonly activeProvider: string | null;
   readonly providers: ReadonlyArray<ProviderInfo>;
   readonly activeMode: string | null;
+  /**
+   * Presentation hint for the active mode, when it advertises one (see
+   * {@link ModeBadge}). Lets channels render a persistent accent badge for
+   * autonomous modes like goal mode. `null` when no mode is active or the
+   * active mode declares no badge.
+   */
+  readonly activeModeBadge: ModeBadge | null;
   readonly modes: ReadonlyArray<string>;
   readonly tools: ReadonlyArray<ToolInfo>;
   readonly skills: ReadonlyArray<SkillInfo>;
@@ -105,6 +112,11 @@ export interface SessionInfo {
   readonly hasTranscriber: boolean;
   /** Name of the active transcriber, or null. Lets a thin client proxy STT. */
   readonly activeTranscriber: string | null;
+  /** Whether any text-to-speech backend is registered. */
+  readonly hasSynthesizer: boolean;
+  /** Name of the active synthesizer, or null. Lets a thin client proxy TTS
+   *  (the desktop routes "Read aloud" through it; null → OS voice fallback). */
+  readonly activeSynthesizer: string | null;
 }
 
 /**

package/src/synthesizer.ts

@@ -0,0 +1,67 @@
+/**
+ * Synthesizers convert text into spoken audio (text-to-speech). They are the
+ * symmetric counterpart to {@link Transcriber} (speech-to-text): a separate,
+ * swappable capability so a user can pair any text provider with a dedicated
+ * TTS backend (the OS voice, ElevenLabs, OpenAI, a local engine, …).
+ *
+ * Plugins register a `SynthesizerDef` via `PluginSpec.synthesizers`. The core
+ * `SynthesizerRegistry` holds them; surfaces that read aloud (the desktop's
+ * "Read aloud" button, a future voice channel) call
+ * `session.synthesizers.getActive()` (or `tryGetActive()`) to turn text into
+ * audio. There is at most one *active* synthesizer at a time; when none is
+ * active the caller degrades gracefully (the desktop falls back to the
+ * browser/OS `speechSynthesis`).
+ *
+ * The user swaps the active synthesizer by asking the agent ("use ElevenLabs
+ * for read-aloud") — there is no settings UI. The agent activates a registered
+ * synthesizer (a plugin it authored/installed self-activates in `onInit`, or it
+ * calls a set-active tool); switching back to the OS voice deactivates it.
+ */
+
+export interface SynthesizeOptions {
+  /** Preferred voice id/name, when the backend supports selection. */
+  readonly voice?: string;
+  /** BCP-47 language hint (e.g. `en`, `pl`). */
+  readonly language?: string;
+  /** Speaking rate multiplier (1.0 = normal), when supported. */
+  readonly rate?: number;
+  /** Cancellation signal — callers propagate a stop/abort here. */
+  readonly signal?: AbortSignal;
+}
+
+export interface SynthesisResult {
+  /** Encoded audio bytes ready to play (e.g. an mp3/wav/ogg payload). */
+  readonly audio: Uint8Array;
+  /** MIME type of {@link audio}, e.g. `audio/mpeg`, `audio/wav`, `audio/ogg`. */
+  readonly mimeType: string;
+}
+
+export interface Synthesizer {
+  /** Short stable name, e.g. `elevenlabs`. */
+  readonly name: string;
+  synthesize(text: string, opts?: SynthesizeOptions): Promise<SynthesisResult>;
+}
+
+/**
+ * Context handed to {@link SynthesizerDef.create} when the registry activates a
+ * synthesizer. `getSecret` reads a vault secret (the API key) the same way a
+ * tool handler does via `ctx.getSecret` — so an authored TTS plugin never has
+ * to touch `process.env`, and the key never enters the model's context.
+ */
+export interface SynthesizerCreateContext {
+  /** Per-synthesizer config (e.g. from `session.synthesizers.setActive(name, config)`). */
+  readonly config: Record<string, unknown>;
+  /** Read a named vault secret, or null if unset / no vault is wired. */
+  readonly getSecret?: (name: string) => Promise<string | null>;
+}
+
+/**
+ * Plugin-side definition. Mirrors `TranscriberDef`: a `create(ctx)` factory the
+ * registry calls when this synthesizer is activated.
+ */
+export interface SynthesizerDef {
+  readonly name: string;
+  /** Optional human-readable label for UI surfaces. */
+  readonly displayName?: string;
+  create(ctx: SynthesizerCreateContext): Synthesizer;
+}

package/src/token-efficiency.test.ts

@@ -457,3 +457,61 @@ describe('applyLazyTools', () => {
     expect(messages).toBe(baseMsgs); // same reference → byte-stable
   });
 });
+
+describe('attachment projection in projectMessagesFromLog', () => {
+  it('projects an image attachment to an image content block', () => {
+    const msgs = projectMessagesFromLog({
+      log: reader([
+        event(0, {
+          type: 'user_prompt',
+          turnId: t1,
+          source: 'user',
+          text: 'what is this?',
+          attachments: [{ kind: 'image', content: 'AAAA', mediaType: 'image/png', name: 'pic.png' }],
+        }),
+      ]),
+    });
+    expect(msgs[0].content).toEqual([
+      { type: 'text', text: 'what is this?' },
+      { type: 'image', mediaType: 'image/png', data: 'AAAA' },
+    ]);
+  });
+
+  it('projects a document attachment to a native document content block', () => {
+    const msgs = projectMessagesFromLog({
+      log: reader([
+        event(0, {
+          type: 'user_prompt',
+          turnId: t1,
+          source: 'user',
+          text: 'summarize this',
+          attachments: [
+            { kind: 'document', content: 'JVBERi0=', mediaType: 'application/pdf', name: 'r.pdf' },
+          ],
+        }),
+      ]),
+    });
+    expect(msgs[0].content).toEqual([
+      { type: 'text', text: 'summarize this' },
+      { type: 'document', mediaType: 'application/pdf', data: 'JVBERi0=', name: 'r.pdf' },
+    ]);
+  });
+
+  it('inlines a file attachment as a labeled text block', () => {
+    const msgs = projectMessagesFromLog({
+      log: reader([
+        event(0, {
+          type: 'user_prompt',
+          turnId: t1,
+          source: 'user',
+          text: 'review',
+          attachments: [{ kind: 'file', content: 'const x = 1;', name: 'a.ts' }],
+        }),
+      ]),
+    });
+    expect(msgs[0].content).toEqual([
+      { type: 'text', text: 'review' },
+      { type: 'text', text: '[file a.ts]\nconst x = 1;' },
+    ]);
+  });
+});

package/src/tool.ts

@@ -127,6 +127,19 @@ export interface ToolContext {
    * `caps.subprocess` + optional `caps.commands` allowlist.
    */
   readonly exec?: BrokeredExec;
+  /**
+   * Read a named secret from the host's secret store (the vault) at call
+   * time. Present when the session is wired with a vault-backed resolver
+   * (the CLI/runner does this). Returns the plaintext value, or `null` if
+   * no such secret exists.
+   *
+   * This is the blessed way for a tool/plugin to consume an API key or
+   * token: the plaintext only ever reaches the handler that asks for it —
+   * it is never placed in the model's context or in `process.env`. Store
+   * the secret once via `/vault set NAME …` (or the desktop Secrets UI),
+   * then read it here, e.g. `const key = await ctx.getSecret('ELEVENLABS_API_KEY')`.
+   */
+  readonly getSecret?: (name: string) => Promise<string | null>;
 }
 
 /**