@k2b/cloud 0.6.0 → 0.8.0

package/src/ai/transcription.ts

@@ -0,0 +1,202 @@
+import { openAICompatibleTranscription, type TranscriptionProvider } from "@k2b/nessi/ai";
+import { coreSettings } from "../services";
+import { logger, type TraceContext, trace } from "../services/logging";
+import { AI_AUDIO_MAX_BYTES, inspectAiAudio } from "./audio-format";
+import { getAiCredential } from "./credentials";
+import { beginAiCall, finishAiCall } from "./inference-calls";
+import { readAiSettingsState, selectAiModelProfile } from "./settings";
+import { type AiUsageAttribution, safelyRecordStructuredRun } from "./structured-runs";
+import type { AiDataBoundary, AiModelProfile } from "./types";
+
+export const AI_AUDIO_MODEL_SETTING_KEY = "ai.audio_model_id";
+/** Matches the existing chat turn execution budget; callers may cancel sooner. */
+export const AI_TRANSCRIPTION_TIMEOUT_MS = 10 * 60_000;
+
+export type AiResolvedAudioModel = { profile: AiModelProfile; provider: TranscriptionProvider };
+
+export const createAiTranscriptionProvider = (profile: AiModelProfile, apiKey?: string): TranscriptionProvider => {
+  if (
+    profile.capabilities.length !== 1 ||
+    !profile.capabilities.includes("transcription") ||
+    (profile.provider !== "openai" && profile.provider !== "openai-compatible")
+  ) {
+    throw new Error("Choose an audio transcription profile with an OpenAI-compatible endpoint.");
+  }
+  const baseURL = profile.baseURL ?? (profile.provider === "openai" ? "https://api.openai.com/v1" : undefined);
+  if (!baseURL) throw new Error("Audio transcription requires a base URL.");
+  return openAICompatibleTranscription(profile.model, { name: profile.provider, baseURL, apiKey });
+};
+
+export const resolveAiAudioModel = async (
+  input: { requestedModelId?: string; allowedDataBoundaries?: AiDataBoundary[] } = {},
+): Promise<AiResolvedAudioModel> => {
+  const state = await readAiSettingsState();
+  if (!state.ok) throw Object.assign(new Error(state.error.message), { aiError: state.error });
+  if (!state.enabled) throw new Error("AI is disabled.");
+  const modelId = input.requestedModelId ?? String((await coreSettings.get<string>(AI_AUDIO_MODEL_SETTING_KEY)) ?? "").trim();
+  if (!modelId) throw new Error("No audio transcription model is configured.");
+  const profile = selectAiModelProfile(state, {
+    kind: "locked",
+    modelId,
+    requiredCapabilities: ["transcription"],
+    allowedDataBoundaries: input.allowedDataBoundaries,
+  });
+  const credential = await getAiCredential(profile.id);
+  if (profile.provider === "openai" && !credential?.trim()) throw new Error("The audio model is missing provider credentials.");
+  return { profile, provider: createAiTranscriptionProvider(profile, credential?.trim() || undefined) };
+};
+
+export type RunAiTranscriptionInput = {
+  task: string;
+  file: Blob;
+  filename: string;
+  language?: string;
+  prompt?: string;
+  requestedModelId?: string;
+  allowedDataBoundaries?: AiDataBoundary[];
+  signal?: AbortSignal;
+  appId?: string;
+  traceParent?: TraceContext;
+  /** Attribution is metadata; the caller must authorize the source before calling. */
+  attribution?: AiUsageAttribution;
+  resolveModel?: () => Promise<AiResolvedAudioModel>;
+};
+
+/** One provider call. Durable execution and input authorization belong to the caller. */
+export const runAiTranscription = async (input: RunAiTranscriptionInput): Promise<{ text: string; modelProfileId: string }> => {
+  return trace.withSpan(
+    {
+      name: `ai.transcription.${input.task}`,
+      source: `ai:transcription:${input.task}`,
+      category: "ai",
+      appId: input.appId,
+      parent: input.traceParent,
+    },
+    async (span) => {
+      const startedAt = Date.now();
+      let failure: AiTranscriptionError | undefined;
+      let stage: TranscriptionStage = "input";
+      let resolved: AiResolvedAudioModel | undefined;
+      let succeeded = false;
+      let callId: string | undefined;
+      const timeout = AbortSignal.timeout(AI_TRANSCRIPTION_TIMEOUT_MS);
+      const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout;
+      try {
+        signal.throwIfAborted();
+        if (input.file.size > AI_AUDIO_MAX_BYTES) throw new Error("Audio transcription accepts files up to 25 MB.");
+        const format = inspectAiAudio(new Uint8Array(await input.file.arrayBuffer()));
+        stage = "configuration";
+        resolved = await (input.resolveModel?.() ?? resolveAiAudioModel(input));
+        const filename =
+          input.filename
+            .split(/[\\/]/)
+            .at(-1)
+            ?.replace(/\.[^.]*$/, "") || "audio";
+        stage = "provider";
+        callId = (
+          await beginAiCall(
+            { ...resolved.profile, pricing: undefined },
+            { kind: "background", task: input.task, appId: input.appId, traceId: span.traceId, ...input.attribution },
+            0,
+          )
+        ).id;
+        const result = await resolved.provider.transcribe({
+          file: new Blob([input.file], { type: format.mediaType }),
+          filename: `${filename}.${format.extension}`,
+          language: input.language,
+          prompt: input.prompt,
+          signal,
+        });
+        signal.throwIfAborted();
+        succeeded = true;
+        return { text: result.text, modelProfileId: resolved.profile.id };
+      } catch (error) {
+        failure = describeTranscriptionFailure(error, stage, signal.aborted);
+        logger("ai:transcription")[input.signal?.aborted ? "info" : "error"](failure.message, {
+          traceId: span.traceId,
+          task: input.task,
+          modelProfileId: resolved?.profile.id ?? input.requestedModelId,
+          providerModel: resolved?.profile.model,
+          stage,
+          errorCode: failure.code,
+          httpStatus: failure.httpStatus,
+        });
+        throw failure;
+      } finally {
+        if (callId) {
+          try {
+            await finishAiCall(callId, undefined, succeeded ? "ok" : "failed");
+          } catch {
+            logger("ai:transcription").error("AI cost booking failed", { callId });
+          }
+        }
+        await safelyRecordStructuredRun({
+          traceId: span.traceId,
+          task: input.task,
+          appId: input.appId,
+          attribution: input.attribution,
+          modelProfileId: resolved?.profile.id,
+          providerModel: resolved?.profile.model,
+          mode: "transcription",
+          attempts: resolved ? 1 : 0,
+          status: succeeded ? "ok" : "failed",
+          durationMs: Date.now() - startedAt,
+          // Provider error bodies may echo source data. Keep accounting metadata-only.
+          errorCode: failure?.code,
+          error: failure?.message,
+        });
+      }
+    },
+  );
+};
+
+export type TranscriptionStage = "input" | "configuration" | "provider";
+/** Only classified metadata escapes the adapter: response bodies can contain private source data. */
+export class AiTranscriptionError extends Error {
+  constructor(
+    readonly code: string,
+    message: string,
+    readonly retryable = false,
+    readonly httpStatus?: number,
+  ) {
+    super(message);
+    this.name = "AiTranscriptionError";
+  }
+}
+export const describeTranscriptionFailure = (error: unknown, stage: TranscriptionStage, aborted = false): AiTranscriptionError => {
+  if (error instanceof AiTranscriptionError) return error;
+  if (aborted) return new AiTranscriptionError("transcription_aborted", "Audio transcription was canceled or timed out.");
+  const message = error instanceof Error ? error.message : "";
+  const match = /^(?:openai|openai-compatible) (\d{3}): /.exec(message);
+  if (match) {
+    const status = Number(match[1]);
+    const hint =
+      status === 401 || status === 403
+        ? "Check provider credentials and model access."
+        : status === 404
+          ? "Check the provider base URL and model name."
+          : status === 429
+            ? "The provider rate limit was reached."
+            : "The provider rejected the audio transcription request.";
+    return new AiTranscriptionError(
+      `transcription_http_${status}`,
+      `Audio provider HTTP ${status}. ${hint}`,
+      [408, 409, 425, 429].includes(status) || status >= 500,
+      status,
+    );
+  }
+  if (/^(?:openai|openai-compatible) connection failed: /.test(message))
+    return new AiTranscriptionError(
+      "transcription_connection_failed",
+      "Could not connect to the audio provider. Check its endpoint and network availability.",
+      true,
+    );
+  return new AiTranscriptionError(
+    `transcription_${stage}_failed`,
+    stage === "configuration"
+      ? "Audio model configuration could not be resolved. Check the audio profile, base URL, credentials and access policy."
+      : stage === "input"
+        ? "Audio input is invalid or exceeds the supported size."
+        : "Audio provider returned an invalid response or an unclassified error.",
+  );
+};

package/src/ai/turn-timing.ts

@@ -0,0 +1,114 @@
+import type { LoopAggregate, OutboundEvent } from "@k2b/nessi";
+import { sql } from "bun";
+
+export type TimingInterval = { start: number; end: number };
+const union = (intervals: TimingInterval[]): TimingInterval[] => {
+  const result: TimingInterval[] = [];
+  for (const interval of intervals.filter((x) => x.end > x.start).sort((a, b) => a.start - b.start)) {
+    const previous = result.at(-1);
+    if (previous && interval.start <= previous.end) previous.end = Math.max(previous.end, interval.end);
+    else result.push({ ...interval });
+  }
+  return result;
+};
+const duration = (intervals: TimingInterval[]) => union(intervals).reduce((sum, x) => sum + x.end - x.start, 0);
+const excluding = (intervals: TimingInterval[], removed: TimingInterval[]) => {
+  let result = union(intervals);
+  for (const cut of union(removed))
+    result = result.flatMap((x) =>
+      cut.end <= x.start || cut.start >= x.end
+        ? [x]
+        : [
+            { start: x.start, end: Math.min(x.end, cut.start) },
+            { start: Math.max(x.start, cut.end), end: x.end },
+          ].filter((x) => x.end > x.start),
+    );
+  return result;
+};
+
+/** Exclusive phase durations; parallel tools and nested approvals never double-count wall time. */
+export function summarizeTurnTiming(input: {
+  start: number;
+  end: number;
+  generation: TimingInterval[];
+  tools: TimingInterval[];
+  waits: TimingInterval[];
+  outputTokens?: number;
+}): NonNullable<LoopAggregate["timing"]> {
+  const clip = (rows: TimingInterval[]) => rows.map((x) => ({ start: Math.max(input.start, x.start), end: Math.min(input.end, x.end) }));
+  const generation = clip(input.generation),
+    waits = excluding(clip(input.waits), generation);
+  const generationMs = duration(generation),
+    actionWaitMs = duration(waits);
+  const toolExecutionMs = duration(excluding(clip(input.tools), [...generation, ...waits]));
+  const wallMs = Math.max(0, input.end - input.start);
+  return {
+    wallMs,
+    totalElapsedMs: wallMs,
+    generationMs,
+    actionWaitMs,
+    toolExecutionMs,
+    ...(generationMs > 0 && input.outputTokens !== undefined ? { outputTokensPerSecond: input.outputTokens / (generationMs / 1000) } : {}),
+  };
+}
+
+/** One recorder per executor attempt; rows survive client actions and process resumption. */
+export function createTurnTimingRecorder(turnId: string, db = sql) {
+  let requestId: string | null = null;
+  const finishGeneration = async () => {
+    if (!requestId) return;
+    const id = requestId;
+    requestId = null;
+    await db`UPDATE ai.turn_generation_timings SET ended_at=now() WHERE turn_id=${turnId}::uuid AND request_id=${id} AND ended_at IS NULL`;
+  };
+  return {
+    finishGeneration,
+    async event(event: OutboundEvent) {
+      if (event.type === "turn_start" && !event.resumed) {
+        await finishGeneration();
+        requestId = event.turnId;
+        await db`INSERT INTO ai.turn_generation_timings(turn_id,request_id) VALUES(${turnId}::uuid,${requestId}) ON CONFLICT DO NOTHING`;
+      } else if (
+        event.type === "usage" ||
+        event.type === "tool_execution_start" ||
+        event.type === "turn_end" ||
+        event.type === "loop_end"
+      ) {
+        await finishGeneration();
+      }
+    },
+  };
+}
+
+export async function withDurableTurnTiming(turnId: string, aggregate: LoopAggregate, db = sql): Promise<LoopAggregate> {
+  const [turn] = await db<
+    { start: Date; end: Date }[]
+  >`SELECT created_at AS start,COALESCE(completed_at,now()) AS end FROM ai.turns WHERE id=${turnId}::uuid`;
+  const generation = await db<
+    { start: Date; end: Date | null }[]
+  >`SELECT started_at AS start,ended_at AS end FROM ai.turn_generation_timings WHERE turn_id=${turnId}::uuid`;
+  // An older/incomplete trace cannot justify a speed computed from the whole loop's tokens.
+  if (!turn || generation.length < aggregate.assistantMessageCount || generation.some((x) => !x.end)) {
+    const { timing: _, ...rest } = aggregate;
+    return rest;
+  }
+  const tools = await db<
+    { start: Date; end: Date }[]
+  >`SELECT started_at AS start,COALESCE(completed_at,now()) AS end FROM ai.tool_calls WHERE turn_id=${turnId}::uuid AND started_at IS NOT NULL`;
+  const waits = await db<{ start: Date; end: Date }[]>`SELECT a.created_at AS start,
+    CASE WHEN a.kind='client_tool' THEN LEAST(COALESCE(t.started_at,a.resolved_at,now()),COALESCE(a.resolved_at,now())) ELSE COALESCE(a.resolved_at,now()) END AS end
+    FROM ai.pending_actions a LEFT JOIN ai.tool_calls t ON t.turn_id=a.turn_id AND t.call_id=a.call_id WHERE a.turn_id=${turnId}::uuid`;
+  const intervals = (rows: { start: Date; end: Date | null }[]) =>
+    rows.flatMap((x) => (x.end ? [{ start: new Date(x.start).getTime(), end: new Date(x.end).getTime() }] : []));
+  return {
+    ...aggregate,
+    timing: summarizeTurnTiming({
+      start: new Date(turn.start).getTime(),
+      end: new Date(turn.end).getTime(),
+      generation: intervals(generation),
+      tools: intervals(tools),
+      waits: intervals(waits),
+      outputTokens: aggregate.usage?.output,
+    }),
+  };
+}

package/src/ai/types.ts

@@ -16,7 +16,7 @@ import type { CapabilityActionReview, CloudResourceRef } from "../contracts/capa
 import type { RequestActor } from "../server";
 import type { AiTurnBlock } from "./protocol";
 
-export const AI_MODEL_CAPABILITIES = ["streaming", "tools", "vision"] as const;
+export const AI_MODEL_CAPABILITIES = ["streaming", "tools", "vision", "transcription"] as const;
 export type AiModelCapability = (typeof AI_MODEL_CAPABILITIES)[number];
 
 export const AI_IMAGE_MEDIA_TYPES = ["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"] as const;
@@ -49,8 +49,7 @@ export type AiModelProfile = {
   maxLoadedTools?: number;
   /** Tool-using model rounds per chat turn. Missing or <= 0 is unlimited. */
   maxToolRounds?: number;
-  creditsPerInputToken?: number;
-  creditsPerOutputToken?: number;
+  pricing?: import("../shared/ai-costs").AiModelPricing;
 };
 
 export type AiPublicModelProfile = Pick<
@@ -118,8 +117,9 @@ export type AiResolvedModel = {
 
 export type AiDraftContentPart =
   | { type: "text"; text: string }
-  | { type: "resource"; ref: CloudResourceRef; title?: string; icon?: string; href?: string }
-  | { type: "file"; path: string; mediaType: string; size: number; version: number };
+  | { type: "resource"; ref: CloudResourceRef; title?: string; icon?: string; href?: string; inline?: boolean }
+  | { type: "file"; path: string; mediaType: string; size: number; version: number; inline?: boolean }
+  | { type: "project-file"; path: string; inline?: boolean };
 
 export type AiConversationDraft = {
   content: AiDraftContentPart[];
@@ -129,10 +129,12 @@ export type AiConversationDraft = {
 
 export type AiConversationFieldSource = "default" | "auto" | "user";
 export type AiConversationTitleSource = AiConversationFieldSource;
-export type AiConversationRunStatus = "idle" | "queued" | "running" | "needs_attention" | "failed";
-export type AiConversationStatusFilter = Exclude<AiConversationRunStatus, "idle" | "queued"> | "unread";
+export type AiConversationRunStatus = "idle" | "queued" | "running" | "needs_attention" | "waiting_for_browser" | "failed";
+export type AiConversationStatusFilter = Exclude<AiConversationRunStatus, "idle" | "queued" | "waiting_for_browser"> | "unread";
 
 export type AiConversation = {
+  activity?: { completed: number; total: number; step: string | null; tool: string | null };
+  todoPlan?: { seq: number; todos: import("./todo-contracts").AiTodoPlan["todos"] };
   id: string;
   shortId: string;
   title: string;
@@ -146,6 +148,11 @@ export type AiConversation = {
   keywords: string[];
   pinnedAt: string | null;
   archivedAt: string | null;
+  /** null: automatic after seven days without use; false: explicitly active; true: finished. */
+  done: boolean | null;
+  isDone: boolean;
+  hasActiveSchedule?: boolean;
+  lastUsedAt: string;
   runStatus: AiConversationRunStatus;
   /** Error from the latest turn when `runStatus` is `failed`. */
   runError: string | null;
@@ -247,7 +254,7 @@ export type AiConversationPage = {
 
 export type AiConversationProjectUpdateResult =
   | { ok: true; conversation: AiConversation }
-  | { ok: false; reason: "not_found" | "active_turn" };
+  | { ok: false; reason: "not_found" | "active_turn" | "already_assigned" };
 
 export type AiStoredMessage = {
   id: string;
@@ -270,7 +277,10 @@ export type AiStoredMessage = {
   compactedAt: string | null;
   /** UI metadata (e.g. how many messages a compaction summary replaced). */
   meta: {
+    /** The atomically consumed composer revision, used to confirm a local send. */
+    submittedDraftRevision?: number;
     compactedCount?: number;
+    todoPlan?: import("./todo-contracts").AiTodoPlan;
     steerId?: string;
     agentMessage?: {
       id: string;
@@ -336,6 +346,9 @@ export type AiTurnFinalizedEvent = {
 };
 
 export type AiTurn = {
+  deadline?: string | null;
+  runBudgetMs?: number | null;
+  cancelRequestedAt?: string | null;
   id: string;
   shortId: string;
   conversationId: string;
@@ -419,7 +432,17 @@ export type AiPendingTurnActionRecord = {
 
 export type AiTurnToolSource = { kind: "none" } | { kind: "default"; appTools?: boolean };
 
-export type AiClientToolId = "local_bash";
+export type AiClientToolId =
+  | "local_bash"
+  | "code_run"
+  | "code_action"
+  | "code_inspect"
+  | "code_interact"
+  | "code_stop"
+  | "code_open"
+  | "code_present"
+  | "code_export"
+  | "code_secret";
 
 /** Immutable project instructions and context manifest captured for one turn. */
 export type AiProjectPromptSnapshot = {
@@ -517,6 +540,7 @@ export type AiConversationSourceObservation = {
 };
 
 export type AiConversationFileSnapshotEntry = {
+  dictationRecordedAt?: string;
   path: string;
   size: number;
   mediaType: string;
@@ -531,7 +555,15 @@ export type AiConversationFileSnapshot = {
   total: number;
 };
 
+export type AiBackgroundRunContext = {
+  taskId: string;
+  occurrenceId: string;
+  context: Array<{ seq: number; kind: "message" | "summary"; message: Message }>;
+};
+
 export type AiChatTurnRunConfig = {
+  background?: AiBackgroundRunContext;
+  selectedSkillIds?: string[];
   /** Server-owned marker: model grants apply only to interactive Assistant chat. */
   assistantChat?: true;
   kind?: "chat";
@@ -614,16 +646,18 @@ export type AiConversationService = {
     search?: string;
     refs?: CloudResourceRef[];
     archived?: boolean;
+    done?: boolean;
     status?: AiConversationStatusFilter;
     projectId?: string;
     unassigned?: boolean;
     limit?: number;
   }): Promise<AiConversation[]>;
-  listSidebarConversations(input: { ownerUserId: string; unassignedLimit?: number; perProjectLimit?: number }): Promise<AiConversation[]>;
+  listSidebarConversations(input: { ownerUserId: string }): Promise<AiConversation[]>;
   listConversationsPage(input: {
     ownerUserId: string;
     search?: string;
     archived?: boolean;
+    done?: boolean;
     status?: AiConversationStatusFilter;
     projectId?: string;
     unassigned?: boolean;
@@ -712,8 +746,14 @@ export type AiConversationService = {
     conversationId: string;
     ownerUserId?: string;
     projectId: string | null;
+    onlyUnassigned?: boolean;
   }): Promise<AiConversationProjectUpdateResult>;
   setConversationPinned(input: { conversationId: string; ownerUserId?: string; pinned: boolean }): Promise<AiConversation | null>;
+  setConversationDone(input: {
+    conversationId: string;
+    ownerUserId: string;
+    done: boolean | null;
+  }): Promise<{ ok: true; conversation: AiConversation } | { ok: false; reason: "not_found" | "active_turn" }>;
   archiveConversation(input: { conversationId: string; ownerUserId?: string }): Promise<boolean>;
   restoreConversation(input: { conversationId: string; ownerUserId?: string }): Promise<AiConversation | null>;
   markConversationViewed(input: { conversationId: string; ownerUserId?: string }): Promise<boolean>;
@@ -788,6 +828,7 @@ export type AiConversationService = {
   }): Promise<void>;
   compactMessages(input: {
     conversationId: string;
+    turnId?: string;
     checkpointSeq: number;
     summary: Message;
     modelProfileId?: string | null;
@@ -803,6 +844,7 @@ export type AiConversationService = {
   createCompactionTurn(input: { conversationId: string; modelProfileId: string; runConfig: AiCompactionTurnRunConfig }): Promise<AiTurn>;
   /** Persist the user message and create its turn in one transaction. */
   submitChatTurn(input: {
+    queuedMessageId?: string;
     conversationId: string;
     modelProfileId: string;
     runConfig: AiChatTurnRunConfig;
@@ -819,6 +861,7 @@ export type AiConversationService = {
     retrySourceTurnId?: string;
   }): Promise<{ turn: AiTurn; message: AiStoredMessage }>;
   getTurnRunConfig(input: { conversationId: string; turnId: string }): Promise<AiTurnRunConfig | null>;
+  getLatestTurn(input: { conversationId: string }): Promise<AiTurn | null>;
   getTurn(input: { conversationId: string; turnId: string }): Promise<AiTurn | null>;
   getTurnByShortId(input: { conversationId: string; shortId: string }): Promise<AiTurn | null>;
   getActiveTurn(input: { conversationId: string }): Promise<{ turn: AiTurn; liveBlocks: AiTurnBlock[]; liveSeq: number } | null>;
@@ -888,6 +931,9 @@ export type AiConversationService = {
   /** Atomically make pending steers visible to the model and return them in order. */
   takePendingTurnSteers(input: { conversationId: string; turnId: string; leaseOwner: string }): Promise<AiTurnSteer[]>;
   createSessionStore(input: {
+    background?: AiBackgroundRunContext;
+    /** Called after a durable append, for incremental usage/history projection. */
+    onMessage?: (message: AiStoredMessage) => Promise<void>;
     conversationId: string;
     modelProfileId?: string | null;
     turnId?: string | null;
@@ -956,6 +1002,8 @@ export type AiToolRuntime<TInput extends z.ZodType = z.ZodType, TOutput extends
           selectedModel?: AiResolvedModel;
           locale?: string;
           timeZone?: string;
+          /** Localized short status; never include payloads or secrets. */
+          reportProgress?: (message: string) => Promise<void>;
         },
       ): Promise<z.infer<TOutput>>;
     }

package/src/ai/ui.tsx

@@ -1,3 +1,5 @@
+export { AiTurnBlockView } from "./chat/blocks";
+
 import { latestLoopUsage, latestUsage, latestUsageSnapshot, textFromMessage } from "./chat/message-utils";
 
 export {
@@ -9,6 +11,7 @@ export {
   aiChatAttachments,
   aiChatModelOptions,
   aiComposerAttachmentRecords,
+  aiComposerDraft,
   aiComposerFileAccept,
   aiComposerSendInput,
   createAiPastedTextFile,