crewly 1.20.213 → 1.20.215

package/dist/backend/backend/src/services/talk/talk-transcribe.service.d.ts

@@ -0,0 +1,344 @@
+/**
+ * Talk transcription — speech-to-text for the portal Talk page on the
+ * owner's own machine (specs/2026-10-04-talk-whisper-transcribe.md, #1074).
+ *
+ * The phone records a short clip and sends it over the relay to the machine
+ * of the agent being talked to; this service turns it into text with
+ * whisper.cpp and hands the text back. It never sends anything to an agent:
+ * the owner reads, edits and confirms the text on the phone first.
+ *
+ * Engine: the `transcribe-audio` skill's — same `whisper-cli` lookup order,
+ * same large-v3-turbo model file, same ffmpeg normalisation to 16 kHz mono
+ * WAV. Its setup block is what installs the pieces ({@link startSetup}).
+ *
+ * Properties:
+ * - **One at a time.** whisper uses most of the CPU; requests queue behind
+ *   the running one (at most {@link TALK_TRANSCRIBE_CONSTANTS.MAX_QUEUE}
+ *   waiting) and share one deadline that includes the wait.
+ * - **Nothing kept.** Audio is written to a private temp dir that is removed
+ *   in `finally`; neither audio nor transcript is ever logged.
+ * - **Clear failures.** Every refusal carries a code the phone acts on by
+ *   falling back to on-device recognition: `whisper_unavailable`,
+ *   `too_long`, `timeout`, `failed` (and `busy`, `invalid_audio`).
+ *
+ * @module services/talk/talk-transcribe.service
+ */
+import { type ComponentLogger } from '../core/logger.service.js';
+import type { RunCommand } from '../harness/harness.types.js';
+declare const C: {
+    readonly SKILL_ID: "transcribe-audio";
+    readonly CAPABILITY: "talk_whisper";
+    readonly MODEL_FILENAME: "ggml-large-v3-turbo-q5_0.bin";
+    readonly MODEL_DIRS: readonly [".flopost/whisper", ".cache/whisper-models"];
+    readonly HOME_BIN_CANDIDATES: readonly [".flopost/whisper/whisper-cli"];
+    readonly WHISPER_BINARY: "whisper-cli";
+    readonly FFMPEG_BINARY: "ffmpeg";
+    readonly MAX_AUDIO_BYTES: number;
+    readonly MAX_DURATION_SEC: 90;
+    readonly REQUEST_TIMEOUT_MS: 45000;
+    readonly MAX_QUEUE: 3;
+    readonly SAMPLE_RATE: 16000;
+    readonly WAV_BYTES_PER_SEC: 32000;
+    readonly WAV_HEADER_BYTES: 44;
+    readonly THREADS_RESERVED: 2;
+    readonly MIN_THREADS: 4;
+    readonly PROMPT_PREFIX: "以下是普通话的句子。";
+    readonly GLOSSARY: readonly ["Crewly", "CE", "Crewly Cloud", "Slack"];
+    readonly MAX_VOCAB_CHARS: 400;
+    readonly LANGUAGES: readonly ["auto", "zh", "en"];
+    readonly MIME_EXTENSIONS: Readonly<Record<string, string>>;
+    readonly TEMP_PREFIX: "crewly-talk-stt-";
+    readonly CODES: {
+        readonly WHISPER_UNAVAILABLE: "whisper_unavailable";
+        readonly TOO_LONG: "too_long";
+        readonly TIMEOUT: "timeout";
+        readonly FAILED: "failed";
+        readonly BUSY: "busy";
+        readonly INVALID_AUDIO: "invalid_audio";
+        readonly OWNER_ONLY: "owner_only";
+    };
+};
+/** Error code returned to the phone. */
+export type TalkTranscribeCode = (typeof C.CODES)[keyof typeof C.CODES];
+/** HTTP status for each refusal. */
+export declare const TALK_TRANSCRIBE_HTTP_STATUS: Readonly<Record<TalkTranscribeCode, number>>;
+/** A refusal with a machine-readable code. Its message never contains audio or transcript. */
+export declare class TalkTranscribeError extends Error {
+    readonly code: TalkTranscribeCode;
+    /**
+     * @param code - What went wrong (the phone falls back on any of them)
+     * @param message - Short English explanation
+     */
+    constructor(code: TalkTranscribeCode, message: string);
+    /** HTTP status for this refusal. */
+    get httpStatus(): number;
+}
+/** Where the engine's pieces are (null = missing). */
+export interface WhisperEngine {
+    whisperBin: string | null;
+    model: string | null;
+    ffmpeg: string | null;
+}
+/** The setup job started from the phone, as far as this service knows. */
+export interface TalkSetupState {
+    state: 'running' | 'succeeded' | 'failed';
+    jobId?: string;
+    message?: string;
+}
+/** `GET /api/talk/transcribe/status` data. */
+export interface TalkTranscribeStatus {
+    /** whisper-cli, the model and ffmpeg are all present */
+    whisperReady: boolean;
+    /** Missing pieces, by the skill's setup step ids (`ffmpeg`, `whisper-cli`, `whisper-model`) */
+    missing: string[];
+    /** This machine's name (for "Whisper · <machine>") */
+    deviceName: string;
+    /** A transcription is running */
+    busy: boolean;
+    /** Requests waiting behind it */
+    queued: number;
+    maxDurationSec: number;
+    maxAudioBytes: number;
+    /** Setup started from the phone, if any */
+    setup?: TalkSetupState;
+}
+/** Input to {@link TalkTranscribeService.transcribe}. */
+export interface TranscribeInput {
+    /** Base64 audio (a `data:` URL prefix is tolerated) */
+    audio: unknown;
+    /** Container type from MediaRecorder, e.g. `audio/mp4` or `audio/webm;codecs=opus` */
+    mimeType: unknown;
+    /** `auto` (default), `zh` or `en` */
+    language?: unknown;
+}
+/** A finished transcription. */
+export interface TranscribeResult {
+    text: string;
+    /** Language whisper detected (or was told) */
+    language: string;
+    durationSec: number;
+    engine: 'whisper.cpp';
+    deviceName: string;
+    /** Time spent, including any wait in the queue (ms) */
+    elapsedMs: number;
+}
+/** Injectable dependencies (all optional). */
+export interface TalkTranscribeDeps {
+    run?: RunCommand;
+    env?: NodeJS.ProcessEnv;
+    homeDir?: () => string;
+    crewlyHome?: () => string;
+    isExecutable?: (file: string) => boolean;
+    isFile?: (file: string) => boolean;
+    /** Team / agent / project names for the vocabulary prompt */
+    vocabulary?: () => Promise<string[]>;
+    deviceName?: () => Promise<string>;
+    /** Start the transcribe-audio skill setup in the background */
+    startSetup?: () => Promise<TalkSetupState>;
+    /** Current state of a setup job */
+    setupJob?: (jobId: string) => Promise<TalkSetupState | null>;
+    tmpDir?: () => string;
+    cpuCount?: () => number;
+    now?: () => number;
+    logger?: ComponentLogger;
+    /** Overrides of the limits (tests) */
+    limits?: Partial<{
+        maxAudioBytes: number;
+        maxDurationSec: number;
+        requestTimeoutMs: number;
+        maxQueue: number;
+    }>;
+}
+/**
+ * Temp-file extension for a MediaRecorder type (`audio/webm;codecs=opus` → `webm`).
+ *
+ * @param mimeType - Container type
+ * @returns Extension, or null when the type is not audio we take
+ */
+export declare function extensionForMime(mimeType: string): string | null;
+/**
+ * Decode the base64 body, refusing anything malformed or over the size cap
+ * before decoding it.
+ *
+ * @param audio - Base64 string, optionally a `data:` URL
+ * @param maxBytes - Size cap of the decoded audio
+ * @returns The audio bytes
+ * @throws TalkTranscribeError `invalid_audio` or `too_long`
+ */
+export declare function decodeAudio(audio: unknown, maxBytes: number): Buffer;
+/**
+ * The initial prompt: the Simplified-Chinese bias sentence, then the owner
+ * glossary and this machine's team / agent / project names, so names like
+ * "CE" are not heard as "侧". Bounded so whisper keeps all of it.
+ *
+ * @param names - Vocabulary from this machine
+ * @param maxChars - Bound on the vocabulary part
+ * @returns Prompt text
+ *
+ * @example
+ * ```ts
+ * buildVocabularyPrompt(['Marketing', 'Ella']);
+ * // '以下是普通话的句子。Crewly、CE、Crewly Cloud、Slack、Marketing、Ella。'
+ * ```
+ */
+export declare function buildVocabularyPrompt(names: readonly string[], maxChars?: number): string;
+/**
+ * Join whisper segments: a space only between two Latin-script edges, so
+ * Chinese stays unspaced and English words stay apart.
+ *
+ * @param segments - Segment texts
+ * @returns The transcript
+ */
+export declare function joinSegments(segments: readonly string[]): string;
+/** Talk speech-to-text with whisper.cpp. */
+export declare class TalkTranscribeService {
+    private readonly run;
+    private readonly env;
+    private readonly homeDir;
+    private readonly crewlyHome;
+    private readonly isExecutable;
+    private readonly isFile;
+    private readonly vocabulary;
+    private readonly deviceNameFn;
+    private readonly startSetupFn;
+    private readonly setupJobFn;
+    private readonly tmpDir;
+    private readonly cpuCount;
+    private readonly now;
+    private readonly logger;
+    private readonly maxAudioBytes;
+    private readonly maxDurationSec;
+    private readonly requestTimeoutMs;
+    private readonly maxQueue;
+    private running;
+    private readonly waiters;
+    private lastSetup;
+    /**
+     * @param deps - Injectable dependencies
+     */
+    constructor(deps?: TalkTranscribeDeps);
+    /**
+     * PATH searched for commands: `$CREWLY_HOME/bin`, the process PATH, then
+     * the usual system and Homebrew dirs (a backend started by launchd often
+     * lacks /opt/homebrew/bin) — the skill setup runner's search path.
+     *
+     * @returns PATH string
+     */
+    private searchPath;
+    /**
+     * Find whisper-cli, the model and ffmpeg. Checked on every call (cheap
+     * file checks), so a finished setup is seen at once.
+     *
+     * Order, as in `transcribe-audio/execute.sh`: `FLOPOST_WHISPER_BIN`,
+     * `~/.flopost/whisper/whisper-cli`, `$CREWLY_HOME/bin`, PATH, Homebrew;
+     * model: `FLOPOST_WHISPER_MODEL`, `~/.flopost/whisper/`, `~/.cache/whisper-models/`.
+     *
+     * @returns Paths (null = missing)
+     */
+    resolveEngine(): WhisperEngine;
+    /**
+     * Missing pieces, named by the skill's setup step ids.
+     *
+     * @param engine - Resolved engine
+     * @returns e.g. `['whisper-model']`
+     */
+    private missingParts;
+    /**
+     * Whether a transcription can run here now.
+     *
+     * @returns True when whisper-cli, the model and ffmpeg are present
+     */
+    isReady(): boolean;
+    /**
+     * Capabilities for the Cloud heartbeat: `talk_whisper` while ready.
+     *
+     * @returns Capability list
+     */
+    capabilities(): string[];
+    /**
+     * This machine's name, never failing.
+     *
+     * @returns Device name
+     */
+    private deviceName;
+    /**
+     * Status for the phone: ready or not, what is missing, queue, setup.
+     *
+     * @returns Status
+     */
+    getStatus(): Promise<TalkTranscribeStatus>;
+    /**
+     * The last setup job, refreshed from the job service.
+     *
+     * @returns State, or null when none was started here
+     */
+    private currentSetup;
+    /**
+     * Install ffmpeg, whisper-cli and the model with the transcribe-audio
+     * skill's setup block, in the background (unattended; the skill is
+     * official). Returns at once.
+     *
+     * @returns The setup state (`succeeded` when everything was already there)
+     * @throws TalkTranscribeError `failed` when setup cannot be started
+     */
+    startSetup(): Promise<TalkSetupState>;
+    /**
+     * Take the one-at-a-time slot, waiting until `deadline` at most.
+     *
+     * @param deadline - Absolute ms
+     * @throws TalkTranscribeError `busy` (queue full) or `timeout`
+     */
+    private acquire;
+    /** Hand the slot to the next waiter, or free it. */
+    private release;
+    /**
+     * Transcribe one clip.
+     *
+     * Validation (type, size, engine present) happens before queueing; the
+     * deadline covers the wait, ffmpeg and whisper. Temp files are removed in
+     * `finally`. Logs carry only sizes, durations and codes.
+     *
+     * @param input - Base64 audio, container type, language hint
+     * @returns The transcript
+     * @throws TalkTranscribeError with a code the phone falls back on
+     *
+     * @example
+     * ```ts
+     * const r = await svc.transcribe({ audio: b64, mimeType: 'audio/mp4' });
+     * // { text: '帮我看一下 CE 的网站', language: 'zh', durationSec: 3.2, ... }
+     * ```
+     */
+    transcribe(input: TranscribeInput): Promise<TranscribeResult>;
+    /**
+     * ffmpeg → WAV, duration check, whisper-cli → JSON → text.
+     *
+     * @param engine - Resolved paths (all present)
+     * @param workDir - Private temp dir
+     * @param bytes - Encoded audio
+     * @param ext - Input extension
+     * @param language - `auto` / `zh` / `en`
+     * @param deadline - Absolute ms
+     * @returns Text, language, duration
+     */
+    private runEngine;
+}
+/**
+ * The process-wide service, created with the default dependencies on first use.
+ *
+ * @returns The service
+ */
+export declare function getTalkTranscribeService(): TalkTranscribeService;
+/**
+ * Replace the process-wide service (tests; null resets).
+ *
+ * @param service - Service or null
+ */
+export declare function setTalkTranscribeService(service: TalkTranscribeService | null): void;
+/**
+ * Capabilities for the Cloud heartbeat (`talk_whisper` while Whisper is ready).
+ *
+ * @returns Capability list
+ */
+export declare function talkTranscribeCapabilities(): string[];
+export {};
+//# sourceMappingURL=talk-transcribe.service.d.ts.map

package/dist/backend/backend/src/services/talk/talk-transcribe.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"talk-transcribe.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/talk/talk-transcribe.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAE9D,QAAA,MAAM,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAA4B,CAAC;AAEpC,wCAAwC;AACxC,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAExE,oCAAoC;AACpC,eAAO,MAAM,2BAA2B,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAQpF,CAAC;AAEF,8FAA8F;AAC9F,qBAAa,mBAAoB,SAAQ,KAAK;aAM5B,IAAI,EAAE,kBAAkB;IALzC;;;OAGG;gBAEc,IAAI,EAAE,kBAAkB,EACxC,OAAO,EAAE,MAAM;IAMhB,oCAAoC;IACpC,IAAI,UAAU,IAAI,MAAM,CAEvB;CACD;AAED,sDAAsD;AACtD,MAAM,WAAW,aAAa;IAC7B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACpC,wDAAwD;IACxD,YAAY,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,IAAI,EAAE,OAAO,CAAC;IACd,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,KAAK,CAAC,EAAE,cAAc,CAAC;CACvB;AAED,yDAAyD;AACzD,MAAM,WAAW,eAAe;IAC/B,uDAAuD;IACvD,KAAK,EAAE,OAAO,CAAC;IACf,sFAAsF;IACtF,QAAQ,EAAE,OAAO,CAAC;IAClB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,gCAAgC;AAChC,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,8CAA8C;AAC9C,MAAM,WAAW,kBAAkB;IAClC,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IACzC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IACnC,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrC,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3C,mCAAmC;IACnC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAC7D,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,sCAAsC;IACtC,MAAM,CAAC,EAAE,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAChH;AA+BD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGhE;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAkBpE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,GAAE,MAA0B,GAAG,MAAM,CAe5G;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAShE;AAQD,4CAA4C;AAC5C,qBAAa,qBAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IACjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAe;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA4B;IACzD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4B;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA0B;IACrD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAyC;IACtE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6D;IACxF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;IACjD,OAAO,CAAC,SAAS,CAA+B;IAEhD;;OAEG;gBACS,IAAI,GAAE,kBAAuB;IAyBzC;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IASlB;;;;;;;;;OASG;IACH,aAAa,IAAI,aAAa;IA0B9B;;;;;OAKG;IACH,OAAO,CAAC,YAAY;IAQpB;;;;OAIG;IACH,OAAO,IAAI,OAAO;IAIlB;;;;OAIG;IACH,YAAY,IAAI,MAAM,EAAE;IAIxB;;;;OAIG;YACW,UAAU;IAQxB;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAehD;;;;OAIG;YACW,YAAY;IAS1B;;;;;;;OAOG;IACG,UAAU,IAAI,OAAO,CAAC,cAAc,CAAC;IAqB3C;;;;;OAKG;IACH,OAAO,CAAC,OAAO;IAsBf,oDAAoD;IACpD,OAAO,CAAC,OAAO;IAMf;;;;;;;;;;;;;;;;OAgBG;IACG,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiDnE;;;;;;;;;;OAUG;YACW,SAAS;CAiEvB;AAQD;;;;GAIG;AACH,wBAAgB,wBAAwB,IAAI,qBAAqB,CAGhE;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,GAAG,IAAI,GAAG,IAAI,CAEpF;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,EAAE,CAMrD"}