omnirush 0.8.6 → 0.9.0

package/assets/extensions/omnirush/voice/core/keyterms.ts

@@ -0,0 +1,117 @@
+/** Coding vocabulary sent as a hint with every recording. */
+export const DEV_KEYTERMS = [
+  "OmniRush",
+  "TypeScript",
+  "JavaScript",
+  "JSON",
+  "YAML",
+  "OAuth",
+  "GitHub",
+  "localhost",
+  "regex",
+  "gRPC",
+  "API",
+  "CLI",
+  "MCP",
+  "npm",
+  "pnpm",
+  "worktree",
+  "subagent",
+  "async",
+  "README",
+] as const;
+
+const MAX_TERMS = 50;
+const MAX_CHARS = 1024;
+
+/** `feat/voice-mode_v2`, `useVoiceDictation`, `voice-core.ts` → their words. */
+export function splitIdentifier(value: string): string[] {
+  return value
+    .replace(/\.[a-z0-9]{1,5}$/i, "")
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .split(/[^\p{L}\p{N}]+/u)
+    .filter((word) => word.length >= 3 && !/^\d+$/.test(word));
+}
+
+export type KeytermContext = {
+  /** The project folder or repository name. */
+  repo?: string | null;
+  branch?: string | null;
+  /** Recently used file paths or names. */
+  files?: readonly string[];
+  extra?: readonly string[];
+};
+
+/**
+ * At most 50 terms and 1024 characters: the dev list, then the repository,
+ * branch and file names (whole and split into words), de-duplicated
+ * case-insensitively, first occurrence wins.
+ */
+export function buildKeyterms(context: KeytermContext = {}): string[] {
+  const candidates: string[] = [...DEV_KEYTERMS, ...(context.extra ?? [])];
+  if (context.repo?.trim()) candidates.push(context.repo.trim(), ...splitIdentifier(context.repo));
+  if (context.branch?.trim() && !/^(main|master|HEAD)$/.test(context.branch.trim())) candidates.push(...splitIdentifier(context.branch));
+  for (const file of context.files ?? []) {
+    const name = file.split(/[\\/]/).pop() ?? "";
+    if (!name) continue;
+    candidates.push(name, ...splitIdentifier(name));
+  }
+  const seen = new Set<string>();
+  const out: string[] = [];
+  let chars = 0;
+  for (const raw of candidates) {
+    const term = raw.trim();
+    const key = term.toLowerCase();
+    if (!term || term.length > 64 || seen.has(key)) continue;
+    if (out.length >= MAX_TERMS || chars + term.length + 1 > MAX_CHARS) break;
+    seen.add(key);
+    out.push(term);
+    chars += term.length + 1;
+  }
+  return out;
+}
+
+/** Spoken forms of written dev terms. Conservative: only phrases no one says for anything else. */
+const SPOKEN_FORMS: ReadonlyArray<[RegExp, string]> = [
+  [/\bo(?:h)?[ -]auth\b/gi, "OAuth"],
+  [/\bgit[ -]?hub\b/gi, "GitHub"],
+  [/\bget[ -]hub\b/gi, "GitHub"],
+  [/\btype[ -]script\b/gi, "TypeScript"],
+  [/\bjava[ -]script\b/gi, "JavaScript"],
+  [/\bnode[ .]?js\b/gi, "Node.js"],
+  [/\bnext[ .]?js\b/gi, "Next.js"],
+  [/\blocal[ -]host\b/gi, "localhost"],
+  [/\bwork[ -]tree(s?)\b/gi, "worktree$1"],
+  [/\bsub[ -]agent(s?)\b/gi, "subagent$1"],
+  [/\bread[ -]me\b/gi, "README"],
+  [/\bp[ -]?npm\b/gi, "pnpm"],
+  [/\bomni[ -]rush\b/gi, "OmniRush"],
+];
+
+const EXTENSIONS = "ts|tsx|js|jsx|mjs|cjs|py|rs|go|md|json|yaml|yml|toml|css|html|sh|sql|txt";
+const DOT_EXTENSION = new RegExp(`\\b([\\p{L}\\p{N}_-]+) dot (${EXTENSIONS})\\b`, "giu");
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Light post-correction of a transcript: spoken dev terms to their written
+ * form, "auth dot ts" to "auth.ts", and project terms with inner capitals
+ * ("omni rush" / "omnirush" → "OmniRush") restored from the keyterms.
+ */
+export function applyKeyterms(text: string, keyterms: readonly string[] = []): string {
+  let out = text;
+  for (const [pattern, replacement] of SPOKEN_FORMS) out = out.replace(pattern, replacement);
+  out = out.replace(DOT_EXTENSION, (_match, name: string, extension: string) => `${name}.${extension.toLowerCase()}`);
+  for (const term of keyterms) {
+    // Only terms whose casing carries meaning, never plain words.
+    if (term.length < 4 || term === term.toLowerCase() || /[^\p{L}\p{N}]/u.test(term)) continue;
+    const words = splitIdentifier(term);
+    // The words must spell the whole term ("gRPC" splits to "RPC": skipped).
+    if (words.join("") !== term) continue;
+    const spoken = words.map(escapeRegExp).join("[ -]?");
+    out = out.replace(new RegExp(`\\b${spoken}\\b`, "gi"), term);
+  }
+  return out;
+}

package/assets/extensions/omnirush/voice/core/resample.ts

@@ -0,0 +1,63 @@
+import { VOICE_SAMPLE_RATE } from "./types";
+
+function toInt16(value: number): number {
+  const clamped = Math.max(-1, Math.min(1, value));
+  return clamped < 0 ? Math.round(clamped * 32768) : Math.round(clamped * 32767);
+}
+
+/**
+ * A streaming converter from float audio at `inputRate` to 16 kHz Int16.
+ * Downsampling averages every input sample that falls inside an output
+ * sample's window (a box filter: cheap, and enough against aliasing for
+ * speech); upsampling interpolates linearly. State carries across calls, so
+ * chunk boundaries leave no clicks.
+ */
+export class Downsampler {
+  private readonly ratio: number;
+  /** Position of the next output sample, in input samples from the start of the pending chunk. */
+  private position = 0;
+  private carry: Float32Array = new Float32Array(0);
+
+  constructor(readonly inputRate: number, readonly outputRate = VOICE_SAMPLE_RATE) {
+    if (!(inputRate > 0) || !(outputRate > 0)) throw new Error("Sample rates must be positive");
+    this.ratio = inputRate / outputRate;
+  }
+
+  push(input: Float32Array): Int16Array {
+    if (this.inputRate === this.outputRate) {
+      const out = new Int16Array(input.length);
+      for (let index = 0; index < input.length; index += 1) out[index] = toInt16(input[index]!);
+      return out;
+    }
+    const data = new Float32Array(this.carry.length + input.length);
+    data.set(this.carry);
+    data.set(input, this.carry.length);
+    const out: number[] = [];
+    if (this.ratio >= 1) {
+      while (this.position + this.ratio <= data.length) {
+        const start = Math.floor(this.position);
+        const end = Math.floor(this.position + this.ratio);
+        let sum = 0;
+        for (let index = start; index < end; index += 1) sum += data[index]!;
+        out.push(toInt16(sum / Math.max(1, end - start)));
+        this.position += this.ratio;
+      }
+    } else {
+      while (this.position + 1 < data.length) {
+        const index = Math.floor(this.position);
+        const fraction = this.position - index;
+        out.push(toInt16(data[index]! * (1 - fraction) + data[index + 1]! * fraction));
+        this.position += this.ratio;
+      }
+    }
+    const consumed = Math.floor(this.position);
+    this.carry = data.slice(consumed);
+    this.position -= consumed;
+    return Int16Array.from(out);
+  }
+}
+
+/** Whole-buffer conversion (file input). */
+export function resampleTo16k(samples: Float32Array, inputRate: number): Int16Array {
+  return new Downsampler(inputRate).push(samples);
+}

package/assets/extensions/omnirush/voice/core/segmenter.ts

@@ -0,0 +1,231 @@
+import { VOICE_SAMPLE_RATE } from "./types";
+
+/** RMS of Int16 samples, normalized to 0..1. */
+export function rms(samples: Int16Array): number {
+  if (samples.length === 0) return 0;
+  let sum = 0;
+  for (let index = 0; index < samples.length; index += 1) {
+    const value = samples[index]! / 32768;
+    sum += value * value;
+  }
+  return Math.sqrt(sum / samples.length);
+}
+
+/** A display level 0..1 for a normalized RMS: square-root scaled so quiet speech still moves the bars. */
+export function levelFromRms(value: number): number {
+  return Math.min(1, Math.sqrt((value * 32768) / 2000));
+}
+
+/** The rolling 16-bar level history the waveform draws. */
+export class LevelMeter {
+  private readonly history: number[];
+
+  constructor(readonly bars = 16) {
+    this.history = new Array<number>(bars).fill(0);
+  }
+
+  push(value: number): void {
+    this.history.push(levelFromRms(value));
+    if (this.history.length > this.bars) this.history.shift();
+  }
+
+  levels(): number[] {
+    return [...this.history];
+  }
+}
+
+export type Segment = {
+  index: number;
+  /** 16 kHz mono PCM, including a short lead-in before the first speech frame. */
+  samples: Int16Array;
+  startMs: number;
+  endMs: number;
+  /** Frames classified as speech, in ms. */
+  speechMs: number;
+  /** Highest frame RMS in the segment, 0..1. */
+  peakRms: number;
+  /** The speech threshold when the segment closed, for the hallucination filter. */
+  threshold: number;
+};
+
+export type SegmenterOptions = {
+  sampleRate?: number;
+  /** Analysis frame length. */
+  frameMs?: number;
+  /** A pause at least this long ends a segment. */
+  pauseMs?: number;
+  /** A segment is cut here even mid-speech. */
+  maxSegmentMs?: number;
+  /** A segment shorter than this is held open across a normal pause (short segments transcribe badly). */
+  minSegmentMs?: number;
+  /** A pause this long ends even a short segment. */
+  longPauseMs?: number;
+  /** Less speech than this (a click, a cough) is dropped as noise. */
+  minSpeechMs?: number;
+  /** Audio kept before the first speech frame, so onsets are not clipped. */
+  prerollMs?: number;
+  /** Silence kept after the last speech frame. */
+  tailMs?: number;
+  /** Absolute speech floor (normalized RMS); the adaptive threshold never goes below it. */
+  minThreshold?: number;
+  /** Speech is this many times above the running noise floor. */
+  noiseMultiplier?: number;
+};
+
+const DEFAULTS: Required<SegmenterOptions> = {
+  sampleRate: VOICE_SAMPLE_RATE,
+  frameMs: 20,
+  pauseMs: 600,
+  maxSegmentMs: 12_000,
+  minSegmentMs: 1_000,
+  longPauseMs: 1_500,
+  minSpeechMs: 120,
+  prerollMs: 200,
+  tailMs: 250,
+  minThreshold: 0.01,
+  noiseMultiplier: 3,
+};
+
+/**
+ * Energy VAD with an adaptive noise floor that splits a PCM stream into
+ * speech segments at pauses. Silence outside a segment is discarded as it
+ * arrives: a silent recording produces no segment at all, so nothing is ever
+ * uploaded for it.
+ */
+export class Segmenter {
+  readonly options: Required<SegmenterOptions>;
+  private readonly frameSize: number;
+  private pendingFrame: Int16Array = new Int16Array(0);
+  private noiseFloor = 0.004;
+  private frames: Int16Array[] = [];
+  private frameRms: number[] = [];
+  /** Frames before the first speech of the next segment, capped at the preroll. */
+  private preroll: Int16Array[] = [];
+  private inSegment = false;
+  private speechFrames = 0;
+  private silenceRun = 0;
+  private segmentStartFrame = 0;
+  private framesSeen = 0;
+  private nextIndex = 0;
+  /** Highest frame RMS of the whole stream. */
+  maxRms = 0;
+  /** Speech frames of the whole stream (including dropped noise bursts). */
+  totalSpeechFrames = 0;
+
+  constructor(options: SegmenterOptions = {}) {
+    this.options = { ...DEFAULTS, ...options };
+    this.frameSize = Math.round((this.options.sampleRate * this.options.frameMs) / 1000);
+  }
+
+  get threshold(): number {
+    return Math.max(this.options.minThreshold, this.noiseFloor * this.options.noiseMultiplier);
+  }
+
+  /** Feeds PCM; returns segments that closed. `onFrame` sees every analysis frame's RMS (the level meter). */
+  push(samples: Int16Array, onFrame?: (frameRms: number) => void): Segment[] {
+    const closed: Segment[] = [];
+    let data = samples;
+    if (this.pendingFrame.length > 0) {
+      data = new Int16Array(this.pendingFrame.length + samples.length);
+      data.set(this.pendingFrame);
+      data.set(samples, this.pendingFrame.length);
+    }
+    let offset = 0;
+    for (; offset + this.frameSize <= data.length; offset += this.frameSize) {
+      const frame = data.slice(offset, offset + this.frameSize);
+      const value = rms(frame);
+      onFrame?.(value);
+      const segment = this.frame(frame, value);
+      if (segment) closed.push(segment);
+    }
+    this.pendingFrame = data.slice(offset);
+    return closed;
+  }
+
+  /** Ends the stream: the open segment, if it holds enough speech. */
+  flush(): Segment | null {
+    this.pendingFrame = new Int16Array(0);
+    if (!this.inSegment) return null;
+    return this.close(true);
+  }
+
+  private frame(frame: Int16Array, value: number): Segment | null {
+    this.framesSeen += 1;
+    this.maxRms = Math.max(this.maxRms, value);
+    const speech = value > this.threshold;
+    if (speech) this.totalSpeechFrames += 1;
+    else this.noiseFloor = Math.min(0.05, Math.max(0.001, this.noiseFloor * 0.95 + value * 0.05));
+    const { frameMs } = this.options;
+
+    if (!this.inSegment) {
+      if (!speech) {
+        this.preroll.push(frame);
+        if (this.preroll.length * frameMs > this.options.prerollMs) this.preroll.shift();
+        return null;
+      }
+      this.inSegment = true;
+      this.frames = [...this.preroll, frame];
+      this.frameRms = [...this.preroll.map(rms), value];
+      this.segmentStartFrame = this.framesSeen - this.frames.length;
+      this.preroll = [];
+      this.speechFrames = 1;
+      this.silenceRun = 0;
+      return null;
+    }
+
+    this.frames.push(frame);
+    this.frameRms.push(value);
+    if (speech) {
+      this.speechFrames += 1;
+      this.silenceRun = 0;
+    } else {
+      this.silenceRun += 1;
+    }
+    const lengthMs = this.frames.length * frameMs;
+    const silenceMs = this.silenceRun * frameMs;
+    if (lengthMs >= this.options.maxSegmentMs) return this.close(false);
+    if (silenceMs >= this.options.longPauseMs) return this.close(true);
+    if (silenceMs >= this.options.pauseMs && lengthMs - silenceMs >= this.options.minSegmentMs) return this.close(true);
+    return null;
+  }
+
+  private close(trimSilence: boolean): Segment | null {
+    const { frameMs } = this.options;
+    let frames = this.frames;
+    let frameRms = this.frameRms;
+    if (trimSilence && this.silenceRun > 0) {
+      const keep = Math.ceil(this.options.tailMs / frameMs);
+      const drop = Math.max(0, this.silenceRun - keep);
+      frames = frames.slice(0, frames.length - drop);
+      frameRms = frameRms.slice(0, frameRms.length - drop);
+      // The trimmed silence may lead into the next segment's preroll.
+      this.preroll = this.frames.slice(this.frames.length - Math.min(drop, Math.ceil(this.options.prerollMs / frameMs)));
+    } else {
+      this.preroll = [];
+    }
+    const speechMs = this.speechFrames * frameMs;
+    const startFrame = this.segmentStartFrame;
+    this.inSegment = false;
+    this.frames = [];
+    this.frameRms = [];
+    this.speechFrames = 0;
+    this.silenceRun = 0;
+    if (speechMs < this.options.minSpeechMs) return null;
+    const length = frames.reduce((total, frame) => total + frame.length, 0);
+    const samples = new Int16Array(length);
+    let at = 0;
+    for (const frame of frames) {
+      samples.set(frame, at);
+      at += frame.length;
+    }
+    return {
+      index: this.nextIndex++,
+      samples,
+      startMs: startFrame * frameMs,
+      endMs: (startFrame + frames.length) * frameMs,
+      speechMs,
+      peakRms: Math.max(0, ...frameRms),
+      threshold: this.threshold,
+    };
+  }
+}