@startsimpli/llm 0.4.29 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -5
- package/bin/startsimpli-speech.js +22 -0
- package/package.json +10 -3
- package/src/__tests__/fixtures/captions-golden.json +1881 -0
- package/src/__tests__/fixtures/eleven-v3-tagged-alignment.json +1 -0
- package/src/__tests__/fixtures/voices-golden.json +614 -0
- package/src/__tests__/refine.test.ts +193 -0
- package/src/__tests__/speech-captions-cli.test.ts +37 -0
- package/src/__tests__/speech-captions.test.ts +136 -0
- package/src/__tests__/speech-cli.test.ts +67 -0
- package/src/__tests__/speech-synthesize.test.ts +277 -0
- package/src/__tests__/speech-voices-cli.test.ts +45 -0
- package/src/__tests__/speech-voices.test.ts +109 -0
- package/src/index.ts +3 -0
- package/src/refine/errors.ts +23 -0
- package/src/refine/gates.ts +55 -0
- package/src/refine/index.ts +11 -0
- package/src/refine/llm-step.ts +35 -0
- package/src/refine/refine.ts +81 -0
- package/src/refine/types.ts +54 -0
- package/src/speech/alignment.ts +48 -0
- package/src/speech/bin.ts +12 -0
- package/src/speech/captions/cues.ts +108 -0
- package/src/speech/captions/formats.ts +33 -0
- package/src/speech/captions/index.ts +6 -0
- package/src/speech/captions/types.ts +18 -0
- package/src/speech/captions/words.ts +91 -0
- package/src/speech/captions-cli.ts +50 -0
- package/src/speech/cli.ts +140 -0
- package/src/speech/http.ts +11 -4
- package/src/speech/index.ts +8 -2
- package/src/speech/interface.ts +7 -5
- package/src/speech/providers/elevenlabs-models.ts +34 -0
- package/src/speech/providers/elevenlabs-tts.ts +88 -0
- package/src/speech/providers/elevenlabs.ts +23 -5
- package/src/speech/providers/mock.ts +23 -3
- package/src/speech/providers/openai.ts +34 -3
- package/src/speech/repr.ts +14 -0
- package/src/speech/service.ts +25 -14
- package/src/speech/types.ts +55 -1
- package/src/speech/voices/cast-voices.json +127 -0
- package/src/speech/voices/cast-voices.ts +169 -0
- package/src/speech/voices/emotion.ts +51 -0
- package/src/speech/voices/errors.ts +14 -0
- package/src/speech/voices/index.ts +12 -0
- package/src/speech/voices/schema.ts +53 -0
- package/src/speech/voices/tags.ts +11 -0
- package/src/speech/voices-cli.ts +43 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { TokenUsage } from '../types';
|
|
2
|
+
|
|
3
|
+
/** One line a gate reports. A bare string is a defect; `level: 'info'` is information and never fails the gate. */
|
|
4
|
+
export type GateFinding = string | { text: string; level?: 'defect' | 'info' };
|
|
5
|
+
|
|
6
|
+
/** Gates see the step's result and a caller-owned context. They may be async (a render check, a validator service). */
|
|
7
|
+
export type RefineGate<T, C = unknown> = (result: T, context: C) => GateFinding[] | Promise<GateFinding[]>;
|
|
8
|
+
|
|
9
|
+
/** The failures one gate found; a report is the list of gates that failed, empty when every gate passed. */
|
|
10
|
+
export interface GateFailure { gate: string; failures: string[] }
|
|
11
|
+
|
|
12
|
+
export interface GateNote { gate: string; text: string }
|
|
13
|
+
|
|
14
|
+
/** What the step is handed on every try. Data only: the step decides how to put it in a prompt. */
|
|
15
|
+
export interface StepInput<T> {
|
|
16
|
+
/** 1-based try number. */
|
|
17
|
+
n: number;
|
|
18
|
+
/** The previous try's result, or null on the first try and after a try that produced none. */
|
|
19
|
+
previous: T | null;
|
|
20
|
+
/** The previous try's gate report; empty on the first try. */
|
|
21
|
+
failures: GateFailure[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface StepOutput<T> {
|
|
25
|
+
/** The revised result, or null when the reply could not be read. */
|
|
26
|
+
result: T | null;
|
|
27
|
+
/** What the step rejected in the reply (parse / schema errors). They count as failures of the `rejectedGate` gate. */
|
|
28
|
+
errors?: string[];
|
|
29
|
+
/** Model that answered, for the usage tally. */
|
|
30
|
+
model?: string;
|
|
31
|
+
usage?: TokenUsage;
|
|
32
|
+
/** USD this try's calls cost, when the step prices them. */
|
|
33
|
+
costUsd?: number;
|
|
34
|
+
/** Step calls this try made (more than one only for a step that regenerates parts). Default 1. */
|
|
35
|
+
calls?: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** One call of the model (or several, for a step that regenerates parts of a result). The client is closed over, so it is injectable. */
|
|
39
|
+
export type RefineStep<T> = (input: StepInput<T>) => Promise<StepOutput<T>>;
|
|
40
|
+
|
|
41
|
+
export interface RefineCost { usd: number; promptTokens: number; completionTokens: number; cachedTokens: number }
|
|
42
|
+
|
|
43
|
+
export interface RefineAttempt<T> {
|
|
44
|
+
n: number;
|
|
45
|
+
result: T | null;
|
|
46
|
+
/** Empty = every gate passed. */
|
|
47
|
+
report: GateFailure[];
|
|
48
|
+
/** Informational gate lines; never sent back to the step. */
|
|
49
|
+
notes: GateNote[];
|
|
50
|
+
cost: RefineCost;
|
|
51
|
+
passed: boolean;
|
|
52
|
+
seconds: number;
|
|
53
|
+
calls: number;
|
|
54
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { AlignedCharacter, TranscriptWord } from './types';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Character alignment -> word timings, split on whitespace.
|
|
5
|
+
*
|
|
6
|
+
* `[tag]` spans (v3-style audio tags such as `[excited]`) are delivery
|
|
7
|
+
* directions the vendor aligns like speech but nobody hears as words, so they
|
|
8
|
+
* are skipped by default. Skipping removes the characters, not the time: the
|
|
9
|
+
* next word keeps its true offset, so a ~0.2s tag leaves a gap rather than
|
|
10
|
+
* shifting everything left. A `[` with no closing `]` is ordinary text.
|
|
11
|
+
*/
|
|
12
|
+
export function wordsFromCharacters(
|
|
13
|
+
characters: AlignedCharacter[],
|
|
14
|
+
options: { keepTags?: boolean } = {},
|
|
15
|
+
): TranscriptWord[] {
|
|
16
|
+
const words: TranscriptWord[] = [];
|
|
17
|
+
let text = '';
|
|
18
|
+
let start = 0;
|
|
19
|
+
let end = 0;
|
|
20
|
+
const flush = () => {
|
|
21
|
+
if (text) words.push({ text, start: round(start), end: round(end) });
|
|
22
|
+
text = '';
|
|
23
|
+
};
|
|
24
|
+
for (let i = 0; i < characters.length; i += 1) {
|
|
25
|
+
const c = characters[i];
|
|
26
|
+
if (!options.keepTags && c.text === '[' && !text) {
|
|
27
|
+
const close = closingBracket(characters, i);
|
|
28
|
+
if (close > 0) { i = close; continue; }
|
|
29
|
+
}
|
|
30
|
+
if (/\s/.test(c.text)) { flush(); continue; }
|
|
31
|
+
if (!text) start = c.start;
|
|
32
|
+
text += c.text;
|
|
33
|
+
end = c.end;
|
|
34
|
+
}
|
|
35
|
+
flush();
|
|
36
|
+
return words;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Index of the `]` closing a tag opened at `open`, or -1 (no close, or whitespace-only run is not a tag). */
|
|
40
|
+
function closingBracket(characters: AlignedCharacter[], open: number): number {
|
|
41
|
+
for (let j = open + 1; j < characters.length; j += 1) {
|
|
42
|
+
if (characters[j].text === ']') return j > open + 1 ? j : -1;
|
|
43
|
+
if (characters[j].text === '[') return -1;
|
|
44
|
+
}
|
|
45
|
+
return -1;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const round = (n: number) => Math.round(n * 1000) / 1000;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { runCli } from './cli';
|
|
2
|
+
|
|
3
|
+
async function readStdin(): Promise<string> {
|
|
4
|
+
const chunks: Buffer[] = [];
|
|
5
|
+
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
|
6
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
runCli(process.argv.slice(2), { readStdin }).then(({ exitCode, output }) => {
|
|
10
|
+
process.stdout.write(`${JSON.stringify(output)}\n`);
|
|
11
|
+
process.exitCode = exitCode;
|
|
12
|
+
});
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type { CaptionWord, Cue, CueOptions } from './types';
|
|
2
|
+
|
|
3
|
+
const CLAUSE_END = [',', ';', ':', '.', '?', '!'];
|
|
4
|
+
const endsClause = (token: string) => CLAUSE_END.some((c) => token.endsWith(c));
|
|
5
|
+
/** Length in code points, as Python counts. */
|
|
6
|
+
const len = (s: string) => [...s].length;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Index k of the last token (not the final one) that ends a clause with at least two tokens before the cut, else null.
|
|
10
|
+
* A cue that has to end early ends there instead of in the middle of a clause.
|
|
11
|
+
*/
|
|
12
|
+
function clauseCut(tokens: string[]): number | null {
|
|
13
|
+
if (endsClause(tokens[tokens.length - 1])) return null;
|
|
14
|
+
for (let k = tokens.length - 2; k > 0; k -= 1) if (endsClause(tokens[k])) return k;
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Words -> cues of at most `maxWords` words and `maxChars` characters, also split at a silence over `maxGap`. */
|
|
19
|
+
export function cuesFromWords(words: CaptionWord[], options: CueOptions = {}): Cue[] {
|
|
20
|
+
const { maxWords = 7, maxChars = 42, maxGap = 0.6, clauseBreaks = false } = options;
|
|
21
|
+
const cues: Cue[] = [];
|
|
22
|
+
let cur: CaptionWord[] = [];
|
|
23
|
+
const flush = (upto?: number) => {
|
|
24
|
+
const part = upto ? cur.slice(0, upto) : [...cur];
|
|
25
|
+
if (part.length) {
|
|
26
|
+
cues.push({ start: part[0].start, end: part[part.length - 1].end, text: part.map((w) => w.text).join(' ') });
|
|
27
|
+
cur = cur.slice(part.length);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
for (const w of words) {
|
|
31
|
+
if (cur.length) {
|
|
32
|
+
const tooLong = len([...cur, w].map((x) => x.text).join(' ')) > maxChars;
|
|
33
|
+
const gap = w.start - cur[cur.length - 1].end;
|
|
34
|
+
if (cur.length >= maxWords || tooLong || gap > maxGap) {
|
|
35
|
+
const k = clauseBreaks && gap <= maxGap ? clauseCut(cur.map((x) => x.text)) : null;
|
|
36
|
+
flush(k !== null ? k + 1 : undefined);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
cur.push(w);
|
|
40
|
+
}
|
|
41
|
+
flush();
|
|
42
|
+
return cues;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Text with no timings: spread evenly over [0, duration], each chunk weighted by its character count. */
|
|
46
|
+
export function cuesFromText(text: string, duration: number, options: CueOptions = {}): Cue[] {
|
|
47
|
+
const { maxWords = 7, maxChars = 42, clauseBreaks = false } = options;
|
|
48
|
+
const tokens = text.split(/\s+/).filter(Boolean);
|
|
49
|
+
if (!tokens.length) return [];
|
|
50
|
+
const chunks: string[][] = [];
|
|
51
|
+
let cur: string[] = [];
|
|
52
|
+
for (const t of tokens) {
|
|
53
|
+
if (cur.length && (cur.length >= maxWords || len([...cur, t].join(' ')) > maxChars)) {
|
|
54
|
+
const k = clauseBreaks ? clauseCut(cur) : null;
|
|
55
|
+
chunks.push(k !== null ? cur.slice(0, k + 1) : cur);
|
|
56
|
+
cur = k !== null ? cur.slice(k + 1) : [];
|
|
57
|
+
}
|
|
58
|
+
cur.push(t);
|
|
59
|
+
}
|
|
60
|
+
chunks.push(cur);
|
|
61
|
+
const texts = chunks.map((c) => c.join(' '));
|
|
62
|
+
const total = texts.reduce((n, t) => n + len(t), 0);
|
|
63
|
+
let t0 = 0;
|
|
64
|
+
return texts.map((s, i) => {
|
|
65
|
+
const t1 = i === texts.length - 1 ? duration : t0 + (duration * len(s)) / total;
|
|
66
|
+
const cue = { start: t0, end: t1, text: s };
|
|
67
|
+
t0 = t1;
|
|
68
|
+
return cue;
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** One beat's cues: from its word timings when it has them, else from its text; shifted by `offset` seconds. */
|
|
73
|
+
export function cuesForBeat(
|
|
74
|
+
words: CaptionWord[] | null | undefined, text: string | null | undefined, duration: number,
|
|
75
|
+
options: CueOptions & { offset?: number } = {},
|
|
76
|
+
): Cue[] {
|
|
77
|
+
const { offset = 0, ...cueOptions } = options;
|
|
78
|
+
const cues = words?.length ? cuesFromWords(words, cueOptions) : cuesFromText(text ?? '', duration, cueOptions);
|
|
79
|
+
return cues.map((c) => ({ start: c.start + offset, end: c.end + offset, text: c.text }));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Cues with each end moved to the next cue's start when the silence between them is at most `maxGapS`, so a caption stays
|
|
84
|
+
* up across a short pause. Starts never move; 0 returns the cues unchanged. The input is not mutated.
|
|
85
|
+
*/
|
|
86
|
+
export function holdGaps(cues: Cue[], maxGapS: number): Cue[] {
|
|
87
|
+
const out = cues.map((c) => ({ ...c }));
|
|
88
|
+
if (maxGapS > 0) {
|
|
89
|
+
for (let i = 0; i + 1 < out.length; i += 1) {
|
|
90
|
+
const gap = out[i + 1].start - out[i].end;
|
|
91
|
+
if (gap > 0 && gap <= maxGapS) out[i].end = out[i + 1].start;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Round half to even, as Python's `round`. */
|
|
98
|
+
function roundHalfEven(x: number): number {
|
|
99
|
+
const floor = Math.floor(x);
|
|
100
|
+
const diff = x - floor;
|
|
101
|
+
if (diff === 0.5) return floor % 2 === 0 ? floor : floor + 1;
|
|
102
|
+
return Math.round(x);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** [first frame, end frame exclusive]: frame i is shown at time i / fps. */
|
|
106
|
+
export function cueFrameRange(cue: Pick<Cue, 'start' | 'end'>, fps: number): [number, number] {
|
|
107
|
+
return [roundHalfEven(cue.start * fps), roundHalfEven(cue.end * fps)];
|
|
108
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Cue } from './types';
|
|
2
|
+
|
|
3
|
+
function stamp(ms: number, sep: ',' | '.'): string {
|
|
4
|
+
const pad = (n: number, w = 2) => String(n).padStart(w, '0');
|
|
5
|
+
return `${pad(Math.floor(ms / 3_600_000))}:${pad(Math.floor(ms / 60_000) % 60)}:${pad(Math.floor(ms / 1000) % 60)}${sep}${pad(ms % 1000, 3)}`;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/** Millisecond start/end; an end that rounds onto its start is lifted one millisecond so the cue stays valid. */
|
|
9
|
+
function span(c: Cue): [number, number] {
|
|
10
|
+
const start = Math.max(0, Math.round(c.start * 1000));
|
|
11
|
+
return [start, Math.max(start + 1, Math.round(c.end * 1000))];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const oneBlock = (text: string) => text.split(/\n\s*\n/).map((s) => s.trim()).filter(Boolean).join('\n');
|
|
15
|
+
|
|
16
|
+
/** SubRip. Cue text keeps its line breaks; blank lines inside a cue are dropped (they would end it). */
|
|
17
|
+
export function toSrt(cues: Cue[]): string {
|
|
18
|
+
return cues.map((c, i) => {
|
|
19
|
+
const [a, b] = span(c);
|
|
20
|
+
return `${i + 1}\n${stamp(a, ',')} --> ${stamp(b, ',')}\n${oneBlock(c.text)}\n`;
|
|
21
|
+
}).join('\n');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const vttText = (text: string) => oneBlock(text).replace(/&/g, '&').replace(/</g, '<').replace(/-->/g, '-->');
|
|
25
|
+
|
|
26
|
+
/** WebVTT. `&`, `<` and `-->` in cue text are escaped as the spec requires. */
|
|
27
|
+
export function toVtt(cues: Cue[]): string {
|
|
28
|
+
const body = cues.map((c) => {
|
|
29
|
+
const [a, b] = span(c);
|
|
30
|
+
return `${stamp(a, '.')} --> ${stamp(b, '.')}\n${vttText(c.text)}\n`;
|
|
31
|
+
}).join('\n');
|
|
32
|
+
return `WEBVTT\n\n${body}`;
|
|
33
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Caption cues from word timings (or bare text), word-alignment checks, SRT / VTT output. Pure: no vendor, no I/O. */
|
|
2
|
+
export { cueFrameRange, cuesForBeat, cuesFromText, cuesFromWords, holdGaps } from './cues';
|
|
3
|
+
export { toSrt, toVtt } from './formats';
|
|
4
|
+
export type { CaptionWord, Cue, CueOptions } from './types';
|
|
5
|
+
export { InvalidWordsError, assertValidWords, captionTiming, splitWhitespaceTokens, wordsAreProportional } from './words';
|
|
6
|
+
export type { CaptionTiming } from './words';
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { TranscriptWord } from '../types';
|
|
2
|
+
|
|
3
|
+
/** A word with an optional vendor flag: `alignment: 'proportional'` says the times are guesses spread by character count. */
|
|
4
|
+
export type CaptionWord = TranscriptWord & { alignment?: string };
|
|
5
|
+
|
|
6
|
+
/** A caption on the video timeline, seconds. */
|
|
7
|
+
export interface Cue { start: number; end: number; text: string }
|
|
8
|
+
|
|
9
|
+
export interface CueOptions {
|
|
10
|
+
/** Most words in one cue. Default 7. */
|
|
11
|
+
maxWords?: number;
|
|
12
|
+
/** Most characters in one cue. Default 42. */
|
|
13
|
+
maxChars?: number;
|
|
14
|
+
/** A silence longer than this (seconds) starts a new cue. Default 0.6. */
|
|
15
|
+
maxGap?: number;
|
|
16
|
+
/** A cue that must end early ends at its last clause end (comma, full stop, ...) when it holds one. Default off. */
|
|
17
|
+
clauseBreaks?: boolean;
|
|
18
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { pyFloat, repr } from '../repr';
|
|
2
|
+
import type { CaptionWord, Cue } from './types';
|
|
3
|
+
|
|
4
|
+
/** A transcript that cannot be captioned: empty, or its times run backwards. Callers stop; nothing substitutes another. */
|
|
5
|
+
export class InvalidWordsError extends Error {
|
|
6
|
+
constructor(message: string) { super(message); this.name = 'InvalidWordsError'; }
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** Throws `InvalidWordsError` unless there is at least one word and the words are in time order, each ending after it starts. */
|
|
10
|
+
export function assertValidWords<T extends CaptionWord>(words: T[]): T[] {
|
|
11
|
+
if (!words.length) throw new InvalidWordsError('transcription returned no words');
|
|
12
|
+
let prev = -1.0;
|
|
13
|
+
words.forEach((w, i) => {
|
|
14
|
+
if (w.end < w.start) throw new InvalidWordsError(`word ${i} (${repr(w.text)}) ends before it starts (${pyFloat(w.start)} > ${pyFloat(w.end)})`);
|
|
15
|
+
if (w.start < prev) {
|
|
16
|
+
throw new InvalidWordsError(`words are not in time order: word ${i} (${repr(w.text)}) starts at ${pyFloat(w.start)}, before ${pyFloat(prev)}`);
|
|
17
|
+
}
|
|
18
|
+
prev = w.start;
|
|
19
|
+
});
|
|
20
|
+
return words;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const round3 = (n: number) => Math.round(n * 1000) / 1000;
|
|
24
|
+
|
|
25
|
+
/** A token holding spaces becomes several words sharing its time evenly; an all-space token is dropped. */
|
|
26
|
+
export function splitWhitespaceTokens(words: CaptionWord[]): CaptionWord[] {
|
|
27
|
+
const out: CaptionWord[] = [];
|
|
28
|
+
for (const w of words) {
|
|
29
|
+
const parts = w.text.split(/\s+/).filter(Boolean);
|
|
30
|
+
if (parts.length <= 1) {
|
|
31
|
+
if (parts.length) out.push({ ...w, text: parts[0] });
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
const step = (w.end - w.start) / parts.length;
|
|
35
|
+
parts.forEach((p, i) => out.push({ ...w, text: p, start: round3(w.start + i * step), end: round3(w.start + (i + 1) * step) }));
|
|
36
|
+
}
|
|
37
|
+
return out;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const median = (xs: number[]) => {
|
|
41
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
42
|
+
const m = s.length >> 1;
|
|
43
|
+
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* True when the words say so (`alignment: 'proportional'`) or, unflagged, when seconds-per-character barely changes from one
|
|
48
|
+
* word to the next (median relative step). Real vendor alignments step by 30% and more; characters spread evenly by 0%.
|
|
49
|
+
*/
|
|
50
|
+
export function wordsAreProportional(words: CaptionWord[], options: { stepMax?: number; minWords?: number } = {}): boolean {
|
|
51
|
+
const { stepMax = 0.02, minWords = 8 } = options;
|
|
52
|
+
const flags = new Set(words.map((w) => w.alignment).filter(Boolean));
|
|
53
|
+
if (flags.size) return flags.has('proportional');
|
|
54
|
+
const rate = words.filter((w) => w.end > w.start).map((w) => (w.end - w.start) / Math.max(1, [...w.text].length));
|
|
55
|
+
if (rate.length < minWords) return false;
|
|
56
|
+
const mean = rate.reduce((a, b) => a + b, 0) / rate.length;
|
|
57
|
+
if (mean <= 0) return false;
|
|
58
|
+
const steps = rate.slice(1).map((r, i) => Math.abs(r - rate[i]));
|
|
59
|
+
return steps.length > 0 && median(steps) / mean < stepMax;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export type CaptionTiming =
|
|
63
|
+
| { measured: false; pass: null; n: number; reason: string }
|
|
64
|
+
| { measured: true; n: number; max_abs_start_err_ms: number; median_abs_start_err_ms: number; max_abs_end_err_ms: number; pass: boolean };
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Each caption covers the next `text.split().length` words; compare its start and end with those words'. Proportional words
|
|
68
|
+
* make the error 0 by construction, which is not a pass: it is `measured: false`. Report keys match the Python review reports.
|
|
69
|
+
*/
|
|
70
|
+
export function captionTiming(captions: Cue[], words: CaptionWord[], options: { maxStartErrMs?: number } = {}): CaptionTiming {
|
|
71
|
+
const { maxStartErrMs = 250 } = options;
|
|
72
|
+
if (wordsAreProportional(words)) {
|
|
73
|
+
return { measured: false, pass: null, n: captions.length, reason: 'word times are proportional guesses (no forced alignment), so caption timing cannot be checked against them' };
|
|
74
|
+
}
|
|
75
|
+
let i = 0;
|
|
76
|
+
const startErr: number[] = [];
|
|
77
|
+
const endErr: number[] = [];
|
|
78
|
+
for (const c of captions) {
|
|
79
|
+
const k = Math.max(1, c.text.split(/\s+/).filter(Boolean).length);
|
|
80
|
+
const part = words.slice(i, i + k);
|
|
81
|
+
i += k;
|
|
82
|
+
if (!part.length) break;
|
|
83
|
+
startErr.push(Math.abs(c.start - part[0].start) * 1000);
|
|
84
|
+
endErr.push(Math.abs(c.end - part[part.length - 1].end) * 1000);
|
|
85
|
+
}
|
|
86
|
+
if (!startErr.length) return { measured: true, n: 0, max_abs_start_err_ms: 0, median_abs_start_err_ms: 0, max_abs_end_err_ms: 0, pass: false };
|
|
87
|
+
return {
|
|
88
|
+
measured: true, n: startErr.length, max_abs_start_err_ms: Math.max(...startErr), median_abs_start_err_ms: median(startErr),
|
|
89
|
+
max_abs_end_err_ms: Math.max(...endErr), pass: Math.max(...startErr) <= maxStartErrMs,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { writeFile } from 'node:fs/promises';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
InvalidWordsError, assertValidWords, cueFrameRange, cuesForBeat, holdGaps, toSrt, toVtt,
|
|
5
|
+
type CaptionWord,
|
|
6
|
+
} from './captions';
|
|
7
|
+
|
|
8
|
+
export interface CaptionsOutcome { exitCode: 0 | 1 | 2; output: unknown }
|
|
9
|
+
|
|
10
|
+
const FORMATS = ['json', 'srt', 'vtt'];
|
|
11
|
+
|
|
12
|
+
/** The Python side names a word's text `word`; either spelling is read. */
|
|
13
|
+
const toWord = (w: Record<string, any>): CaptionWord => {
|
|
14
|
+
const { word, text, ...rest } = w;
|
|
15
|
+
return { ...rest, text: text ?? word } as CaptionWord;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* `startsimpli-speech captions [--format json|srt|vtt] [--out FILE]` — caption cues from word timings or bare text:
|
|
20
|
+
* {"words":[{"text"|"word","start","end"}] | "text" + "duration", "offset"?, "clauseBreaks"?, "maxWords"?, "maxChars"?,
|
|
21
|
+
* "maxGap"?, "holdGapS"?, "fps"?}
|
|
22
|
+
* Output `{format, cues, content?, frames?, path?}`: `content` is the SRT / VTT text (also written to --out), `frames` the
|
|
23
|
+
* [first, end) frame of each cue when `fps` is given. Unusable words are `{error: {code: "INVALID_WORDS"}}`, exit 1.
|
|
24
|
+
*/
|
|
25
|
+
export async function runCaptions(body: Record<string, any>, flags: Record<string, string>): Promise<CaptionsOutcome> {
|
|
26
|
+
const usage = (message: string): CaptionsOutcome => ({ exitCode: 2, output: { error: { code: 'USAGE', message } } });
|
|
27
|
+
const format = flags.format ?? 'json';
|
|
28
|
+
if (!FORMATS.includes(format)) return usage(`--format must be ${FORMATS.join(', ')}, got "${format}"`);
|
|
29
|
+
const words: CaptionWord[] | undefined = Array.isArray(body.words) ? body.words.map(toWord) : undefined;
|
|
30
|
+
if (!words?.length && !(typeof body.text === 'string' && typeof body.duration === 'number')) {
|
|
31
|
+
return usage('captions needs "words" (timed), or "text" and "duration" in the request');
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
if (words?.length) assertValidWords(words);
|
|
35
|
+
let cues = cuesForBeat(words, body.text, body.duration ?? 0, {
|
|
36
|
+
offset: body.offset, clauseBreaks: body.clauseBreaks, maxWords: body.maxWords, maxChars: body.maxChars, maxGap: body.maxGap,
|
|
37
|
+
});
|
|
38
|
+
if (typeof body.holdGapS === 'number') cues = holdGaps(cues, body.holdGapS);
|
|
39
|
+
const output: Record<string, unknown> = { format, cues };
|
|
40
|
+
if (typeof body.fps === 'number') output.frames = cues.map((c) => cueFrameRange(c, body.fps));
|
|
41
|
+
if (format !== 'json') {
|
|
42
|
+
output.content = format === 'srt' ? toSrt(cues) : toVtt(cues);
|
|
43
|
+
if (flags.out) { await writeFile(flags.out, output.content as string); output.path = flags.out; }
|
|
44
|
+
}
|
|
45
|
+
return { exitCode: 0, output };
|
|
46
|
+
} catch (error) {
|
|
47
|
+
if (error instanceof InvalidWordsError) return { exitCode: 1, output: { error: { code: 'INVALID_WORDS', message: error.message } } };
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `startsimpli-speech` — the speech service for callers that are not TypeScript.
|
|
3
|
+
*
|
|
4
|
+
* startsimpli-speech synthesize --in request.json --out speech.mp3 > result.json
|
|
5
|
+
* startsimpli-speech transcribe --in request.json > result.json
|
|
6
|
+
*
|
|
7
|
+
* JSON in (a file with --in, else stdin), JSON out on stdout, audio as a file.
|
|
8
|
+
* Keys come from the environment only (ELEVENLABS_API_KEY, ELEVENLABS_VOICE_ID,
|
|
9
|
+
* OPENAI_API_KEY, SPEECH_PROVIDER). Nothing here knows who is calling.
|
|
10
|
+
*
|
|
11
|
+
* Failures print `{"error": {code, message, retryable, statusCode, provider}}`
|
|
12
|
+
* and exit 1; a malformed command line exits 2. Success exits 0.
|
|
13
|
+
*/
|
|
14
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
15
|
+
import { extname } from 'node:path';
|
|
16
|
+
|
|
17
|
+
import { LLMProviderError } from '../errors';
|
|
18
|
+
import { SpeechService } from './service';
|
|
19
|
+
import { runCaptions } from './captions-cli';
|
|
20
|
+
import { runVoices } from './voices-cli';
|
|
21
|
+
import type { SynthesizeRequest, TranscribeRequest } from './types';
|
|
22
|
+
|
|
23
|
+
export interface CliIO {
|
|
24
|
+
/** Reads all of stdin; only called when --in is absent. */
|
|
25
|
+
readStdin: () => Promise<string>;
|
|
26
|
+
/** Builds the service; a seam for tests. */
|
|
27
|
+
makeService?: () => SpeechService;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface CliOutcome {
|
|
31
|
+
exitCode: 0 | 1 | 2;
|
|
32
|
+
/** JSON document for stdout. */
|
|
33
|
+
output: unknown;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const MIME_BY_EXT: Record<string, string> = {
|
|
37
|
+
'.webm': 'audio/webm', '.ogg': 'audio/ogg', '.mp4': 'audio/mp4', '.m4a': 'audio/mp4', '.mp3': 'audio/mpeg',
|
|
38
|
+
'.wav': 'audio/wav', '.flac': 'audio/flac',
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const FLAGS = new Set(['in', 'out', 'provider', 'model', 'voice', 'format', 'voice-settings', 'language', 'library', 'expiring', 'today']);
|
|
42
|
+
|
|
43
|
+
const USAGE = 'usage: startsimpli-speech <synthesize|transcribe|voices resolve|captions> [--in request.json] [--out audio-file] '
|
|
44
|
+
+ '[--provider NAME] [--model ID] [--voice ID] [--format FMT] [--voice-settings JSON] [--language CODE]';
|
|
45
|
+
|
|
46
|
+
class UsageError extends Error {}
|
|
47
|
+
|
|
48
|
+
function parseFlags(args: string[]): Record<string, string> {
|
|
49
|
+
const flags: Record<string, string> = {};
|
|
50
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
51
|
+
const name = args[i].startsWith('--') ? args[i].slice(2) : '';
|
|
52
|
+
if (!FLAGS.has(name)) throw new UsageError(`unknown argument "${args[i]}"`);
|
|
53
|
+
const value = args[i + 1];
|
|
54
|
+
if (value === undefined) throw new UsageError(`--${name} needs a value`);
|
|
55
|
+
flags[name] = value;
|
|
56
|
+
i += 1;
|
|
57
|
+
}
|
|
58
|
+
return flags;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function readRequest(flags: Record<string, string>, io: CliIO): Promise<Record<string, any>> {
|
|
62
|
+
const raw = flags.in ? await readFile(flags.in, 'utf8') : await io.readStdin();
|
|
63
|
+
try {
|
|
64
|
+
const parsed = JSON.parse(raw || '{}');
|
|
65
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object');
|
|
66
|
+
return parsed;
|
|
67
|
+
} catch (error) {
|
|
68
|
+
throw new UsageError(`request is not a JSON object: ${(error as Error).message}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function runCli(argv: string[], io: CliIO): Promise<CliOutcome> {
|
|
73
|
+
try {
|
|
74
|
+
const [command, ...rest] = argv;
|
|
75
|
+
if (command === 'voices') {
|
|
76
|
+
const [sub, ...flagArgs] = rest;
|
|
77
|
+
const flags = parseFlags(flagArgs);
|
|
78
|
+
return runVoices(sub, sub === 'resolve' ? await readRequest(flags, io) : {}, flags);
|
|
79
|
+
}
|
|
80
|
+
if (command === 'captions') {
|
|
81
|
+
const flags = parseFlags(rest);
|
|
82
|
+
return runCaptions(await readRequest(flags, io), flags);
|
|
83
|
+
}
|
|
84
|
+
if (command !== 'synthesize' && command !== 'transcribe') throw new UsageError(USAGE);
|
|
85
|
+
const flags = parseFlags(rest);
|
|
86
|
+
const body = await readRequest(flags, io);
|
|
87
|
+
const service = io.makeService ? io.makeService() : new SpeechService(flags.provider ? { preferredProvider: flags.provider } : {});
|
|
88
|
+
return { exitCode: 0, output: command === 'synthesize' ? await synthesize(service, body, flags) : await transcribe(service, body, flags) };
|
|
89
|
+
} catch (error) {
|
|
90
|
+
if (error instanceof UsageError) return { exitCode: 2, output: { error: { code: 'USAGE', message: error.message } } };
|
|
91
|
+
if (error instanceof LLMProviderError) {
|
|
92
|
+
const { code, message, retryable, statusCode, provider } = error;
|
|
93
|
+
return { exitCode: 1, output: { error: { code, message, retryable, statusCode, provider } } };
|
|
94
|
+
}
|
|
95
|
+
return { exitCode: 1, output: { error: { code: 'UNKNOWN_ERROR', message: (error as Error)?.message ?? String(error), retryable: false } } };
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function synthesize(service: SpeechService, body: Record<string, any>, flags: Record<string, string>) {
|
|
100
|
+
const out: string | undefined = flags.out ?? body.out;
|
|
101
|
+
if (!out) throw new UsageError('synthesize needs --out <audio file> (or "out" in the request)');
|
|
102
|
+
const request: SynthesizeRequest = {
|
|
103
|
+
...body,
|
|
104
|
+
...(flags.model ? { model: flags.model } : {}),
|
|
105
|
+
...(flags.voice ? { voiceId: flags.voice } : {}),
|
|
106
|
+
...(flags.format ? { outputFormat: flags.format } : {}),
|
|
107
|
+
...(flags.language ? { language: flags.language } : {}),
|
|
108
|
+
...(flags['voice-settings'] ? { voiceSettings: parseSettings(flags['voice-settings']) } : {}),
|
|
109
|
+
} as SynthesizeRequest;
|
|
110
|
+
delete (request as any).out;
|
|
111
|
+
delete (request as any).provider;
|
|
112
|
+
const result = await service.synthesize(request);
|
|
113
|
+
await writeFile(out, result.audio);
|
|
114
|
+
const { audio, ...rest } = result;
|
|
115
|
+
return { ...rest, audioPath: out, bytes: audio.byteLength };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function parseSettings(text: string): Record<string, number | boolean> {
|
|
119
|
+
try {
|
|
120
|
+
const parsed = JSON.parse(text);
|
|
121
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed;
|
|
122
|
+
} catch { /* fall through */ }
|
|
123
|
+
throw new UsageError('--voice-settings must be a JSON object, e.g. \'{"stability":0.4}\'');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async function transcribe(service: SpeechService, body: Record<string, any>, flags: Record<string, string>) {
|
|
127
|
+
const audioPath: string | undefined = body.audioPath;
|
|
128
|
+
if (!audioPath) throw new UsageError('transcribe needs "audioPath" in the request');
|
|
129
|
+
const audio = new Uint8Array(await readFile(audioPath));
|
|
130
|
+
const { audioPath: _drop, provider: _p, ...rest } = body;
|
|
131
|
+
const request: TranscribeRequest = {
|
|
132
|
+
timestamps: 'word',
|
|
133
|
+
...rest,
|
|
134
|
+
audio,
|
|
135
|
+
mimeType: body.mimeType ?? MIME_BY_EXT[extname(audioPath).toLowerCase()] ?? 'audio/mpeg',
|
|
136
|
+
...(flags.model ? { model: flags.model } : {}),
|
|
137
|
+
...(flags.language ? { language: flags.language } : {}),
|
|
138
|
+
};
|
|
139
|
+
return service.transcribe(request);
|
|
140
|
+
}
|
package/src/speech/http.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { LLMProviderError } from '../errors';
|
|
2
2
|
import type { LLMErrorCode } from '../errors';
|
|
3
|
-
import type { TranscribeRequest } from './types';
|
|
3
|
+
import type { SynthesizeRequest, TranscribeRequest } from './types';
|
|
4
4
|
|
|
5
5
|
/** Container extension a vendor can recognise, from the recorder's MIME type. */
|
|
6
6
|
const EXTENSIONS: Record<string, string> = {
|
|
@@ -34,8 +34,14 @@ export function assertAudio(request: Pick<TranscribeRequest, 'audio'>, provider:
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
export function assertText(request: Pick<SynthesizeRequest, 'text'>, provider: string): void {
|
|
38
|
+
if (!request.text?.trim()) {
|
|
39
|
+
throw new LLMProviderError('There is no text to speak.', 'INVALID_REQUEST', false, undefined, provider);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
37
43
|
/** One HTTP-status -> error-code mapping, so every vendor classifies failures the same way. */
|
|
38
|
-
export function speechHttpError(status: number, body: string, provider: string): LLMProviderError {
|
|
44
|
+
export function speechHttpError(status: number, body: string, provider: string, operation = 'transcription'): LLMProviderError {
|
|
39
45
|
const code: LLMErrorCode =
|
|
40
46
|
status === 401 || status === 403 ? 'AUTHENTICATION_ERROR'
|
|
41
47
|
: status === 429 ? 'RATE_LIMITED'
|
|
@@ -45,7 +51,7 @@ export function speechHttpError(status: number, body: string, provider: string):
|
|
|
45
51
|
: 'UNKNOWN_ERROR';
|
|
46
52
|
const retryable = code === 'RATE_LIMITED' || code === 'SERVICE_UNAVAILABLE' || code === 'TIMEOUT';
|
|
47
53
|
return new LLMProviderError(
|
|
48
|
-
`${provider}
|
|
54
|
+
`${provider} ${operation} failed (${status}): ${body.slice(0, 300)}`,
|
|
49
55
|
code, retryable, status, provider,
|
|
50
56
|
);
|
|
51
57
|
}
|
|
@@ -57,10 +63,11 @@ export async function speechFetch(
|
|
|
57
63
|
init: RequestInit,
|
|
58
64
|
timeoutMs = 240_000,
|
|
59
65
|
fetchImpl: typeof fetch = fetch,
|
|
66
|
+
operation = 'transcription',
|
|
60
67
|
): Promise<Response> {
|
|
61
68
|
try {
|
|
62
69
|
const response = await fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
63
|
-
if (!response.ok) throw speechHttpError(response.status, await response.text().catch(() => ''), provider);
|
|
70
|
+
if (!response.ok) throw speechHttpError(response.status, await response.text().catch(() => ''), provider, operation);
|
|
64
71
|
return response;
|
|
65
72
|
} catch (error) {
|
|
66
73
|
if (error instanceof LLMProviderError) throw error;
|
package/src/speech/index.ts
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `@startsimpli/llm/speech` — pluggable speech-to-text.
|
|
2
|
+
* `@startsimpli/llm/speech` — pluggable speech-to-text and text-to-speech.
|
|
3
3
|
*
|
|
4
4
|
* const speech = new SpeechService(); // SPEECH_PROVIDER, else the cheapest vendor with a key
|
|
5
5
|
* const { text } = await speech.transcribe({ audio, mimeType: 'audio/webm' });
|
|
6
6
|
*
|
|
7
|
+
* const { audio, words } = await speech.synthesize({ text: 'Hello there.' });
|
|
8
|
+
*
|
|
7
9
|
* A new vendor: implement `ISpeechProvider`, then `registerSpeechProvider(name, factory)`.
|
|
8
10
|
*/
|
|
9
11
|
export { SpeechService, registerSpeechProvider, speechProviders } from './service';
|
|
10
12
|
export type { ISpeechProvider } from './interface';
|
|
11
13
|
export type {
|
|
12
|
-
SpeechServiceConfig,
|
|
14
|
+
AlignedCharacter, SpeechServiceConfig, SpeechStyle, SpeechUsage, SynthesizeRequest, SynthesizeResult,
|
|
15
|
+
TranscribeRequest, TranscriptResult, TranscriptWord,
|
|
13
16
|
} from './types';
|
|
14
17
|
export { ElevenLabsSpeechProvider, createElevenLabsSpeechProvider } from './providers/elevenlabs';
|
|
15
18
|
export { OpenAISpeechProvider, createOpenAISpeechProvider } from './providers/openai';
|
|
16
19
|
export type { OpenAISpeechOptions } from './providers/openai';
|
|
17
20
|
export { MockSpeechProvider, setMockSpeechBehavior, resetMockSpeechBehavior } from './providers/mock';
|
|
18
21
|
export { audioExtension, speechHttpError } from './http';
|
|
22
|
+
export { wordsFromCharacters } from './alignment';
|
|
23
|
+
export * from './voices';
|
|
24
|
+
export * from './captions';
|