@softure-ai/marketing-kit 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.md +119 -16
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +92 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +12 -1
- package/dist/footage/failure.d.ts.map +1 -1
- package/dist/footage/failure.js +10 -1
- package/dist/footage/failure.js.map +1 -1
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +25 -6
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +37 -16
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +12 -2
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +8 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +8 -2
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +54 -45
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +51 -10
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +90 -15
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +9 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +55 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +215 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +123 -22
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +14 -1
- package/src/footage/fake.ts +28 -4
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +62 -22
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +25 -3
- package/src/footage/pipeline.ts +61 -46
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +129 -16
- package/src/index.ts +41 -1
- package/src/og/index.ts +23 -0
- package/src/og/photo-cover.ts +280 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
export const PROVIDER_MESSAGE_LENGTH = 300;
|
|
2
|
+
const REQUEST_ID_HEADERS = ["request-id", "x-request-id"];
|
|
3
|
+
/** The kind of an HTTP status the provider answered (not 2xx). */
|
|
4
|
+
export function getProviderFailureKind(status) {
|
|
5
|
+
if (status === 429)
|
|
6
|
+
return "rate_limited";
|
|
7
|
+
if (status === 408)
|
|
8
|
+
return "timeout";
|
|
9
|
+
if (status >= 500)
|
|
10
|
+
return "unavailable";
|
|
11
|
+
return "refused";
|
|
12
|
+
}
|
|
13
|
+
const isRetryableKind = (kind) => kind !== "refused";
|
|
14
|
+
const truncate = (text) => {
|
|
15
|
+
const trimmed = text.trim();
|
|
16
|
+
return trimmed.length === 0 ? null : trimmed.slice(0, PROVIDER_MESSAGE_LENGTH);
|
|
17
|
+
};
|
|
18
|
+
const readString = (value) => (typeof value === "string" && value.trim().length > 0 ? value.trim() : null);
|
|
19
|
+
/** The provider's code, message and request id from an error body: `{ detail: { code | status, message, request_id } }`, `{ detail: "…" }` or text. */
|
|
20
|
+
export function readProviderErrorBody(text) {
|
|
21
|
+
let body;
|
|
22
|
+
try {
|
|
23
|
+
body = JSON.parse(text);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return { code: null, message: truncate(text), requestId: null };
|
|
27
|
+
}
|
|
28
|
+
const root = typeof body === "object" && body !== null ? body : {};
|
|
29
|
+
const detail = root.detail ?? root.error ?? root;
|
|
30
|
+
if (typeof detail === "string")
|
|
31
|
+
return { code: null, message: truncate(detail), requestId: readString(root.request_id) };
|
|
32
|
+
const fields = typeof detail === "object" && detail !== null ? detail : {};
|
|
33
|
+
const message = readString(fields.message);
|
|
34
|
+
return {
|
|
35
|
+
code: readString(fields.code) ?? readString(fields.status) ?? readString(fields.type),
|
|
36
|
+
message: message === null ? null : truncate(message),
|
|
37
|
+
requestId: readString(fields.request_id) ?? readString(root.request_id),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/** The failure of a response that is not 2xx, from its status, headers and body text. */
|
|
41
|
+
export function getProviderFailure(response, text) {
|
|
42
|
+
const body = readProviderErrorBody(text);
|
|
43
|
+
const header = REQUEST_ID_HEADERS.map((name) => readString(response.headers.get(name))).find((value) => value !== null) ?? null;
|
|
44
|
+
const kind = getProviderFailureKind(response.status);
|
|
45
|
+
return { kind, status: response.status, code: body.code, message: body.message, requestId: header ?? body.requestId, isRetryable: isRetryableKind(kind) };
|
|
46
|
+
}
|
|
47
|
+
/** The failure of a request that got no answer: `timeout` when it timed out, `connection` otherwise. */
|
|
48
|
+
export function getConnectionFailure(error) {
|
|
49
|
+
const kind = error instanceof Error && error.name === "TimeoutError" ? "timeout" : "connection";
|
|
50
|
+
return { kind, status: null, code: null, message: null, requestId: null, isRetryable: true };
|
|
51
|
+
}
|
|
52
|
+
/** A failure the kit decides before any request (e.g. a missing key): `refused`, with the kit's own code. */
|
|
53
|
+
export function getLocalRefusal(code) {
|
|
54
|
+
return { kind: "refused", status: null, code, message: null, requestId: null, isRetryable: false };
|
|
55
|
+
}
|
|
56
|
+
/** The failure in one line for a log: kind, status, code, request id and message. */
|
|
57
|
+
export function describeProviderFailure(failure) {
|
|
58
|
+
const parts = [failure.kind];
|
|
59
|
+
if (failure.status !== null)
|
|
60
|
+
parts.push(String(failure.status));
|
|
61
|
+
if (failure.code !== null)
|
|
62
|
+
parts.push(failure.code);
|
|
63
|
+
if (failure.requestId !== null)
|
|
64
|
+
parts.push(`request ${failure.requestId}`);
|
|
65
|
+
const head = parts.join(", ");
|
|
66
|
+
return failure.message === null ? head : `${head}: ${failure.message}`;
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=provider-failure.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider-failure.js","sourceRoot":"","sources":["../../src/voice/provider-failure.ts"],"names":[],"mappings":"AAuBA,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;AAE1D,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC;IAC1C,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACrC,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,aAAa,CAAC;IACxC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,eAAe,GAAG,CAAC,IAAyB,EAAW,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;AAEnF,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAiB,EAAE;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,KAAc,EAAiB,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAQnI,uJAAuJ;AACvJ,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAE,IAAgC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChG,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;IACzH,MAAM,MAAM,GAAG,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC,CAAE,MAAkC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxG,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3C,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;QACrF,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;QACpD,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC;KACxE,CAAC;AACJ,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,kBAAkB,CAAC,QAAkB,EAAE,IAAY;IACjE,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;IAChI,MAAM,IAAI,GAAG,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,WAAW,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5J,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,oBAAoB,CAAC,KAAc;IACjD,MAAM,IAAI,GAAwB,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;IACrH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AAC/F,CAAC;AAED,6GAA6G;AAC7G,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;AACrG,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,uBAAuB,CAAC,OAAwB;IAC9D,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,OAAO,CAAC,IAAI,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,OAAO,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;AACzE,CAAC"}
|
package/dist/voice/provider.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ProviderFailure } from "./provider-failure.js";
|
|
1
2
|
import type { TimedWord } from "./voiceover.js";
|
|
2
3
|
/** What a provider speaks: the text in one voice, model and language. */
|
|
3
4
|
export interface TtsInput {
|
|
@@ -22,12 +23,17 @@ export interface TtsEstimate {
|
|
|
22
23
|
maxCost: number;
|
|
23
24
|
unit: string;
|
|
24
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* A provider's answer. A failure carries `error`, the sentence for logs, and, when the provider answered (or the kit
|
|
28
|
+
* refused before asking), `providerFailure`: what an app logs and reads to decide whether to ask again (#1141).
|
|
29
|
+
*/
|
|
25
30
|
export type TtsResult<T> = {
|
|
26
31
|
ok: true;
|
|
27
32
|
value: T;
|
|
28
33
|
} | {
|
|
29
34
|
ok: false;
|
|
30
35
|
error: string;
|
|
36
|
+
providerFailure?: ProviderFailure;
|
|
31
37
|
};
|
|
32
38
|
/**
|
|
33
39
|
* A text-to-speech service. `estimate` is free and needs no credentials; `synthesize` may spend
|
|
@@ -39,5 +45,5 @@ export interface TtsProvider {
|
|
|
39
45
|
synthesize(input: TtsInput): Promise<TtsResult<TtsRecording>>;
|
|
40
46
|
}
|
|
41
47
|
export declare const ok: <T>(value: T) => TtsResult<T>;
|
|
42
|
-
export declare const err: <T>(error: string) => TtsResult<T>;
|
|
48
|
+
export declare const err: <T>(error: string, providerFailure?: ProviderFailure) => TtsResult<T>;
|
|
43
49
|
//# sourceMappingURL=provider.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/voice/provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,yEAAyE;AACzE,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,sFAAsF;AACtF,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/voice/provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,yEAAyE;AACzE,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,sFAAsF;AACtF,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,eAAe,CAAC,EAAE,eAAe,CAAA;CAAE,CAAC;AAEpH;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,WAAW,CAAC;IACvC,UAAU,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC;CAC/D;AAED,eAAO,MAAM,EAAE,GAAI,CAAC,EAAE,OAAO,CAAC,KAAG,SAAS,CAAC,CAAC,CAA0B,CAAC;AACvE,eAAO,MAAM,GAAG,GAAI,CAAC,EAAE,OAAO,MAAM,EAAE,kBAAkB,eAAe,KAAG,SAAS,CAAC,CAAC,CAAmG,CAAC"}
|
package/dist/voice/provider.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export const ok = (value) => ({ ok: true, value });
|
|
2
|
-
export const err = (error) => ({ ok: false, error });
|
|
2
|
+
export const err = (error, providerFailure) => (providerFailure === undefined ? { ok: false, error } : { ok: false, error, providerFailure });
|
|
3
3
|
//# sourceMappingURL=provider.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/voice/provider.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/voice/provider.ts"],"names":[],"mappings":"AA6CA,MAAM,CAAC,MAAM,EAAE,GAAG,CAAI,KAAQ,EAAgB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACvE,MAAM,CAAC,MAAM,GAAG,GAAG,CAAI,KAAa,EAAE,eAAiC,EAAgB,EAAE,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@softure-ai/marketing-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "SOFTURE AI marketing kit: a CLI that turns a project's real app into a vertical film (cached voiceover, frame-by-frame recording, HTML composition, hyperframes render) plus per-platform post copy and OG images.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -51,6 +51,7 @@
|
|
|
51
51
|
"@resvg/resvg-js": "^2.6.2",
|
|
52
52
|
"gsap": "^3.15.0",
|
|
53
53
|
"hyperframes": "0.9.6",
|
|
54
|
+
"jpeg-js": "^0.4.4",
|
|
54
55
|
"playwright": "^1.63.0",
|
|
55
56
|
"satori": "0.63.2",
|
|
56
57
|
"tsx": "^4.23.0",
|
|
@@ -3069,6 +3069,12 @@
|
|
|
3069
3069
|
"type": "number",
|
|
3070
3070
|
"minimum": 0.2,
|
|
3071
3071
|
"maximum": 3
|
|
3072
|
+
},
|
|
3073
|
+
"audioBitrateKbps": {
|
|
3074
|
+
"description": "The finished sound's AAC bitrate in kbps (192 to 512, default 192): the film's only lossy audio encode.",
|
|
3075
|
+
"type": "integer",
|
|
3076
|
+
"minimum": 192,
|
|
3077
|
+
"maximum": 512
|
|
3072
3078
|
}
|
|
3073
3079
|
},
|
|
3074
3080
|
"additionalProperties": false
|
package/src/config/schema.ts
CHANGED
|
@@ -643,6 +643,13 @@ const footageSchema = z
|
|
|
643
643
|
.describe(
|
|
644
644
|
"A near-miss band in LU (0.2 to 3, at least tolerance): a mix outside target ± tolerance but inside target ± warnTolerance, under the true peak limit, finishes with a warning instead of failing.",
|
|
645
645
|
),
|
|
646
|
+
audioBitrateKbps: z
|
|
647
|
+
.number()
|
|
648
|
+
.int()
|
|
649
|
+
.min(192)
|
|
650
|
+
.max(512)
|
|
651
|
+
.optional()
|
|
652
|
+
.describe("The finished sound's AAC bitrate in kbps (192 to 512, default 192): the film's only lossy audio encode."),
|
|
646
653
|
})
|
|
647
654
|
.refine((mix) => mix.warnTolerance === undefined || mix.warnTolerance >= (mix.tolerance ?? 1), { path: ["warnTolerance"], message: "must be at least tolerance (default 1)" }),
|
|
648
655
|
])
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { dirname, extname, join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import type { FootageClip } from "./film.js";
|
|
5
|
+
import { isSilent, readLoudnessMeasurement, type LoudnessMeasurement } from "./loudness.js";
|
|
6
|
+
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
7
|
+
import { runFfmpeg, runProcess } from "./process.js";
|
|
8
|
+
import type { FootageProbe } from "./probe.js";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Each clip's sound for the speech chain (issue #1139): read from the clip's source with the normalize step's own seek
|
|
12
|
+
* and length (so the EDL's times hold), brought to 48 kHz stereo, raised by one linear pre-gain to about
|
|
13
|
+
* `SPEECH_PRE_GAIN.target` with its true peak kept under `SPEECH_PRE_GAIN.peakCeiling`, and written as 24-bit PCM WAV.
|
|
14
|
+
* Quiet phone speech is then never encoded lossily at its quiet level: the voice isolator, the speech to text, the cut,
|
|
15
|
+
* the declick and the loudness pass all work on lossless sound, and the film's final AAC is the only lossy encode.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Where the pre-gain brings a talking clip's sound: integrated loudness in LUFS, and the true peak's ceiling in dBTP. */
|
|
19
|
+
export const SPEECH_PRE_GAIN = { target: -23, peakCeiling: -3 } as const;
|
|
20
|
+
|
|
21
|
+
/** The loudness range `loudnorm` measures with; the pre-gain itself is a plain linear `volume`. */
|
|
22
|
+
const MEASURE_RANGE = 11;
|
|
23
|
+
|
|
24
|
+
/** The clip as the normalize step reads it: its trim, its source file and what the probe found. */
|
|
25
|
+
export interface ClipSoundInput {
|
|
26
|
+
clip: Pick<FootageClip, "id" | "role" | "trim">;
|
|
27
|
+
/** The source as read (a URL's downloaded copy). */
|
|
28
|
+
source: string;
|
|
29
|
+
probe: Pick<FootageProbe, "hasAudio">;
|
|
30
|
+
/** Seconds after the trim. */
|
|
31
|
+
duration: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** A clip's lossless, pre-gained sound and the gain it got in dB. */
|
|
35
|
+
export interface ClipSound {
|
|
36
|
+
file: string;
|
|
37
|
+
gainDb: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const toFormat = `aformat=sample_rates=${String(FOOTAGE_SAMPLE_RATE)}:channel_layouts=stereo`;
|
|
41
|
+
|
|
42
|
+
/** The seek and the length of the trimmed clip, as `buildNormalizeArgs` reads its source. */
|
|
43
|
+
function getTrimArgs(input: ClipSoundInput): { seek: string[]; length: string[] } {
|
|
44
|
+
return { seek: input.clip.trim.start > 0 ? ["-ss", String(input.clip.trim.start)] : [], length: ["-t", input.duration.toFixed(3)] };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The measuring pass over the trimmed clip's sound, after the same format change the extraction makes; writes nothing. */
|
|
48
|
+
export function buildClipSoundMeasureArgs(input: ClipSoundInput): string[] {
|
|
49
|
+
const { seek, length } = getTrimArgs(input);
|
|
50
|
+
const measure = `loudnorm=I=${String(SPEECH_PRE_GAIN.target)}:TP=${String(SPEECH_PRE_GAIN.peakCeiling)}:LRA=${String(MEASURE_RANGE)}:print_format=json`;
|
|
51
|
+
return ["-hide_banner", "-nostats", ...seek, "-i", input.source, ...length, "-map", "0:a:0", "-vn", "-af", `${toFormat},${measure}`, "-f", "null", "-"];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The pre-gain in dB for a measured clip: up (or down) to `SPEECH_PRE_GAIN.target`, but never past the gain that puts
|
|
56
|
+
* its true peak at `SPEECH_PRE_GAIN.peakCeiling`; 0 for silence. Rounded to a hundredth of a dB.
|
|
57
|
+
*/
|
|
58
|
+
export function getPreGainDb(measurement: LoudnessMeasurement | null): number {
|
|
59
|
+
if (measurement === null || isSilent(measurement)) return 0;
|
|
60
|
+
const toTarget = SPEECH_PRE_GAIN.target - measurement.inputI;
|
|
61
|
+
const toCeiling = Number.isFinite(measurement.inputTp) ? SPEECH_PRE_GAIN.peakCeiling - measurement.inputTp : toTarget;
|
|
62
|
+
return Math.round(Math.min(toTarget, toCeiling) * 100) / 100 || 0;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The trimmed clip's sound, gained by `gainDb`, as a 24-bit 48 kHz stereo WAV; silence as long as the clip when it has none. */
|
|
66
|
+
export function buildClipSoundArgs(input: ClipSoundInput & { gainDb: number; output: string }): string[] {
|
|
67
|
+
const codec = ["-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2", "-c:a", "pcm_s24le", input.output];
|
|
68
|
+
const { seek, length } = getTrimArgs(input);
|
|
69
|
+
if (!input.probe.hasAudio) return ["-v", "error", "-y", "-f", "lavfi", "-i", `anullsrc=r=${String(FOOTAGE_SAMPLE_RATE)}:cl=stereo`, ...length, ...codec];
|
|
70
|
+
return ["-v", "error", "-y", ...seek, "-i", input.source, ...length, "-map", "0:a:0", "-vn", "-af", `${toFormat},volume=${String(input.gainDb)}dB`, ...codec];
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Where a clip's sound goes in the build folder. */
|
|
74
|
+
export function getClipSoundPath(buildDir: string, clipId: string): string {
|
|
75
|
+
return join(buildDir, "audio", `${clipId}.wav`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Measures one clip's sound; null when the clip has none. Throws naming the clip when ffmpeg fails. */
|
|
79
|
+
async function measureClipSound(input: ClipSoundInput, signal: AbortSignal | undefined): Promise<LoudnessMeasurement | null> {
|
|
80
|
+
if (!input.probe.hasAudio) return null;
|
|
81
|
+
const step = `Measuring the sound of clip ${input.clip.id}`;
|
|
82
|
+
const measured = await runProcess({ command: "ffmpeg", args: buildClipSoundMeasureArgs(input), ...(signal === undefined ? {} : { signal }) });
|
|
83
|
+
if (measured.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${measured.error}).`);
|
|
84
|
+
if (measured.code !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
|
|
85
|
+
const measurement = readLoudnessMeasurement(measured.stderr);
|
|
86
|
+
if (measurement === null) throw new Error(`${step}: ffmpeg printed no loudnorm measurement for ${input.source}.`);
|
|
87
|
+
return measurement;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Every clip's lossless, pre-gained sound under `<buildDir>/audio/<id>.wav`, by clip id. A talking clip is gained by its
|
|
92
|
+
* own measurement (`getPreGainDb`); any other clip by the mean of the talking clips' gains (0 dB without one), so a
|
|
93
|
+
* b-roll's own sound under the speech keeps its relation to it. Throws naming the clip when ffmpeg fails.
|
|
94
|
+
*/
|
|
95
|
+
export async function prepareClipSounds(clips: readonly ClipSoundInput[], options: { buildDir: string; signal?: AbortSignal }): Promise<Record<string, ClipSound>> {
|
|
96
|
+
const { signal } = options;
|
|
97
|
+
const gains = new Map<string, number>();
|
|
98
|
+
for (const clip of clips) {
|
|
99
|
+
if (clip.clip.role !== "talk") continue;
|
|
100
|
+
gains.set(clip.clip.id, getPreGainDb(await measureClipSound(clip, signal)));
|
|
101
|
+
}
|
|
102
|
+
const talking = [...gains.values()];
|
|
103
|
+
const sharedGain = talking.length === 0 ? 0 : Math.round((talking.reduce((sum, gain) => sum + gain, 0) / talking.length) * 100) / 100 || 0;
|
|
104
|
+
const sounds: Record<string, ClipSound> = {};
|
|
105
|
+
for (const clip of clips) {
|
|
106
|
+
const gainDb = gains.get(clip.clip.id) ?? sharedGain;
|
|
107
|
+
const output = getClipSoundPath(options.buildDir, clip.clip.id);
|
|
108
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
109
|
+
await runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildClipSoundArgs({ ...clip, gainDb, output }), { ...(signal === undefined ? {} : { signal }), output });
|
|
110
|
+
sounds[clip.clip.id] = { file: output, gainDb };
|
|
111
|
+
}
|
|
112
|
+
return sounds;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The audio codec for an intermediate file by its extension: `.wav` 32-bit float PCM (no stage clips between stages),
|
|
117
|
+
* `.flac` FLAC, anything else MP3 (an app's own `.mp3` names keep working).
|
|
118
|
+
*/
|
|
119
|
+
export function getAudioCodecArgs(output: string): string[] {
|
|
120
|
+
const extension = extname(output).toLowerCase();
|
|
121
|
+
if (extension === ".wav") return ["-c:a", "pcm_f32le"];
|
|
122
|
+
if (extension === ".flac") return ["-c:a", "flac"];
|
|
123
|
+
return ["-c:a", "libmp3lame", "-q:a", "2"];
|
|
124
|
+
}
|
package/src/footage/declick.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { extname } from "node:path";
|
|
3
2
|
|
|
4
3
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
4
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
5
5
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
6
|
import { runFfmpeg } from "./process.js";
|
|
7
7
|
|
|
@@ -295,10 +295,9 @@ export function buildDeclickDecodeArgs(input: string, output: string): string[]
|
|
|
295
295
|
return ["-v", "error", "-y", "-i", input, "-vn", "-c:a", "pcm_f32le", ...PCM_FORMAT, output];
|
|
296
296
|
}
|
|
297
297
|
|
|
298
|
-
/** The declicked PCM encoded
|
|
298
|
+
/** The declicked PCM encoded by the output's extension (`getAudioCodecArgs`: `.wav` lossless float, as the chain writes it). */
|
|
299
299
|
export function buildDeclickEncodeArgs(input: string, output: string): string[] {
|
|
300
|
-
|
|
301
|
-
return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...codec, output];
|
|
300
|
+
return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...getAudioCodecArgs(output), output];
|
|
302
301
|
}
|
|
303
302
|
|
|
304
303
|
/** Raw little-endian float32 bytes as samples (copied, so the view is aligned). */
|
package/src/footage/draft.ts
CHANGED
|
@@ -12,18 +12,35 @@ import { chooseFootageDraftFont } from "./draft-font.js";
|
|
|
12
12
|
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
13
13
|
import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
|
|
14
14
|
import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
|
|
15
|
+
import { buildFilmMixArgs } from "./film-mix.js";
|
|
16
|
+
import { DEFAULT_LOUDNESS, formatDb, getLoudnessVerdict, measureLoudness, normalizeLoudness, type MixLoudnessTarget } from "./loudness.js";
|
|
15
17
|
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
16
18
|
import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
|
|
17
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
findFootageMusic,
|
|
21
|
+
getFfmpegReporting,
|
|
22
|
+
getFrame,
|
|
23
|
+
getSpeechChainFiles,
|
|
24
|
+
mixFootage,
|
|
25
|
+
type FinishedLoudness,
|
|
26
|
+
type FinishedMix,
|
|
27
|
+
type FootageBrand,
|
|
28
|
+
type FootageResult,
|
|
29
|
+
type FootageRunOptions,
|
|
30
|
+
type PreparedClip,
|
|
31
|
+
type PreparedMix,
|
|
32
|
+
type TranscribeOutcome,
|
|
33
|
+
} from "./pipeline.js";
|
|
18
34
|
import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
|
|
19
35
|
import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
20
36
|
|
|
21
37
|
/**
|
|
22
38
|
* A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
|
|
23
39
|
* captions burned in (its caption style, in the brand's body font and caption colours), its zoom cuts (eased as in the
|
|
24
|
-
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, the
|
|
25
|
-
*
|
|
26
|
-
* render.
|
|
40
|
+
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and the render's sound chain: the
|
|
41
|
+
* speech track's stages (cut, declick, loudness pass), the ducked music when it costs nothing, and the mix brought to
|
|
42
|
+
* `mixLoudness` as the render brings it (issue #1143). No end card, cover or sound effects. What an app shows the owner
|
|
43
|
+
* before the paid render.
|
|
27
44
|
*/
|
|
28
45
|
|
|
29
46
|
/** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
|
|
@@ -241,6 +258,11 @@ export interface FootageDraftArgsInput {
|
|
|
241
258
|
speech: string;
|
|
242
259
|
/** The ducked music track (`buildMusicArgs`, as long as the cut), mixed under the speech; null or absent for none. */
|
|
243
260
|
music?: string | null;
|
|
261
|
+
/**
|
|
262
|
+
* The film's mix, already brought to its target and encoded to AAC (`mixFootage`): the draft copies it as its sound
|
|
263
|
+
* and reads neither `speech` nor `music`. Null or absent: the draft mixes `speech` and `music` and encodes them.
|
|
264
|
+
*/
|
|
265
|
+
mix?: string | null;
|
|
244
266
|
/** The film's full frame (the normalized clips' size). */
|
|
245
267
|
frame: Frame;
|
|
246
268
|
/** The draft's width; default 540. */
|
|
@@ -330,7 +352,8 @@ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
|
|
|
330
352
|
/**
|
|
331
353
|
* The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
|
|
332
354
|
* draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
|
|
333
|
-
*
|
|
355
|
+
* film's mix copied under it (or the speech and music tracks mixed and encoded to AAC); H.264 yuv420p with fast start,
|
|
356
|
+
* exactly the cut's length.
|
|
334
357
|
*/
|
|
335
358
|
export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
336
359
|
const { edl } = input;
|
|
@@ -378,10 +401,13 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
378
401
|
} else {
|
|
379
402
|
parts.push(`[${joined}]null[draft]`);
|
|
380
403
|
}
|
|
381
|
-
const
|
|
382
|
-
const
|
|
383
|
-
|
|
404
|
+
const soundInput = edl.segments.length;
|
|
405
|
+
const mix = input.mix ?? null;
|
|
406
|
+
const music = mix === null ? (input.music ?? null) : null;
|
|
407
|
+
if (music !== null) parts.push(`[${soundInput}:a][${soundInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
|
|
384
408
|
const seconds = getEdlDuration(edl);
|
|
409
|
+
// A prepared mix is copied: it was measured as encoded, and a second encode would move its loudness and true peak.
|
|
410
|
+
const audioCodec = mix === null ? ["-c:a", "aac", "-b:a", DRAFT_AUDIO_BITRATE] : ["-c:a", "copy"];
|
|
385
411
|
return [
|
|
386
412
|
"-v",
|
|
387
413
|
"error",
|
|
@@ -389,14 +415,14 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
389
415
|
...threads.global,
|
|
390
416
|
...inputs,
|
|
391
417
|
"-i",
|
|
392
|
-
input.speech,
|
|
418
|
+
mix ?? input.speech,
|
|
393
419
|
...(music === null ? [] : ["-i", music]),
|
|
394
420
|
"-filter_complex",
|
|
395
421
|
parts.join(";"),
|
|
396
422
|
"-map",
|
|
397
423
|
"[draft]",
|
|
398
424
|
"-map",
|
|
399
|
-
music === null ? `${
|
|
425
|
+
music === null ? `${soundInput}:a` : "[sound]",
|
|
400
426
|
"-t",
|
|
401
427
|
String(seconds),
|
|
402
428
|
"-c:v",
|
|
@@ -407,10 +433,7 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
407
433
|
String(DRAFT_CRF),
|
|
408
434
|
"-pix_fmt",
|
|
409
435
|
"yuv420p",
|
|
410
|
-
|
|
411
|
-
"aac",
|
|
412
|
-
"-b:a",
|
|
413
|
-
DRAFT_AUDIO_BITRATE,
|
|
436
|
+
...audioCodec,
|
|
414
437
|
"-movflags",
|
|
415
438
|
"+faststart",
|
|
416
439
|
...threads.output,
|
|
@@ -453,6 +476,13 @@ export interface FootageDraft {
|
|
|
453
476
|
bytes: number;
|
|
454
477
|
/** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
|
|
455
478
|
declick: DeclickReport | null;
|
|
479
|
+
/**
|
|
480
|
+
* The draft's sound as measured, as the render measures its finished film (`warning` on a near miss inside
|
|
481
|
+
* `mixLoudness.warnTolerance`); null for a silent draft.
|
|
482
|
+
*/
|
|
483
|
+
loudness: FinishedLoudness | null;
|
|
484
|
+
/** What the mix stage did, as in the render; null when it did not run (`mixLoudness: null`, a silent mix). */
|
|
485
|
+
mix: FinishedMix | null;
|
|
456
486
|
}
|
|
457
487
|
|
|
458
488
|
/** The draft's music track: a file to duck, or why there is none. */
|
|
@@ -505,15 +535,24 @@ async function runRenderFootageDraft(
|
|
|
505
535
|
rmSync(draftDir, { recursive: true, force: true });
|
|
506
536
|
mkdirSync(draftDir, { recursive: true });
|
|
507
537
|
const declickLevel = getDeclickLevel(film);
|
|
508
|
-
const
|
|
538
|
+
const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
|
|
539
|
+
const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
|
|
540
|
+
const hasMusic = music.value.kind === "file";
|
|
541
|
+
const items = [
|
|
542
|
+
"speech",
|
|
543
|
+
...(declickLevel === "off" ? [] : ["declick"]),
|
|
544
|
+
...(loudness === null ? [] : ["loudness"]),
|
|
545
|
+
...(hasMusic ? ["music"] : []),
|
|
546
|
+
...(mixLoudness === null ? [] : ["mix"]),
|
|
547
|
+
"film",
|
|
548
|
+
];
|
|
509
549
|
const report = (item: string, done: number): void => {
|
|
510
550
|
const itemIndex = items.indexOf(item) + 1;
|
|
511
551
|
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
|
|
512
552
|
};
|
|
513
553
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
|
|
554
|
+
// The render's speech chain and its file names, so the draft's sound is made as the render's is.
|
|
555
|
+
const { rawSpeechFile, declickedFile, speechFile } = getSpeechChainFiles(draftDir, { isDeclick: declickLevel !== "off", isLoudness: loudness !== null });
|
|
517
556
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
518
557
|
const frame = getFrame(film);
|
|
519
558
|
const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
|
|
@@ -538,13 +577,22 @@ async function runRenderFootageDraft(
|
|
|
538
577
|
const output = resolve(options.output);
|
|
539
578
|
mkdirSync(dirname(output), { recursive: true });
|
|
540
579
|
report("speech", 0);
|
|
541
|
-
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech,
|
|
580
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: rawSpeechFile, onSeconds: (done) => report("speech", done) });
|
|
542
581
|
let declick: DeclickReport | null = null;
|
|
543
582
|
if (declickLevel !== "off") {
|
|
544
583
|
report("declick", 0);
|
|
545
|
-
declick = await declickSpeech({ input:
|
|
584
|
+
declick = await declickSpeech({ input: rawSpeechFile, output: declickedFile, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
|
|
546
585
|
log(describeDeclick(declick));
|
|
547
586
|
}
|
|
587
|
+
if (loudness !== null) {
|
|
588
|
+
report("loudness", 0);
|
|
589
|
+
const outcome = await normalizeLoudness(declickedFile, speechFile, loudness, { ...aborting, onSeconds: (done) => report("loudness", done) });
|
|
590
|
+
log(
|
|
591
|
+
outcome.kind === "silent"
|
|
592
|
+
? "loudness: the speech track is silent, left as it is."
|
|
593
|
+
: `loudness: the speech track measured ${outcome.measured.toFixed(1)} LUFS, normalized to ${loudness.target} LUFS (true peak ${loudness.truePeak} dBTP).`,
|
|
594
|
+
);
|
|
595
|
+
}
|
|
548
596
|
let musicFile: string | null = null;
|
|
549
597
|
if (music.value.kind === "file") {
|
|
550
598
|
musicFile = join(draftDir, "music.mp3");
|
|
@@ -555,14 +603,27 @@ async function runRenderFootageDraft(
|
|
|
555
603
|
} else if (music.value.kind === "not_cached") {
|
|
556
604
|
log(`draft: no music, ${music.value.reason}.`);
|
|
557
605
|
}
|
|
606
|
+
let prepared: PreparedMix | null = null;
|
|
607
|
+
if (mixLoudness !== null) {
|
|
608
|
+
report("mix", 0);
|
|
609
|
+
// The render's mix stage over the draft's tracks (the draft has no sound effects): the same target, ceiling and attempts.
|
|
610
|
+
const mixSource = join(draftDir, "mix.wav");
|
|
611
|
+
const args = buildFilmMixArgs({ speech: { file: speechFile, seconds }, music: musicFile === null ? null : { file: musicFile, seconds }, sfx: [], seconds, output: mixSource });
|
|
612
|
+
await runFfmpeg("The draft's mix", args, { ...aborting, output: mixSource, onSeconds: (done) => report("mix", done) });
|
|
613
|
+
const mixed = await mixFootage(mixLoudness, { mixSource, output: join(draftDir, "mix.m4a") }, { log, ...aborting });
|
|
614
|
+
rmSync(mixSource, { force: true });
|
|
615
|
+
if (!mixed.ok) return mixed;
|
|
616
|
+
prepared = mixed.value;
|
|
617
|
+
}
|
|
558
618
|
report("film", 0);
|
|
559
619
|
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
560
620
|
const args = buildFootageDraftArgs({
|
|
561
621
|
film,
|
|
562
622
|
edl,
|
|
563
623
|
clips: clipFiles,
|
|
564
|
-
speech,
|
|
624
|
+
speech: speechFile,
|
|
565
625
|
music: musicFile,
|
|
626
|
+
mix: prepared?.audioFile ?? null,
|
|
566
627
|
frame,
|
|
567
628
|
...(options.width === undefined ? {} : { width: options.width }),
|
|
568
629
|
...(options.threads === undefined ? {} : { threads: options.threads }),
|
|
@@ -571,11 +632,51 @@ async function runRenderFootageDraft(
|
|
|
571
632
|
output,
|
|
572
633
|
});
|
|
573
634
|
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
|
|
635
|
+
const measured = await measureDraftLoudness(output, mixLoudness, prepared !== null, aborting);
|
|
636
|
+
if (!measured.ok) return measured;
|
|
637
|
+
const draftLoudness = measured.value;
|
|
638
|
+
if (draftLoudness !== null) {
|
|
639
|
+
log(`loudness: the draft measures ${formatDb(draftLoudness.integrated)} LUFS, true peak ${formatDb(draftLoudness.truePeak)} dBTP.`);
|
|
640
|
+
if (draftLoudness.warning !== null) log(`loudness warning: ${draftLoudness.warning}`);
|
|
641
|
+
}
|
|
574
642
|
report("film", seconds);
|
|
575
643
|
const draft = getFootageDraftFrame(frame, options.width);
|
|
576
644
|
const bytes = statSync(output).size;
|
|
577
645
|
const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
|
|
578
646
|
const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
|
|
579
647
|
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
|
|
580
|
-
return ok({
|
|
648
|
+
return ok({
|
|
649
|
+
output,
|
|
650
|
+
music: draftMusic,
|
|
651
|
+
captionFont: captions === null ? null : font.family,
|
|
652
|
+
seconds,
|
|
653
|
+
width: draft.width,
|
|
654
|
+
height: draft.height,
|
|
655
|
+
bytes,
|
|
656
|
+
declick,
|
|
657
|
+
loudness: draftLoudness,
|
|
658
|
+
mix: prepared?.mix ?? null,
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* The written draft's sound measured as `finishFootage` measures the finished film: held to `mixLoudness` when the mix
|
|
664
|
+
* stage made it (a miss fails, a near miss inside `warnTolerance` carries `warning`), reported otherwise; null when
|
|
665
|
+
* the draft is silent.
|
|
666
|
+
*/
|
|
667
|
+
async function measureDraftLoudness(
|
|
668
|
+
output: string,
|
|
669
|
+
mixLoudness: MixLoudnessTarget | null,
|
|
670
|
+
isMixed: boolean,
|
|
671
|
+
aborting: { signal?: AbortSignal },
|
|
672
|
+
): Promise<FootageResult<FinishedLoudness | null>> {
|
|
673
|
+
const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the footage draft", ...aborting });
|
|
674
|
+
if (measured.kind === "silent") return ok(null);
|
|
675
|
+
const sound = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
|
|
676
|
+
const verdict = mixLoudness === null || !isMixed ? { kind: "pass" as const } : getLoudnessVerdict(sound, mixLoudness);
|
|
677
|
+
if (verdict.kind === "fail") {
|
|
678
|
+
return err(`the footage draft ${output} measures ${formatDb(sound.integrated)} LUFS integrated, ${formatDb(sound.truePeak)} dBTP true peak: ${verdict.problems.join("; ")}.`, { kind: "loudness", loudness: sound });
|
|
679
|
+
}
|
|
680
|
+
return ok({ ...sound, warning: verdict.kind === "warn" ? `the footage draft ${output}: ${verdict.warning}` : null });
|
|
581
681
|
}
|
|
682
|
+
|
|
@@ -5,6 +5,7 @@ import { z } from "zod";
|
|
|
5
5
|
|
|
6
6
|
import { ELEVENLABS_API_KEY_ENV } from "../voice/elevenlabs.js";
|
|
7
7
|
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
8
|
+
import { getConnectionFailure, getLocalRefusal, getProviderFailure } from "../voice/provider-failure.js";
|
|
8
9
|
import { ELEVENLABS_API_URL, type TimedWord } from "../voice/voiceover.js";
|
|
9
10
|
import type { IsolationProvider, MusicProvider, PaidAudio, SttProvider, SttTranscript } from "./providers.js";
|
|
10
11
|
|
|
@@ -57,17 +58,17 @@ function fileBlob(path: string): Blob {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
async function send(options: ElevenLabsFootageOptions, label: string, path: string, init: RequestInit): Promise<TtsResult<Response>> {
|
|
60
|
-
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment
|
|
61
|
+
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment.`, getLocalRefusal("missing_api_key"));
|
|
61
62
|
const request = options.fetch ?? globalThis.fetch;
|
|
62
63
|
let response: Response;
|
|
63
64
|
try {
|
|
64
65
|
response = await request(`${ELEVENLABS_API_URL}${path}`, { ...init, headers: { ...init.headers, "xi-api-key": options.apiKey } });
|
|
65
66
|
} catch (error) {
|
|
66
|
-
return err(`ElevenLabs ${label}: the connection failed (${String(error)})
|
|
67
|
+
return err(`ElevenLabs ${label}: the connection failed (${String(error)}).`, getConnectionFailure(error));
|
|
67
68
|
}
|
|
68
69
|
if (!response.ok) {
|
|
69
70
|
const detail = await response.text().catch((error: unknown) => `(unreadable body: ${String(error)})`);
|
|
70
|
-
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}
|
|
71
|
+
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}`, getProviderFailure(response, detail));
|
|
71
72
|
}
|
|
72
73
|
return ok(response);
|
|
73
74
|
}
|
|
@@ -118,9 +119,16 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
|
|
|
118
119
|
};
|
|
119
120
|
}
|
|
120
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The shortest input the ElevenLabs Voice Isolator accepts, in seconds: production renders saw every range of 4.18 s
|
|
124
|
+
* or less refused on every attempt (`audio_too_short`) and every range of 4.64 s or more accepted (issue #1140).
|
|
125
|
+
*/
|
|
126
|
+
export const ELEVENLABS_ISOLATION_MIN_SECONDS = 4.6;
|
|
127
|
+
|
|
121
128
|
export function createElevenLabsIsolationProvider(options: ElevenLabsFootageOptions): IsolationProvider {
|
|
122
129
|
return {
|
|
123
130
|
id: "elevenlabs",
|
|
131
|
+
minSeconds: ELEVENLABS_ISOLATION_MIN_SECONDS,
|
|
124
132
|
async isolate(input) {
|
|
125
133
|
const form = new FormData();
|
|
126
134
|
form.append("audio", fileBlob(input.audio), basename(input.audio));
|