hyperframes 0.8.185 → 0.8.186
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{add-ODTNPRGG.js → add-UGK7AZI6.js} +11 -11
- package/dist/{animatedGifPrep-PEP5GWOJ.js → animatedGifPrep-HPZPSB43.js} +5 -5
- package/dist/api.js +14 -14
- package/dist/{asset-EZG42XPA.js → asset-X2GTFWOY.js} +2 -2
- package/dist/{assetCataloger-OHAJFCA7.js → assetCataloger-5NWEG4IU.js} +5 -5
- package/dist/audio-7PBM5F3H.js +2317 -0
- package/dist/{auth-VP2SIZFN.js → auth-WABSMO46.js} +1 -1
- package/dist/{autoUpdate-RVSUX4SS.js → autoUpdate-6SCMLTSC.js} +2 -2
- package/dist/{backgroundChecks-LEHK74NS.js → backgroundChecks-XOGPAAR3.js} +3 -3
- package/dist/backgroundChecksWorker.js +3 -3
- package/dist/{batchRender-3X6CL7PK.js → batchRender-VR6IA77K.js} +3 -3
- package/dist/{beats-66VPQLSC.js → beats-3IWXLMSC.js} +5 -5
- package/dist/{benchmark-PBJVTOJP.js → benchmark-2SSYATCX.js} +8 -8
- package/dist/{browser-TPL5MWGC.js → browser-PMRN345D.js} +3 -3
- package/dist/{bundleWithLocalizedFonts-67MLTHKM.js → bundleWithLocalizedFonts-TT32MZBO.js} +2 -2
- package/dist/{capture-F3GNMRZM.js → capture-4VD6AKUJ.js} +8 -8
- package/dist/{capture-YYRE7EV3.js → capture-BDGYTE57.js} +2 -2
- package/dist/{captureCompositionFrame-PXV2ISHD.js → captureCompositionFrame-5FVLAETA.js} +13 -9
- package/dist/{catalog-6SDD3OHF.js → catalog-HUYZO5D7.js} +11 -11
- package/dist/{catch-up-CWIVA5VG.js → catch-up-SSGTR2ZT.js} +4 -4
- package/dist/{check-YQ6REK2S.js → check-JWUWHGM2.js} +29 -29
- package/dist/{checkBrowser-5XZCBRXM.js → checkBrowser-X7G6YNR7.js} +26 -26
- package/dist/{chunk-RUEGYANA.js → chunk-22QKQDEE.js} +10 -10
- package/dist/{chunk-IPDKZ5IN.js → chunk-27H2363S.js} +1 -1
- package/dist/{chunk-2TAS5O4P.js → chunk-2II34JPM.js} +108 -20
- package/dist/{chunk-KQ4JEJLF.js → chunk-2PZZMAME.js} +8 -7
- package/dist/{chunk-7RAY4WZP.js → chunk-3YHGQJI6.js} +3 -3
- package/dist/{chunk-BRAPBUPQ.js → chunk-4AFNEKU7.js} +1 -1
- package/dist/{chunk-CX3USX23.js → chunk-4NTW45W5.js} +3 -3
- package/dist/{chunk-K2ZKXTVL.js → chunk-4XFFJAWS.js} +1 -1
- package/dist/{chunk-6GPR3G3C.js → chunk-5ZMIVTHN.js} +1 -1
- package/dist/{chunk-TAX2ZJAP.js → chunk-5ZMXGNGC.js} +9 -9
- package/dist/{chunk-H5SM5PJI.js → chunk-6IOBAOU7.js} +1 -1
- package/dist/{chunk-WQLKDL35.js → chunk-77QM6YRQ.js} +1 -1
- package/dist/{chunk-SQVRXHOW.js → chunk-7AUMTARQ.js} +3 -3
- package/dist/{chunk-I3IH5MV5.js → chunk-7M7TMM4S.js} +77 -2
- package/dist/{chunk-37CNYTQQ.js → chunk-7MLFYMFB.js} +1 -1
- package/dist/{chunk-55YELV7B.js → chunk-7MVODWQF.js} +1 -1
- package/dist/{chunk-X6L6MD5B.js → chunk-ACLABKEF.js} +6 -6
- package/dist/{chunk-Q47FAYYK.js → chunk-BRTDLVK3.js} +1 -1
- package/dist/{chunk-OAG47RE6.js → chunk-DQZMJS2P.js} +1 -1
- package/dist/{chunk-G2UANIDX.js → chunk-DTXDQ6OP.js} +1 -1
- package/dist/{chunk-I45HL3VP.js → chunk-EZINX6HH.js} +2 -2
- package/dist/{chunk-IVEFOUZI.js → chunk-FCWFV4P7.js} +36 -12
- package/dist/{chunk-ISTRDGIX.js → chunk-FRLTKTOR.js} +1 -1
- package/dist/{chunk-6RIZY7OP.js → chunk-GTMBII7X.js} +1 -1
- package/dist/{chunk-FX6RUJUY.js → chunk-HOFX63F2.js} +3 -3
- package/dist/{chunk-5GUKZRQN.js → chunk-HZ7M2MZA.js} +1 -1
- package/dist/{chunk-HXUTX5GR.js → chunk-ID5AJ4L3.js} +2 -2
- package/dist/{chunk-PHMBMOXF.js → chunk-K6KBOORN.js} +48 -4
- package/dist/{chunk-F3OPUWQX.js → chunk-KNLSYCI5.js} +7 -7
- package/dist/{chunk-I7QMXZSB.js → chunk-LDW3KVGK.js} +2 -2
- package/dist/{chunk-5UDRPPN2.js → chunk-MY6CVBDD.js} +7 -4
- package/dist/{chunk-XWC7CNPY.js → chunk-NPDPDEPP.js} +1 -1
- package/dist/{chunk-7PS6RVWD.js → chunk-PEIV2SOG.js} +1 -1
- package/dist/{chunk-H6UY725W.js → chunk-R6IGQ5NO.js} +2 -2
- package/dist/{chunk-A6QCO474.js → chunk-RFJ6SX4U.js} +5 -5
- package/dist/{chunk-KROXPHXX.js → chunk-RNZEQTWZ.js} +2 -8
- package/dist/{chunk-6ASIXMFP.js → chunk-ROWEDRVX.js} +1 -1
- package/dist/{chunk-HDIRZGK4.js → chunk-RPUIMFHF.js} +1 -1
- package/dist/{chunk-EMSDWBG3.js → chunk-RVZNSKRH.js} +2 -2
- package/dist/{chunk-IDGLGXPA.js → chunk-T4RP55P2.js} +1 -1
- package/dist/{chunk-EMAWIZ54.js → chunk-TH7J3JZ5.js} +57 -20
- package/dist/{chunk-VHZ6DL3M.js → chunk-TMK4Q7JA.js} +10 -7
- package/dist/{chunk-UPR3MU6Z.js → chunk-TQ2RRB5V.js} +1 -1
- package/dist/{chunk-IDFNKFPL.js → chunk-U4BMRIE6.js} +16 -7
- package/dist/{chunk-HHSJJFVM.js → chunk-U7QTMRLW.js} +2 -2
- package/dist/{chunk-TOI35GN2.js → chunk-UAJCTEK2.js} +1 -1
- package/dist/{chunk-ESBMUZXZ.js → chunk-UJZT6HNN.js} +1 -1
- package/dist/chunk-UOZX5W5T.js +3852 -0
- package/dist/{chunk-FFJ5TDRD.js → chunk-VHBBV5DH.js} +10 -3
- package/dist/{chunk-SKYPMHQC.js → chunk-WHMFW5ET.js} +2 -2
- package/dist/{chunk-LYKSTRAY.js → chunk-WLHNIIXU.js} +3 -3
- package/dist/{chunk-RTBSA3EA.js → chunk-Y2M6XSNK.js} +1 -1
- package/dist/{chunk-4CTNDBE3.js → chunk-YNH6ERDL.js} +2 -2
- package/dist/{chunk-D42NU3DZ.js → chunk-YQICCAVA.js} +1 -1
- package/dist/{chunk-PMMQRJDE.js → chunk-YRAJDFIN.js} +1 -1
- package/dist/{chunk-BH7TTIPN.js → chunk-Z5233OAN.js} +4 -4
- package/dist/{chunk-K73RJB23.js → chunk-ZKBW7NU2.js} +2 -2
- package/dist/{chunk-J5XEZ24I.js → chunk-ZVBEGBII.js} +3 -3
- package/dist/{clean-7J6ZCCGV.js → clean-IM2JIAUT.js} +13 -13
- package/dist/cli.js +53 -53
- package/dist/{cloud-7IJCJCDD.js → cloud-6IPBCJSX.js} +4 -4
- package/dist/{cloudrun-UNZRIIDB.js → cloudrun-MWZ6S2I2.js} +4 -4
- package/dist/{compare-Q7VW5JZT.js → compare-WD7A2TV5.js} +23 -23
- package/dist/{compiler-KPHYGFXL.js → compiler-2EKSMC3I.js} +2 -2
- package/dist/{component-UUBREJKN.js → component-JKALWH36.js} +3 -3
- package/dist/{compositions-T4FTKIDP.js → compositions-643NAG4A.js} +9 -9
- package/dist/{delete-WZQF2IIA.js → delete-EASPXKHQ.js} +2 -2
- package/dist/{deploy-JU7A3XNC.js → deploy-CDKCQ5MX.js} +1 -1
- package/dist/{dist-ULGTPSU4.js → dist-F6FOKRSU.js} +6 -4
- package/dist/{doctor-VSNGRKXJ.js → doctor-7XUVYDJQ.js} +9 -9
- package/dist/{events-CRW2DYUT.js → events-5TTPYBKG.js} +2 -2
- package/dist/{events-E63YBUL2.js → events-TULV6D3G.js} +3 -3
- package/dist/{feedback-OSZWVR6N.js → feedback-4FTPWM6P.js} +6 -6
- package/dist/{ffmpeg-JLH3Z7G7.js → ffmpeg-MP7ZVYLZ.js} +5 -5
- package/dist/{figma-2BQNNR53.js → figma-B7LFWYKO.js} +3 -3
- package/dist/fontLocalizeCli.js +7 -7
- package/dist/{get-P2XI53QW.js → get-F7W4AQ2V.js} +2 -2
- package/dist/{gpuPolicy-NTKQJY75.js → gpuPolicy-NSNVDFXC.js} +5 -5
- package/dist/{grade-compare-OYASIHIB.js → grade-compare-6QXDTLS2.js} +24 -24
- package/dist/{help-HK6K2N4G.js → help-RJAA4O74.js} +1 -1
- package/dist/{history-JMTYKLXW.js → history-OOQBDVV5.js} +6 -6
- package/dist/hyperframe.manifest.json +1 -1
- package/dist/hyperframe.runtime.iife.js +65 -65
- package/dist/hyperframes-player.global.js +1 -1
- package/dist/{info-AW7CIVRE.js → info-5OGGYYLO.js} +7 -7
- package/dist/{init-6KIDG4LB.js → init-DM3TJKW2.js} +63 -45
- package/dist/{inspect-B7BLYMRZ.js → inspect-FECQCLS5.js} +24 -24
- package/dist/{installer-ZP2LQZUQ.js → installer-WOG4VORV.js} +4 -4
- package/dist/{keyframes-EQ36A5JE.js → keyframes-SZHWUF5Y.js} +6 -6
- package/dist/{lambda-E6VURLIS.js → lambda-JG2ZRGMC.js} +6 -6
- package/dist/{layout-XYC64QQQ.js → layout-7ACF7ZKC.js} +24 -24
- package/dist/{lint-V5DJYWBX.js → lint-ZCQGMILV.js} +6 -6
- package/dist/{list-RUGVRTVD.js → list-MH2X3Y2T.js} +2 -2
- package/dist/{login-ZTKGKXGS.js → login-CK46WX5U.js} +4 -4
- package/dist/{manager-JUP7PM6N.js → manager-GXTPNJVW.js} +6 -6
- package/dist/{media-treatment-VIUPK4OR.js → media-treatment-EH2AT34Z.js} +11 -11
- package/dist/{models-LJS2HBOZ.js → models-D5XP2UK6.js} +9 -9
- package/dist/{motionShot-3UAXZ7SN.js → motionShot-UR7D3SRQ.js} +12 -12
- package/dist/{normalize-4UDISLPL.js → normalize-3AYFSN2F.js} +1 -1
- package/dist/{normalize-audio-SBYI4YIH.js → normalize-audio-635JOR5W.js} +4 -4
- package/dist/{open-S27TUD3A.js → open-6WE5WFVM.js} +11 -11
- package/dist/{parakeetRunner-ETLMLJ4L.js → parakeetRunner-STHZPG32.js} +10 -10
- package/dist/{pipeline-I2M22M24.js → pipeline-LVEKRETU.js} +6 -6
- package/dist/{play-COJU6VJK.js → play-W6TDW4FE.js} +16 -16
- package/dist/{preflight-FZ24NCHH.js → preflight-LIMQXLR7.js} +6 -6
- package/dist/{present-YAHKLC5D.js → present-RX5QX6UR.js} +12 -12
- package/dist/{preview-KQOEU5YE.js → preview-KIWTWK63.js} +16 -16
- package/dist/{producer-DTNNG4T6.js → producer-TSNKSCUM.js} +1 -1
- package/dist/{projectConfig-4A76VYUP.js → projectConfig-NCP7JRH4.js} +6 -6
- package/dist/{publication-SQPMC2W3.js → publication-5W5UZWGC.js} +2 -2
- package/dist/{publish-QUEJBGGT.js → publish-7WR4XJ6I.js} +7 -7
- package/dist/{remove-background-FP2ETAQH.js → remove-background-43JGBRWQ.js} +7 -7
- package/dist/{render-BIFIL4U6.js → render-456HCO6O.js} +26 -26
- package/dist/{render-P664SMC7.js → render-7T44HIM6.js} +3 -3
- package/dist/{render-GQFXJWLV.js → render-WHMLYIOI.js} +13 -13
- package/dist/{render-batch-ZCPMYH6Z.js → render-batch-LURBJTBV.js} +3 -3
- package/dist/{resolver-BUC2GFZQ.js → resolver-MBIW4VC4.js} +4 -4
- package/dist/{sherpa-HFT6XEP6.js → sherpa-5J2AALED.js} +9 -9
- package/dist/{skills-3WEVO3NU.js → skills-UHOXOGGF.js} +5 -5
- package/dist/{skillsUpdateCheck-BFPWFOK5.js → skillsUpdateCheck-PDZIFUJE.js} +3 -3
- package/dist/{snapshot-WDQ6S7CJ.js → snapshot-PFV7ALLH.js} +33 -30
- package/dist/{src-RZ574OKP.js → src-DCVSEODX.js} +4 -4
- package/dist/{src-NPR2JZOJ.js → src-XK5YIR6L.js} +6 -6
- package/dist/{staticProjectServer-FTGTEZI4.js → staticProjectServer-4IE4G4JO.js} +12 -12
- package/dist/studio/assets/{hyperframes-player-CoOdf435.js → hyperframes-player-B_skzvYC.js} +1 -1
- package/dist/studio/assets/{index-CR85xg2x.js → index-CgLbvCUo.js} +1 -1
- package/dist/studio/assets/{index-C_6rTpaB.js → index-Dn5FAk2e.js} +99 -99
- package/dist/studio/index.html +1 -1
- package/dist/{studio-api-TAWB7B23.js → studio-api-JMWYU7KF.js} +6 -6
- package/dist/{studioServer-HRQGJTBU.js → studioServer-BWLSTGIG.js} +43 -42
- package/dist/{telemetry-QZ6HJF4W.js → telemetry-JCU3AV7W.js} +3 -3
- package/dist/{timeline-IKUEW42T.js → timeline-BCIFQYOM.js} +14 -14
- package/dist/{tokens-AUGUEWHS.js → tokens-7IXKALMQ.js} +2 -2
- package/dist/{transcribe-7734CG3N.js → transcribe-HW7HGJ4V.js} +15 -15
- package/dist/{transcribe-FWU6EMEO.js → transcribe-ZUQD3B32.js} +8 -8
- package/dist/{updateCheck-VIPQ3FPW.js → updateCheck-Z6LNEK7B.js} +2 -2
- package/dist/{upgrade-4VNDQTVO.js → upgrade-JYH72JGL.js} +2 -2
- package/dist/{urlDownloader-6CXL4CL7.js → urlDownloader-WKLFHSUH.js} +4 -4
- package/dist/{validate-QWCMVWID.js → validate-QCC27YYI.js} +18 -18
- package/dist/{video-245Q5WN4.js → video-EZ7ONY6L.js} +5 -5
- package/dist/warmCheck.js +31 -31
- package/package.json +1 -1
- package/dist/audio-65R4T74I.js +0 -1154
- package/dist/chunk-3NPOFC43.js +0 -3805
|
@@ -0,0 +1,2317 @@
|
|
|
1
|
+
import { createRequire as __hf_createRequire } from "node:module";
|
|
2
|
+
import { fileURLToPath as __hf_fileURLToPath } from "node:url";
|
|
3
|
+
import { dirname as __hf_dirname } from "node:path";
|
|
4
|
+
var require = __hf_createRequire(import.meta.url);
|
|
5
|
+
var __filename = __hf_fileURLToPath(import.meta.url);
|
|
6
|
+
var __dirname = __hf_dirname(__filename);
|
|
7
|
+
import {
|
|
8
|
+
uploadAssetViaDirectUpload
|
|
9
|
+
} from "./chunk-MY6CVBDD.js";
|
|
10
|
+
import {
|
|
11
|
+
HyperframesCloudClient
|
|
12
|
+
} from "./chunk-5UBZPG74.js";
|
|
13
|
+
import {
|
|
14
|
+
describeProject
|
|
15
|
+
} from "./chunk-Z5233OAN.js";
|
|
16
|
+
import "./chunk-LLMSX6V6.js";
|
|
17
|
+
import {
|
|
18
|
+
bandFramesOf,
|
|
19
|
+
decodeAudio,
|
|
20
|
+
envelope,
|
|
21
|
+
medianInterval,
|
|
22
|
+
probeAudio,
|
|
23
|
+
rhythm,
|
|
24
|
+
runLadder,
|
|
25
|
+
signalDuration,
|
|
26
|
+
signalProvider,
|
|
27
|
+
snapper,
|
|
28
|
+
tempoOf
|
|
29
|
+
} from "./chunk-VHBBV5DH.js";
|
|
30
|
+
import {
|
|
31
|
+
refreshIfNeeded,
|
|
32
|
+
resolveCloudBaseUrl
|
|
33
|
+
} from "./chunk-HRPWRZGF.js";
|
|
34
|
+
import {
|
|
35
|
+
buildAuthHeaders,
|
|
36
|
+
tryResolveCredential
|
|
37
|
+
} from "./chunk-7KVVNF7B.js";
|
|
38
|
+
import {
|
|
39
|
+
downloadToFile
|
|
40
|
+
} from "./chunk-CZUAX3BV.js";
|
|
41
|
+
import {
|
|
42
|
+
loadTranscript
|
|
43
|
+
} from "./chunk-TMK4Q7JA.js";
|
|
44
|
+
import {
|
|
45
|
+
resolveProject
|
|
46
|
+
} from "./chunk-HZ7M2MZA.js";
|
|
47
|
+
import "./chunk-6JR2XUEE.js";
|
|
48
|
+
import "./chunk-DTXDQ6OP.js";
|
|
49
|
+
import "./chunk-K2FLNJVB.js";
|
|
50
|
+
import {
|
|
51
|
+
ensureDOMParser
|
|
52
|
+
} from "./chunk-442JNXRM.js";
|
|
53
|
+
import "./chunk-EOVYQ3IG.js";
|
|
54
|
+
import "./chunk-77QM6YRQ.js";
|
|
55
|
+
import "./chunk-6IOBAOU7.js";
|
|
56
|
+
import "./chunk-VWG7OQKU.js";
|
|
57
|
+
import {
|
|
58
|
+
assertKnownFlags
|
|
59
|
+
} from "./chunk-HTIUNMTP.js";
|
|
60
|
+
import {
|
|
61
|
+
defineCommand
|
|
62
|
+
} from "./chunk-S3S4TGEH.js";
|
|
63
|
+
import "./chunk-54HJ63Q2.js";
|
|
64
|
+
import {
|
|
65
|
+
CliUsageError
|
|
66
|
+
} from "./chunk-JVBRJMOP.js";
|
|
67
|
+
import "./chunk-ZTZA4UWZ.js";
|
|
68
|
+
import "./chunk-7M7TMM4S.js";
|
|
69
|
+
import "./chunk-IZCW2NKT.js";
|
|
70
|
+
import "./chunk-RNZEQTWZ.js";
|
|
71
|
+
import "./chunk-RQVDTPGD.js";
|
|
72
|
+
import "./chunk-BRTDLVK3.js";
|
|
73
|
+
import "./chunk-UOZX5W5T.js";
|
|
74
|
+
import "./chunk-VC76R6W2.js";
|
|
75
|
+
import "./chunk-UJKQWGFF.js";
|
|
76
|
+
import "./chunk-B2A56S5C.js";
|
|
77
|
+
import "./chunk-OFQWTKPW.js";
|
|
78
|
+
import "./chunk-DPVXBVLL.js";
|
|
79
|
+
import "./chunk-3GXQPCJC.js";
|
|
80
|
+
import "./chunk-634HSJ3J.js";
|
|
81
|
+
import "./chunk-FXMKHHPO.js";
|
|
82
|
+
import "./chunk-T6AGUTZJ.js";
|
|
83
|
+
import "./chunk-XT6MXC7A.js";
|
|
84
|
+
import "./chunk-4UMPIPCQ.js";
|
|
85
|
+
import "./chunk-Q447CX26.js";
|
|
86
|
+
import {
|
|
87
|
+
findFfBinary
|
|
88
|
+
} from "./chunk-VAS6HQX3.js";
|
|
89
|
+
import {
|
|
90
|
+
resolveAuthoredAssetFile
|
|
91
|
+
} from "./chunk-6BNTGNGD.js";
|
|
92
|
+
import "./chunk-RUOG4VB7.js";
|
|
93
|
+
import "./chunk-MMQ52NIZ.js";
|
|
94
|
+
import "./chunk-BSQ7XANM.js";
|
|
95
|
+
import "./chunk-SDYDNHEE.js";
|
|
96
|
+
import "./chunk-IXLUTE6N.js";
|
|
97
|
+
import "./chunk-I5JRXMCV.js";
|
|
98
|
+
import "./chunk-W2LGJSEN.js";
|
|
99
|
+
import {
|
|
100
|
+
withMeta
|
|
101
|
+
} from "./chunk-FRLTKTOR.js";
|
|
102
|
+
import "./chunk-K4WIOZFP.js";
|
|
103
|
+
import "./chunk-XUXGIA7H.js";
|
|
104
|
+
import "./chunk-WERLJGIC.js";
|
|
105
|
+
import "./chunk-ISN5GRYI.js";
|
|
106
|
+
import "./chunk-LDW3KVGK.js";
|
|
107
|
+
import "./chunk-KNU5COPB.js";
|
|
108
|
+
import "./chunk-M2WBOQ2Q.js";
|
|
109
|
+
|
|
110
|
+
// src/commands/audio.ts
|
|
111
|
+
import { join as join7, resolve as resolve2 } from "path";
|
|
112
|
+
|
|
113
|
+
// src/audio/analysis/providers/api.ts
|
|
114
|
+
import { randomUUID } from "crypto";
|
|
115
|
+
|
|
116
|
+
// src/audio/analysis/providers/shared.ts
|
|
117
|
+
import { execFile } from "child_process";
|
|
118
|
+
import { mkdtemp, rm, writeFile } from "fs/promises";
|
|
119
|
+
import { tmpdir } from "os";
|
|
120
|
+
import { join } from "path";
|
|
121
|
+
import { promisify } from "util";
|
|
122
|
+
var runFile = promisify(execFile);
|
|
123
|
+
var unavailable = (reason) => ({ ok: false, reason });
|
|
124
|
+
var AVAILABLE = { ok: true };
|
|
125
|
+
function encodeWav(signal) {
|
|
126
|
+
const channels = signal.channels.length;
|
|
127
|
+
const frames = signal.channels[0]?.length ?? 0;
|
|
128
|
+
const data = frames * channels * 2;
|
|
129
|
+
const wav = Buffer.alloc(44 + data);
|
|
130
|
+
wav.write("RIFF", 0);
|
|
131
|
+
wav.writeUInt32LE(36 + data, 4);
|
|
132
|
+
wav.write("WAVEfmt ", 8);
|
|
133
|
+
wav.writeUInt32LE(16, 16);
|
|
134
|
+
wav.writeUInt16LE(1, 20);
|
|
135
|
+
wav.writeUInt16LE(channels, 22);
|
|
136
|
+
wav.writeUInt32LE(signal.sampleRate, 24);
|
|
137
|
+
wav.writeUInt32LE(signal.sampleRate * channels * 2, 28);
|
|
138
|
+
wav.writeUInt16LE(channels * 2, 32);
|
|
139
|
+
wav.writeUInt16LE(16, 34);
|
|
140
|
+
wav.write("data", 36);
|
|
141
|
+
wav.writeUInt32LE(data, 40);
|
|
142
|
+
let offset = 44;
|
|
143
|
+
for (let i = 0; i < frames; i++)
|
|
144
|
+
for (const channel of signal.channels) {
|
|
145
|
+
const sample = Math.max(-1, Math.min(1, channel[i]));
|
|
146
|
+
wav.writeInt16LE(Math.round(sample * 32767), offset);
|
|
147
|
+
offset += 2;
|
|
148
|
+
}
|
|
149
|
+
return wav;
|
|
150
|
+
}
|
|
151
|
+
async function withWavFile(signal, use) {
|
|
152
|
+
const dir = await mkdtemp(join(tmpdir(), "hf-audio-"));
|
|
153
|
+
try {
|
|
154
|
+
const path = join(dir, "audio.wav");
|
|
155
|
+
await writeFile(path, encodeWav(signal));
|
|
156
|
+
return await use(path);
|
|
157
|
+
} finally {
|
|
158
|
+
await rm(dir, { recursive: true, force: true });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
function perSignal(compute) {
|
|
162
|
+
const answers = /* @__PURE__ */ new WeakMap();
|
|
163
|
+
return (signal) => {
|
|
164
|
+
let answer = answers.get(signal);
|
|
165
|
+
if (!answer) answers.set(signal, answer = compute(signal));
|
|
166
|
+
return answer;
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
function downbeatsWithMeter(beats, downbeats) {
|
|
170
|
+
const bar = medianInterval(downbeats);
|
|
171
|
+
const beat = medianInterval(beats);
|
|
172
|
+
const meter = bar && beat ? Math.round(bar / beat) : null;
|
|
173
|
+
return { times: downbeats, meter: meter && meter > 0 ? meter : null };
|
|
174
|
+
}
|
|
175
|
+
var round = (t) => Math.round(t * 1e3) / 1e3;
|
|
176
|
+
var rounded = (times) => times.map(round);
|
|
177
|
+
var NO_HEYGEN_CREDENTIAL = "no HeyGen credential: set HEYGEN_API_KEY or run `hyperframes auth login --api-key`";
|
|
178
|
+
async function heygenHeaders() {
|
|
179
|
+
try {
|
|
180
|
+
const credential = await tryResolveCredential();
|
|
181
|
+
return credential ? buildAuthHeaders(await refreshIfNeeded(credential)) : NO_HEYGEN_CREDENTIAL;
|
|
182
|
+
} catch (error) {
|
|
183
|
+
return error instanceof Error ? error.message : String(error);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
async function serviceAccess(env, path, defaultMs) {
|
|
187
|
+
const timeout = Number(process.env[env.timeout]);
|
|
188
|
+
const timeoutMs = timeout > 0 ? timeout : defaultMs;
|
|
189
|
+
const url = process.env[env.url]?.trim();
|
|
190
|
+
const key = process.env[env.key]?.trim();
|
|
191
|
+
if (url === "") return `${env.url} is empty, which turns this service off`;
|
|
192
|
+
if (key)
|
|
193
|
+
return {
|
|
194
|
+
url: url || `${resolveCloudBaseUrl()}${path}`,
|
|
195
|
+
headers: { "x-api-key": key },
|
|
196
|
+
timeoutMs
|
|
197
|
+
};
|
|
198
|
+
if (url) return `${env.url} is set but ${env.key} is not`;
|
|
199
|
+
const headers = await heygenHeaders();
|
|
200
|
+
if (typeof headers === "string") return headers;
|
|
201
|
+
return { url: `${resolveCloudBaseUrl()}${path}`, headers, timeoutMs };
|
|
202
|
+
}
|
|
203
|
+
function errorMessageIn(text) {
|
|
204
|
+
try {
|
|
205
|
+
return JSON.parse(text).error?.message;
|
|
206
|
+
} catch {
|
|
207
|
+
return void 0;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async function serviceFailure(response, service) {
|
|
211
|
+
const text = await response.text().catch(() => "");
|
|
212
|
+
const detail = errorMessageIn(text) ?? text.slice(0, 200);
|
|
213
|
+
return new Error(`${service} answered ${response.status}${detail ? `: ${detail}` : ""}`);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// src/audio/analysis/providers/api.ts
|
|
217
|
+
var URL_ENV = "HYPERFRAMES_AUDIO_API_URL";
|
|
218
|
+
var KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY";
|
|
219
|
+
var TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS";
|
|
220
|
+
var DEFAULT_TIMEOUT_MS = 12e4;
|
|
221
|
+
var UPLOAD_RATE = 16e3;
|
|
222
|
+
var SINC_ZEROS = 16;
|
|
223
|
+
var ROLLOFF = 0.9;
|
|
224
|
+
var SINC_STEPS = 512;
|
|
225
|
+
var HANN_SINC = Float32Array.from({ length: SINC_ZEROS * SINC_STEPS + 2 }, (_, k) => {
|
|
226
|
+
const x = k / SINC_STEPS;
|
|
227
|
+
if (x >= SINC_ZEROS) return 0;
|
|
228
|
+
const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
|
|
229
|
+
return sinc * (0.5 + 0.5 * Math.cos(Math.PI * x / SINC_ZEROS));
|
|
230
|
+
});
|
|
231
|
+
function resample(input, from, to) {
|
|
232
|
+
if (from === to) return input;
|
|
233
|
+
const step = from / to;
|
|
234
|
+
const scale = Math.min(1, to / from) * ROLLOFF;
|
|
235
|
+
const reach = SINC_ZEROS / scale;
|
|
236
|
+
const output = new Float32Array(Math.round(input.length / step));
|
|
237
|
+
for (let i = 0; i < output.length; i++) {
|
|
238
|
+
const center = i * step;
|
|
239
|
+
const last = Math.min(input.length - 1, Math.floor(center + reach));
|
|
240
|
+
let sum = 0;
|
|
241
|
+
let weights = 0;
|
|
242
|
+
for (let n = Math.max(0, Math.ceil(center - reach)); n <= last; n++) {
|
|
243
|
+
const at = Math.abs(n - center) * scale * SINC_STEPS;
|
|
244
|
+
const k = Math.floor(at);
|
|
245
|
+
const weight = HANN_SINC[k] + (at - k) * (HANN_SINC[k + 1] - HANN_SINC[k]);
|
|
246
|
+
sum += input[n] * weight;
|
|
247
|
+
weights += weight;
|
|
248
|
+
}
|
|
249
|
+
output[i] = weights ? sum / weights : 0;
|
|
250
|
+
}
|
|
251
|
+
return output;
|
|
252
|
+
}
|
|
253
|
+
function monoInput(channels) {
|
|
254
|
+
const mono = new Float32Array(channels[0]?.length ?? 0);
|
|
255
|
+
let loudest2 = mono;
|
|
256
|
+
let loudestEnergy = 0;
|
|
257
|
+
for (const channel of channels) {
|
|
258
|
+
let energy = 0;
|
|
259
|
+
for (let i = 0; i < mono.length; i++) {
|
|
260
|
+
const value = channel[i];
|
|
261
|
+
mono[i] += value / channels.length;
|
|
262
|
+
energy += value * value;
|
|
263
|
+
}
|
|
264
|
+
if (energy > loudestEnergy) {
|
|
265
|
+
loudest2 = channel;
|
|
266
|
+
loudestEnergy = energy;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
const monoEnergy = mono.reduce((energy, value) => energy + value * value, 0);
|
|
270
|
+
return monoEnergy < loudestEnergy * 0.1 ? loudest2 : mono;
|
|
271
|
+
}
|
|
272
|
+
function uploaded(signal) {
|
|
273
|
+
return {
|
|
274
|
+
sampleRate: UPLOAD_RATE,
|
|
275
|
+
channels: [resample(monoInput(signal.channels), signal.sampleRate, UPLOAD_RATE)]
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
var askedTogether = (capability) => capability === "beats" || capability === "downbeats" ? ["beats", "downbeats"] : [capability];
|
|
279
|
+
var access = () => serviceAccess(
|
|
280
|
+
{ url: URL_ENV, key: KEY_ENV, timeout: TIMEOUT_ENV },
|
|
281
|
+
"/v3/audio/analysis",
|
|
282
|
+
DEFAULT_TIMEOUT_MS
|
|
283
|
+
);
|
|
284
|
+
async function granted() {
|
|
285
|
+
const found = await access();
|
|
286
|
+
if (typeof found === "string") throw new Error(found);
|
|
287
|
+
return found;
|
|
288
|
+
}
|
|
289
|
+
var isNumberList = (value) => Array.isArray(value) && value.every((t) => typeof t === "number" && Number.isFinite(t));
|
|
290
|
+
function checked(capability, body) {
|
|
291
|
+
const value = body[capability];
|
|
292
|
+
if (!value || typeof value !== "object")
|
|
293
|
+
throw new Error(`the service did not answer ${capability}`);
|
|
294
|
+
const valid = {
|
|
295
|
+
beats: () => isNumberList(value.times),
|
|
296
|
+
downbeats: () => isNumberList(value.times),
|
|
297
|
+
sections: () => Array.isArray(value.spans) && value.spans.every((s3) => isNumberList([s3.start, s3.end])),
|
|
298
|
+
describe: () => Array.isArray(value.tags) && value.tags.every(
|
|
299
|
+
(t) => typeof t.label === "string" && typeof t.score === "number"
|
|
300
|
+
)
|
|
301
|
+
}[capability];
|
|
302
|
+
if (!valid())
|
|
303
|
+
throw new Error(`the service answered ${capability} in a shape this CLI cannot read`);
|
|
304
|
+
}
|
|
305
|
+
var uploadedAsset = perSignal(async (analyzed) => {
|
|
306
|
+
const { url, headers, timeoutMs } = await granted();
|
|
307
|
+
const deadline = AbortSignal.timeout(timeoutMs);
|
|
308
|
+
const boundedFetch = (input, init) => fetch(input, {
|
|
309
|
+
...init,
|
|
310
|
+
redirect: "error",
|
|
311
|
+
signal: deadline
|
|
312
|
+
});
|
|
313
|
+
const client = new HyperframesCloudClient({
|
|
314
|
+
baseUrl: new URL(url).origin,
|
|
315
|
+
getAuthHeaders: () => headers,
|
|
316
|
+
fetchImpl: boundedFetch
|
|
317
|
+
});
|
|
318
|
+
const asset = await uploadAssetViaDirectUpload({
|
|
319
|
+
client,
|
|
320
|
+
bytes: new Uint8Array(encodeWav(uploaded(analyzed))),
|
|
321
|
+
filename: "audio.wav",
|
|
322
|
+
contentType: "audio/wav",
|
|
323
|
+
idempotencyKey: randomUUID(),
|
|
324
|
+
fetchImpl: boundedFetch
|
|
325
|
+
});
|
|
326
|
+
return asset.asset_id;
|
|
327
|
+
});
|
|
328
|
+
async function ask(analyzed, capabilities2) {
|
|
329
|
+
const { url, headers, timeoutMs } = await granted();
|
|
330
|
+
const request = {
|
|
331
|
+
version: 1,
|
|
332
|
+
capabilities: capabilities2,
|
|
333
|
+
audio: { type: "asset_id", asset_id: await uploadedAsset(analyzed) }
|
|
334
|
+
};
|
|
335
|
+
const response = await fetch(url, {
|
|
336
|
+
method: "POST",
|
|
337
|
+
redirect: "error",
|
|
338
|
+
headers: {
|
|
339
|
+
...headers,
|
|
340
|
+
"idempotency-key": randomUUID(),
|
|
341
|
+
"content-type": "application/json"
|
|
342
|
+
},
|
|
343
|
+
body: JSON.stringify(request),
|
|
344
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
345
|
+
});
|
|
346
|
+
if (!response.ok) throw await serviceFailure(response, "the service");
|
|
347
|
+
const body = await response.json();
|
|
348
|
+
return body?.data ?? body ?? {};
|
|
349
|
+
}
|
|
350
|
+
var requests = /* @__PURE__ */ new Map();
|
|
351
|
+
function answerFor(signal, capability) {
|
|
352
|
+
const together = askedTogether(capability);
|
|
353
|
+
const key = together.join(",");
|
|
354
|
+
let request = requests.get(key);
|
|
355
|
+
if (!request) requests.set(key, request = perSignal((s3) => ask(s3, together)));
|
|
356
|
+
return request(signal);
|
|
357
|
+
}
|
|
358
|
+
function beatSnapper(signal, beatSeconds) {
|
|
359
|
+
const measured3 = rhythm(signal).beats.times;
|
|
360
|
+
const period = beatSeconds ?? medianInterval(measured3) ?? 0;
|
|
361
|
+
return snapper(measured3, period / 4, "signal beats");
|
|
362
|
+
}
|
|
363
|
+
function snappedBeats(signal, body) {
|
|
364
|
+
const times = body.beats.times;
|
|
365
|
+
const onBeat2 = beatSnapper(signal, medianInterval(times));
|
|
366
|
+
return { times: rounded(times.map(onBeat2.snap)), snap: onBeat2.report() };
|
|
367
|
+
}
|
|
368
|
+
function snappedDownbeats(signal, body) {
|
|
369
|
+
const { times, meter = null } = body.downbeats;
|
|
370
|
+
const beat = body.beats ? medianInterval(body.beats.times) : null;
|
|
371
|
+
const bar = medianInterval(times);
|
|
372
|
+
const onBeat2 = beatSnapper(signal, beat ?? (bar && meter ? bar / meter : null));
|
|
373
|
+
return { times: rounded(times.map(onBeat2.snap)), meter, snap: onBeat2.report() };
|
|
374
|
+
}
|
|
375
|
+
function snappedSections(signal, body) {
|
|
376
|
+
const onBeat2 = beatSnapper(signal, null);
|
|
377
|
+
const end = (signal.channels[0]?.length ?? 0) / signal.sampleRate;
|
|
378
|
+
const move = (t) => t <= 0 || t >= end ? t : onBeat2.snap(t);
|
|
379
|
+
const spans = body.sections.spans.map((s3) => ({
|
|
380
|
+
...s3,
|
|
381
|
+
start: rounded([move(s3.start)])[0],
|
|
382
|
+
end: rounded([move(s3.end)])[0]
|
|
383
|
+
}));
|
|
384
|
+
return { spans, snap: onBeat2.report() };
|
|
385
|
+
}
|
|
386
|
+
function measured(capability, signal, body) {
|
|
387
|
+
const value = capability === "beats" ? snappedBeats(signal, body) : capability === "downbeats" ? snappedDownbeats(signal, body) : capability === "sections" ? snappedSections(signal, body) : body.describe;
|
|
388
|
+
return body.source ? { ...value, source: body.source } : value;
|
|
389
|
+
}
|
|
390
|
+
var API_CAPABILITIES = ["beats", "downbeats", "sections", "describe"];
|
|
391
|
+
var apiProvider = {
|
|
392
|
+
id: "api",
|
|
393
|
+
local: false,
|
|
394
|
+
capabilities: API_CAPABILITIES,
|
|
395
|
+
async available() {
|
|
396
|
+
const found = await access();
|
|
397
|
+
return typeof found === "string" ? unavailable(found) : AVAILABLE;
|
|
398
|
+
},
|
|
399
|
+
async analyze(capability, signal) {
|
|
400
|
+
const asked = capability;
|
|
401
|
+
const body = await answerFor(signal, asked);
|
|
402
|
+
checked(asked, body);
|
|
403
|
+
return measured(asked, signal, body);
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
|
|
407
|
+
// src/audio/analysis/providers/apple.ts
|
|
408
|
+
import { createHash } from "crypto";
|
|
409
|
+
import { existsSync } from "fs";
|
|
410
|
+
import { mkdir, rename, writeFile as writeFile2 } from "fs/promises";
|
|
411
|
+
import { homedir } from "os";
|
|
412
|
+
import { join as join2 } from "path";
|
|
413
|
+
var SOUND_ANALYSIS = "/System/Library/Frameworks/SoundAnalysis.framework";
|
|
414
|
+
var MUSIC_UNDERSTANDING_MACOS = 27;
|
|
415
|
+
var SOUND_TAGGER = `
|
|
416
|
+
import Foundation
|
|
417
|
+
import SoundAnalysis
|
|
418
|
+
|
|
419
|
+
final class Collector: NSObject, SNResultsObserving {
|
|
420
|
+
var sums: [String: Double] = [:]
|
|
421
|
+
var windows = 0
|
|
422
|
+
var failure: Error?
|
|
423
|
+
func request(_ request: SNRequest, didProduce result: SNResult) {
|
|
424
|
+
guard let result = result as? SNClassificationResult else { return }
|
|
425
|
+
windows += 1
|
|
426
|
+
for c in result.classifications { sums[c.identifier, default: 0] += c.confidence }
|
|
427
|
+
}
|
|
428
|
+
func request(_ request: SNRequest, didFailWithError error: Error) { failure = error }
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
do {
|
|
432
|
+
let analyzer = try SNAudioFileAnalyzer(url: URL(fileURLWithPath: CommandLine.arguments[1]))
|
|
433
|
+
let collector = Collector()
|
|
434
|
+
try analyzer.add(try SNClassifySoundRequest(classifierIdentifier: .version1), withObserver: collector)
|
|
435
|
+
analyzer.analyze()
|
|
436
|
+
if let failure = collector.failure { throw failure }
|
|
437
|
+
let count = Double(max(1, collector.windows))
|
|
438
|
+
var tags: [[String: Any]] = []
|
|
439
|
+
for (label, sum) in collector.sums.sorted(by: { $0.value > $1.value }).prefix(10) {
|
|
440
|
+
let score: Double = sum / count
|
|
441
|
+
if score >= 0.01 { tags.append(["label": label, "score": (score * 1000).rounded() / 1000]) }
|
|
442
|
+
}
|
|
443
|
+
let json = try JSONSerialization.data(withJSONObject: ["tags": tags, "windows": collector.windows])
|
|
444
|
+
FileHandle.standardOutput.write(json)
|
|
445
|
+
} catch {
|
|
446
|
+
FileHandle.standardError.write(Data("\\(error)".utf8))
|
|
447
|
+
exit(1)
|
|
448
|
+
}
|
|
449
|
+
`;
|
|
450
|
+
var CLASSIFIER_WINDOW_SECONDS = 3;
|
|
451
|
+
function atLeastOneWindow(signal) {
|
|
452
|
+
const length = Math.ceil(CLASSIFIER_WINDOW_SECONDS * signal.sampleRate);
|
|
453
|
+
if ((signal.channels[0]?.length ?? 0) >= length) return signal;
|
|
454
|
+
return {
|
|
455
|
+
...signal,
|
|
456
|
+
channels: signal.channels.map((channel) => {
|
|
457
|
+
const padded = new Float32Array(length);
|
|
458
|
+
padded.set(channel);
|
|
459
|
+
return padded;
|
|
460
|
+
})
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
var helperDir = () => join2(homedir(), ".cache", "hyperframes", "audio-helpers");
|
|
464
|
+
function helperPath(name, source) {
|
|
465
|
+
const hash = createHash("sha256").update(source).digest("hex").slice(0, 12);
|
|
466
|
+
return join2(helperDir(), `${name}-${hash}`);
|
|
467
|
+
}
|
|
468
|
+
async function hasSwiftCompiler() {
|
|
469
|
+
try {
|
|
470
|
+
await runFile("xcrun", ["--find", "swiftc"], { timeout: 3e4 });
|
|
471
|
+
return true;
|
|
472
|
+
} catch {
|
|
473
|
+
return false;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
var firstLine = (error) => (error.stderr || String(error)).trim().split("\n")[0];
|
|
477
|
+
var builds = /* @__PURE__ */ new Map();
|
|
478
|
+
function compiled(name, source) {
|
|
479
|
+
const binary = helperPath(name, source);
|
|
480
|
+
if (existsSync(binary)) return Promise.resolve(binary);
|
|
481
|
+
let build = builds.get(binary);
|
|
482
|
+
if (!build) {
|
|
483
|
+
build = (async () => {
|
|
484
|
+
await mkdir(helperDir(), { recursive: true });
|
|
485
|
+
const file = `${binary}.swift`;
|
|
486
|
+
await writeFile2(file, source);
|
|
487
|
+
const args = [
|
|
488
|
+
"--sdk",
|
|
489
|
+
"macosx",
|
|
490
|
+
"swiftc",
|
|
491
|
+
"-O",
|
|
492
|
+
"-swift-version",
|
|
493
|
+
"5",
|
|
494
|
+
file,
|
|
495
|
+
"-o",
|
|
496
|
+
`${binary}.tmp`
|
|
497
|
+
];
|
|
498
|
+
await runFile("xcrun", args, { timeout: 5 * 6e4 }).catch((error) => {
|
|
499
|
+
throw new Error(`could not build the ${name} helper: ${firstLine(error)}`);
|
|
500
|
+
});
|
|
501
|
+
await rename(`${binary}.tmp`, binary);
|
|
502
|
+
return binary;
|
|
503
|
+
})();
|
|
504
|
+
builds.set(binary, build);
|
|
505
|
+
}
|
|
506
|
+
return build;
|
|
507
|
+
}
|
|
508
|
+
async function macosMajor() {
|
|
509
|
+
if (process.platform !== "darwin") return null;
|
|
510
|
+
const { stdout } = await runFile("sw_vers", ["-productVersion"], { timeout: 1e4 });
|
|
511
|
+
return Number.parseInt(stdout, 10) || null;
|
|
512
|
+
}
|
|
513
|
+
var appleSoundProvider = {
|
|
514
|
+
id: "apple-sound",
|
|
515
|
+
local: true,
|
|
516
|
+
capabilities: ["describe"],
|
|
517
|
+
async available() {
|
|
518
|
+
if (process.platform !== "darwin") return unavailable("macOS only");
|
|
519
|
+
if (!existsSync(SOUND_ANALYSIS))
|
|
520
|
+
return unavailable("SoundAnalysis.framework is not on this Mac");
|
|
521
|
+
if (existsSync(helperPath("apple-sound", SOUND_TAGGER))) return AVAILABLE;
|
|
522
|
+
if (!await hasSwiftCompiler()) return unavailable("no Swift compiler to build the helper");
|
|
523
|
+
return AVAILABLE;
|
|
524
|
+
},
|
|
525
|
+
async analyze(_capability, signal) {
|
|
526
|
+
const helper = await compiled("apple-sound", SOUND_TAGGER);
|
|
527
|
+
const { stdout } = await withWavFile(
|
|
528
|
+
atLeastOneWindow(signal),
|
|
529
|
+
(wav) => runFile(helper, [wav], { timeout: 5 * 6e4, maxBuffer: 8 * 1024 * 1024 })
|
|
530
|
+
);
|
|
531
|
+
const { tags } = JSON.parse(stdout);
|
|
532
|
+
return {
|
|
533
|
+
tags: tags.map((t) => ({ label: t.label, score: Math.round(t.score * 1e3) / 1e3 }))
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
};
|
|
537
|
+
var appleMuProvider = {
|
|
538
|
+
id: "apple-mu",
|
|
539
|
+
local: true,
|
|
540
|
+
capabilities: ["beats", "downbeats", "sections"],
|
|
541
|
+
async available() {
|
|
542
|
+
const major = await macosMajor();
|
|
543
|
+
if (major === null) return unavailable("macOS only");
|
|
544
|
+
if (major < MUSIC_UNDERSTANDING_MACOS)
|
|
545
|
+
return unavailable(
|
|
546
|
+
`Music Understanding needs macOS ${MUSIC_UNDERSTANDING_MACOS}; this Mac runs ${major}`
|
|
547
|
+
);
|
|
548
|
+
return unavailable("no Music Understanding helper yet: it needs the macOS 27 SDK to build");
|
|
549
|
+
},
|
|
550
|
+
analyze() {
|
|
551
|
+
return Promise.reject(new Error("Music Understanding is not available"));
|
|
552
|
+
}
|
|
553
|
+
};
|
|
554
|
+
|
|
555
|
+
// src/audio/analysis/providers/beatThis.ts
|
|
556
|
+
import { existsSync as existsSync2 } from "fs";
|
|
557
|
+
import { homedir as homedir2 } from "os";
|
|
558
|
+
import { join as join3 } from "path";
|
|
559
|
+
var PYTHON_ENV = "HYPERFRAMES_BEAT_THIS_PYTHON";
|
|
560
|
+
var CHECKPOINT_ENV = "HYPERFRAMES_BEAT_THIS_CHECKPOINT";
|
|
561
|
+
function checkpointPath() {
|
|
562
|
+
const configured = process.env[CHECKPOINT_ENV]?.trim();
|
|
563
|
+
if (configured) return configured;
|
|
564
|
+
const torchHome = process.env.TORCH_HOME ?? join3(homedir2(), ".cache", "torch");
|
|
565
|
+
return join3(torchHome, "hub", "checkpoints", "beat_this-small0.ckpt");
|
|
566
|
+
}
|
|
567
|
+
var FIND_PACKAGE = "import importlib.util,sys; sys.exit(importlib.util.find_spec('beat_this') is None)";
|
|
568
|
+
var TRACK = `
|
|
569
|
+
import json, sys
|
|
570
|
+
from beat_this.inference import File2Beats
|
|
571
|
+
beats, downbeats = File2Beats(checkpoint_path=sys.argv[2], device="cpu", dbn=False)(sys.argv[1])
|
|
572
|
+
print(json.dumps({"beats": [float(t) for t in beats], "downbeats": [float(t) for t in downbeats]}))
|
|
573
|
+
`;
|
|
574
|
+
var installed = /* @__PURE__ */ new Map();
|
|
575
|
+
function hasPackage(python) {
|
|
576
|
+
let found = installed.get(python);
|
|
577
|
+
if (!found) {
|
|
578
|
+
found = runFile(python, ["-c", FIND_PACKAGE], { timeout: 3e4, windowsHide: true }).then(
|
|
579
|
+
() => true,
|
|
580
|
+
() => false
|
|
581
|
+
);
|
|
582
|
+
installed.set(python, found);
|
|
583
|
+
}
|
|
584
|
+
return found;
|
|
585
|
+
}
|
|
586
|
+
var track = perSignal(
|
|
587
|
+
(signal) => withWavFile(signal, async (wav) => {
|
|
588
|
+
const python = process.env[PYTHON_ENV].trim();
|
|
589
|
+
const { stdout } = await runFile(python, ["-c", TRACK, wav, checkpointPath()], {
|
|
590
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
591
|
+
timeout: 10 * 6e4,
|
|
592
|
+
windowsHide: true
|
|
593
|
+
});
|
|
594
|
+
const answer = JSON.parse(stdout.trim().split("\n").pop());
|
|
595
|
+
return { beats: rounded(answer.beats), downbeats: rounded(answer.downbeats) };
|
|
596
|
+
})
|
|
597
|
+
);
|
|
598
|
+
var beatThisProvider = {
|
|
599
|
+
id: "beat-this",
|
|
600
|
+
local: true,
|
|
601
|
+
capabilities: ["beats", "downbeats"],
|
|
602
|
+
async available() {
|
|
603
|
+
const python = process.env[PYTHON_ENV]?.trim();
|
|
604
|
+
if (!python) return unavailable(`${PYTHON_ENV} is not set`);
|
|
605
|
+
if (!existsSync2(python)) return unavailable(`${PYTHON_ENV} names no file`);
|
|
606
|
+
if (!existsSync2(checkpointPath()))
|
|
607
|
+
return unavailable(`no Beat This! weights at ${checkpointPath()}`);
|
|
608
|
+
if (!await hasPackage(python)) return unavailable(`beat_this is not installed for ${python}`);
|
|
609
|
+
return AVAILABLE;
|
|
610
|
+
},
|
|
611
|
+
async analyze(capability, signal) {
|
|
612
|
+
const { beats, downbeats } = await track(signal);
|
|
613
|
+
return capability === "beats" ? { times: beats } : downbeatsWithMeter(beats, downbeats);
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
|
|
617
|
+
// src/audio/analysis/fileTime.ts
|
|
618
|
+
var round2 = (t) => Math.round(t * 1e3) / 1e3;
|
|
619
|
+
function withSnap(value, at) {
|
|
620
|
+
return value.snap ? { ...value, snap: { ...value.snap, unsnapped: value.snap.unsnapped.map(at) } } : value;
|
|
621
|
+
}
|
|
622
|
+
var SHIFTS = {
|
|
623
|
+
moments: (m, at) => Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)])),
|
|
624
|
+
onsets: (o, at) => ({ ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) }),
|
|
625
|
+
silence: (spans, at) => spans.map((s3) => ({ start: at(s3.start), end: at(s3.end) })),
|
|
626
|
+
loudness: (l, at) => ({
|
|
627
|
+
...l,
|
|
628
|
+
clipping: {
|
|
629
|
+
...l.clipping,
|
|
630
|
+
firstAt: l.clipping.firstAt === null ? null : at(l.clipping.firstAt)
|
|
631
|
+
}
|
|
632
|
+
}),
|
|
633
|
+
envelope: (e, _at, offset) => ({ startsAt: round2(offset), ...e }),
|
|
634
|
+
beats: (b, at) => withSnap({ ...b, times: b.times.map(at) }, at),
|
|
635
|
+
downbeats: (b, at) => withSnap({ ...b, times: b.times.map(at) }, at),
|
|
636
|
+
bpm: (tempo, at) => ({
|
|
637
|
+
...tempo,
|
|
638
|
+
curve: tempo.curve.map((p) => ({ ...p, t: at(p.t) }))
|
|
639
|
+
}),
|
|
640
|
+
sections: (s3, at) => withSnap(
|
|
641
|
+
{
|
|
642
|
+
...s3,
|
|
643
|
+
spans: s3.spans.map((span) => ({ ...span, start: at(span.start), end: at(span.end) }))
|
|
644
|
+
},
|
|
645
|
+
at
|
|
646
|
+
)
|
|
647
|
+
};
|
|
648
|
+
function shiftedTimes(capability, value, offset) {
|
|
649
|
+
if (value === null || typeof value !== "object") return value;
|
|
650
|
+
const shift = SHIFTS[capability];
|
|
651
|
+
return shift ? shift(value, (t) => round2(t + offset), offset) : value;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// src/audio/analysis/analyze.ts
|
|
655
|
+
var DEFAULT_CAPABILITIES = [
|
|
656
|
+
"kind",
|
|
657
|
+
"moments",
|
|
658
|
+
"onsets",
|
|
659
|
+
"loudness",
|
|
660
|
+
"silence",
|
|
661
|
+
"envelope"
|
|
662
|
+
];
|
|
663
|
+
function audioAnalysisProviders() {
|
|
664
|
+
return [apiProvider, appleMuProvider, beatThisProvider, appleSoundProvider, signalProvider];
|
|
665
|
+
}
|
|
666
|
+
function envelopeHopMs(seconds, asked) {
|
|
667
|
+
if (asked !== void 0) return asked;
|
|
668
|
+
return Math.max(5, Math.ceil(seconds * 1e3 / 2e3));
|
|
669
|
+
}
|
|
670
|
+
var round3 = (t) => Math.round(t * 1e3) / 1e3;
|
|
671
|
+
function within(value, from, to) {
|
|
672
|
+
if (!value || typeof value !== "object" || !("events" in value)) return value;
|
|
673
|
+
const onsets = value;
|
|
674
|
+
return { ...onsets, events: onsets.events.filter((e) => e.t >= from && e.t < to) };
|
|
675
|
+
}
|
|
676
|
+
function frame(signal, context) {
|
|
677
|
+
const before = context?.before ?? 0;
|
|
678
|
+
const skip = Math.round(before * signal.sampleRate);
|
|
679
|
+
const length = signal.channels[0]?.length ?? 0;
|
|
680
|
+
const end = Math.max(skip, length - Math.round((context?.after ?? 0) * signal.sampleRate));
|
|
681
|
+
const cut = skip > 0 || end < length;
|
|
682
|
+
const windowed = cut ? { ...signal, channels: signal.channels.map((c) => c.subarray(skip, end)) } : signal;
|
|
683
|
+
return { windowed, before, end: end / signal.sampleRate, cut };
|
|
684
|
+
}
|
|
685
|
+
function optionsFor(capability, request, seconds) {
|
|
686
|
+
const options = { ...request.options?.[capability] };
|
|
687
|
+
if (capability === "envelope")
|
|
688
|
+
options.hopMs = envelopeHopMs(seconds, options.hopMs);
|
|
689
|
+
return options;
|
|
690
|
+
}
|
|
691
|
+
async function analyzeSignal(signal, request, providers = audioAnalysisProviders()) {
|
|
692
|
+
const quality = request.quality ?? "local";
|
|
693
|
+
const { beats, bpm } = request.provider ?? {};
|
|
694
|
+
if (beats && bpm && beats !== bpm) throw new Error("beats and bpm must use the same provider");
|
|
695
|
+
const beatProvider = beats ?? bpm;
|
|
696
|
+
const framing = frame(signal, request.context);
|
|
697
|
+
const analysis = {
|
|
698
|
+
analyzed: round3(signalDuration(framing.windowed)),
|
|
699
|
+
values: {},
|
|
700
|
+
provenance: {},
|
|
701
|
+
ladder: {},
|
|
702
|
+
warnings: []
|
|
703
|
+
};
|
|
704
|
+
const ladders = /* @__PURE__ */ new Map();
|
|
705
|
+
const ladderFor = (capability) => {
|
|
706
|
+
let result = ladders.get(capability);
|
|
707
|
+
if (!result) {
|
|
708
|
+
const withContext = capability === "onsets" && framing.cut;
|
|
709
|
+
result = runLadder(capability, withContext ? signal : framing.windowed, providers, {
|
|
710
|
+
quality,
|
|
711
|
+
provider: capability === "beats" ? beatProvider : request.provider?.[capability],
|
|
712
|
+
options: optionsFor(capability, request, analysis.analyzed)
|
|
713
|
+
});
|
|
714
|
+
ladders.set(capability, result);
|
|
715
|
+
}
|
|
716
|
+
return result;
|
|
717
|
+
};
|
|
718
|
+
for (const capability of request.capabilities) {
|
|
719
|
+
const withContext = capability === "onsets" && framing.cut;
|
|
720
|
+
const result = capability === "bpm" ? tempoFromBeats(await ladderFor("beats")) : await ladderFor(capability);
|
|
721
|
+
const value = withContext ? within(result.value, framing.before, framing.end) : result.value;
|
|
722
|
+
const offset = (request.fileOffset ?? 0) - (withContext ? framing.before : 0);
|
|
723
|
+
analysis.values[capability] = shiftedTimes(capability, value, offset);
|
|
724
|
+
analysis.provenance[capability] = result.provenance;
|
|
725
|
+
analysis.ladder[capability] = result.steps;
|
|
726
|
+
const said = new Set(analysis.warnings);
|
|
727
|
+
analysis.warnings.push(...result.warnings.filter((w) => !said.has(w)));
|
|
728
|
+
}
|
|
729
|
+
analysis.warnings.push(...serviceHint(quality, analysis, request.capabilities));
|
|
730
|
+
return analysis;
|
|
731
|
+
}
|
|
732
|
+
function serviceHint(quality, analysis, capabilities2) {
|
|
733
|
+
const leftToService = (capability) => analysis.values[capability] == null && (analysis.ladder[capability] ?? []).some((s3) => s3.outcome === "excluded");
|
|
734
|
+
return quality === "local" && capabilities2.some(leftToService) ? ["a service could have answered what is missing: --quality best uploads the audio to HeyGen"] : [];
|
|
735
|
+
}
|
|
736
|
+
function tempoFromBeats(beats) {
|
|
737
|
+
const times = beats.value?.times;
|
|
738
|
+
return { ...beats, value: times ? tempoOf(times) : null };
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// src/audio/analysis/targets.ts
|
|
742
|
+
import { dirname, join as join4, resolve } from "path";
|
|
743
|
+
var round4 = (t) => Math.round(t * 1e3) / 1e3;
|
|
744
|
+
function placementOf(row) {
|
|
745
|
+
return {
|
|
746
|
+
start: row.absStart,
|
|
747
|
+
end: row.absEnd,
|
|
748
|
+
dataStart: row.start,
|
|
749
|
+
mediaStart: row.mediaStart ?? 0,
|
|
750
|
+
playbackRate: row.playbackRate ?? 1,
|
|
751
|
+
volume: row.volume ?? 1
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
function filmTime(placement, fileTime2) {
|
|
755
|
+
return round4(placement.start + (fileTime2 - placement.mediaStart) / placement.playbackRate);
|
|
756
|
+
}
|
|
757
|
+
function fileTime(placement, filmTime2) {
|
|
758
|
+
return round4(placement.mediaStart + (filmTime2 - placement.start) * placement.playbackRate);
|
|
759
|
+
}
|
|
760
|
+
var soundingRows = (timeline) => timeline.tracks.filter((track2) => track2.kind === "audio" || track2.kind === "video").flatMap((track2) => track2.rows).filter((row) => row.src && row.volume !== 0 && /^(audio|video)$/.test(row.kind)).sort((a, b) => a.absStart - b.absStart);
|
|
761
|
+
function matches(row, query) {
|
|
762
|
+
switch (query.by) {
|
|
763
|
+
case "all":
|
|
764
|
+
case "element":
|
|
765
|
+
return true;
|
|
766
|
+
case "at":
|
|
767
|
+
return row.absStart <= query.t && query.t < row.absEnd;
|
|
768
|
+
case "range":
|
|
769
|
+
return row.absStart < query.end && query.start < row.absEnd;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
function targetOf(projectDir, row, query) {
|
|
773
|
+
const placement = placementOf(row);
|
|
774
|
+
const from = query.by === "range" ? Math.max(placement.start, query.start) : placement.start;
|
|
775
|
+
const to = query.by === "range" ? Math.min(placement.end, query.end) : placement.end;
|
|
776
|
+
return {
|
|
777
|
+
path: resolveAuthoredAssetFile(projectDir, row.file, row.src) ?? resolve(dirname(join4(projectDir, row.file)), row.src),
|
|
778
|
+
file: row.src,
|
|
779
|
+
ref: row.ref || null,
|
|
780
|
+
kind: row.kind,
|
|
781
|
+
placement,
|
|
782
|
+
window: {
|
|
783
|
+
from: fileTime(placement, from),
|
|
784
|
+
seconds: round4((to - from) * placement.playbackRate)
|
|
785
|
+
}
|
|
786
|
+
};
|
|
787
|
+
}
|
|
788
|
+
function filmTargets(projectDir, timeline, query) {
|
|
789
|
+
if (query.by === "element") {
|
|
790
|
+
const id = query.ref.startsWith("#") ? query.ref.slice(1) : null;
|
|
791
|
+
const found = timeline.tracks.flatMap((track2) => track2.rows).filter((row2) => row2.ref === query.ref || id !== null && row2.id === id);
|
|
792
|
+
if (found.length !== 1)
|
|
793
|
+
throw new Error(
|
|
794
|
+
`${query.ref} ${found.length ? `matches ${found.length} clips` : "was not found"}; use a ref from hyperframes timeline`
|
|
795
|
+
);
|
|
796
|
+
const row = found[0];
|
|
797
|
+
if (!row.src || !/^(audio|video)$/.test(row.kind))
|
|
798
|
+
throw new Error(`${query.ref} is not an audio or video clip`);
|
|
799
|
+
return [targetOf(projectDir, row, query)];
|
|
800
|
+
}
|
|
801
|
+
return soundingRows(timeline).filter((row) => matches(row, query)).map((row) => targetOf(projectDir, row, query));
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
// src/audio/analysis/format.ts
|
|
805
|
+
var BARS = "\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588";
|
|
806
|
+
var SPARK_WIDTH = 48;
|
|
807
|
+
var ONSETS_SHOWN = 12;
|
|
808
|
+
var s = (t) => t.toFixed(3);
|
|
809
|
+
function sparkline(envelope2, width = SPARK_WIDTH) {
|
|
810
|
+
const { values } = envelope2;
|
|
811
|
+
if (values.length === 0) return "";
|
|
812
|
+
const per = Math.max(1, Math.ceil(values.length / width));
|
|
813
|
+
const slices = [];
|
|
814
|
+
for (let i = 0; i < values.length; i += per) slices.push(Math.max(...values.slice(i, i + per)));
|
|
815
|
+
const top = Math.max(...slices);
|
|
816
|
+
const floor = Math.max(Math.min(...slices), top - 60);
|
|
817
|
+
if (top - floor < 1e-9) return BARS[0].repeat(slices.length);
|
|
818
|
+
return slices.map(
|
|
819
|
+
(v) => BARS[Math.round((Math.max(v, floor) - floor) / (top - floor) * (BARS.length - 1))]
|
|
820
|
+
).join("");
|
|
821
|
+
}
|
|
822
|
+
function momentsLine(moments, at) {
|
|
823
|
+
return ` onset ${at(moments.onset)} steepest ${at(moments.steepest)} peak ${at(moments.peak)} tail end ${at(moments.tailEnd)}`;
|
|
824
|
+
}
|
|
825
|
+
function hitsLine(onsets, at) {
|
|
826
|
+
if (!onsets.events.length) return null;
|
|
827
|
+
const shown = onsets.events.slice(0, ONSETS_SHOWN).map((e) => `${at(e.t)}(${e.strength.toFixed(2)})`);
|
|
828
|
+
const more = onsets.events.length - shown.length;
|
|
829
|
+
return ` hits ${shown.join(" ")}${more > 0 ? ` +${more}` : ""}`;
|
|
830
|
+
}
|
|
831
|
+
function loudnessLine(loudness, at) {
|
|
832
|
+
const lufs = loudness.lufsIntegrated === null ? "below gate" : `${loudness.lufsIntegrated} LUFS`;
|
|
833
|
+
const { count, firstAt } = loudness.clipping;
|
|
834
|
+
const clip = count && firstAt !== null ? `, clips ${count}\xD7 from ${at(firstAt)}` : "";
|
|
835
|
+
return ` loudness ${lufs}, true peak ${loudness.truePeakDb} dBTP${clip}`;
|
|
836
|
+
}
|
|
837
|
+
function silenceLine(silence, at) {
|
|
838
|
+
return silence.length ? ` silent ${silence.map((g) => `${at(g.start)}\u2013${at(g.end)}`).join(" ")}` : null;
|
|
839
|
+
}
|
|
840
|
+
function snapNote(snap) {
|
|
841
|
+
if (!snap) return "";
|
|
842
|
+
const total = snap.snapped + snap.unsnapped.length;
|
|
843
|
+
const median2 = snap.medianShiftMs === null ? "" : `, median ${snap.medianShiftMs} ms`;
|
|
844
|
+
return `; ${snap.snapped}/${total} snapped to ${snap.reference}${median2}`;
|
|
845
|
+
}
|
|
846
|
+
var listed = (times, at) => {
|
|
847
|
+
const shown = times.slice(0, ONSETS_SHOWN).map(at).join(" ");
|
|
848
|
+
return times.length > ONSETS_SHOWN ? `${shown} +${times.length - ONSETS_SHOWN}` : shown;
|
|
849
|
+
};
|
|
850
|
+
function beatsLine(beats, at) {
|
|
851
|
+
if (!beats.times.length) return ` beats none${beats.reason ? ` (${beats.reason})` : ""}`;
|
|
852
|
+
return ` beats ${beats.times.length}: ${listed(beats.times, at)}${snapNote(beats.snap)}`;
|
|
853
|
+
}
|
|
854
|
+
function downbeatsLine(downbeats, at) {
|
|
855
|
+
if (!downbeats.times.length) return null;
|
|
856
|
+
const meter = downbeats.meter ? ` in ${downbeats.meter}` : "";
|
|
857
|
+
const low = downbeats.confidence === "low" ? ", a guess" : "";
|
|
858
|
+
return ` downbeats${meter}${low}: ${listed(downbeats.times, at)}${snapNote(downbeats.snap)}`;
|
|
859
|
+
}
|
|
860
|
+
function tempoLine(tempo) {
|
|
861
|
+
if (tempo.value === null) return null;
|
|
862
|
+
const bpms = tempo.curve.map((p) => p.bpm);
|
|
863
|
+
const spread = bpms.length && Math.max(...bpms) - Math.min(...bpms) >= 2 ? ` (${Math.min(...bpms)}\u2013${Math.max(...bpms)})` : "";
|
|
864
|
+
return ` tempo ${tempo.value} BPM${spread}`;
|
|
865
|
+
}
|
|
866
|
+
function sectionsLine(sections, at) {
|
|
867
|
+
const spans = sections.spans.map((s3) => `${s3.label} ${at(s3.start)}\u2013${at(s3.end)}`).join(" ");
|
|
868
|
+
return ` sections ${spans}${snapNote(sections.snap)}`;
|
|
869
|
+
}
|
|
870
|
+
function describeLine(description) {
|
|
871
|
+
const tags = description.tags.map((t) => `${t.label} ${t.score.toFixed(2)}`).join(", ");
|
|
872
|
+
return ` sounds like ${[description.text, tags].filter(Boolean).join(" \u2014 ")}`;
|
|
873
|
+
}
|
|
874
|
+
var LINES = [
|
|
875
|
+
["moments", momentsLine],
|
|
876
|
+
["onsets", hitsLine],
|
|
877
|
+
["loudness", loudnessLine],
|
|
878
|
+
["silence", silenceLine],
|
|
879
|
+
["envelope", (envelope2) => ` ${sparkline(envelope2)}`],
|
|
880
|
+
["describe", describeLine],
|
|
881
|
+
["bpm", tempoLine],
|
|
882
|
+
["beats", beatsLine],
|
|
883
|
+
["downbeats", downbeatsLine],
|
|
884
|
+
["sections", sectionsLine]
|
|
885
|
+
];
|
|
886
|
+
function headline(item, where) {
|
|
887
|
+
const kind = item.kind;
|
|
888
|
+
const span = item.placement ? `film ${s(item.placement.start)}\u2013${s(item.placement.end)}, file from ${s(item.placement.mediaStart)}` : `${s(item.duration ?? 0)} s`;
|
|
889
|
+
return `${where} ${span}${kind ? ` ${kind.value} (${kind.confidence})` : ""}`;
|
|
890
|
+
}
|
|
891
|
+
function formatItem(item) {
|
|
892
|
+
const where = item.ref ? `${item.file} ${item.ref}` : item.file;
|
|
893
|
+
if (item.error) return [`${where}: ${item.error}`];
|
|
894
|
+
const at = (t) => s(item.placement ? filmTime(item.placement, t) : t);
|
|
895
|
+
const lines = [headline(item, where)];
|
|
896
|
+
for (const [capability, line] of LINES) {
|
|
897
|
+
const value = item[capability];
|
|
898
|
+
const text = value ? line(value, at) : null;
|
|
899
|
+
if (text) lines.push(text);
|
|
900
|
+
}
|
|
901
|
+
for (const warning of item.warnings) lines.push(` ! ${warning}`);
|
|
902
|
+
return lines;
|
|
903
|
+
}
|
|
904
|
+
function formatReport(report) {
|
|
905
|
+
if (report.items.length === 0) return "No audio matched.";
|
|
906
|
+
return report.items.map((item) => formatItem(item).join("\n")).join("\n\n");
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
// src/audio/analysis/run.ts
|
|
910
|
+
import { existsSync as existsSync3 } from "fs";
|
|
911
|
+
var round5 = (t) => Math.round(t * 1e3) / 1e3;
|
|
912
|
+
var EDGE_CONTEXT_SECONDS = 0.1;
|
|
913
|
+
async function previewTarget(path, file, at, mediaStart = 0) {
|
|
914
|
+
if (at === null) return { path, file, ref: null, kind: "file", placement: null };
|
|
915
|
+
const stream = existsSync3(path) ? await probeAudio(path) : null;
|
|
916
|
+
if (stream && mediaStart >= stream.duration)
|
|
917
|
+
throw new Error(
|
|
918
|
+
`skipping ${mediaStart} s leaves nothing of ${file}, which is ${round5(stream.duration)} s long`
|
|
919
|
+
);
|
|
920
|
+
const length = stream ? stream.duration - mediaStart : 0;
|
|
921
|
+
return {
|
|
922
|
+
path,
|
|
923
|
+
file,
|
|
924
|
+
ref: null,
|
|
925
|
+
kind: "file",
|
|
926
|
+
placement: {
|
|
927
|
+
start: at,
|
|
928
|
+
end: round5(at + length),
|
|
929
|
+
dataStart: at,
|
|
930
|
+
mediaStart,
|
|
931
|
+
playbackRate: 1,
|
|
932
|
+
volume: 1
|
|
933
|
+
},
|
|
934
|
+
window: { from: mediaStart, seconds: length }
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
var emptyItem = (target) => ({
|
|
938
|
+
file: target.file,
|
|
939
|
+
ref: target.ref,
|
|
940
|
+
element: target.kind === "file" ? null : target.kind,
|
|
941
|
+
placement: target.placement,
|
|
942
|
+
timebase: "file",
|
|
943
|
+
window: null,
|
|
944
|
+
duration: null,
|
|
945
|
+
sampleRate: null,
|
|
946
|
+
channels: null,
|
|
947
|
+
provenance: {},
|
|
948
|
+
warnings: []
|
|
949
|
+
});
|
|
950
|
+
function unreadable(target) {
|
|
951
|
+
if (/^[a-z][a-z\d+.-]*:\/\//i.test(target.file)) return "remote sources are not analyzed";
|
|
952
|
+
if (!existsSync3(target.path)) return "file not found";
|
|
953
|
+
return null;
|
|
954
|
+
}
|
|
955
|
+
function decodeSpan(target, stream) {
|
|
956
|
+
const from = target.window?.from ?? 0;
|
|
957
|
+
const seconds = target.window?.seconds ?? stream.duration - from;
|
|
958
|
+
const before = Math.min(from, EDGE_CONTEXT_SECONDS);
|
|
959
|
+
const after = Math.max(0, Math.min(EDGE_CONTEXT_SECONDS, stream.duration - from - seconds));
|
|
960
|
+
return {
|
|
961
|
+
from,
|
|
962
|
+
read: { from: from - before, seconds: seconds + before + after },
|
|
963
|
+
context: { before, after }
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
var loudest = (signal) => Math.max(0, ...envelope(signal, { hopMs: 10, scale: "linear" }).values);
|
|
967
|
+
function retained(signal, before, after) {
|
|
968
|
+
const skip = Math.round(before * signal.sampleRate);
|
|
969
|
+
const length = signal.channels[0]?.length ?? 0;
|
|
970
|
+
const end = Math.max(skip, length - Math.round(after * signal.sampleRate));
|
|
971
|
+
if (!skip && end >= length) return signal;
|
|
972
|
+
return { ...signal, channels: signal.channels.map((channel) => channel.subarray(skip, end)) };
|
|
973
|
+
}
|
|
974
|
+
async function skippedPeak(target, kept, stream) {
|
|
975
|
+
const skipped = target.kind === "file" ? target.window?.from ?? 0 : 0;
|
|
976
|
+
if (skipped <= 0) return [];
|
|
977
|
+
const head = await decodeAudio(target.path, { from: 0, seconds: skipped }, stream);
|
|
978
|
+
if (!head || loudest(head.signal) <= loudest(kept)) return [];
|
|
979
|
+
return [
|
|
980
|
+
`the loudest part of ${target.file} is in the first ${round5(skipped)} s, which is skipped`
|
|
981
|
+
];
|
|
982
|
+
}
|
|
983
|
+
async function analyzeTarget(target, request, full) {
|
|
984
|
+
const item = emptyItem(target);
|
|
985
|
+
const problem = unreadable(target);
|
|
986
|
+
if (problem) return { ...item, error: problem };
|
|
987
|
+
const stream = await probeAudio(target.path);
|
|
988
|
+
if (!stream) return { ...item, error: "no audio stream" };
|
|
989
|
+
const { from, read: read2, context } = decodeSpan(target, stream);
|
|
990
|
+
const decoded = await decodeAudio(target.path, read2, stream);
|
|
991
|
+
if (!decoded) return { ...item, error: "no audio stream" };
|
|
992
|
+
const analysis = await analyzeSignal(decoded.signal, { ...request, fileOffset: from, context });
|
|
993
|
+
return {
|
|
994
|
+
...item,
|
|
995
|
+
window: { from: round5(from), to: round5(from + analysis.analyzed) },
|
|
996
|
+
duration: round5(stream.duration),
|
|
997
|
+
sampleRate: stream.sampleRate,
|
|
998
|
+
channels: stream.channels,
|
|
999
|
+
...analysis.values,
|
|
1000
|
+
provenance: analysis.provenance,
|
|
1001
|
+
...full ? { ladder: analysis.ladder } : {},
|
|
1002
|
+
warnings: [
|
|
1003
|
+
...analysis.warnings,
|
|
1004
|
+
...await skippedPeak(
|
|
1005
|
+
target,
|
|
1006
|
+
retained(decoded.signal, context.before, context.after),
|
|
1007
|
+
stream
|
|
1008
|
+
)
|
|
1009
|
+
]
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
async function analyzeTargets(targets, request, full = false) {
|
|
1013
|
+
const items = [];
|
|
1014
|
+
for (const target of targets) {
|
|
1015
|
+
try {
|
|
1016
|
+
items.push(await analyzeTarget(target, request, full));
|
|
1017
|
+
} catch (error) {
|
|
1018
|
+
const reason = error instanceof Error ? error.message.split("\n")[0] : String(error);
|
|
1019
|
+
items.push({ ...emptyItem(target), error: `could not read: ${reason}` });
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
return { version: 1, items };
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
// src/audio/read/signal.ts
|
|
1026
|
+
var READ_RATE = 22050;
|
|
1027
|
+
var PITCH_RATE = 8e3;
|
|
1028
|
+
var SILENT_DB = -120;
|
|
1029
|
+
async function decodeMono(path, sampleRate = READ_RATE) {
|
|
1030
|
+
const decoded = await decodeAudio(path, {}, void 0, { sampleRate, mono: true });
|
|
1031
|
+
if (!decoded) throw new Error(`${path} has no audio stream`);
|
|
1032
|
+
return decoded.signal.channels[0];
|
|
1033
|
+
}
|
|
1034
|
+
var asSignal = (samples, sampleRate = READ_RATE) => ({
|
|
1035
|
+
sampleRate,
|
|
1036
|
+
channels: [samples]
|
|
1037
|
+
});
|
|
1038
|
+
function mixed(tracks) {
|
|
1039
|
+
const length = Math.max(0, ...tracks.map((t) => t.length));
|
|
1040
|
+
const sum = new Float32Array(length);
|
|
1041
|
+
for (const track2 of tracks) for (let i = 0; i < track2.length; i++) sum[i] += track2[i];
|
|
1042
|
+
return sum;
|
|
1043
|
+
}
|
|
1044
|
+
function levels(samples, sampleRate, hop, window = hop * 2) {
|
|
1045
|
+
const hopSamples = Math.max(1, hop * sampleRate);
|
|
1046
|
+
const half = Math.max(1, Math.round(window * sampleRate / 2));
|
|
1047
|
+
const squares = new Float64Array(samples.length + 1);
|
|
1048
|
+
for (let i = 0; i < samples.length; i++) squares[i + 1] = squares[i] + samples[i] ** 2;
|
|
1049
|
+
const frames = Math.ceil(samples.length / hopSamples);
|
|
1050
|
+
const out = new Float32Array(frames);
|
|
1051
|
+
for (let f = 0; f < frames; f++) {
|
|
1052
|
+
const center = Math.round(f * hopSamples);
|
|
1053
|
+
const a = Math.max(0, center - half);
|
|
1054
|
+
const b = Math.min(samples.length, center + half);
|
|
1055
|
+
const power = b > a ? (squares[b] - squares[a]) / (b - a) : 0;
|
|
1056
|
+
out[f] = power > 0 ? Math.max(SILENT_DB, 10 * Math.log10(power)) : SILENT_DB;
|
|
1057
|
+
}
|
|
1058
|
+
return out;
|
|
1059
|
+
}
|
|
1060
|
+
function quantile(values, q) {
|
|
1061
|
+
if (!values.length) return NaN;
|
|
1062
|
+
const sorted = Float64Array.from(values).sort();
|
|
1063
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(q * (sorted.length - 1))))];
|
|
1064
|
+
}
|
|
1065
|
+
var median = (values) => quantile(values, 0.5);
|
|
1066
|
+
var round6 = (n, places = 2) => {
|
|
1067
|
+
const k = 10 ** places;
|
|
1068
|
+
return Math.round(n * k) / k;
|
|
1069
|
+
};
|
|
1070
|
+
|
|
1071
|
+
// src/audio/read/pitch.ts
|
|
1072
|
+
var FRAME = 320;
|
|
1073
|
+
var HOP = 320;
|
|
1074
|
+
var LOW_HZ = 70;
|
|
1075
|
+
var HIGH_HZ = 500;
|
|
1076
|
+
var VOICED_DB = -35;
|
|
1077
|
+
var PERIODIC = 0.15;
|
|
1078
|
+
var MIN_LAG = Math.floor(PITCH_RATE / HIGH_HZ);
|
|
1079
|
+
var MAX_LAG = Math.ceil(PITCH_RATE / LOW_HZ);
|
|
1080
|
+
function voiced(pcm, start) {
|
|
1081
|
+
let energy = 0;
|
|
1082
|
+
for (let i = 0; i < FRAME; i++) energy += pcm[start + i] ** 2;
|
|
1083
|
+
return 10 * Math.log10(energy / FRAME + 1e-12) >= VOICED_DB;
|
|
1084
|
+
}
|
|
1085
|
+
function difference(pcm, start, tau) {
|
|
1086
|
+
let sum = 0;
|
|
1087
|
+
for (let i = 0; i < FRAME; i++) {
|
|
1088
|
+
const delta = pcm[start + i] - pcm[start + i + tau];
|
|
1089
|
+
sum += delta * delta;
|
|
1090
|
+
}
|
|
1091
|
+
return sum;
|
|
1092
|
+
}
|
|
1093
|
+
function periodOf(pcm, start, d) {
|
|
1094
|
+
let running = 0;
|
|
1095
|
+
let lag = -1;
|
|
1096
|
+
for (let tau = 1; tau <= MAX_LAG; tau++) {
|
|
1097
|
+
const sum = difference(pcm, start, tau);
|
|
1098
|
+
running += sum;
|
|
1099
|
+
d[tau] = running > 0 ? sum * tau / running : 1;
|
|
1100
|
+
if (lag === -1) {
|
|
1101
|
+
if (tau >= MIN_LAG && d[tau] < PERIODIC) lag = tau;
|
|
1102
|
+
} else if (d[tau] > d[tau - 1]) break;
|
|
1103
|
+
else if (d[tau] < d[lag]) lag = tau;
|
|
1104
|
+
}
|
|
1105
|
+
return lag;
|
|
1106
|
+
}
|
|
1107
|
+
function pitchTrack(pcm) {
|
|
1108
|
+
const out = [];
|
|
1109
|
+
const d = new Float64Array(MAX_LAG + 1);
|
|
1110
|
+
for (let start = 0; start + FRAME + MAX_LAG <= pcm.length; start += HOP) {
|
|
1111
|
+
if (!voiced(pcm, start)) continue;
|
|
1112
|
+
const lag = periodOf(pcm, start, d);
|
|
1113
|
+
if (lag > 0) out.push({ t: (start + FRAME / 2) / PITCH_RATE, hz: PITCH_RATE / lag });
|
|
1114
|
+
}
|
|
1115
|
+
return out;
|
|
1116
|
+
}
|
|
1117
|
+
function within2(track2, start, end) {
|
|
1118
|
+
let lo = 0;
|
|
1119
|
+
let hi = track2.length;
|
|
1120
|
+
while (lo < hi) {
|
|
1121
|
+
const mid = lo + hi >> 1;
|
|
1122
|
+
if (track2[mid].t < start) lo = mid + 1;
|
|
1123
|
+
else hi = mid;
|
|
1124
|
+
}
|
|
1125
|
+
const out = [];
|
|
1126
|
+
for (let i = lo; i < track2.length && track2[i].t < end; i++) out.push(track2[i].hz);
|
|
1127
|
+
return out;
|
|
1128
|
+
}
|
|
1129
|
+
var semitones = (a, b) => 12 * Math.log2(a / b);
|
|
1130
|
+
|
|
1131
|
+
// src/audio/read/format.ts
|
|
1132
|
+
var PITCH_NOTE_SEMITONES = 3;
|
|
1133
|
+
var KIND_WIDTH = 7;
|
|
1134
|
+
var s2 = (t) => t.toFixed(1);
|
|
1135
|
+
var signed = (db) => `${db >= 0 ? "+" : ""}${db} dB`;
|
|
1136
|
+
function when(line) {
|
|
1137
|
+
return line.end === null ? s2(line.start) : `${s2(line.start)}\u2013${s2(line.end)}`;
|
|
1138
|
+
}
|
|
1139
|
+
function voiceLine(sentence, voice) {
|
|
1140
|
+
const { unit, meanRate, medianPitchHz } = voice;
|
|
1141
|
+
const parts = [sentence.text === null ? "(no transcript)" : `"${sentence.text}"`];
|
|
1142
|
+
if (sentence.rate !== null)
|
|
1143
|
+
parts.push(`${sentence.rate} ${unit}/s${meanRate !== null ? ` (mean ${meanRate})` : ""}`);
|
|
1144
|
+
if (sentence.pitchHz !== null && medianPitchHz !== null && Math.abs(semitones(sentence.pitchHz, medianPitchHz)) >= PITCH_NOTE_SEMITONES)
|
|
1145
|
+
parts.push(`pitch ${sentence.pitchHz} Hz (median ${medianPitchHz})`);
|
|
1146
|
+
if (sentence.ending && sentence.ending !== "level") parts.push(`${sentence.ending} end`);
|
|
1147
|
+
return { start: sentence.start, end: sentence.end, kind: "voice", text: parts.join(" ") };
|
|
1148
|
+
}
|
|
1149
|
+
function placeOf(t, pauses) {
|
|
1150
|
+
const pause = pauses.find((p) => t >= p.start && t < p.end);
|
|
1151
|
+
return pause ? `${(t - pause.start).toFixed(1)} s into the pause` : null;
|
|
1152
|
+
}
|
|
1153
|
+
function standing(effect, musicRead) {
|
|
1154
|
+
if (effect.overMusicDb !== null) return `${signed(effect.overMusicDb)} over music`;
|
|
1155
|
+
return musicRead ? "while the music is out" : `${signed(effect.riseDb)} over what played before`;
|
|
1156
|
+
}
|
|
1157
|
+
function effectLine(effect, pauses, musicRead) {
|
|
1158
|
+
const parts = [
|
|
1159
|
+
`${effect.kind}${effect.count > 1 ? ` \xD7${effect.count}` : ""} (${effect.group}): ${effect.band}`,
|
|
1160
|
+
`attack ${effect.attackMs} ms`,
|
|
1161
|
+
`tail ${effect.tail.toFixed(1)} s`,
|
|
1162
|
+
standing(effect, musicRead)
|
|
1163
|
+
];
|
|
1164
|
+
const place = placeOf(effect.t, pauses);
|
|
1165
|
+
if (place) parts.push(place);
|
|
1166
|
+
return { start: effect.t, end: null, kind: "effect", text: parts.join(", ") };
|
|
1167
|
+
}
|
|
1168
|
+
var ENTRY = {
|
|
1169
|
+
start: "",
|
|
1170
|
+
new: ", first appearance after a break",
|
|
1171
|
+
resumes: ", resumes after a break",
|
|
1172
|
+
returns: ", returns"
|
|
1173
|
+
};
|
|
1174
|
+
function musicLine(segment, analysis) {
|
|
1175
|
+
const piece = analysis.music.pieces.find((p) => p.id === segment.piece);
|
|
1176
|
+
const about = [piece.instruments.join(", "), segment.bpm ? `${segment.bpm} BPM` : null].filter(Boolean).join(" \xB7 ");
|
|
1177
|
+
const fade = segment.fadeFrom !== null ? `, fades from ${s2(segment.fadeFrom)}` : "";
|
|
1178
|
+
return {
|
|
1179
|
+
start: segment.start,
|
|
1180
|
+
end: segment.end,
|
|
1181
|
+
kind: "music",
|
|
1182
|
+
text: `piece ${piece.id}${about ? ` (${about})` : ""}${ENTRY[segment.entry]}${fade}`
|
|
1183
|
+
};
|
|
1184
|
+
}
|
|
1185
|
+
function overlaps(line, range) {
|
|
1186
|
+
if (!range) return true;
|
|
1187
|
+
return line.start < range[1] && (line.end ?? line.start) >= range[0];
|
|
1188
|
+
}
|
|
1189
|
+
function header(analysis) {
|
|
1190
|
+
const { voice, effects, music } = analysis;
|
|
1191
|
+
const separated2 = analysis.separation === "stems" ? `stems: ${analysis.stems.join(", ")}` : "not separated";
|
|
1192
|
+
const lines = [`${analysis.file} \xB7 ${s2(analysis.duration)} s \xB7 ${separated2}`];
|
|
1193
|
+
if (voice) {
|
|
1194
|
+
const rate = voice.meanRate !== null ? `mean ${voice.meanRate} ${voice.unit}/s` : null;
|
|
1195
|
+
const pitch = voice.medianPitchHz !== null ? `median pitch ${voice.medianPitchHz} Hz` : null;
|
|
1196
|
+
lines.push(
|
|
1197
|
+
`voice: ${voice.sentences.length} lines, ${voice.pauses.length} pauses${[rate, pitch].filter(Boolean).map((p) => `, ${p}`).join("")}`
|
|
1198
|
+
);
|
|
1199
|
+
}
|
|
1200
|
+
if (effects) {
|
|
1201
|
+
const top = effects.groups.slice(0, 6).map((g) => `${g.id} ${g.kind} \xD7${g.count}`).join(", ");
|
|
1202
|
+
lines.push(
|
|
1203
|
+
`effects: ${effects.events.length} in ${effects.groups.length} sounds${top ? ` (${top})` : ""}${effects.drumHits ? `; ${effects.drumHits} drum hits on the beat left out` : ""}`
|
|
1204
|
+
);
|
|
1205
|
+
}
|
|
1206
|
+
if (music) lines.push(`music: ${music.pieces.length} pieces, ${music.breaks.length} breaks`);
|
|
1207
|
+
return lines;
|
|
1208
|
+
}
|
|
1209
|
+
function formatRead(analysis, range) {
|
|
1210
|
+
const { voice, effects } = analysis;
|
|
1211
|
+
const pauses = voice?.pauses ?? [];
|
|
1212
|
+
const lines = [
|
|
1213
|
+
...voice?.sentences.map((x) => voiceLine(x, voice)) ?? [],
|
|
1214
|
+
...pauses.map((p) => ({
|
|
1215
|
+
start: p.start,
|
|
1216
|
+
end: p.end,
|
|
1217
|
+
kind: "pause",
|
|
1218
|
+
text: `${(p.end - p.start).toFixed(1)} s`
|
|
1219
|
+
})),
|
|
1220
|
+
...effects?.events.map((e) => effectLine(e, pauses, analysis.music !== null)) ?? [],
|
|
1221
|
+
...analysis.music?.segments.map((m) => musicLine(m, analysis)) ?? [],
|
|
1222
|
+
...analysis.music?.breaks.map((b) => ({
|
|
1223
|
+
start: b.start,
|
|
1224
|
+
end: b.end,
|
|
1225
|
+
kind: "music",
|
|
1226
|
+
text: `break ${(b.end - b.start).toFixed(1)} s`
|
|
1227
|
+
})) ?? []
|
|
1228
|
+
].filter((line) => overlaps(line, range)).sort((a, b) => a.start - b.start || (a.end ?? a.start) - (b.end ?? b.start));
|
|
1229
|
+
const width = Math.max(0, ...lines.map((l) => when(l).length));
|
|
1230
|
+
const body = lines.map((l) => `${when(l).padEnd(width)} ${l.kind.padEnd(KIND_WIDTH)} ${l.text}`);
|
|
1231
|
+
const warnings = analysis.warnings.map((w) => `! ${w}`);
|
|
1232
|
+
return [...header(analysis), "", ...body, ...warnings.length ? ["", ...warnings] : []].join(
|
|
1233
|
+
"\n"
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
// src/audio/read/read.ts
|
|
1238
|
+
import { createHash as createHash2 } from "crypto";
|
|
1239
|
+
import {
|
|
1240
|
+
createReadStream,
|
|
1241
|
+
existsSync as existsSync5,
|
|
1242
|
+
mkdirSync as mkdirSync2,
|
|
1243
|
+
readFileSync as readFileSync2,
|
|
1244
|
+
renameSync,
|
|
1245
|
+
writeFileSync as writeFileSync2
|
|
1246
|
+
} from "fs";
|
|
1247
|
+
import { basename, extname, join as join6 } from "path";
|
|
1248
|
+
|
|
1249
|
+
// src/audio/read/effects.ts
|
|
1250
|
+
var HOP2 = 0.01;
|
|
1251
|
+
var FLOOR_FRAMES = 150;
|
|
1252
|
+
var FLOOR_EVERY = 10;
|
|
1253
|
+
var RISE_DB = 20;
|
|
1254
|
+
var STEP_DB = 6;
|
|
1255
|
+
var AUDIBLE_DB = -60;
|
|
1256
|
+
var MASKED_DB = 20;
|
|
1257
|
+
var MUSIC_OUT_DB = -60;
|
|
1258
|
+
var TAIL_DB = 30;
|
|
1259
|
+
var LONGEST = 4;
|
|
1260
|
+
var SAME_SOUND = 0.95;
|
|
1261
|
+
var TRAIN_GAP = 0.2;
|
|
1262
|
+
var BEAT_SLACK = 0.05;
|
|
1263
|
+
var ON_BEAT_SHARE = 0.7;
|
|
1264
|
+
var DRUM_GROUP_MIN = 8;
|
|
1265
|
+
var LOW_HZ2 = 150;
|
|
1266
|
+
var BAND_FPS = 100;
|
|
1267
|
+
function floors(db) {
|
|
1268
|
+
const out = new Float32Array(db.length);
|
|
1269
|
+
let current = db[0] ?? -120;
|
|
1270
|
+
for (let f = 0; f < db.length; f++) {
|
|
1271
|
+
if (f % FLOOR_EVERY === 0 && f > 0)
|
|
1272
|
+
current = quantile(db.subarray(Math.max(0, f - FLOOR_FRAMES), f), 0.2);
|
|
1273
|
+
out[f] = current;
|
|
1274
|
+
}
|
|
1275
|
+
return out;
|
|
1276
|
+
}
|
|
1277
|
+
var BLOCK = 1e-3;
|
|
1278
|
+
function refined(samples, sampleRate, frame2, floorDb) {
|
|
1279
|
+
const block = Math.max(1, Math.round(BLOCK * sampleRate));
|
|
1280
|
+
const from = Math.max(0, Math.round((frame2 - 2) * HOP2 * sampleRate));
|
|
1281
|
+
const to = Math.min(samples.length, Math.round((frame2 + 2) * HOP2 * sampleRate));
|
|
1282
|
+
for (let a = from; a + block <= to; a += block) {
|
|
1283
|
+
let power = 0;
|
|
1284
|
+
for (let i = a; i < a + block; i++) power += samples[i] ** 2;
|
|
1285
|
+
if (10 * Math.log10(power / block + 1e-12) >= floorDb + RISE_DB / 2) return a / sampleRate;
|
|
1286
|
+
}
|
|
1287
|
+
return frame2 * HOP2;
|
|
1288
|
+
}
|
|
1289
|
+
function extent(db, f, floor) {
|
|
1290
|
+
let onset = f;
|
|
1291
|
+
while (onset > Math.max(0, f - 50) && db[onset - 1] > floor + STEP_DB) onset--;
|
|
1292
|
+
let peak = f;
|
|
1293
|
+
let end = f;
|
|
1294
|
+
const last = Math.min(db.length - 1, f + LONGEST / HOP2);
|
|
1295
|
+
for (let g = f; g <= last; g++) {
|
|
1296
|
+
if (db[g] > db[peak]) peak = g;
|
|
1297
|
+
end = g;
|
|
1298
|
+
if (g > peak && (db[g] < db[peak] - TAIL_DB || db[g] < floor + 3)) break;
|
|
1299
|
+
}
|
|
1300
|
+
return { onset, peak, end };
|
|
1301
|
+
}
|
|
1302
|
+
function startsHere(db, f, floor) {
|
|
1303
|
+
const level = db[f];
|
|
1304
|
+
if (level < AUDIBLE_DB || level - floor < RISE_DB) return false;
|
|
1305
|
+
return level - Math.min(db[f - 1], db[f - 2], db[f - 3], db[f - 4], db[f - 5]) >= STEP_DB;
|
|
1306
|
+
}
|
|
1307
|
+
function detect(db, samples, sampleRate, skip) {
|
|
1308
|
+
const floor = floors(db);
|
|
1309
|
+
const found = [];
|
|
1310
|
+
for (let f = 5; f < db.length; f++) {
|
|
1311
|
+
if (!startsHere(db, f, floor[f])) continue;
|
|
1312
|
+
const { onset, peak, end } = extent(db, f, floor[f]);
|
|
1313
|
+
const t = refined(samples, sampleRate, onset, floor[f]);
|
|
1314
|
+
if (!skip(t))
|
|
1315
|
+
found.push({
|
|
1316
|
+
t,
|
|
1317
|
+
peak: peak * HOP2,
|
|
1318
|
+
end: end * HOP2,
|
|
1319
|
+
peakDb: db[peak],
|
|
1320
|
+
riseDb: db[peak] - floor[f]
|
|
1321
|
+
});
|
|
1322
|
+
f = Math.max(end, onset + 8);
|
|
1323
|
+
}
|
|
1324
|
+
return found;
|
|
1325
|
+
}
|
|
1326
|
+
var BIN_HZ = 22050 / 1024;
|
|
1327
|
+
var EDGES = (() => {
|
|
1328
|
+
const edges = [];
|
|
1329
|
+
for (let hz = 30; hz < 9e3; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / BIN_HZ)));
|
|
1330
|
+
return edges;
|
|
1331
|
+
})();
|
|
1332
|
+
var bandHz = (k) => 30 * 2 ** ((k + 0.5) / 5);
|
|
1333
|
+
var WIDTH = EDGES.slice(0, -1).map((edge, k) => Math.max(edge + 1, EDGES[k + 1]) - edge);
|
|
1334
|
+
var energies = (power) => Array.from(power, (v, k) => Math.expm1(v) * WIDTH[k]);
|
|
1335
|
+
var BANDS_PER_PRINT = 5;
|
|
1336
|
+
function shape(bands, from, to) {
|
|
1337
|
+
const a = Math.max(0, Math.floor(from * BAND_FPS));
|
|
1338
|
+
const b = Math.max(a + 1, Math.min(bands.length, Math.ceil(to * BAND_FPS)));
|
|
1339
|
+
const size = bands[0]?.length ?? 0;
|
|
1340
|
+
const mean = new Float64Array(size);
|
|
1341
|
+
for (let f = a; f < b; f++) for (let k = 0; k < size; k++) mean[k] += bands[f][k] / (b - a);
|
|
1342
|
+
return mean;
|
|
1343
|
+
}
|
|
1344
|
+
function lowShare(power) {
|
|
1345
|
+
const e = energies(power);
|
|
1346
|
+
const total = e.reduce((a, b) => a + b, 0);
|
|
1347
|
+
const low = e.reduce((sum, p, k) => sum + (bandHz(k) < LOW_HZ2 ? p : 0), 0);
|
|
1348
|
+
return total > 0 ? low / total : 0;
|
|
1349
|
+
}
|
|
1350
|
+
function centroidHz(power) {
|
|
1351
|
+
const e = energies(power);
|
|
1352
|
+
const weight = e.reduce((a, b) => a + b, 0);
|
|
1353
|
+
const sum = e.reduce((acc, p, k) => acc + p * Math.log2(bandHz(k)), 0);
|
|
1354
|
+
return weight > 0 ? 2 ** (sum / weight) : 0;
|
|
1355
|
+
}
|
|
1356
|
+
var tonal = (power) => {
|
|
1357
|
+
const sorted = Float64Array.from(power).sort();
|
|
1358
|
+
return sorted[sorted.length - 1] - sorted[Math.floor(sorted.length * 0.75)] >= 2.5;
|
|
1359
|
+
};
|
|
1360
|
+
var KINDS = [
|
|
1361
|
+
[(s3) => s3.length >= 0.8 && s3.attack >= 0.45 * s3.length, () => "riser"],
|
|
1362
|
+
[(s3) => s3.band === "low", (s3) => s3.tail >= 0.6 ? "boom" : "thump"],
|
|
1363
|
+
[(s3) => s3.length <= 0.15, (s3) => s3.centroid >= 3e3 ? "tick" : "click"],
|
|
1364
|
+
[(s3) => s3.tonal && s3.tail >= 0.4, () => "ding"],
|
|
1365
|
+
[(s3) => s3.attack >= 0.06 && s3.length <= 1.5, () => "whoosh"]
|
|
1366
|
+
];
|
|
1367
|
+
function bandOf(low, centroid) {
|
|
1368
|
+
if (low >= 0.5) return "low";
|
|
1369
|
+
return centroid >= 2500 ? "high" : "mid";
|
|
1370
|
+
}
|
|
1371
|
+
function kindOf(found, power) {
|
|
1372
|
+
const centroid = centroidHz(power);
|
|
1373
|
+
const shape2 = {
|
|
1374
|
+
length: found.end - found.t,
|
|
1375
|
+
attack: found.peak - found.t,
|
|
1376
|
+
tail: found.end - found.peak,
|
|
1377
|
+
band: bandOf(lowShare(power), centroid),
|
|
1378
|
+
centroid,
|
|
1379
|
+
tonal: tonal(power)
|
|
1380
|
+
};
|
|
1381
|
+
const kind = KINDS.find(([fits]) => fits(shape2))?.[1](shape2) ?? "hit";
|
|
1382
|
+
return { kind, band: shape2.band };
|
|
1383
|
+
}
|
|
1384
|
+
function printOf(power) {
|
|
1385
|
+
const e = energies(power);
|
|
1386
|
+
const octaves = new Float64Array(Math.ceil(e.length / BANDS_PER_PRINT));
|
|
1387
|
+
e.forEach((p, k) => octaves[Math.floor(k / BANDS_PER_PRINT)] += p);
|
|
1388
|
+
const total = octaves.reduce((a, b) => a + b, 0);
|
|
1389
|
+
return octaves.map((p) => total > 0 ? Math.sqrt(p / total) : 0);
|
|
1390
|
+
}
|
|
1391
|
+
function cosine(a, b) {
|
|
1392
|
+
let dot2 = 0;
|
|
1393
|
+
let na = 0;
|
|
1394
|
+
let nb = 0;
|
|
1395
|
+
for (let k = 0; k < a.length; k++) {
|
|
1396
|
+
dot2 += a[k] * b[k];
|
|
1397
|
+
na += a[k] ** 2;
|
|
1398
|
+
nb += b[k] ** 2;
|
|
1399
|
+
}
|
|
1400
|
+
return na && nb ? dot2 / Math.sqrt(na * nb) : 0;
|
|
1401
|
+
}
|
|
1402
|
+
function grouped(events) {
|
|
1403
|
+
const prints = [];
|
|
1404
|
+
for (const event of events) {
|
|
1405
|
+
const length = event.end - event.t;
|
|
1406
|
+
let best = -1;
|
|
1407
|
+
let bestScore = SAME_SOUND;
|
|
1408
|
+
prints.forEach((p, i) => {
|
|
1409
|
+
const ratio = length / p.length;
|
|
1410
|
+
if (ratio < 0.5 || ratio > 2) return;
|
|
1411
|
+
const score = cosine(
|
|
1412
|
+
event.print,
|
|
1413
|
+
p.sum.map((v) => v / p.n)
|
|
1414
|
+
);
|
|
1415
|
+
if (score >= bestScore) {
|
|
1416
|
+
best = i;
|
|
1417
|
+
bestScore = score;
|
|
1418
|
+
}
|
|
1419
|
+
});
|
|
1420
|
+
if (best < 0) {
|
|
1421
|
+
prints.push({ sum: Float64Array.from(event.print), n: 1, length });
|
|
1422
|
+
best = prints.length - 1;
|
|
1423
|
+
} else {
|
|
1424
|
+
const p = prints[best];
|
|
1425
|
+
event.print.forEach((v, k) => p.sum[k] += v);
|
|
1426
|
+
p.length = (p.length * p.n + length) / (p.n + 1);
|
|
1427
|
+
p.n++;
|
|
1428
|
+
}
|
|
1429
|
+
event.group = best;
|
|
1430
|
+
}
|
|
1431
|
+
return prints.map((p) => p.sum);
|
|
1432
|
+
}
|
|
1433
|
+
function onBeat(t, beats) {
|
|
1434
|
+
let lo = 0;
|
|
1435
|
+
let hi = beats.length;
|
|
1436
|
+
while (lo < hi) {
|
|
1437
|
+
const mid = lo + hi >> 1;
|
|
1438
|
+
if (beats[mid] < t) lo = mid + 1;
|
|
1439
|
+
else hi = mid;
|
|
1440
|
+
}
|
|
1441
|
+
const near = [beats[lo - 1], beats[lo]].filter((b) => b !== void 0);
|
|
1442
|
+
return near.some((b) => Math.abs(b - t) <= BEAT_SLACK);
|
|
1443
|
+
}
|
|
1444
|
+
function majority(kinds) {
|
|
1445
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1446
|
+
for (const k of kinds) counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
1447
|
+
return [...counts].sort((a, b) => b[1] - a[1])[0][0];
|
|
1448
|
+
}
|
|
1449
|
+
function measured2(input) {
|
|
1450
|
+
const db = levels(input.source, input.sampleRate, HOP2);
|
|
1451
|
+
const inSpeech = (t) => input.speech.some((s3) => t >= s3.start && t < s3.end);
|
|
1452
|
+
const heardDb = input.heard ? levels(input.heard, input.sampleRate, HOP2, 0.5) : null;
|
|
1453
|
+
const masked = (f) => {
|
|
1454
|
+
const level = heardDb?.[Math.round(f.peak / HOP2)];
|
|
1455
|
+
return level !== void 0 && f.peakDb < level - MASKED_DB;
|
|
1456
|
+
};
|
|
1457
|
+
const found = detect(db, input.source, input.sampleRate, inSpeech).filter((f) => !masked(f));
|
|
1458
|
+
const bands = found.length ? bandFramesOf(asSignal(input.source, input.sampleRate)).bands : [];
|
|
1459
|
+
const out = found.map((f) => {
|
|
1460
|
+
const power = shape(bands, f.t, Math.min(f.end, f.t + 0.4));
|
|
1461
|
+
return { ...f, ...kindOf(f, power), print: printOf(power), group: -1 };
|
|
1462
|
+
});
|
|
1463
|
+
grouped(out);
|
|
1464
|
+
return out;
|
|
1465
|
+
}
|
|
1466
|
+
function drumGroups(members, beats) {
|
|
1467
|
+
const drums = /* @__PURE__ */ new Set();
|
|
1468
|
+
if (!beats.length) return drums;
|
|
1469
|
+
for (const [id, list] of members) {
|
|
1470
|
+
const onBeats = list.filter((m) => onBeat(m.t, beats)).length;
|
|
1471
|
+
if (list.length >= DRUM_GROUP_MIN && onBeats >= ON_BEAT_SHARE * list.length) drums.add(id);
|
|
1472
|
+
}
|
|
1473
|
+
return drums;
|
|
1474
|
+
}
|
|
1475
|
+
function effectOf(m, group, kind, musicDb) {
|
|
1476
|
+
const musicLevel = musicDb?.[Math.round(m.peak / HOP2)];
|
|
1477
|
+
const musicPlays = musicLevel !== void 0 && musicLevel > MUSIC_OUT_DB;
|
|
1478
|
+
return {
|
|
1479
|
+
t: round6(m.t),
|
|
1480
|
+
end: round6(m.end),
|
|
1481
|
+
kind,
|
|
1482
|
+
group,
|
|
1483
|
+
band: m.band,
|
|
1484
|
+
attackMs: Math.round((m.peak - m.t) * 1e3),
|
|
1485
|
+
tail: round6(m.end - m.peak),
|
|
1486
|
+
riseDb: Math.round(m.riseDb),
|
|
1487
|
+
overMusicDb: musicPlays ? Math.round(m.peakDb - musicLevel) : null,
|
|
1488
|
+
count: 1
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
function findEffects(input) {
|
|
1492
|
+
const sounds = measured2(input);
|
|
1493
|
+
const members = /* @__PURE__ */ new Map();
|
|
1494
|
+
for (const m of sounds) members.set(m.group, [...members.get(m.group) ?? [], m]);
|
|
1495
|
+
const drums = drumGroups(members, input.beats);
|
|
1496
|
+
const kept = [...members].filter(([id]) => !drums.has(id)).sort((a, b) => b[1].length - a[1].length);
|
|
1497
|
+
const names = new Map(kept.map(([id], i) => [id, `S${i + 1}`]));
|
|
1498
|
+
const groups = kept.map(([id, list]) => ({
|
|
1499
|
+
id: names.get(id),
|
|
1500
|
+
kind: majority(list.map((m) => m.kind)),
|
|
1501
|
+
count: list.length
|
|
1502
|
+
}));
|
|
1503
|
+
const kindOfGroup = new Map(groups.map((g) => [g.id, g.kind]));
|
|
1504
|
+
const musicDb = input.music ? levels(input.music, input.sampleRate, HOP2, 0.5) : null;
|
|
1505
|
+
const events = [];
|
|
1506
|
+
for (const m of sounds) {
|
|
1507
|
+
if (drums.has(m.group)) continue;
|
|
1508
|
+
const group = names.get(m.group);
|
|
1509
|
+
const previous = events[events.length - 1];
|
|
1510
|
+
if (previous?.group === group && m.t - previous.end <= TRAIN_GAP) {
|
|
1511
|
+
previous.end = round6(m.end);
|
|
1512
|
+
previous.count++;
|
|
1513
|
+
} else events.push(effectOf(m, group, kindOfGroup.get(group) ?? m.kind, musicDb));
|
|
1514
|
+
}
|
|
1515
|
+
const drumHits = [...drums].reduce((n, id) => n + members.get(id).length, 0);
|
|
1516
|
+
return { events, groups, drumHits };
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
// src/audio/read/music.ts
|
|
1520
|
+
var HOP3 = 0.1;
|
|
1521
|
+
var ABSENT_DB = -55;
|
|
1522
|
+
var CUT_DB = 25;
|
|
1523
|
+
var RUNNING_FRAMES = 200;
|
|
1524
|
+
var LEVEL_WINDOW = 0.4;
|
|
1525
|
+
var CONTEXT = 2;
|
|
1526
|
+
var SHORTEST_BREAK = 0.5;
|
|
1527
|
+
var LONGEST_REST = 1;
|
|
1528
|
+
var SHORTEST_SEGMENT = 1;
|
|
1529
|
+
var SAME_PIECE = 0.92;
|
|
1530
|
+
var SAME_TEMPO = 0.06;
|
|
1531
|
+
var SAME_MIX = 0.25;
|
|
1532
|
+
var RESUMING_GAP = 5;
|
|
1533
|
+
var RESUMING_PIECE = 0.85;
|
|
1534
|
+
var RESUMING_MIX = 0.35;
|
|
1535
|
+
var FADE_DB = 10;
|
|
1536
|
+
var FADE_WINDOW = 3;
|
|
1537
|
+
var ENERGY_WINDOW = 30;
|
|
1538
|
+
var LEADING_SHARE = 0.2;
|
|
1539
|
+
function joined(spans, tooShort2) {
|
|
1540
|
+
const out = [];
|
|
1541
|
+
for (const span of spans) {
|
|
1542
|
+
const last = out[out.length - 1];
|
|
1543
|
+
if (last && (tooShort2(span) || last.quiet === span.quiet)) last.end = span.end;
|
|
1544
|
+
else out.push({ ...span });
|
|
1545
|
+
}
|
|
1546
|
+
return out;
|
|
1547
|
+
}
|
|
1548
|
+
function powerDb(db, start, end) {
|
|
1549
|
+
const a = Math.max(0, Math.round(start / HOP3));
|
|
1550
|
+
const b = Math.min(db.length, Math.round(end / HOP3));
|
|
1551
|
+
let sum = 0;
|
|
1552
|
+
for (let f = a; f < b; f++) sum += 10 ** (db[f] / 10);
|
|
1553
|
+
return b > a ? 10 * Math.log10(sum / (b - a) || 1e-12) : Number.NEGATIVE_INFINITY;
|
|
1554
|
+
}
|
|
1555
|
+
function holdsUp(db, span) {
|
|
1556
|
+
const inside = powerDb(db, span.start, span.end);
|
|
1557
|
+
const around = Math.max(
|
|
1558
|
+
powerDb(db, span.start - CONTEXT, span.start),
|
|
1559
|
+
powerDb(db, span.end, span.end + CONTEXT)
|
|
1560
|
+
);
|
|
1561
|
+
return inside < ABSENT_DB || inside < around - CUT_DB;
|
|
1562
|
+
}
|
|
1563
|
+
var tooShort = (span) => span.end - span.start < (span.quiet ? SHORTEST_BREAK : SHORTEST_SEGMENT);
|
|
1564
|
+
function candidates(quiet, from, to, absorbed = tooShort) {
|
|
1565
|
+
const spans = [];
|
|
1566
|
+
for (let f = from; f < to; f++) {
|
|
1567
|
+
const isQuiet = quiet[f] === 1;
|
|
1568
|
+
const last = spans[spans.length - 1];
|
|
1569
|
+
if (last && last.quiet === isQuiet) last.end = (f + 1) * HOP3;
|
|
1570
|
+
else spans.push({ quiet: isQuiet, start: f * HOP3, end: (f + 1) * HOP3 });
|
|
1571
|
+
}
|
|
1572
|
+
return joined(spans, absorbed);
|
|
1573
|
+
}
|
|
1574
|
+
function silencesIn(db, span) {
|
|
1575
|
+
const from = Math.round(span.start / HOP3);
|
|
1576
|
+
const to = Math.min(db.length, Math.round(span.end / HOP3));
|
|
1577
|
+
const absent = new Uint8Array(db.length);
|
|
1578
|
+
for (let f = from; f < to; f++) absent[f] = db[f] < ABSENT_DB ? 1 : 0;
|
|
1579
|
+
const shortSilence = (s3) => s3.quiet && s3.end - s3.start < LONGEST_REST;
|
|
1580
|
+
return candidates(absent, from, to, shortSilence).map(
|
|
1581
|
+
(s3) => s3.quiet && !holdsUp(db, s3) ? { ...s3, quiet: false } : s3
|
|
1582
|
+
);
|
|
1583
|
+
}
|
|
1584
|
+
function runs(db) {
|
|
1585
|
+
const quiet = new Uint8Array(db.length);
|
|
1586
|
+
const heard = [];
|
|
1587
|
+
for (let f = 0; f < db.length; f++) {
|
|
1588
|
+
const level = db[f];
|
|
1589
|
+
const reference = heard.length ? median(heard.slice(-RUNNING_FRAMES)) : Number.NEGATIVE_INFINITY;
|
|
1590
|
+
quiet[f] = level < ABSENT_DB || level < reference - CUT_DB ? 1 : 0;
|
|
1591
|
+
if (!quiet[f]) heard.push(level);
|
|
1592
|
+
}
|
|
1593
|
+
const merged = candidates(quiet, 0, db.length).flatMap(
|
|
1594
|
+
(span) => span.quiet && !holdsUp(db, span) ? silencesIn(db, span) : [span]
|
|
1595
|
+
);
|
|
1596
|
+
const settled = joined(merged, (s3) => !s3.quiet && s3.end - s3.start < SHORTEST_SEGMENT);
|
|
1597
|
+
return {
|
|
1598
|
+
segments: settled.filter((s3) => !s3.quiet),
|
|
1599
|
+
breaks: settled.filter((s3) => s3.quiet && s3.start > 0 && s3.end < db.length * HOP3)
|
|
1600
|
+
};
|
|
1601
|
+
}
|
|
1602
|
+
function chromaOf(chroma, fps, run) {
|
|
1603
|
+
const a = Math.floor(run.start * fps);
|
|
1604
|
+
const b = Math.max(a + 1, Math.min(chroma.length, Math.ceil(run.end * fps)));
|
|
1605
|
+
const sum = new Float64Array(12);
|
|
1606
|
+
for (let f = a; f < b; f++) chroma[f]?.forEach((v, k) => sum[k] += v);
|
|
1607
|
+
const norm = Math.hypot(...sum);
|
|
1608
|
+
return norm ? sum.map((v) => v / norm) : sum;
|
|
1609
|
+
}
|
|
1610
|
+
var dot = (a, b) => a.reduce((n, v, k) => n + v * b[k], 0);
|
|
1611
|
+
function energyOf(samples, sampleRate, run) {
|
|
1612
|
+
const a = Math.floor(run.start * sampleRate);
|
|
1613
|
+
const b = Math.min(samples.length, Math.floor(run.end * sampleRate));
|
|
1614
|
+
let sum = 0;
|
|
1615
|
+
for (let i = a; i < b; i++) sum += samples[i] ** 2;
|
|
1616
|
+
return b > a ? sum / (b - a) : 0;
|
|
1617
|
+
}
|
|
1618
|
+
function shares(input, run) {
|
|
1619
|
+
const energies2 = Object.entries(input.stems).map(
|
|
1620
|
+
([name, samples]) => [name, energyOf(samples, input.sampleRate, run)]
|
|
1621
|
+
);
|
|
1622
|
+
const total = energies2.reduce((n, [, e]) => n + e, 0);
|
|
1623
|
+
return Object.fromEntries(energies2.map(([name, e]) => [name, total ? e / total : 0]));
|
|
1624
|
+
}
|
|
1625
|
+
function mixDistance(a, b) {
|
|
1626
|
+
return Object.keys(a).reduce((n, k) => n + Math.abs(a[k] - (b[k] ?? 0)), 0) / 2;
|
|
1627
|
+
}
|
|
1628
|
+
function bpmOf(beats, run) {
|
|
1629
|
+
const inside = beats.filter((t) => t >= run.start && t < run.end);
|
|
1630
|
+
const interval = inside.length >= 4 ? medianInterval(inside) : null;
|
|
1631
|
+
return interval ? Math.round(60 / interval) : null;
|
|
1632
|
+
}
|
|
1633
|
+
var sameTempo = (a, b) => a === null || b === null || Math.abs(a - b) / Math.max(a, b) <= SAME_TEMPO;
|
|
1634
|
+
function fadeFrom(db, run) {
|
|
1635
|
+
const a = Math.floor(run.start / HOP3);
|
|
1636
|
+
const b = Math.min(db.length, Math.floor(run.end / HOP3));
|
|
1637
|
+
const body = median(db.subarray(a, b));
|
|
1638
|
+
const last = db[b - 1] ?? body;
|
|
1639
|
+
if (body - last < FADE_DB) return null;
|
|
1640
|
+
for (let f = Math.max(a, b - FADE_WINDOW / HOP3); f < b; f++)
|
|
1641
|
+
if (db[f] < body - 3) return round6(f * HOP3, 1);
|
|
1642
|
+
return null;
|
|
1643
|
+
}
|
|
1644
|
+
var alike = (p, heard, piece, mix) => dot(p.print, heard.print) >= piece && mixDistance(p.mix, heard.mix) <= mix && sameTempo(p.bpm, heard.bpm);
|
|
1645
|
+
function pieceOf(pieces, previous, heard) {
|
|
1646
|
+
const last = previous && pieces.find((p) => p.id === previous.piece);
|
|
1647
|
+
if (last && heard.run.start - previous.end <= RESUMING_GAP && alike(last, heard, RESUMING_PIECE, RESUMING_MIX))
|
|
1648
|
+
return last;
|
|
1649
|
+
return pieces.find((p) => alike(p, heard, SAME_PIECE, SAME_MIX));
|
|
1650
|
+
}
|
|
1651
|
+
function newPiece(id, heard) {
|
|
1652
|
+
return {
|
|
1653
|
+
id,
|
|
1654
|
+
instruments: Object.entries(heard.mix).filter(([, share]) => share >= LEADING_SHARE).sort((a, b) => b[1] - a[1]).map(([name]) => name),
|
|
1655
|
+
bpm: heard.bpm,
|
|
1656
|
+
firstAt: round6(heard.run.start),
|
|
1657
|
+
print: heard.print,
|
|
1658
|
+
mix: heard.mix
|
|
1659
|
+
};
|
|
1660
|
+
}
|
|
1661
|
+
function energyCurve(melodic, sampleRate, duration) {
|
|
1662
|
+
const energy = [];
|
|
1663
|
+
for (let start = 0; start < duration; start += ENERGY_WINDOW) {
|
|
1664
|
+
const end = Math.min(duration, start + ENERGY_WINDOW);
|
|
1665
|
+
const power = energyOf(melodic, sampleRate, { start, end });
|
|
1666
|
+
energy.push({
|
|
1667
|
+
start,
|
|
1668
|
+
end: round6(end),
|
|
1669
|
+
db: power > 0 ? Math.round(10 * Math.log10(power)) : -120
|
|
1670
|
+
});
|
|
1671
|
+
}
|
|
1672
|
+
return energy;
|
|
1673
|
+
}
|
|
1674
|
+
function readMusic(input) {
|
|
1675
|
+
const melodic = mixed(Object.values(input.stems));
|
|
1676
|
+
const db = levels(melodic, input.sampleRate, HOP3, LEVEL_WINDOW);
|
|
1677
|
+
const { segments: found, breaks } = runs(db);
|
|
1678
|
+
const frames = bandFramesOf(asSignal(melodic, input.sampleRate));
|
|
1679
|
+
const beats = rhythm(
|
|
1680
|
+
asSignal(input.drums ? mixed([melodic, input.drums]) : melodic, input.sampleRate)
|
|
1681
|
+
).beats.times;
|
|
1682
|
+
const pieces = [];
|
|
1683
|
+
const segments = [];
|
|
1684
|
+
for (const run of found) {
|
|
1685
|
+
const heard = {
|
|
1686
|
+
run,
|
|
1687
|
+
print: chromaOf(frames.chroma, frames.chromaFps, run),
|
|
1688
|
+
mix: shares(input, run),
|
|
1689
|
+
bpm: bpmOf(beats, run)
|
|
1690
|
+
};
|
|
1691
|
+
const previous = segments[segments.length - 1];
|
|
1692
|
+
let piece = pieceOf(pieces, previous, heard);
|
|
1693
|
+
let entry;
|
|
1694
|
+
if (piece) entry = previous?.piece === piece.id ? "resumes" : "returns";
|
|
1695
|
+
else {
|
|
1696
|
+
piece = newPiece(pieces.length + 1, heard);
|
|
1697
|
+
pieces.push(piece);
|
|
1698
|
+
entry = previous ? "new" : "start";
|
|
1699
|
+
}
|
|
1700
|
+
segments.push({
|
|
1701
|
+
start: round6(run.start),
|
|
1702
|
+
end: round6(run.end),
|
|
1703
|
+
piece: piece.id,
|
|
1704
|
+
entry,
|
|
1705
|
+
bpm: heard.bpm,
|
|
1706
|
+
fadeFrom: fadeFrom(db, run)
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
return {
|
|
1710
|
+
music: {
|
|
1711
|
+
segments,
|
|
1712
|
+
pieces: pieces.map(({ print: _print, mix: _mix, ...piece }) => piece),
|
|
1713
|
+
breaks: breaks.map((b) => ({ start: round6(b.start), end: round6(b.end) })),
|
|
1714
|
+
energy: energyCurve(melodic, input.sampleRate, db.length * HOP3)
|
|
1715
|
+
},
|
|
1716
|
+
beats
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
// src/audio/read/stems.ts
|
|
1721
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1722
|
+
import { existsSync as existsSync4, mkdirSync, readFileSync, statSync, writeFileSync } from "fs";
|
|
1723
|
+
import { mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
|
|
1724
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
1725
|
+
import { join as join5 } from "path";
|
|
1726
|
+
import { setTimeout as sleep } from "timers/promises";
|
|
1727
|
+
var URL_ENV2 = "HYPERFRAMES_AUDIO_STEMS_URL";
|
|
1728
|
+
var KEY_ENV2 = "HYPERFRAMES_AUDIO_API_KEY";
|
|
1729
|
+
var TIMEOUT_ENV2 = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS";
|
|
1730
|
+
var DEFAULT_TIMEOUT_MS2 = 30 * 6e4;
|
|
1731
|
+
var RETRY_MS = 5e3;
|
|
1732
|
+
var DONE = "stems.json";
|
|
1733
|
+
var STEM_NAMES = ["vocals", "drums", "bass", "guitar", "piano", "other"];
|
|
1734
|
+
var VOICE_STEM = "vocals";
|
|
1735
|
+
var DRUM_STEM = "drums";
|
|
1736
|
+
var stemsAccess = () => serviceAccess(
|
|
1737
|
+
{ url: URL_ENV2, key: KEY_ENV2, timeout: TIMEOUT_ENV2 },
|
|
1738
|
+
"/v3/audio/stems",
|
|
1739
|
+
DEFAULT_TIMEOUT_MS2
|
|
1740
|
+
);
|
|
1741
|
+
function cachedStems(dir) {
|
|
1742
|
+
const done = join5(dir, DONE);
|
|
1743
|
+
if (!existsSync4(done)) return null;
|
|
1744
|
+
const names = JSON.parse(readFileSync(done, "utf8"));
|
|
1745
|
+
if (!STEM_NAMES.every((name) => typeof names[name] === "string")) return null;
|
|
1746
|
+
const paths = Object.fromEntries(STEM_NAMES.map((name) => [name, join5(dir, names[name])]));
|
|
1747
|
+
return Object.values(paths).every((p) => existsSync4(p)) ? paths : null;
|
|
1748
|
+
}
|
|
1749
|
+
var StillRunning = class extends Error {
|
|
1750
|
+
};
|
|
1751
|
+
var DROPPED = /* @__PURE__ */ new Set(["UND_ERR_HEADERS_TIMEOUT", "UND_ERR_SOCKET"]);
|
|
1752
|
+
var dropped = (error) => error instanceof TypeError && DROPPED.has(error.cause?.code ?? "");
|
|
1753
|
+
async function stillRunning(response) {
|
|
1754
|
+
if (response.status !== 409) return false;
|
|
1755
|
+
const body = await response.clone().json().catch(() => null);
|
|
1756
|
+
return body?.error?.code === "request_in_progress";
|
|
1757
|
+
}
|
|
1758
|
+
async function job(response) {
|
|
1759
|
+
if (await stillRunning(response)) throw new StillRunning();
|
|
1760
|
+
if (!response.ok) throw await serviceFailure(response, "the stems service");
|
|
1761
|
+
const body = await response.json();
|
|
1762
|
+
const value = body && "data" in body ? body.data : body;
|
|
1763
|
+
if (!value?.id || !value.status) throw new Error("the stems service answered without a job");
|
|
1764
|
+
return value;
|
|
1765
|
+
}
|
|
1766
|
+
async function encoded(path, dir, signal) {
|
|
1767
|
+
const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true });
|
|
1768
|
+
if (!ffmpeg) throw new Error("ffmpeg is required to separate stems; run: npx hyperframes doctor");
|
|
1769
|
+
const out = join5(dir, "upload.mp3");
|
|
1770
|
+
await runFile(
|
|
1771
|
+
ffmpeg,
|
|
1772
|
+
["-v", "error", "-y", "-i", path, "-vn", "-ac", "2", "-ar", "44100", "-b:a", "192k", out],
|
|
1773
|
+
{ windowsHide: true, maxBuffer: 8 * 1024 * 1024, signal }
|
|
1774
|
+
);
|
|
1775
|
+
return out;
|
|
1776
|
+
}
|
|
1777
|
+
async function separated(url, request, signal) {
|
|
1778
|
+
for (; ; ) {
|
|
1779
|
+
try {
|
|
1780
|
+
return await job(await fetch(url, { ...request, signal }));
|
|
1781
|
+
} catch (error) {
|
|
1782
|
+
if (!(error instanceof StillRunning || dropped(error))) throw error;
|
|
1783
|
+
await sleep(RETRY_MS, void 0, { signal });
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
}
|
|
1787
|
+
async function fetchStems(path, dir, { url, headers, timeoutMs }) {
|
|
1788
|
+
const signal = AbortSignal.timeout(timeoutMs);
|
|
1789
|
+
const scratch = await mkdtemp2(join5(tmpdir2(), "hf-stems-"));
|
|
1790
|
+
try {
|
|
1791
|
+
const mp3 = await encoded(path, scratch, signal);
|
|
1792
|
+
const client = new HyperframesCloudClient({
|
|
1793
|
+
baseUrl: new URL(url).origin,
|
|
1794
|
+
getAuthHeaders: () => headers,
|
|
1795
|
+
fetchImpl: (input, init) => fetch(input, { ...init, redirect: "error", signal })
|
|
1796
|
+
});
|
|
1797
|
+
const asset = await uploadAssetViaDirectUpload({
|
|
1798
|
+
client,
|
|
1799
|
+
path: mp3,
|
|
1800
|
+
size: statSync(mp3).size,
|
|
1801
|
+
filename: "audio.mp3",
|
|
1802
|
+
contentType: "audio/mpeg",
|
|
1803
|
+
idempotencyKey: randomUUID2(),
|
|
1804
|
+
signal
|
|
1805
|
+
});
|
|
1806
|
+
const current = await separated(
|
|
1807
|
+
url,
|
|
1808
|
+
{
|
|
1809
|
+
method: "POST",
|
|
1810
|
+
redirect: "error",
|
|
1811
|
+
headers: {
|
|
1812
|
+
...headers,
|
|
1813
|
+
"idempotency-key": randomUUID2(),
|
|
1814
|
+
"content-type": "application/json"
|
|
1815
|
+
},
|
|
1816
|
+
body: JSON.stringify({ version: 1, audio: { type: "asset_id", asset_id: asset.asset_id } })
|
|
1817
|
+
},
|
|
1818
|
+
signal
|
|
1819
|
+
);
|
|
1820
|
+
if (current.status !== "completed")
|
|
1821
|
+
throw new Error(
|
|
1822
|
+
`stem separation failed${current.failure_message ? `: ${current.failure_message}` : ""}`
|
|
1823
|
+
);
|
|
1824
|
+
const stems = current.stems ?? [];
|
|
1825
|
+
const missing = STEM_NAMES.filter((name) => !stems.some((stem) => stem.name === name));
|
|
1826
|
+
if (missing.length)
|
|
1827
|
+
throw new Error(
|
|
1828
|
+
`the stems service returned an incomplete separation: missing ${missing.join(", ")}`
|
|
1829
|
+
);
|
|
1830
|
+
mkdirSync(dir, { recursive: true });
|
|
1831
|
+
const names = {};
|
|
1832
|
+
for (const name of STEM_NAMES) {
|
|
1833
|
+
const stem = stems.find((candidate) => candidate.name === name);
|
|
1834
|
+
if (!stem) continue;
|
|
1835
|
+
const file = `${name}.audio`;
|
|
1836
|
+
await downloadToFile(stem.url, join5(dir, file), { signal });
|
|
1837
|
+
names[name] = file;
|
|
1838
|
+
}
|
|
1839
|
+
writeFileSync(join5(dir, DONE), JSON.stringify(names));
|
|
1840
|
+
return cachedStems(dir);
|
|
1841
|
+
} finally {
|
|
1842
|
+
await rm2(scratch, { recursive: true, force: true });
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
// src/audio/read/types.ts
|
|
1847
|
+
var READ_VERSION = 2;
|
|
1848
|
+
|
|
1849
|
+
// src/audio/read/voice.ts
|
|
1850
|
+
var SENTENCE_END = /[。!?!?…;;.]["'”’」』)]*$/;
|
|
1851
|
+
var SENTENCE_GAP = 0.6;
|
|
1852
|
+
var LONGEST_SENTENCE = 60;
|
|
1853
|
+
var PAUSE_SECONDS = 0.4;
|
|
1854
|
+
var CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu;
|
|
1855
|
+
var LETTERS = new RegExp("\\p{L}", "u");
|
|
1856
|
+
var ENDING_SEMITONES = 2;
|
|
1857
|
+
var MIN_PITCH_FRAMES = 5;
|
|
1858
|
+
function rateUnit(words) {
|
|
1859
|
+
const text = words.map((w) => w.text).join("");
|
|
1860
|
+
return (text.match(CJK)?.length ?? 0) * 2 >= text.replace(/\s/g, "").length ? "chars" : "words";
|
|
1861
|
+
}
|
|
1862
|
+
function spoken(words, unit) {
|
|
1863
|
+
if (unit === "chars") return words.reduce((n, w) => n + (w.text.match(CJK)?.length ?? 0), 0);
|
|
1864
|
+
return words.filter((w) => LETTERS.test(w.text)).length;
|
|
1865
|
+
}
|
|
1866
|
+
function joined2(words, unit) {
|
|
1867
|
+
return unit === "chars" ? words.map((w) => w.text.trim()).join("") : words.map((w) => w.text.trim()).join(" ").replace(/\s+([,.!?;:])/g, "$1");
|
|
1868
|
+
}
|
|
1869
|
+
function sentenceGroups(words) {
|
|
1870
|
+
const groups = [];
|
|
1871
|
+
let current = [];
|
|
1872
|
+
let chars = 0;
|
|
1873
|
+
words.forEach((word, i) => {
|
|
1874
|
+
current.push(word);
|
|
1875
|
+
chars += word.text.length;
|
|
1876
|
+
const next = words[i + 1];
|
|
1877
|
+
if (!next || SENTENCE_END.test(word.text.trim()) || next.start - word.end >= SENTENCE_GAP || chars >= LONGEST_SENTENCE) {
|
|
1878
|
+
groups.push(current);
|
|
1879
|
+
current = [];
|
|
1880
|
+
chars = 0;
|
|
1881
|
+
}
|
|
1882
|
+
});
|
|
1883
|
+
return groups;
|
|
1884
|
+
}
|
|
1885
|
+
function ending(track2, start, end) {
|
|
1886
|
+
const split = start + (end - start) * 0.7;
|
|
1887
|
+
const body = within2(track2, start, split);
|
|
1888
|
+
const tail = within2(track2, split, end);
|
|
1889
|
+
if (body.length < MIN_PITCH_FRAMES || tail.length < 2) return null;
|
|
1890
|
+
const shift = semitones(median(tail), median(body));
|
|
1891
|
+
if (shift >= ENDING_SEMITONES) return "rising";
|
|
1892
|
+
if (shift <= -ENDING_SEMITONES) return "falling";
|
|
1893
|
+
return "level";
|
|
1894
|
+
}
|
|
1895
|
+
function pitchOf(track2, start, end) {
|
|
1896
|
+
const hz = within2(track2, start, end);
|
|
1897
|
+
return hz.length >= MIN_PITCH_FRAMES ? Math.round(median(hz)) : null;
|
|
1898
|
+
}
|
|
1899
|
+
function summary(sentences, pauses, track2) {
|
|
1900
|
+
const timed = sentences.filter((s3) => s3.rate !== null);
|
|
1901
|
+
const seconds = timed.reduce((n, s3) => n + (s3.end - s3.start), 0);
|
|
1902
|
+
const meanRate = seconds ? round6(timed.reduce((n, s3) => n + s3.rate * (s3.end - s3.start), 0) / seconds, 1) : null;
|
|
1903
|
+
return {
|
|
1904
|
+
sentences,
|
|
1905
|
+
pauses,
|
|
1906
|
+
meanRate,
|
|
1907
|
+
medianPitchHz: track2.length ? Math.round(median(track2.map((p) => p.hz))) : null
|
|
1908
|
+
};
|
|
1909
|
+
}
|
|
1910
|
+
function voiceFromWords(words, track2) {
|
|
1911
|
+
const ordered = words.filter((w) => w.end > w.start).sort((a, b) => a.start - b.start);
|
|
1912
|
+
const unit = rateUnit(ordered);
|
|
1913
|
+
const sentences = sentenceGroups(ordered).map((group) => {
|
|
1914
|
+
const start = group[0].start;
|
|
1915
|
+
const end = group[group.length - 1].end;
|
|
1916
|
+
const count = spoken(group, unit);
|
|
1917
|
+
return {
|
|
1918
|
+
start: round6(start),
|
|
1919
|
+
end: round6(end),
|
|
1920
|
+
text: joined2(group, unit),
|
|
1921
|
+
rate: count && end > start ? round6(count / (end - start), 1) : null,
|
|
1922
|
+
pitchHz: pitchOf(track2, start, end),
|
|
1923
|
+
ending: ending(track2, start, end)
|
|
1924
|
+
};
|
|
1925
|
+
});
|
|
1926
|
+
const pauses = [];
|
|
1927
|
+
for (let i = 1; i < ordered.length; i++) {
|
|
1928
|
+
const gap = ordered[i].start - ordered[i - 1].end;
|
|
1929
|
+
if (gap >= PAUSE_SECONDS)
|
|
1930
|
+
pauses.push({ start: round6(ordered[i - 1].end), end: round6(ordered[i].start) });
|
|
1931
|
+
}
|
|
1932
|
+
return { source: "transcript", unit, ...summary(sentences, pauses, track2) };
|
|
1933
|
+
}
|
|
1934
|
+
var ACTIVITY_HOP = 0.05;
|
|
1935
|
+
var SHORTEST_SPEECH = 0.3;
|
|
1936
|
+
function voiceFromActivity(vocals, sampleRate, track2) {
|
|
1937
|
+
const db = levels(vocals, sampleRate, ACTIVITY_HOP);
|
|
1938
|
+
const threshold = Math.max(-45, quantile(db, 0.9) - 30);
|
|
1939
|
+
const spans = [];
|
|
1940
|
+
let start = null;
|
|
1941
|
+
let quietSince = null;
|
|
1942
|
+
db.forEach((level, f) => {
|
|
1943
|
+
const t = f * ACTIVITY_HOP;
|
|
1944
|
+
if (level >= threshold) {
|
|
1945
|
+
if (start === null) start = t;
|
|
1946
|
+
quietSince = null;
|
|
1947
|
+
} else if (start !== null) {
|
|
1948
|
+
quietSince ??= t;
|
|
1949
|
+
if (t - quietSince >= PAUSE_SECONDS) {
|
|
1950
|
+
if (quietSince - start >= SHORTEST_SPEECH) spans.push({ start, end: quietSince });
|
|
1951
|
+
start = null;
|
|
1952
|
+
quietSince = null;
|
|
1953
|
+
}
|
|
1954
|
+
}
|
|
1955
|
+
});
|
|
1956
|
+
const end = db.length * ACTIVITY_HOP;
|
|
1957
|
+
if (start !== null && (quietSince ?? end) - start >= SHORTEST_SPEECH)
|
|
1958
|
+
spans.push({ start, end: quietSince ?? end });
|
|
1959
|
+
const sentences = spans.map(
|
|
1960
|
+
(s3) => ({
|
|
1961
|
+
start: round6(s3.start),
|
|
1962
|
+
end: round6(s3.end),
|
|
1963
|
+
text: null,
|
|
1964
|
+
rate: null,
|
|
1965
|
+
pitchHz: pitchOf(track2, s3.start, s3.end),
|
|
1966
|
+
ending: ending(track2, s3.start, s3.end)
|
|
1967
|
+
})
|
|
1968
|
+
);
|
|
1969
|
+
const pauses = spans.slice(1).map((s3, i) => ({ start: round6(spans[i].end), end: round6(s3.start) }));
|
|
1970
|
+
return { source: "activity", unit: "chars", ...summary(sentences, pauses, track2) };
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
// src/audio/read/read.ts
|
|
1974
|
+
async function sha256(path) {
|
|
1975
|
+
const hash = createHash2("sha256");
|
|
1976
|
+
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
|
1977
|
+
return hash.digest("hex");
|
|
1978
|
+
}
|
|
1979
|
+
function transcriptFor(path, given) {
|
|
1980
|
+
if (given) return given;
|
|
1981
|
+
const sidecar = `${path}.transcript.json`;
|
|
1982
|
+
return existsSync5(sidecar) ? sidecar : null;
|
|
1983
|
+
}
|
|
1984
|
+
function wordsOf(transcript) {
|
|
1985
|
+
return loadTranscript(transcript).words.map(({ text, start, end }) => ({ text, start, end }));
|
|
1986
|
+
}
|
|
1987
|
+
var fingerprint = (path) => path ? createHash2("sha256").update(readFileSync2(path)).digest("hex").slice(0, 16) : "none";
|
|
1988
|
+
function writeAtomic(path, content) {
|
|
1989
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
1990
|
+
writeFileSync2(temporary, content);
|
|
1991
|
+
renameSync(temporary, path);
|
|
1992
|
+
}
|
|
1993
|
+
async function stemsFor(path, dir, quality, warnings) {
|
|
1994
|
+
const cached = cachedStems(dir);
|
|
1995
|
+
if (cached) return cached;
|
|
1996
|
+
if (quality !== "best") {
|
|
1997
|
+
warnings.push(
|
|
1998
|
+
`not separated into stems: quality ${quality} keeps audio on this machine; --quality best separates it through HeyGen`
|
|
1999
|
+
);
|
|
2000
|
+
return null;
|
|
2001
|
+
}
|
|
2002
|
+
const access2 = await stemsAccess();
|
|
2003
|
+
if (typeof access2 === "string") {
|
|
2004
|
+
warnings.push(`not separated into stems: ${access2}`);
|
|
2005
|
+
return null;
|
|
2006
|
+
}
|
|
2007
|
+
try {
|
|
2008
|
+
return await fetchStems(path, dir, access2);
|
|
2009
|
+
} catch (error) {
|
|
2010
|
+
warnings.push(`not separated into stems: ${error instanceof Error ? error.message : error}`);
|
|
2011
|
+
return null;
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
async function fromStems(path, stemPaths) {
|
|
2015
|
+
const pitched = {};
|
|
2016
|
+
for (const [name, stemPath] of Object.entries(stemPaths))
|
|
2017
|
+
if (name !== VOICE_STEM && name !== DRUM_STEM) pitched[name] = await decodeMono(stemPath);
|
|
2018
|
+
const drums = stemPaths[DRUM_STEM] ? await decodeMono(stemPaths[DRUM_STEM]) : null;
|
|
2019
|
+
const bed = Object.keys(pitched).length ? mixed(Object.values(pitched)) : null;
|
|
2020
|
+
const { music, beats } = bed ? readMusic({ stems: pitched, drums, sampleRate: READ_RATE }) : { music: null, beats: [] };
|
|
2021
|
+
const effects = findEffects({
|
|
2022
|
+
source: drums ?? bed ?? new Float32Array(0),
|
|
2023
|
+
sampleRate: READ_RATE,
|
|
2024
|
+
music: bed,
|
|
2025
|
+
heard: await decodeMono(path),
|
|
2026
|
+
beats,
|
|
2027
|
+
speech: []
|
|
2028
|
+
});
|
|
2029
|
+
return { effects, music };
|
|
2030
|
+
}
|
|
2031
|
+
var transcribeHint = (file) => `run hyperframes transcribe ${file} and pass --transcript`;
|
|
2032
|
+
async function analyze(path, file, sha, transcript, stemDir, quality) {
|
|
2033
|
+
const warnings = [];
|
|
2034
|
+
const stemPaths = await stemsFor(path, stemDir, quality, warnings);
|
|
2035
|
+
const words = transcript ? wordsOf(transcript) : [];
|
|
2036
|
+
const voicePath = stemPaths?.[VOICE_STEM] ?? path;
|
|
2037
|
+
const voiceSamples = await decodeMono(voicePath);
|
|
2038
|
+
const about = base(file, sha, round6(voiceSamples.length / READ_RATE, 3), stemPaths, transcript);
|
|
2039
|
+
if (!stemPaths && !words.length) {
|
|
2040
|
+
warnings.push(
|
|
2041
|
+
`no transcript and no stems: voice and effects are not read, because the mix alone does not tell speech from music; ${transcribeHint(file)}`
|
|
2042
|
+
);
|
|
2043
|
+
return { ...about, voice: null, effects: null, music: null, warnings };
|
|
2044
|
+
}
|
|
2045
|
+
if (!words.length)
|
|
2046
|
+
warnings.push(`no transcript: voice lines carry no words; ${transcribeHint(file)}`);
|
|
2047
|
+
const pitch = pitchTrack(await decodeMono(voicePath, PITCH_RATE));
|
|
2048
|
+
const voice = words.length ? voiceFromWords(words, pitch) : voiceFromActivity(voiceSamples, READ_RATE, pitch);
|
|
2049
|
+
if (stemPaths) return { ...about, voice, ...await fromStems(path, stemPaths), warnings };
|
|
2050
|
+
warnings.push("without stems, effects under speech are not found and music is not read");
|
|
2051
|
+
const effects = findEffects({
|
|
2052
|
+
source: voiceSamples,
|
|
2053
|
+
sampleRate: READ_RATE,
|
|
2054
|
+
music: null,
|
|
2055
|
+
heard: null,
|
|
2056
|
+
beats: [],
|
|
2057
|
+
speech: voice.sentences
|
|
2058
|
+
});
|
|
2059
|
+
return { ...about, voice, effects, music: null, warnings };
|
|
2060
|
+
}
|
|
2061
|
+
function base(file, sha, duration, stemPaths, transcript) {
|
|
2062
|
+
return {
|
|
2063
|
+
version: READ_VERSION,
|
|
2064
|
+
file,
|
|
2065
|
+
sha256: sha,
|
|
2066
|
+
duration,
|
|
2067
|
+
separation: stemPaths ? "stems" : "mix",
|
|
2068
|
+
stems: stemPaths ? Object.keys(stemPaths).sort() : [],
|
|
2069
|
+
transcript: transcript ? basename(transcript) : null
|
|
2070
|
+
};
|
|
2071
|
+
}
|
|
2072
|
+
async function readAudio(path, file, options) {
|
|
2073
|
+
const quality = options.quality ?? "local";
|
|
2074
|
+
const sha = await sha256(path);
|
|
2075
|
+
const transcript = transcriptFor(path, options.transcript);
|
|
2076
|
+
const transcriptKey = fingerprint(transcript);
|
|
2077
|
+
const stem = `${basename(file, extname(file)).replace(/[^\w.-]/g, "_")}-${sha.slice(0, 12)}`;
|
|
2078
|
+
mkdirSync2(options.cacheDir, { recursive: true });
|
|
2079
|
+
const jsonPath = join6(options.cacheDir, `${stem}.json`);
|
|
2080
|
+
const textPath = join6(options.cacheDir, `${stem}.txt`);
|
|
2081
|
+
const stemDir = join6(options.cacheDir, "stems", sha.slice(0, 12));
|
|
2082
|
+
if (existsSync5(jsonPath) && existsSync5(textPath)) {
|
|
2083
|
+
const entry = JSON.parse(readFileSync2(jsonPath, "utf8"));
|
|
2084
|
+
const stemsNowPossible = entry.analysis.separation === "mix" && quality === "best" && (cachedStems(stemDir) !== null || typeof await stemsAccess() !== "string");
|
|
2085
|
+
if ((entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) && entry.analysis.version === READ_VERSION && entry.transcriptKey === transcriptKey && !stemsNowPossible)
|
|
2086
|
+
return { analysis: entry.analysis, textPath, jsonPath, cached: true };
|
|
2087
|
+
}
|
|
2088
|
+
const analysis = await analyze(path, file, sha, transcript, stemDir, quality);
|
|
2089
|
+
writeAtomic(jsonPath, JSON.stringify({ analysis, transcriptKey }));
|
|
2090
|
+
writeAtomic(textPath, formatRead(analysis));
|
|
2091
|
+
return { analysis, textPath, jsonPath, cached: false };
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
// src/commands/audio.ts
|
|
2095
|
+
var examples = [
|
|
2096
|
+
["Where a sound effect lands, its hits and loudness", "hyperframes audio sfx/whoosh.mp3"],
|
|
2097
|
+
[
|
|
2098
|
+
"Where a file not in the film yet would land if it started at 4.2 s",
|
|
2099
|
+
"hyperframes audio sfx/hit.wav --place-at 4.2"
|
|
2100
|
+
],
|
|
2101
|
+
["Every sound playing at 12 s in the film", "hyperframes audio --at 12"],
|
|
2102
|
+
["One clip of the film, as JSON", "hyperframes audio --element '#riser' --json"],
|
|
2103
|
+
[
|
|
2104
|
+
"The beat grid, tempo and sections of a music bed",
|
|
2105
|
+
"hyperframes audio music.mp3 --only beats,downbeats,bpm,sections"
|
|
2106
|
+
],
|
|
2107
|
+
[
|
|
2108
|
+
"A whole recording as voice, effect and music lines, 60\u201370 s shown",
|
|
2109
|
+
"hyperframes audio refs/episode.mp4 --read --range 60,70"
|
|
2110
|
+
],
|
|
2111
|
+
[
|
|
2112
|
+
"Only onsets, from 60\u2013200 Hz, finer",
|
|
2113
|
+
"hyperframes audio drums.wav --only onsets --band 60,200 --hop-ms 2"
|
|
2114
|
+
]
|
|
2115
|
+
];
|
|
2116
|
+
var CAPABILITIES = /* @__PURE__ */ new Set([
|
|
2117
|
+
...DEFAULT_CAPABILITIES,
|
|
2118
|
+
"beats",
|
|
2119
|
+
"downbeats",
|
|
2120
|
+
"bpm",
|
|
2121
|
+
"sections",
|
|
2122
|
+
"describe"
|
|
2123
|
+
]);
|
|
2124
|
+
var QUALITIES = /* @__PURE__ */ new Set(["best", "local", "fast"]);
|
|
2125
|
+
function number(name, raw) {
|
|
2126
|
+
if (raw === void 0) return void 0;
|
|
2127
|
+
const value = Number(raw);
|
|
2128
|
+
if (!Number.isFinite(value)) throw new CliUsageError(`--${name} must be a number, got ${raw}`);
|
|
2129
|
+
return value;
|
|
2130
|
+
}
|
|
2131
|
+
function pair(name, raw) {
|
|
2132
|
+
if (raw === void 0) return void 0;
|
|
2133
|
+
const parts = raw.split(",").map(Number);
|
|
2134
|
+
if (parts.length !== 2 || parts.some((n) => !Number.isFinite(n)) || parts[0] >= parts[1])
|
|
2135
|
+
throw new CliUsageError(`--${name} takes two increasing numbers, like 2.5,4`);
|
|
2136
|
+
return parts;
|
|
2137
|
+
}
|
|
2138
|
+
function capabilities(raw) {
|
|
2139
|
+
if (!raw) return [...DEFAULT_CAPABILITIES];
|
|
2140
|
+
const asked = raw.split(",").map((c) => c.trim()).filter(Boolean);
|
|
2141
|
+
if (asked.includes("words"))
|
|
2142
|
+
throw new CliUsageError("words come from hyperframes transcribe, not hyperframes audio");
|
|
2143
|
+
const unknown = asked.filter((c) => !CAPABILITIES.has(c));
|
|
2144
|
+
if (unknown.length)
|
|
2145
|
+
throw new CliUsageError(
|
|
2146
|
+
`--only does not know ${unknown.join(", ")}; use ${[...CAPABILITIES].join(", ")}`
|
|
2147
|
+
);
|
|
2148
|
+
return asked;
|
|
2149
|
+
}
|
|
2150
|
+
function pins(raw) {
|
|
2151
|
+
if (!raw) return void 0;
|
|
2152
|
+
return Object.fromEntries(
|
|
2153
|
+
raw.split(",").map((entry) => {
|
|
2154
|
+
const [capability, id] = entry.split("=").map((s3) => s3.trim());
|
|
2155
|
+
if (!capability || !id || !CAPABILITIES.has(capability))
|
|
2156
|
+
throw new CliUsageError(`--provider takes capability=provider pairs, like onsets=signal`);
|
|
2157
|
+
return [capability, id];
|
|
2158
|
+
})
|
|
2159
|
+
);
|
|
2160
|
+
}
|
|
2161
|
+
function filmQuery(args) {
|
|
2162
|
+
const given = ["element", "at", "range"].filter((k) => args[k] !== void 0);
|
|
2163
|
+
if (given.length > 1) throw new CliUsageError(`use one of --element, --at, --range`);
|
|
2164
|
+
if (args.element) return { by: "element", ref: args.element };
|
|
2165
|
+
if (args.at !== void 0) return { by: "at", t: number("at", args.at) };
|
|
2166
|
+
const range = pair("range", args.range);
|
|
2167
|
+
return range ? { by: "range", start: range[0], end: range[1] } : null;
|
|
2168
|
+
}
|
|
2169
|
+
function parseQuality(flags) {
|
|
2170
|
+
const quality = flags.quality ?? "local";
|
|
2171
|
+
if (!QUALITIES.has(quality)) throw new CliUsageError("--quality is local, best or fast");
|
|
2172
|
+
return quality;
|
|
2173
|
+
}
|
|
2174
|
+
function analyzeRequest(flags) {
|
|
2175
|
+
const quality = parseQuality(flags);
|
|
2176
|
+
const method = flags["onset-method"];
|
|
2177
|
+
if (method && method !== "spectral-flux" && method !== "energy")
|
|
2178
|
+
throw new CliUsageError("--onset-method is spectral-flux or energy");
|
|
2179
|
+
const band = pair("band", flags.band);
|
|
2180
|
+
return {
|
|
2181
|
+
capabilities: capabilities(flags.only),
|
|
2182
|
+
quality,
|
|
2183
|
+
provider: pins(flags.provider),
|
|
2184
|
+
options: {
|
|
2185
|
+
onsets: { method, band, threshold: number("threshold", flags.threshold) },
|
|
2186
|
+
envelope: { band, hopMs: number("hop-ms", flags["hop-ms"]) },
|
|
2187
|
+
moments: {
|
|
2188
|
+
onsetRatio: number("onset-ratio", flags["onset-ratio"]),
|
|
2189
|
+
tailDb: number("tail-db", flags["tail-db"])
|
|
2190
|
+
},
|
|
2191
|
+
silence: { thresholdDb: number("silence-db", flags["silence-db"]) }
|
|
2192
|
+
}
|
|
2193
|
+
};
|
|
2194
|
+
}
|
|
2195
|
+
async function fileTargets(files, flags) {
|
|
2196
|
+
const start = number("place-at", flags["place-at"]);
|
|
2197
|
+
if (start !== void 0 && files.length !== 1)
|
|
2198
|
+
throw new CliUsageError("--place-at places exactly one file");
|
|
2199
|
+
const skip = number("skip-file-seconds", flags["skip-file-seconds"]);
|
|
2200
|
+
if (skip !== void 0 && start === void 0)
|
|
2201
|
+
throw new CliUsageError("--skip-file-seconds only works with --place-at");
|
|
2202
|
+
const mediaStart = skip ?? 0;
|
|
2203
|
+
const cwd = process.cwd();
|
|
2204
|
+
return Promise.all(
|
|
2205
|
+
files.map((file) => previewTarget(resolve2(cwd, file), file, start ?? null, mediaStart))
|
|
2206
|
+
);
|
|
2207
|
+
}
|
|
2208
|
+
async function clipTargets(flags, query) {
|
|
2209
|
+
const project = resolveProject(flags.dir);
|
|
2210
|
+
ensureDOMParser();
|
|
2211
|
+
const timeline = await describeProject(project.indexPath);
|
|
2212
|
+
return filmTargets(project.dir, timeline, query ?? { by: "all" });
|
|
2213
|
+
}
|
|
2214
|
+
function readQuality(files, flags) {
|
|
2215
|
+
if (files.length !== 1) throw new CliUsageError("--read reads exactly one file");
|
|
2216
|
+
if (flags.element || flags.at || flags["place-at"])
|
|
2217
|
+
throw new CliUsageError("--read takes a file and --range, not a film query");
|
|
2218
|
+
return parseQuality(flags);
|
|
2219
|
+
}
|
|
2220
|
+
async function read(files, flags) {
|
|
2221
|
+
const quality = readQuality(files, flags);
|
|
2222
|
+
const range = pair("range", flags.range);
|
|
2223
|
+
const result = await readAudio(resolve2(process.cwd(), files[0]), files[0], {
|
|
2224
|
+
cacheDir: join7(resolve2(flags.dir ?? process.cwd()), ".hyperframes", "read-audio"),
|
|
2225
|
+
transcript: flags.transcript ? resolve2(process.cwd(), flags.transcript) : void 0,
|
|
2226
|
+
quality
|
|
2227
|
+
});
|
|
2228
|
+
if (flags.json) {
|
|
2229
|
+
console.log(JSON.stringify(withMeta(result)));
|
|
2230
|
+
return;
|
|
2231
|
+
}
|
|
2232
|
+
console.log(`${formatRead(result.analysis, range)}
|
|
2233
|
+
|
|
2234
|
+
whole recording: ${result.textPath}`);
|
|
2235
|
+
}
|
|
2236
|
+
var audio_default = defineCommand({
|
|
2237
|
+
meta: {
|
|
2238
|
+
name: "audio",
|
|
2239
|
+
description: "Measure how sound moves in time: where it lands, its hits, loudness and silence"
|
|
2240
|
+
},
|
|
2241
|
+
args: {
|
|
2242
|
+
files: { type: "positional", description: "Audio or video files to analyze", required: false },
|
|
2243
|
+
dir: { type: "string", description: "Project directory, for film queries (default: current)" },
|
|
2244
|
+
element: {
|
|
2245
|
+
type: "string",
|
|
2246
|
+
description: "One audio or video clip of the film: #id, hf:id or audio/N"
|
|
2247
|
+
},
|
|
2248
|
+
at: { type: "string", description: "Every sound playing at this film second" },
|
|
2249
|
+
range: { type: "string", description: "Every sound playing in start,end film seconds" },
|
|
2250
|
+
"place-at": {
|
|
2251
|
+
type: "string",
|
|
2252
|
+
description: "For one file not in the film yet: the film second it would start at"
|
|
2253
|
+
},
|
|
2254
|
+
"skip-file-seconds": {
|
|
2255
|
+
type: "string",
|
|
2256
|
+
description: "With --place-at: seconds cut from the start of the file"
|
|
2257
|
+
},
|
|
2258
|
+
only: {
|
|
2259
|
+
type: "string",
|
|
2260
|
+
description: `Capabilities to measure (default ${DEFAULT_CAPABILITIES.join(",")}; also beats,downbeats,bpm,sections,describe)`
|
|
2261
|
+
},
|
|
2262
|
+
quality: {
|
|
2263
|
+
type: "string",
|
|
2264
|
+
description: "local (this machine, the default), best (also uploads to HeyGen's audio services, which may bill your account), fast (signal only)"
|
|
2265
|
+
},
|
|
2266
|
+
provider: { type: "string", description: "Pin capabilities to providers: onsets=signal,..." },
|
|
2267
|
+
"onset-method": { type: "string", description: "spectral-flux (default) or energy" },
|
|
2268
|
+
threshold: { type: "string", description: "Onset strength floor, 0\u20131 (default 0.1)" },
|
|
2269
|
+
band: { type: "string", description: "Measure onsets and envelope in lo,hi Hz only" },
|
|
2270
|
+
"hop-ms": {
|
|
2271
|
+
type: "string",
|
|
2272
|
+
description: "Envelope hop in ms (default: about 2,000 values per item)"
|
|
2273
|
+
},
|
|
2274
|
+
"onset-ratio": {
|
|
2275
|
+
type: "string",
|
|
2276
|
+
description: "Moments: onset is where RMS first reaches this share of the peak (0.2)"
|
|
2277
|
+
},
|
|
2278
|
+
"tail-db": {
|
|
2279
|
+
type: "string",
|
|
2280
|
+
description: "Moments: the tail ends this many dB under the peak (-40)"
|
|
2281
|
+
},
|
|
2282
|
+
"silence-db": { type: "string", description: "Silence below this level in dBFS (-50)" },
|
|
2283
|
+
full: {
|
|
2284
|
+
type: "boolean",
|
|
2285
|
+
description: "Include each capability's provider ladder",
|
|
2286
|
+
default: false
|
|
2287
|
+
},
|
|
2288
|
+
read: {
|
|
2289
|
+
type: "boolean",
|
|
2290
|
+
description: "Read one whole recording as voice, effect and music lines (--range picks what is shown)",
|
|
2291
|
+
default: false
|
|
2292
|
+
},
|
|
2293
|
+
transcript: {
|
|
2294
|
+
type: "string",
|
|
2295
|
+
description: "With --read: the recording's transcript (default <file>.transcript.json)"
|
|
2296
|
+
},
|
|
2297
|
+
json: { type: "boolean", description: "Output as JSON", default: false }
|
|
2298
|
+
},
|
|
2299
|
+
setup({ rawArgs, cmd }) {
|
|
2300
|
+
assertKnownFlags({ args: cmd.args }, rawArgs);
|
|
2301
|
+
},
|
|
2302
|
+
async run({ args }) {
|
|
2303
|
+
const flags = args;
|
|
2304
|
+
const files = args._.map(String);
|
|
2305
|
+
if (flags.read) return read(files, flags);
|
|
2306
|
+
if (flags.transcript) throw new CliUsageError("--transcript only works with --read");
|
|
2307
|
+
const query = filmQuery(flags);
|
|
2308
|
+
if (files.length && query) throw new CliUsageError("give files or a film query, not both");
|
|
2309
|
+
const targets = files.length ? await fileTargets(files, flags) : await clipTargets(flags, query);
|
|
2310
|
+
const report = await analyzeTargets(targets, analyzeRequest(flags), Boolean(args.full));
|
|
2311
|
+
console.log(args.json ? JSON.stringify(withMeta(report)) : formatReport(report));
|
|
2312
|
+
}
|
|
2313
|
+
});
|
|
2314
|
+
export {
|
|
2315
|
+
audio_default as default,
|
|
2316
|
+
examples
|
|
2317
|
+
};
|