@libraz/libsonare 1.7.2 → 1.8.0

package/src/public_types_spectral.ts

@@ -58,6 +58,464 @@ export interface NoteMoveOptions {
   targetOnsetSample?: number;
 }
 
+/** Segmentation tuning for `extractNotes`. All fields are optional; 0 or absent takes the default. */
+export interface NoteExtractorOptions {
+  /** Cents of pitch change that start a new note. Default 50. */
+  segmentationThresholdCents?: number;
+  /** Shortest span kept as a note, in ms. Default 30. */
+  minNoteMs?: number;
+  /** Reference pitch the `medianCents` of each note is measured against. Default 440. */
+  referenceHz?: number;
+  /**
+   * Value of `voicedProb` at or above which a frame counts as voiced, in
+   * `[0, 1]`. Read only when `voiced` is omitted. Default 0.5.
+   *
+   * pYIN's `voicedProb` is a frame's voiced observation mass and rises with F0
+   * for a fixed frame length, so this default silently drops low registers —
+   * pass `pitchPyin`'s `voicedFlag` through `voiced` instead.
+   */
+  voicedThreshold?: number;
+}
+
+/**
+ * A pending, non-destructive change to one note. `extractNotes` attaches the
+ * identity edit (no move, no transpose, unity gain and stretch, unmuted) to
+ * every note it returns; `renderNotes` applies whatever the caller has changed.
+ */
+export interface NoteEdit {
+  /**
+   * Moves the note along the timeline; negative moves it earlier. Where the note
+   * lands is not bounds-checked, so a moved note may overwrite a neighbour.
+   */
+  timeOffsetSamples: number;
+  /** Transpose applied to the note's span. */
+  pitchShiftSemitones: number;
+  /** Level change applied to the note's span. */
+  gainDb: number;
+  /** `>1` lengthens the note, `<1` shortens it; pitch is preserved. 0 reads as 1. */
+  timeStretchRatio: number;
+  /**
+   * Moves the spectral envelope, in semitones, on top of whatever the pitch
+   * shift already did to it.
+   *
+   * 0 runs no warp at all, so a pitch-only edit is not charged for an LPC
+   * analysis-resynthesis round it did not ask for. A pitch shift drags the
+   * formants with it, so holding them still is `-pitchShiftSemitones` and the
+   * chipmunk is the default. Saturates near -10.3 and +8.7 semitones rather
+   * than being rejected.
+   */
+  formantShiftSemitones: number;
+  /**
+   * Scales the vibrato measured over the note, stated as a change from it: 0
+   * keeps it, -1 flattens it, +1 doubles it.
+   *
+   * Applying it needs the note's own pitch curve, so `renderNotes` must be given
+   * the `f0Hz` track and the note its `frameStart` / `frameEnd` / `medianHz`; a
+   * note carrying no usable pitch is rejected rather than left alone. The curve
+   * is split at the request's `vibratoCutoffHz`.
+   */
+  vibratoDepthChange: number;
+  /** The same, for the slow drift around the note's centre pitch. */
+  driftChange: number;
+  /** Silences the note's span; the other fields then do not apply. */
+  muted: boolean;
+  /**
+   * Per-frame linear gain over the note's span, on top of `gainDb`.
+   *
+   * A set of gain points rather than a signal: it is stretched over whatever
+   * length the note renders at, so it survives a time stretch and need not match
+   * the note's frame count. One entry is a constant gain, and an empty array is
+   * no envelope. Every value must be finite and non-negative.
+   */
+  amplitudeEnvelope: Float32Array;
+}
+
+/**
+ * A {@link NoteEdit} as supplied to `renderNotes`, `splitNote` or `mergeNotes`.
+ * Every field is optional and an omitted one is the identity, so `{}` leaves the
+ * note untouched.
+ *
+ * `amplitudeEnvelope` is the one field that widens on the way in: a host
+ * building an envelope in JS naturally ends up with a plain array, and the
+ * points are copied into WASM memory either way. What comes back on a
+ * {@link NoteEdit} is always a `Float32Array`.
+ */
+export type NoteEditInput = Omit<Partial<NoteEdit>, 'amplitudeEnvelope'> & {
+  amplitudeEnvelope?: Float32Array | readonly number[];
+};
+
+/**
+ * One editable note, returned by `extractNotes` and by
+ * `PolyphonicAnalysis.notes()`.
+ *
+ * Sample bounds are half-open into the source audio. Frame bounds are half-open
+ * into whichever framing found the note: the caller's own `f0Hz` track through
+ * `extractNotes`, and the analysis's own STFT framing through a
+ * `PolyphonicAnalysis`, whose `f0Hz` curve therefore comes from `noteF0(note)`
+ * rather than from an array the caller holds.
+ *
+ * The per-note F0 curve is deliberately not repeated here — through the by-value
+ * door it is `f0Hz.subarray(frameStart, frameEnd)`, and through a handle it has its
+ * own accessor.
+ *
+ * `onsetSample` and `offsetSample` are 64-bit on the core side and arrive as JS
+ * numbers, which are exact up to `Number.MAX_SAFE_INTEGER`.
+ */
+export interface NoteObject {
+  /** First sample of the note's span. */
+  onsetSample: number;
+  /** One past the last sample of the span. */
+  offsetSample: number;
+  /** First frame of the span, in the framing that found the note. */
+  frameStart: number;
+  /** One past the last frame of the span. */
+  frameEnd: number;
+  /** Median measured pitch over the span, in Hz. */
+  medianHz: number;
+  /** Median pitch in cents above the request's `referenceHz`. */
+  medianCents: number;
+  /** Pitch steadiness in `[0, 1]`; 1 is perfectly steady. */
+  f0Stability: number;
+  /** One RMS value per frame of the span (`frameEnd - frameStart` entries). */
+  amplitude: Float32Array;
+  /** This note's pending edit; the identity as returned. */
+  edit: NoteEdit;
+}
+
+/**
+ * A note handed to `renderNotes`. Only the span, the edit and — for a
+ * `vibratoDepthChange` or `driftChange` edit — the frame bounds and the centre
+ * are read, so a {@link NoteObject} straight from `extractNotes` can be passed
+ * back with its `edit` changed and nothing else.
+ */
+export interface NoteObjectInput {
+  /** First sample of the note's span. */
+  onsetSample: number;
+  /** One past the last sample of the span. */
+  offsetSample: number;
+  /**
+   * First frame of the span in the request's `f0Hz` track. Read only when a
+   * track is given; a curve edit acts on `f0Hz.subarray(frameStart, frameEnd)`.
+   */
+  frameStart?: number;
+  /** One past the last frame of the span, under the same rule. */
+  frameEnd?: number;
+  /** The note's centre pitch in Hz, which a curve edit measures its cents against. */
+  medianHz?: number;
+  /** Omit for the identity edit. */
+  edit?: NoteEditInput;
+}
+
+/**
+ * A note handed to `splitNote` or `mergeNotes`. Both re-derive every note in the
+ * set from the audio and the track, so only the frame bounds and the edit are
+ * read — and the frame bounds are therefore what a note is identified by.
+ */
+export interface NoteSetEntry {
+  /** First frame of the span in the request's `f0Hz` track. */
+  frameStart: number;
+  /** One past the last frame of the span. */
+  frameEnd: number;
+  /** Omit for the identity edit. */
+  edit?: NoteEditInput;
+}
+
+/**
+ * One note of a reference melody, as `noteTargetsFromSmf` returns it and
+ * `assignNoteTargets` reads it.
+ *
+ * Times are seconds from the start of the audio the notes being corrected were
+ * extracted from — a reference is lined up against a take by its own clock, not
+ * by a frame index into either one. Every field must be finite.
+ */
+export interface NoteTarget {
+  /** First second of the target's span. */
+  startSec: number;
+  /** One past the last second of the span. */
+  endSec: number;
+  /** The pitch that stretch of the part is supposed to be, as a MIDI number. */
+  targetMidi: number;
+}
+
+/**
+ * What `assignNoteTargets` does with a note that has a measurable pitch and no
+ * target.
+ *
+ * A note carrying no pitch at all is a different case and no policy reaches it:
+ * there is nothing to correct from, so it is never assigned and never edited.
+ */
+export type NoteTargetUnmatchedPolicy =
+  /** Leave the edit alone; the note renders as recorded. Default. */
+  | 'leave'
+  /** Mute the note's span. */
+  | 'mute'
+  /** Take the nearest target in time, however far away it is. */
+  | 'nearest';
+
+/** What `assignNoteTargets` returns. */
+export interface NoteTargetAssignResult {
+  /**
+   * The note set with each assigned note's `edit.pitchShiftSemitones` — and,
+   * under `'mute'`, its `edit.muted` — rewritten. A new array: the notes handed
+   * in are not touched, and every other field of a note comes back as it went.
+   */
+  notes: NoteObject[];
+  /**
+   * How many notes received a target. Zero is a legitimate answer — a reference
+   * that does not line up with the take — which is why it is reported rather
+   * than left for the caller to infer from the edits.
+   */
+  assignedCount: number;
+}
+
+/**
+ * One note's pitch curve split into a centre, a slow drift and a vibrato by
+ * `decomposeNotePitch`.
+ *
+ * `driftCents[i] + vibratoCents[i]` is the note's own pitch at frame `i`, in
+ * cents above `centreHz`, to within float rounding, so the three parts
+ * reconstruct the curve. The drift filter is zero phase, so neither curve is
+ * shifted in time against the audio.
+ */
+export interface PitchDecompositionResult {
+  /**
+   * The note's steady pitch in Hz. 0 when the note carries no usable pitch, and
+   * then both curves are empty.
+   */
+  centreHz: number;
+  /** Slow deviation from `centreHz` in cents, one entry per frame. */
+  driftCents: Float32Array;
+  /** Fast deviation in cents, over the same frames. */
+  vibratoCents: Float32Array;
+}
+
+/**
+ * Tuning for `analyzePolyphonic`. Every field is optional and 0 or absent takes
+ * the documented default.
+ *
+ * Four fields accept 0 as a value as well as reading it as their default, and are
+ * marked below: **pass a negative number to select 0 on those**. Each rejects a
+ * negative otherwise, so the two meanings cannot collide.
+ *
+ * The window function and the centred framing are not settable. Every span, claim
+ * and mask offset in this chain is derived against one framing, so a second way to
+ * state it would be a second thing to keep in agreement.
+ */
+export interface PolyphonicAnalysisOptions {
+  /** STFT size the whole chain runs in. Default 4096, the size it is tuned at. */
+  nFft?: number;
+  /** STFT hop in samples. Default 512. */
+  hopLength?: number;
+  /** Window length in samples. Defaults to `nFft`. */
+  winLength?: number;
+  /** Bottom of the cent axis the salience is folded onto, in Hz. Default 55. */
+  centRefHz?: number;
+  /** Cent-axis resolution. Default 100/3; finer than 1 cent is rejected. */
+  centsPerBin?: number;
+  /** Top of the cent axis, in Hz. Default 8000. */
+  centMaxHz?: number;
+  /** Stops weighting bins by tonality, which is on. Default `false`. */
+  tonalityOff?: boolean;
+  /** Partials summed per F0 candidate. Default 20, at most 128. */
+  salienceHarmonics?: number;
+  /** Lowest F0 a candidate may take, in Hz. Default 55. */
+  f0MinHz?: number;
+  /** Highest F0 a candidate may take, in Hz. Default 1760. */
+  f0MaxHz?: number;
+  /** Harmonic weighting offset, in Hz. Default 27. */
+  salienceAlphaHz?: number;
+  /** Harmonic weighting scale, in Hz. Default 320. */
+  salienceBetaHz?: number;
+  /**
+   * Partial-series stretch assumed while scoring a candidate, `B` in
+   * `f_h = h*f0*sqrt(1 + B*h^2)`. 0 is the default and also a stretch of zero, so
+   * it needs no sentinel.
+   */
+  salienceInharmonicity?: number;
+  /** Voices one frame may hold. Default 4, at most 64. */
+  maxPolyphony?: number;
+  /**
+   * Stops the per-frame iteration below this share of the frame's first peak.
+   * Default 0.20; **negative selects 0**.
+   */
+  minFramePeakRatio?: number;
+  /** Closest two candidates of one frame may sit. Default 50; **negative selects 0**. */
+  minSeparationCents?: number;
+  /** Share of a found voice removed before the next iteration. Default 1. */
+  subtractionFactor?: number;
+  /** A larger move between two frames breaks the ridge. Default 50 cents. */
+  maxJumpCents?: number;
+  /**
+   * A fade below this share of the ridge's own running peak breaks it. Default
+   * 0.10; **negative selects 0**.
+   */
+  minRidgePeakRatio?: number;
+  /** Shorter ridges are dropped. Default 140 ms; **negative selects 0**. */
+  minRidgeDurationMs?: number;
+  /** Partials claimed per note. Default 20, at most 128. */
+  maskHarmonics?: number;
+  /** Claim half-width in Hann main lobes. Default 1. */
+  claimLobes?: number;
+  /**
+   * Stretch of the claimed partial series, `B` in `f_h = h*f0*sqrt(1 + B*h^2)`.
+   * Default 0, which is also a value.
+   *
+   * Leaving it at 0 for stretched material costs more than a widened claim would:
+   * at a piano's `1e-4` the highest partial of a twenty-harmonic claim sits
+   * outside the claim entirely, and a partial outside every claim is residual —
+   * carried unedited, so it keeps sounding at the old pitch after its note moves.
+   */
+  inharmonicity?: number;
+  /**
+   * Fits a stretch per note from the spectrum instead of spending
+   * {@link PolyphonicAnalysisOptions.inharmonicity} on every one of them. Default
+   * `false`.
+   *
+   * Off by default because of what it reaches rather than what it costs: at the
+   * default framing the fit takes an isolated note in the middle register and
+   * refuses a chord. A refused note keeps the declared stretch, so the fit only
+   * ever replaces a guess with a measurement —
+   * `PolyphonicAnalysis.noteInharmonicity()` reports which notes it reached.
+   */
+  estimateInharmonicity?: boolean;
+  /** Usable partials one fit needs before its result is believed. Default 3. */
+  inharmonicityMinPartials?: number;
+  /** Largest per-partial misfit a fit may leave, in STFT bins. Default 0.5. */
+  inharmonicityMaxResidualBins?: number;
+  /** A fitted stretch above this is refused. Default 0.03125. */
+  inharmonicityMaxStretch?: number;
+  /** Frames per apportionment fit. Default 8, between 4 and 64. */
+  windowFrames?: number;
+  /** Radians per frame two claimed partials must differ by to be fitted. Default 0.01. */
+  minPartialSeparation?: number;
+  /** Relative misfit ceiling above which the fit refuses the bin. Default 0.02. */
+  maxFitResidual?: number;
+  /**
+   * Ceiling on one weight's modulus. A weight is a fitted component over the
+   * observed bin, so where two partials nearly cancel it exceeds one and the
+   * residual carries several times the input there. While every edit is identity
+   * that is inaudible — the notes and the residual still sum to the input.
+   * Lowering it trades separation for a quieter residual. Default 8.
+   */
+  maxWeightModulus?: number;
+  /** Highest partial usable to refine an F0, in Hz. Default 0, which derives one. */
+  maxRefineHz?: number;
+  /** Worst F0 error tolerated by the fit, in cents. Default 50. */
+  f0ToleranceCents?: number;
+  /**
+   * Scales each note's reported `f0Stability` only. Every tracked ridge is one
+   * note, so a mid-ridge pitch jump is never split into two. Default 50.
+   */
+  segmentationThresholdCents?: number;
+  /**
+   * Not read: every tracked ridge is one note, so nothing filters a short note
+   * here. Use `minRidgeDurationMs` to drop short ridges instead. Default 30.
+   */
+  minNoteMs?: number;
+  /** Reference pitch each note's `medianCents` is measured against. Default 440. */
+  referenceHz?: number;
+}
+
+/** Options for `PolyphonicAnalysis.render`. All fields are optional. */
+export interface PolyphonicRenderOptions {
+  /**
+   * Equal-power cross-fade at each edited note's edges. Default 5 ms; a hard cut
+   * is deliberately not selectable, because the seam it leaves is a click.
+   */
+  fadeMs?: number;
+  /**
+   * Boundary between the drift and the vibrato that `vibratoDepthChange` and
+   * `driftChange` act on, in Hz. Default 3 Hz, and it has to be whatever a curve
+   * edit was drawn at.
+   */
+  vibratoCutoffHz?: number;
+}
+
+/**
+ * A pending, non-destructive change to one percussive event.
+ * `extractPercussiveEvents` attaches the identity edit (no move, unity gain,
+ * unmuted) to every event it returns; `renderPercussiveEvents` applies whatever
+ * the caller has changed.
+ *
+ * A struck sound has no steady pitch to edit, so the axes are time and amplitude
+ * and there is deliberately nothing else here.
+ */
+export interface PercussiveEventEdit {
+  /**
+   * Moves the hit along the timeline; negative moves it earlier. Where the hit
+   * lands is not bounds-checked, so a moved event may be written over a
+   * neighbour, and a shift past either end is truncated there.
+   */
+  timeOffsetSamples: number;
+  /** Level change applied to the hit, which is the span's percussive component. */
+  gainDb: number;
+  /** Silences the hit; the other fields then do not apply. */
+  muted: boolean;
+}
+
+/**
+ * A {@link PercussiveEventEdit} as supplied to `renderPercussiveEvents`. Every
+ * field is optional and an omitted one is the identity, so `{}` leaves the event
+ * untouched.
+ */
+export type PercussiveEventEditInput = Partial<PercussiveEventEdit>;
+
+/**
+ * One editable percussive event returned by `extractPercussiveEvents`.
+ *
+ * A struck sound located in time: a span in source samples, three measured
+ * figures, and a pending edit. It carries no pitch and is never associated with
+ * a {@link NoteObject} — the two models are produced by separate calls and do
+ * not refer to each other.
+ *
+ * `onsetSample` and `offsetSample` are 64-bit on the core side and arrive as JS
+ * numbers, which are exact up to `Number.MAX_SAFE_INTEGER`.
+ */
+export interface PercussiveEvent {
+  /** First sample of the span, backtracked to in front of the transient. */
+  onsetSample: number;
+  /** One past the last sample of the span; the next onset, or the cap. */
+  offsetSample: number;
+  /**
+   * Detector strength at the onset, on the onset envelope's own scale. It orders
+   * events against each other and carries no absolute meaning.
+   */
+  strength: number;
+  /**
+   * Peak absolute sample of the percussive component over the span, linear.
+   * Measured on the signal `gainDb` scales rather than on the source.
+   */
+  peakAmplitude: number;
+  /**
+   * Share of the span's energy the separation assigned to percussion, in
+   * `[0, 1]`; 0 when the span is silent.
+   *
+   * It describes the span rather than the onset that opened it. An isolated hit
+   * sits near 1, but a real hit over a loud sustain sits near 0, because the
+   * sustain owns the span's energy. So it separates a hit from a note attack
+   * only where nothing is sustaining through both, and it is not a test for
+   * whether a hit is there.
+   */
+  percussiveRatio: number;
+  /** This event's pending edit; the identity as returned. */
+  edit: PercussiveEventEdit;
+}
+
+/**
+ * An event handed to `renderPercussiveEvents`. Only the span and the edit are
+ * read — the three measured figures are ignored — so a {@link PercussiveEvent}
+ * straight from `extractPercussiveEvents` can be passed back with its `edit`
+ * changed and nothing else.
+ */
+export interface PercussiveEventInput {
+  /** First sample of the span. */
+  onsetSample: number;
+  /** One past the last sample of the span. */
+  offsetSample: number;
+  /** Omit for the identity edit. */
+  edit?: PercussiveEventEditInput;
+}
+
 /** How a `spectralEdit` region op modifies the masked bins. */
 export type SpectralEditMode = 'gain' | 'attenuate' | 'mute' | 'heal';
 
@@ -82,7 +540,7 @@ export interface SpectralRegionOp {
 
 /** STFT + heal parameters for `spectralEdit`. All fields are optional. */
 export interface SpectralEditOptions {
-  /** FFT size; must be a power of two (>= 2). Default 2048. */
+  /** FFT size; a power of two in `[2, 262144]`. Default 2048. */
   nFft?: number;
   /** Hop length; must satisfy 0 < hop <= nFft/2. Default 512. */
   hopLength?: number;
@@ -189,7 +647,24 @@ export interface ChromaResult {
  */
 export interface PitchResult {
   f0: Float32Array;
+  /**
+   * pYIN's per-frame voiced **observation mass**, exactly as librosa returns
+   * it: the summed probability of the frame's voiced pitch hypotheses.
+   *
+   * This is NOT a signal-quality confidence and NOT a correction weight. The
+   * mass depends on how many periods of the pitch fit inside `frameLength`,
+   * because the CMNDF troughs of a long period measured over a short frame are
+   * shallower. For a fixed `frameLength` it therefore rises with F0 even when
+   * the signal is unchanged: a steady three-harmonic tone at 2048 samples /
+   * 48 kHz averages well under 0.1 at C2 and about 0.5 at C5, with every frame
+   * flagged voiced throughout.
+   *
+   * Use {@link voicedFlag} for any voicing decision. In particular, thresholding
+   * this value at a fixed 0.5 (`noteSegments`' default) drops entire low
+   * registers.
+   */
   voicedProb: Float32Array;
+  /** Per-frame voicing decision from the Viterbi path — the voicing oracle. */
   voicedFlag: boolean[];
   nFrames: number;
   medianF0: number;
@@ -245,6 +720,17 @@ export interface LufsResult {
   loudnessRange: number;
 }
 
+/**
+ * The two per-block loudness series a multi-channel measurement builds, in
+ * LUFS. A signal shorter than a window yields an empty series for it.
+ */
+export interface LufsSeriesResult {
+  /** 400 ms momentary series. */
+  momentary: Float32Array;
+  /** 3 s short-term series. */
+  shortTerm: Float32Array;
+}
+
 /**
  * HPSS (Harmonic-Percussive Source Separation) result
  */