vizcore 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +544 -9
- data/docs/.nojekyll +0 -0
- data/docs/assets/site.css +744 -0
- data/docs/assets/vizcore-demo.gif +0 -0
- data/docs/assets/vizcore-poster.png +0 -0
- data/docs/assets/vj-tunnel.js +159 -0
- data/docs/index.html +224 -0
- data/examples/README.md +59 -0
- data/examples/assets/README.md +19 -0
- data/examples/audio_inspector.rb +34 -0
- data/examples/club_intro_drop.rb +78 -0
- data/examples/kansai_rubykaigi_visual.rb +70 -0
- data/examples/live_coding_minimal.rb +22 -0
- data/examples/midi_controller_show.rb +78 -0
- data/examples/midi_scene_switch.rb +3 -1
- data/examples/parser_visualizer.rb +48 -0
- data/examples/readme_demo.rb +17 -0
- data/examples/rhythm_geometry.rb +34 -0
- data/examples/ruby_crystal_show.rb +35 -0
- data/examples/shader_playground.rb +18 -0
- data/examples/unyo_liquid.rb +59 -0
- data/examples/vj_ambient_chill_room.rb +124 -0
- data/examples/vj_dnb_jungle.rb +170 -0
- data/examples/vj_festival_mainstage.rb +245 -0
- data/examples/vj_festival_mainstage.yml +17 -0
- data/examples/vj_glitch_industrial.rb +164 -0
- data/examples/vj_hiphop_cipher.rb +167 -0
- data/examples/vj_jpop_idol_live.rb +210 -0
- data/examples/vj_synthwave_retro.rb +173 -0
- data/examples/vj_techno_warehouse.rb +195 -0
- data/frontend/index.html +468 -2
- data/frontend/src/audio-inspector.js +40 -0
- data/frontend/src/live-controls.js +131 -0
- data/frontend/src/main.js +792 -16
- data/frontend/src/midi-learn.js +194 -0
- data/frontend/src/performance-monitor.js +183 -0
- data/frontend/src/plugin-runtime.js +130 -0
- data/frontend/src/projector-mode.js +56 -0
- data/frontend/src/renderer/engine.js +148 -3
- data/frontend/src/renderer/layer-manager.js +428 -30
- data/frontend/src/renderer/shader-manager.js +26 -0
- data/frontend/src/runtime-control-preset.js +11 -0
- data/frontend/src/shader-error-overlay.js +29 -0
- data/frontend/src/shader-param-controls.js +93 -0
- data/frontend/src/shaders/builtins.js +380 -2
- data/frontend/src/shaders/post-effects.js +52 -0
- data/frontend/src/visual-regression.js +67 -0
- data/frontend/src/visual-settings-preset.js +103 -0
- data/frontend/src/visuals/geometry.js +268 -0
- data/frontend/src/visuals/image-renderer.js +291 -0
- data/frontend/src/visuals/particle-system.js +56 -10
- data/frontend/src/visuals/spectrogram-renderer.js +226 -0
- data/frontend/src/visuals/text-renderer.js +112 -11
- data/frontend/src/websocket-client.js +12 -1
- data/lib/vizcore/analysis/adaptive_normalizer.rb +70 -0
- data/lib/vizcore/analysis/beat_detector.rb +4 -2
- data/lib/vizcore/analysis/bpm_estimator.rb +8 -0
- data/lib/vizcore/analysis/feature_recorder.rb +159 -0
- data/lib/vizcore/analysis/feature_replay.rb +84 -0
- data/lib/vizcore/analysis/pipeline.rb +235 -11
- data/lib/vizcore/analysis/tap_tempo.rb +74 -0
- data/lib/vizcore/analysis.rb +4 -0
- data/lib/vizcore/audio/dummy_sine_input.rb +1 -1
- data/lib/vizcore/audio/fixture_input.rb +65 -0
- data/lib/vizcore/audio/input_manager.rb +4 -2
- data/lib/vizcore/audio/mic_input.rb +24 -8
- data/lib/vizcore/audio/portaudio_ffi.rb +106 -1
- data/lib/vizcore/audio.rb +1 -0
- data/lib/vizcore/cli/doctor.rb +159 -0
- data/lib/vizcore/cli/dsl_reference.rb +99 -0
- data/lib/vizcore/cli/layer_docs.rb +46 -0
- data/lib/vizcore/cli/scene_diagnostics.rb +23 -0
- data/lib/vizcore/cli/scene_inspector.rb +136 -0
- data/lib/vizcore/cli/scene_validator.rb +245 -0
- data/lib/vizcore/cli/shader_template.rb +68 -0
- data/lib/vizcore/cli/shader_uniform_docs.rb +54 -0
- data/lib/vizcore/cli.rb +689 -18
- data/lib/vizcore/config.rb +103 -2
- data/lib/vizcore/control_preset.rb +68 -0
- data/lib/vizcore/dsl/engine.rb +277 -5
- data/lib/vizcore/dsl/layer_builder.rb +491 -22
- data/lib/vizcore/dsl/layer_group_builder.rb +112 -0
- data/lib/vizcore/dsl/mapping_resolver.rb +132 -3
- data/lib/vizcore/dsl/mapping_transform_builder.rb +71 -0
- data/lib/vizcore/dsl/reaction_builder.rb +44 -0
- data/lib/vizcore/dsl/scene_builder.rb +61 -5
- data/lib/vizcore/dsl/shader_source_resolver.rb +67 -6
- data/lib/vizcore/dsl/style_builder.rb +68 -0
- data/lib/vizcore/dsl/timeline_builder.rb +138 -0
- data/lib/vizcore/dsl/transition_controller.rb +77 -0
- data/lib/vizcore/dsl.rb +5 -1
- data/lib/vizcore/layer_catalog.rb +273 -0
- data/lib/vizcore/project_manifest.rb +152 -0
- data/lib/vizcore/renderer/png_writer.rb +57 -0
- data/lib/vizcore/renderer/render_sequence.rb +153 -0
- data/lib/vizcore/renderer/scene_frame_source.rb +119 -0
- data/lib/vizcore/renderer/scene_serializer.rb +36 -3
- data/lib/vizcore/renderer/snapshot.rb +38 -0
- data/lib/vizcore/renderer/snapshot_renderer.rb +446 -0
- data/lib/vizcore/renderer.rb +5 -0
- data/lib/vizcore/server/frame_broadcaster.rb +91 -5
- data/lib/vizcore/server/gallery_app.rb +155 -0
- data/lib/vizcore/server/gallery_page.rb +100 -0
- data/lib/vizcore/server/gallery_runner.rb +48 -0
- data/lib/vizcore/server/rack_app.rb +203 -4
- data/lib/vizcore/server/runner.rb +370 -22
- data/lib/vizcore/server/scene_dependency_watcher.rb +79 -0
- data/lib/vizcore/server/websocket_handler.rb +60 -10
- data/lib/vizcore/server.rb +4 -0
- data/lib/vizcore/sync/osc_message.rb +103 -0
- data/lib/vizcore/sync/osc_receiver.rb +68 -0
- data/lib/vizcore/sync.rb +4 -0
- data/lib/vizcore/templates/midi_control_scene.rb +3 -1
- data/lib/vizcore/templates/plugin_layer.rb +20 -0
- data/lib/vizcore/templates/plugin_readme.md +23 -0
- data/lib/vizcore/templates/plugin_renderer.js +43 -0
- data/lib/vizcore/templates/plugin_scene.rb +14 -0
- data/lib/vizcore/templates/project_readme.md +7 -23
- data/lib/vizcore/templates/rubykaigi_scene.rb +30 -0
- data/lib/vizcore/version.rb +1 -1
- data/lib/vizcore.rb +27 -0
- data/scripts/browser_capture.mjs +75 -0
- data/sig/vizcore.rbs +362 -0
- metadata +83 -3
- data/docs/GETTING_STARTED.md +0 -105
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
require "pathname"
|
|
6
|
+
require_relative "../audio/file_input"
|
|
7
|
+
require_relative "../audio/input_manager"
|
|
8
|
+
require_relative "pipeline"
|
|
9
|
+
|
|
10
|
+
module Vizcore
|
|
11
|
+
module Analysis
|
|
12
|
+
# Records deterministic audio analysis features from a file source.
|
|
13
|
+
class FeatureRecorder
|
|
14
|
+
VERSION = "vizcore.features.v1"
|
|
15
|
+
DEFAULT_FRAME_COUNT = 300
|
|
16
|
+
DEFAULT_FRAME_RATE = 30.0
|
|
17
|
+
|
|
18
|
+
def initialize(
|
|
19
|
+
audio_file:,
|
|
20
|
+
frames: DEFAULT_FRAME_COUNT,
|
|
21
|
+
fps: DEFAULT_FRAME_RATE,
|
|
22
|
+
noise_gate: Pipeline::DEFAULT_NOISE_GATE,
|
|
23
|
+
audio_normalize: nil,
|
|
24
|
+
bpm: nil,
|
|
25
|
+
bpm_lock: false
|
|
26
|
+
)
|
|
27
|
+
@audio_file = Pathname.new(audio_file.to_s).expand_path
|
|
28
|
+
@frames = normalize_frame_count(frames)
|
|
29
|
+
@fps = normalize_frame_rate(fps)
|
|
30
|
+
@noise_gate = Float(noise_gate)
|
|
31
|
+
@audio_normalize = audio_normalize
|
|
32
|
+
@bpm = bpm
|
|
33
|
+
@bpm_lock = bpm_lock
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @param out [String, Pathname] JSON output path
|
|
37
|
+
# @return [Hash] recorder metadata
|
|
38
|
+
def write(out:)
|
|
39
|
+
output_path = Pathname.new(out.to_s).expand_path
|
|
40
|
+
FileUtils.mkdir_p(output_path.dirname)
|
|
41
|
+
payload = record
|
|
42
|
+
output_path.write("#{JSON.pretty_generate(payload)}\n")
|
|
43
|
+
{
|
|
44
|
+
path: output_path,
|
|
45
|
+
frames: @frames,
|
|
46
|
+
fps: @fps,
|
|
47
|
+
sample_rate: payload.fetch("metadata").fetch("sample_rate")
|
|
48
|
+
}
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def record
|
|
54
|
+
validate_audio_file!
|
|
55
|
+
input = Vizcore::Audio::FileInput.new(path: @audio_file.to_s)
|
|
56
|
+
raise ArgumentError, input.last_error.message if input.last_error
|
|
57
|
+
|
|
58
|
+
input.start
|
|
59
|
+
sample_rate = input.stream_sample_rate
|
|
60
|
+
capture_size = capture_size_for(sample_rate)
|
|
61
|
+
pipeline = build_pipeline(sample_rate)
|
|
62
|
+
features = @frames.times.map do |index|
|
|
63
|
+
{
|
|
64
|
+
"index" => index,
|
|
65
|
+
"time" => (index / @fps).round(6),
|
|
66
|
+
"audio" => serializable(pipeline.call(input.read(capture_size)))
|
|
67
|
+
}
|
|
68
|
+
end
|
|
69
|
+
payload(sample_rate: sample_rate, capture_size: capture_size, features: features)
|
|
70
|
+
ensure
|
|
71
|
+
input&.stop
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def payload(sample_rate:, capture_size:, features:)
|
|
75
|
+
{
|
|
76
|
+
"version" => VERSION,
|
|
77
|
+
"metadata" => {
|
|
78
|
+
"audio_file" => @audio_file.to_s,
|
|
79
|
+
"frames" => @frames,
|
|
80
|
+
"fps" => @fps,
|
|
81
|
+
"sample_rate" => sample_rate,
|
|
82
|
+
"capture_size" => capture_size,
|
|
83
|
+
"noise_gate" => @noise_gate,
|
|
84
|
+
"bpm" => @bpm,
|
|
85
|
+
"bpm_lock" => @bpm_lock,
|
|
86
|
+
"audio_normalize" => serializable(@audio_normalize)
|
|
87
|
+
},
|
|
88
|
+
"features" => features
|
|
89
|
+
}
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def build_pipeline(sample_rate)
|
|
93
|
+
Pipeline.new(
|
|
94
|
+
sample_rate: sample_rate,
|
|
95
|
+
fft_size: supported_fft_size(Vizcore::Audio::InputManager::DEFAULT_FRAME_SIZE),
|
|
96
|
+
noise_gate: @noise_gate,
|
|
97
|
+
audio_normalize: @audio_normalize,
|
|
98
|
+
bpm: @bpm,
|
|
99
|
+
bpm_lock: @bpm_lock
|
|
100
|
+
)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def validate_audio_file!
|
|
104
|
+
raise ArgumentError, "Audio file not found: #{@audio_file}" unless @audio_file.file?
|
|
105
|
+
return if Vizcore::Audio::FileInput::SUPPORTED_EXTENSIONS.include?(@audio_file.extname.downcase)
|
|
106
|
+
|
|
107
|
+
raise ArgumentError, "Unsupported audio format: #{@audio_file.extname.downcase}"
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def capture_size_for(sample_rate)
|
|
111
|
+
[(sample_rate.to_f / @fps).round, 1].max
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def supported_fft_size(size)
|
|
115
|
+
value = Integer(size)
|
|
116
|
+
return value if value.positive? && (value & (value - 1)).zero?
|
|
117
|
+
|
|
118
|
+
Vizcore::Audio::InputManager::DEFAULT_FRAME_SIZE
|
|
119
|
+
rescue StandardError
|
|
120
|
+
Vizcore::Audio::InputManager::DEFAULT_FRAME_SIZE
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def serializable(value)
|
|
124
|
+
case value
|
|
125
|
+
when Hash
|
|
126
|
+
value.each_with_object({}) do |(key, entry), output|
|
|
127
|
+
output[key.to_s] = serializable(entry)
|
|
128
|
+
end
|
|
129
|
+
when Array
|
|
130
|
+
value.map { |entry| serializable(entry) }
|
|
131
|
+
when Float
|
|
132
|
+
value.finite? ? value.round(6) : 0.0
|
|
133
|
+
when Symbol
|
|
134
|
+
value.to_s
|
|
135
|
+
else
|
|
136
|
+
value
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def normalize_frame_count(value)
|
|
141
|
+
count = Integer(value)
|
|
142
|
+
raise ArgumentError, "frames must be positive" unless count.positive?
|
|
143
|
+
|
|
144
|
+
count
|
|
145
|
+
rescue ArgumentError, TypeError
|
|
146
|
+
raise ArgumentError, "frames must be a positive integer"
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def normalize_frame_rate(value)
|
|
150
|
+
rate = Float(value)
|
|
151
|
+
raise ArgumentError, "fps must be positive" unless rate.positive?
|
|
152
|
+
|
|
153
|
+
rate
|
|
154
|
+
rescue ArgumentError, TypeError
|
|
155
|
+
raise ArgumentError, "fps must be a positive number"
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "pathname"
|
|
5
|
+
require_relative "feature_recorder"
|
|
6
|
+
|
|
7
|
+
module Vizcore
|
|
8
|
+
module Analysis
|
|
9
|
+
# Replays recorded analysis features as a pipeline-compatible source.
|
|
10
|
+
class FeatureReplay
|
|
11
|
+
attr_reader :metadata
|
|
12
|
+
|
|
13
|
+
def initialize(path:)
|
|
14
|
+
@path = Pathname.new(path.to_s).expand_path
|
|
15
|
+
payload = load_payload
|
|
16
|
+
@metadata = deep_symbolize(payload.fetch("metadata", {}))
|
|
17
|
+
@features = load_features(payload)
|
|
18
|
+
@cursor = 0
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @param _samples [Array<Float>, nil] ignored; replay data already contains analyzed features
|
|
22
|
+
# @return [Hash<Symbol, Object>] recorded audio analysis for the next frame
|
|
23
|
+
def call(_samples = nil)
|
|
24
|
+
audio = @features.fetch(@cursor)
|
|
25
|
+
@cursor = (@cursor + 1) % @features.length
|
|
26
|
+
deep_dup(audio)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def frame_count
|
|
30
|
+
@features.length
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def load_payload
|
|
36
|
+
raise ArgumentError, "Feature file not found: #{@path}" unless @path.file?
|
|
37
|
+
|
|
38
|
+
payload = JSON.parse(@path.read)
|
|
39
|
+
version = payload["version"]
|
|
40
|
+
return payload if version == FeatureRecorder::VERSION
|
|
41
|
+
|
|
42
|
+
raise ArgumentError, "Unsupported feature file version: #{version.inspect}"
|
|
43
|
+
rescue JSON::ParserError => e
|
|
44
|
+
raise ArgumentError, "Invalid feature file JSON: #{e.message}"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def load_features(payload)
|
|
48
|
+
features = Array(payload["features"]).map.with_index do |entry, index|
|
|
49
|
+
audio = Hash(entry).fetch("audio", nil)
|
|
50
|
+
raise ArgumentError, "Feature frame #{index} is missing audio data" unless audio.is_a?(Hash)
|
|
51
|
+
|
|
52
|
+
deep_symbolize(audio)
|
|
53
|
+
end
|
|
54
|
+
raise ArgumentError, "Feature file contains no frames" if features.empty?
|
|
55
|
+
|
|
56
|
+
features
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def deep_symbolize(value)
|
|
60
|
+
case value
|
|
61
|
+
when Hash
|
|
62
|
+
value.each_with_object({}) do |(key, entry), output|
|
|
63
|
+
output[key.to_s.to_sym] = deep_symbolize(entry)
|
|
64
|
+
end
|
|
65
|
+
when Array
|
|
66
|
+
value.map { |entry| deep_symbolize(entry) }
|
|
67
|
+
else
|
|
68
|
+
value
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def deep_dup(value)
|
|
73
|
+
case value
|
|
74
|
+
when Hash
|
|
75
|
+
value.each_with_object({}) { |(key, entry), output| output[key] = deep_dup(entry) }
|
|
76
|
+
when Array
|
|
77
|
+
value.map { |entry| deep_dup(entry) }
|
|
78
|
+
else
|
|
79
|
+
value
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -4,6 +4,12 @@ module Vizcore
|
|
|
4
4
|
module Analysis
|
|
5
5
|
# End-to-end analysis pipeline from PCM samples to renderer-ready features.
|
|
6
6
|
class Pipeline
|
|
7
|
+
BEAT_PULSE_DECAY = 0.86
|
|
8
|
+
BEAT_PULSE_FLOOR = 0.001
|
|
9
|
+
DEFAULT_NOISE_GATE = 0.01
|
|
10
|
+
DEFAULT_AUDIO_NORMALIZE = { mode: :off }.freeze
|
|
11
|
+
SILENCE_RESET_FRAMES = 90
|
|
12
|
+
|
|
7
13
|
attr_reader :fft_processor, :band_splitter, :beat_detector, :bpm_estimator, :smoother
|
|
8
14
|
|
|
9
15
|
# @param sample_rate [Integer]
|
|
@@ -12,38 +18,248 @@ module Vizcore
|
|
|
12
18
|
# @param beat_detector [Vizcore::Analysis::BeatDetector, nil]
|
|
13
19
|
# @param bpm_estimator [Vizcore::Analysis::BPMEstimator, nil]
|
|
14
20
|
# @param smoother [Vizcore::Analysis::Smoother, nil]
|
|
15
|
-
|
|
21
|
+
# @param noise_gate [Numeric] RMS threshold below which input is treated as silence
|
|
22
|
+
# @param audio_normalize [Hash, nil] optional audio normalization settings
|
|
23
|
+
# @param bpm [Numeric, nil] fixed BPM value used when bpm_lock is true
|
|
24
|
+
# @param bpm_lock [Boolean] true when BPM output should stay fixed
|
|
25
|
+
def initialize(sample_rate: 44_100, fft_size: 1024, window: :hamming, beat_detector: nil, bpm_estimator: nil, smoother: nil, noise_gate: DEFAULT_NOISE_GATE, audio_normalize: nil, bpm: nil, bpm_lock: false)
|
|
16
26
|
@fft_processor = FFTProcessor.new(sample_rate: sample_rate, fft_size: fft_size, window: window)
|
|
17
27
|
@band_splitter = BandSplitter.new(sample_rate: sample_rate, fft_size: fft_size)
|
|
18
28
|
@beat_detector = beat_detector || BeatDetector.new
|
|
19
|
-
|
|
20
|
-
@bpm_estimator = bpm_estimator || BPMEstimator.new(frame_rate:
|
|
29
|
+
@analysis_frame_rate = sample_rate.to_f / fft_size.to_f
|
|
30
|
+
@bpm_estimator = bpm_estimator || BPMEstimator.new(frame_rate: @analysis_frame_rate)
|
|
21
31
|
@smoother = smoother || Smoother.new(alpha: 0.35)
|
|
32
|
+
@noise_gate = normalize_noise_gate(noise_gate)
|
|
33
|
+
self.bpm_lock = { bpm: bpm, locked: bpm_lock }
|
|
34
|
+
self.audio_normalize = audio_normalize
|
|
35
|
+
@beat_pulse = 0.0
|
|
36
|
+
@last_bpm = 0.0
|
|
37
|
+
@silent_frame_count = 0
|
|
38
|
+
@previous_onset_amplitude = 0.0
|
|
39
|
+
@previous_onset_bands = {}
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @param settings [Hash, nil]
|
|
43
|
+
# @return [Hash] normalized settings
|
|
44
|
+
def audio_normalize=(settings)
|
|
45
|
+
@audio_normalize = normalize_audio_normalize(settings)
|
|
46
|
+
@normalizer = build_normalizer(@audio_normalize)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @param settings [Hash]
|
|
50
|
+
# @return [Float, nil]
|
|
51
|
+
def bpm_lock=(settings)
|
|
52
|
+
values = symbolize_hash(settings)
|
|
53
|
+
@locked_bpm = normalize_locked_bpm(values[:bpm], bpm_lock: values[:locked])
|
|
54
|
+
@last_bpm = @locked_bpm if @locked_bpm
|
|
22
55
|
end
|
|
23
56
|
|
|
24
57
|
# @param samples [Array<Numeric>] audio frame samples
|
|
25
58
|
# @return [Hash] normalized analysis payload consumed by frame broadcaster
|
|
26
59
|
def call(samples)
|
|
60
|
+
amplitude = rms(samples)
|
|
61
|
+
if silence?(amplitude)
|
|
62
|
+
track_silent_frame(samples)
|
|
63
|
+
return silent_frame(reset_tempo: sustained_silence?)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
@silent_frame_count = 0
|
|
67
|
+
|
|
27
68
|
fft = @fft_processor.call(samples)
|
|
28
69
|
bands = @band_splitter.call(fft[:magnitudes])
|
|
29
70
|
beat = @beat_detector.call(samples)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
71
|
+
beat_detected = beat[:beat]
|
|
72
|
+
confidence = beat_confidence(beat)
|
|
73
|
+
@beat_pulse = beat_detected ? 1.0 : @beat_pulse * BEAT_PULSE_DECAY
|
|
74
|
+
@beat_pulse = 0.0 if @beat_pulse < BEAT_PULSE_FLOOR
|
|
75
|
+
bpm = resolve_bpm(beat_detected)
|
|
76
|
+
normalized = normalize_features(
|
|
77
|
+
amplitude: amplitude,
|
|
78
|
+
bands: bands,
|
|
79
|
+
fft: preview_spectrum(fft[:magnitudes])
|
|
80
|
+
)
|
|
81
|
+
onsets = detect_onsets(amplitude: normalized[:amplitude], bands: normalized[:bands])
|
|
82
|
+
drums = detect_drum_sources(bands: normalized[:bands], onsets: onsets[:bands])
|
|
33
83
|
|
|
34
84
|
{
|
|
35
|
-
amplitude: @smoother.smooth(:amplitude, amplitude),
|
|
36
|
-
bands: @smoother.smooth_hash(bands, namespace: :bands),
|
|
37
|
-
fft: @smoother.smooth_array(
|
|
38
|
-
|
|
85
|
+
amplitude: @smoother.smooth(:amplitude, normalized[:amplitude]),
|
|
86
|
+
bands: @smoother.smooth_hash(normalized[:bands], namespace: :bands),
|
|
87
|
+
fft: @smoother.smooth_array(normalized[:fft], namespace: :fft),
|
|
88
|
+
onset: onsets[:amplitude],
|
|
89
|
+
onsets: onsets[:bands],
|
|
90
|
+
drums: drums,
|
|
91
|
+
beat: beat_detected,
|
|
92
|
+
beat_confidence: confidence,
|
|
93
|
+
beat_pulse: @beat_pulse,
|
|
39
94
|
beat_count: beat[:beat_count],
|
|
40
|
-
bpm:
|
|
95
|
+
bpm: bpm,
|
|
41
96
|
peak_frequency: fft[:peak_frequency]
|
|
42
97
|
}
|
|
43
98
|
end
|
|
44
99
|
|
|
45
100
|
private
|
|
46
101
|
|
|
102
|
+
def silence?(amplitude)
|
|
103
|
+
amplitude < @noise_gate
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def normalize_noise_gate(value)
|
|
107
|
+
Float(value).clamp(0.0, 1.0)
|
|
108
|
+
rescue ArgumentError, TypeError
|
|
109
|
+
DEFAULT_NOISE_GATE
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def normalize_audio_normalize(value)
|
|
113
|
+
settings = DEFAULT_AUDIO_NORMALIZE.merge(symbolize_hash(value))
|
|
114
|
+
mode = settings[:mode].to_s.strip.to_sym
|
|
115
|
+
raise ArgumentError, "unsupported audio_normalize mode: #{settings[:mode]}" unless %i[off adaptive].include?(mode)
|
|
116
|
+
|
|
117
|
+
settings.merge(mode: mode)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def normalize_locked_bpm(value, bpm_lock:)
|
|
121
|
+
return nil unless bpm_lock
|
|
122
|
+
|
|
123
|
+
numeric = Float(value)
|
|
124
|
+
raise ArgumentError, "bpm must be positive when bpm_lock is enabled" unless numeric.positive?
|
|
125
|
+
|
|
126
|
+
numeric
|
|
127
|
+
rescue ArgumentError, TypeError
|
|
128
|
+
raise ArgumentError, "bpm must be a positive number when bpm_lock is enabled"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def build_normalizer(settings)
|
|
132
|
+
return nil unless settings[:mode] == :adaptive
|
|
133
|
+
|
|
134
|
+
AdaptiveNormalizer.new(
|
|
135
|
+
window_size: normalization_window_size(settings),
|
|
136
|
+
target: settings.fetch(:target, AdaptiveNormalizer::DEFAULT_TARGET),
|
|
137
|
+
floor: settings.fetch(:floor, AdaptiveNormalizer::DEFAULT_FLOOR)
|
|
138
|
+
)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def normalization_window_size(settings)
|
|
142
|
+
return settings[:window_size] if settings.key?(:window_size)
|
|
143
|
+
|
|
144
|
+
seconds = settings.fetch(:window, nil)
|
|
145
|
+
return AdaptiveNormalizer::DEFAULT_WINDOW_SIZE if seconds.nil?
|
|
146
|
+
|
|
147
|
+
(Float(seconds) * @analysis_frame_rate).round.clamp(1, 10_000)
|
|
148
|
+
rescue ArgumentError, TypeError
|
|
149
|
+
AdaptiveNormalizer::DEFAULT_WINDOW_SIZE
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def normalize_features(amplitude:, bands:, fft:)
|
|
153
|
+
return { amplitude: amplitude, bands: bands, fft: fft } unless @normalizer
|
|
154
|
+
|
|
155
|
+
@normalizer.call(amplitude: amplitude, bands: bands, fft: fft)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def detect_onsets(amplitude:, bands:)
|
|
159
|
+
current_amplitude = Float(amplitude).clamp(0.0, 1.0)
|
|
160
|
+
current_bands = Hash(bands).transform_values { |value| Float(value).clamp(0.0, 1.0) }
|
|
161
|
+
|
|
162
|
+
amplitude_onset = positive_delta(current_amplitude, @previous_onset_amplitude)
|
|
163
|
+
band_onsets = current_bands.each_with_object({}) do |(key, value), output|
|
|
164
|
+
output[key] = positive_delta(value, @previous_onset_bands[key].to_f)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
@previous_onset_amplitude = current_amplitude
|
|
168
|
+
@previous_onset_bands = current_bands
|
|
169
|
+
|
|
170
|
+
{ amplitude: amplitude_onset, bands: band_onsets }
|
|
171
|
+
rescue ArgumentError, TypeError
|
|
172
|
+
{ amplitude: 0.0, bands: {} }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def positive_delta(current, previous)
|
|
176
|
+
[current - previous, 0.0].max.clamp(0.0, 1.0)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def detect_drum_sources(bands:, onsets:)
|
|
180
|
+
band_values = Hash(bands)
|
|
181
|
+
onset_values = Hash(onsets)
|
|
182
|
+
|
|
183
|
+
{
|
|
184
|
+
kick: drum_confidence([:sub, :low], band_values, onset_values),
|
|
185
|
+
snare: drum_confidence([:mid], band_values, onset_values),
|
|
186
|
+
hihat: drum_confidence([:high], band_values, onset_values)
|
|
187
|
+
}
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def drum_confidence(keys, bands, onsets)
|
|
191
|
+
level = keys.map { |key| Float(bands[key] || 0.0) }.max || 0.0
|
|
192
|
+
rise = keys.map { |key| Float(onsets[key] || 0.0) }.max || 0.0
|
|
193
|
+
(level * rise).clamp(0.0, 1.0)
|
|
194
|
+
rescue ArgumentError, TypeError
|
|
195
|
+
0.0
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def silent_frame(reset_tempo:)
|
|
199
|
+
@beat_pulse = 0.0
|
|
200
|
+
reset_tempo_state if reset_tempo
|
|
201
|
+
@smoother.reset if @smoother.respond_to?(:reset)
|
|
202
|
+
@previous_onset_amplitude = 0.0
|
|
203
|
+
@previous_onset_bands = {}
|
|
204
|
+
|
|
205
|
+
{
|
|
206
|
+
amplitude: 0.0,
|
|
207
|
+
bands: { sub: 0.0, low: 0.0, mid: 0.0, high: 0.0 },
|
|
208
|
+
fft: Array.new(32, 0.0),
|
|
209
|
+
onset: 0.0,
|
|
210
|
+
onsets: { sub: 0.0, low: 0.0, mid: 0.0, high: 0.0 },
|
|
211
|
+
drums: { kick: 0.0, snare: 0.0, hihat: 0.0 },
|
|
212
|
+
beat: false,
|
|
213
|
+
beat_confidence: 0.0,
|
|
214
|
+
beat_pulse: 0.0,
|
|
215
|
+
beat_count: current_beat_count,
|
|
216
|
+
bpm: @last_bpm,
|
|
217
|
+
peak_frequency: 0.0
|
|
218
|
+
}
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def reset_tempo_state
|
|
222
|
+
@last_bpm = @locked_bpm || 0.0
|
|
223
|
+
@bpm_estimator.reset if @bpm_estimator.respond_to?(:reset)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def track_silent_frame(samples)
|
|
227
|
+
@silent_frame_count += 1
|
|
228
|
+
@beat_detector.call(samples) if @beat_detector.respond_to?(:call)
|
|
229
|
+
@last_bpm = @bpm_estimator.call(beat: false).to_f if @bpm_estimator.respond_to?(:call)
|
|
230
|
+
rescue StandardError
|
|
231
|
+
nil
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def sustained_silence?
|
|
235
|
+
@silent_frame_count == SILENCE_RESET_FRAMES
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def current_beat_count
|
|
239
|
+
return Integer(@beat_detector.beat_count) if @beat_detector.respond_to?(:beat_count)
|
|
240
|
+
|
|
241
|
+
0
|
|
242
|
+
rescue StandardError
|
|
243
|
+
0
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def beat_confidence(beat)
|
|
247
|
+
threshold = Float(beat[:threshold])
|
|
248
|
+
instant_energy = Float(beat[:instant_energy])
|
|
249
|
+
return beat[:beat] ? 1.0 : 0.0 unless threshold.positive?
|
|
250
|
+
|
|
251
|
+
(instant_energy / threshold).clamp(0.0, 1.0)
|
|
252
|
+
rescue ArgumentError, TypeError
|
|
253
|
+
beat[:beat] ? 1.0 : 0.0
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def resolve_bpm(beat_detected)
|
|
257
|
+
return @last_bpm = @locked_bpm if @locked_bpm
|
|
258
|
+
|
|
259
|
+
bpm = @bpm_estimator.call(beat: beat_detected)
|
|
260
|
+
@last_bpm = @smoother.smooth(:bpm, bpm, alpha: 0.2).to_f
|
|
261
|
+
end
|
|
262
|
+
|
|
47
263
|
def preview_spectrum(magnitudes, bins: 32)
|
|
48
264
|
values = Array(magnitudes)
|
|
49
265
|
return Array.new(bins, 0.0) if values.empty?
|
|
@@ -67,6 +283,14 @@ module Vizcore
|
|
|
67
283
|
rescue ArgumentError, TypeError
|
|
68
284
|
0.0
|
|
69
285
|
end
|
|
286
|
+
|
|
287
|
+
def symbolize_hash(value)
|
|
288
|
+
Hash(value).each_with_object({}) do |(key, entry), output|
|
|
289
|
+
output[key.to_sym] = entry
|
|
290
|
+
end
|
|
291
|
+
rescue StandardError
|
|
292
|
+
{}
|
|
293
|
+
end
|
|
70
294
|
end
|
|
71
295
|
end
|
|
72
296
|
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Vizcore
|
|
4
|
+
module Analysis
|
|
5
|
+
# Estimates a fixed BPM from manual tap timestamps.
|
|
6
|
+
class TapTempo
|
|
7
|
+
DEFAULT_MIN_BPM = 40.0
|
|
8
|
+
DEFAULT_MAX_BPM = 240.0
|
|
9
|
+
DEFAULT_HISTORY_SIZE = 4
|
|
10
|
+
DEFAULT_RESET_AFTER_MS = 2_500.0
|
|
11
|
+
|
|
12
|
+
def initialize(
|
|
13
|
+
min_bpm: DEFAULT_MIN_BPM,
|
|
14
|
+
max_bpm: DEFAULT_MAX_BPM,
|
|
15
|
+
history_size: DEFAULT_HISTORY_SIZE,
|
|
16
|
+
reset_after_ms: DEFAULT_RESET_AFTER_MS
|
|
17
|
+
)
|
|
18
|
+
@min_bpm = Float(min_bpm)
|
|
19
|
+
@max_bpm = Float(max_bpm)
|
|
20
|
+
@history_size = Integer(history_size).clamp(1, 16)
|
|
21
|
+
@reset_after_ms = Float(reset_after_ms)
|
|
22
|
+
@last_tap_ms = nil
|
|
23
|
+
@intervals = []
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @param timestamp_ms [Numeric] tap timestamp in milliseconds
|
|
27
|
+
# @return [Float, nil] estimated BPM after at least two taps
|
|
28
|
+
def tap(timestamp_ms:)
|
|
29
|
+
current = Float(timestamp_ms)
|
|
30
|
+
reset_if_stale(current)
|
|
31
|
+
return remember_first_tap(current) unless @last_tap_ms
|
|
32
|
+
|
|
33
|
+
interval = current - @last_tap_ms
|
|
34
|
+
@last_tap_ms = current
|
|
35
|
+
return nil unless valid_interval?(interval)
|
|
36
|
+
|
|
37
|
+
@intervals << interval
|
|
38
|
+
@intervals.shift while @intervals.length > @history_size
|
|
39
|
+
bpm_from_intervals
|
|
40
|
+
rescue ArgumentError, TypeError
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def reset_if_stale(current)
|
|
47
|
+
return unless @last_tap_ms
|
|
48
|
+
return unless current - @last_tap_ms > @reset_after_ms
|
|
49
|
+
|
|
50
|
+
@last_tap_ms = nil
|
|
51
|
+
@intervals.clear
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def remember_first_tap(current)
|
|
55
|
+
@last_tap_ms = current
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def valid_interval?(interval)
|
|
60
|
+
return false unless interval.positive?
|
|
61
|
+
|
|
62
|
+
bpm = 60_000.0 / interval
|
|
63
|
+
bpm.between?(@min_bpm, @max_bpm)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def bpm_from_intervals
|
|
67
|
+
return nil if @intervals.empty?
|
|
68
|
+
|
|
69
|
+
average_interval = @intervals.sum / @intervals.length.to_f
|
|
70
|
+
(60_000.0 / average_interval).clamp(@min_bpm, @max_bpm)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
data/lib/vizcore/analysis.rb
CHANGED
|
@@ -7,8 +7,12 @@ module Vizcore
|
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
require_relative "analysis/band_splitter"
|
|
10
|
+
require_relative "analysis/adaptive_normalizer"
|
|
10
11
|
require_relative "analysis/beat_detector"
|
|
11
12
|
require_relative "analysis/bpm_estimator"
|
|
12
13
|
require_relative "analysis/fft_processor"
|
|
13
14
|
require_relative "analysis/pipeline"
|
|
15
|
+
require_relative "analysis/feature_recorder"
|
|
16
|
+
require_relative "analysis/feature_replay"
|
|
14
17
|
require_relative "analysis/smoother"
|
|
18
|
+
require_relative "analysis/tap_tempo"
|
|
@@ -4,7 +4,7 @@ require_relative "base_input"
|
|
|
4
4
|
|
|
5
5
|
module Vizcore
|
|
6
6
|
module Audio
|
|
7
|
-
# Deterministic sine-wave generator used
|
|
7
|
+
# Deterministic sine-wave generator used for the explicit dummy source.
|
|
8
8
|
class DummySineInput < BaseInput
|
|
9
9
|
# Default oscillator amplitude.
|
|
10
10
|
DEFAULT_AMPLITUDE = 0.45
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base_input"
|
|
4
|
+
|
|
5
|
+
module Vizcore
|
|
6
|
+
module Audio
|
|
7
|
+
# Deterministic sample-frame input for tests and repeatable development checks.
|
|
8
|
+
class FixtureInput < BaseInput
|
|
9
|
+
# @param frames [Array<Array<Numeric>>] sample frames returned in order
|
|
10
|
+
# @param sample_rate [Integer]
|
|
11
|
+
# @param loop [Boolean] whether to repeat frames after the last one
|
|
12
|
+
def initialize(frames:, sample_rate: 44_100, loop: true)
|
|
13
|
+
super(sample_rate: sample_rate)
|
|
14
|
+
@frames = normalize_frames(frames)
|
|
15
|
+
@loop = !!loop
|
|
16
|
+
@index = 0
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# @param frame_size [Integer]
|
|
20
|
+
# @return [Array<Float>]
|
|
21
|
+
def read(frame_size)
|
|
22
|
+
count = Integer(frame_size)
|
|
23
|
+
return Array.new(count, 0.0) unless running?
|
|
24
|
+
|
|
25
|
+
frame = next_frame
|
|
26
|
+
return Array.new(count, 0.0) unless frame
|
|
27
|
+
|
|
28
|
+
normalize_frame_size(frame, count)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @return [void]
|
|
32
|
+
def reset
|
|
33
|
+
@index = 0
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def next_frame
|
|
39
|
+
return nil if @frames.empty?
|
|
40
|
+
return nil if @index >= @frames.length && !@loop
|
|
41
|
+
|
|
42
|
+
frame = @frames[@index % @frames.length]
|
|
43
|
+
@index += 1
|
|
44
|
+
frame
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def normalize_frames(frames)
|
|
48
|
+
Array(frames).map do |frame|
|
|
49
|
+
values = Array(frame).map { |sample| Float(sample) }
|
|
50
|
+
raise ArgumentError, "fixture frames must not be empty" if values.empty?
|
|
51
|
+
|
|
52
|
+
values
|
|
53
|
+
end
|
|
54
|
+
rescue ArgumentError, TypeError
|
|
55
|
+
raise ArgumentError, "fixture frames must contain numeric samples"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def normalize_frame_size(frame, count)
|
|
59
|
+
return frame.first(count) if frame.length >= count
|
|
60
|
+
|
|
61
|
+
frame + Array.new(count - frame.length, 0.0)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|