vizcore 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +544 -9
- data/docs/.nojekyll +0 -0
- data/docs/assets/site.css +744 -0
- data/docs/assets/vizcore-demo.gif +0 -0
- data/docs/assets/vizcore-poster.png +0 -0
- data/docs/assets/vj-tunnel.js +159 -0
- data/docs/index.html +224 -0
- data/examples/README.md +59 -0
- data/examples/assets/README.md +19 -0
- data/examples/audio_inspector.rb +34 -0
- data/examples/club_intro_drop.rb +78 -0
- data/examples/kansai_rubykaigi_visual.rb +70 -0
- data/examples/live_coding_minimal.rb +22 -0
- data/examples/midi_controller_show.rb +78 -0
- data/examples/midi_scene_switch.rb +3 -1
- data/examples/parser_visualizer.rb +48 -0
- data/examples/readme_demo.rb +17 -0
- data/examples/rhythm_geometry.rb +34 -0
- data/examples/ruby_crystal_show.rb +35 -0
- data/examples/shader_playground.rb +18 -0
- data/examples/unyo_liquid.rb +59 -0
- data/examples/vj_ambient_chill_room.rb +124 -0
- data/examples/vj_dnb_jungle.rb +170 -0
- data/examples/vj_festival_mainstage.rb +245 -0
- data/examples/vj_festival_mainstage.yml +17 -0
- data/examples/vj_glitch_industrial.rb +164 -0
- data/examples/vj_hiphop_cipher.rb +167 -0
- data/examples/vj_jpop_idol_live.rb +210 -0
- data/examples/vj_synthwave_retro.rb +173 -0
- data/examples/vj_techno_warehouse.rb +195 -0
- data/frontend/index.html +468 -2
- data/frontend/src/audio-inspector.js +40 -0
- data/frontend/src/live-controls.js +131 -0
- data/frontend/src/main.js +792 -16
- data/frontend/src/midi-learn.js +194 -0
- data/frontend/src/performance-monitor.js +183 -0
- data/frontend/src/plugin-runtime.js +130 -0
- data/frontend/src/projector-mode.js +56 -0
- data/frontend/src/renderer/engine.js +148 -3
- data/frontend/src/renderer/layer-manager.js +428 -30
- data/frontend/src/renderer/shader-manager.js +26 -0
- data/frontend/src/runtime-control-preset.js +11 -0
- data/frontend/src/shader-error-overlay.js +29 -0
- data/frontend/src/shader-param-controls.js +93 -0
- data/frontend/src/shaders/builtins.js +380 -2
- data/frontend/src/shaders/post-effects.js +52 -0
- data/frontend/src/visual-regression.js +67 -0
- data/frontend/src/visual-settings-preset.js +103 -0
- data/frontend/src/visuals/geometry.js +268 -0
- data/frontend/src/visuals/image-renderer.js +291 -0
- data/frontend/src/visuals/particle-system.js +56 -10
- data/frontend/src/visuals/spectrogram-renderer.js +226 -0
- data/frontend/src/visuals/text-renderer.js +112 -11
- data/frontend/src/websocket-client.js +12 -1
- data/lib/vizcore/analysis/adaptive_normalizer.rb +70 -0
- data/lib/vizcore/analysis/beat_detector.rb +4 -2
- data/lib/vizcore/analysis/bpm_estimator.rb +8 -0
- data/lib/vizcore/analysis/feature_recorder.rb +159 -0
- data/lib/vizcore/analysis/feature_replay.rb +84 -0
- data/lib/vizcore/analysis/pipeline.rb +235 -11
- data/lib/vizcore/analysis/tap_tempo.rb +74 -0
- data/lib/vizcore/analysis.rb +4 -0
- data/lib/vizcore/audio/dummy_sine_input.rb +1 -1
- data/lib/vizcore/audio/fixture_input.rb +65 -0
- data/lib/vizcore/audio/input_manager.rb +4 -2
- data/lib/vizcore/audio/mic_input.rb +24 -8
- data/lib/vizcore/audio/portaudio_ffi.rb +106 -1
- data/lib/vizcore/audio.rb +1 -0
- data/lib/vizcore/cli/doctor.rb +159 -0
- data/lib/vizcore/cli/dsl_reference.rb +99 -0
- data/lib/vizcore/cli/layer_docs.rb +46 -0
- data/lib/vizcore/cli/scene_diagnostics.rb +23 -0
- data/lib/vizcore/cli/scene_inspector.rb +136 -0
- data/lib/vizcore/cli/scene_validator.rb +245 -0
- data/lib/vizcore/cli/shader_template.rb +68 -0
- data/lib/vizcore/cli/shader_uniform_docs.rb +54 -0
- data/lib/vizcore/cli.rb +689 -18
- data/lib/vizcore/config.rb +103 -2
- data/lib/vizcore/control_preset.rb +68 -0
- data/lib/vizcore/dsl/engine.rb +277 -5
- data/lib/vizcore/dsl/layer_builder.rb +491 -22
- data/lib/vizcore/dsl/layer_group_builder.rb +112 -0
- data/lib/vizcore/dsl/mapping_resolver.rb +132 -3
- data/lib/vizcore/dsl/mapping_transform_builder.rb +71 -0
- data/lib/vizcore/dsl/reaction_builder.rb +44 -0
- data/lib/vizcore/dsl/scene_builder.rb +61 -5
- data/lib/vizcore/dsl/shader_source_resolver.rb +67 -6
- data/lib/vizcore/dsl/style_builder.rb +68 -0
- data/lib/vizcore/dsl/timeline_builder.rb +138 -0
- data/lib/vizcore/dsl/transition_controller.rb +77 -0
- data/lib/vizcore/dsl.rb +5 -1
- data/lib/vizcore/layer_catalog.rb +273 -0
- data/lib/vizcore/project_manifest.rb +152 -0
- data/lib/vizcore/renderer/png_writer.rb +57 -0
- data/lib/vizcore/renderer/render_sequence.rb +153 -0
- data/lib/vizcore/renderer/scene_frame_source.rb +119 -0
- data/lib/vizcore/renderer/scene_serializer.rb +36 -3
- data/lib/vizcore/renderer/snapshot.rb +38 -0
- data/lib/vizcore/renderer/snapshot_renderer.rb +446 -0
- data/lib/vizcore/renderer.rb +5 -0
- data/lib/vizcore/server/frame_broadcaster.rb +91 -5
- data/lib/vizcore/server/gallery_app.rb +155 -0
- data/lib/vizcore/server/gallery_page.rb +100 -0
- data/lib/vizcore/server/gallery_runner.rb +48 -0
- data/lib/vizcore/server/rack_app.rb +203 -4
- data/lib/vizcore/server/runner.rb +370 -22
- data/lib/vizcore/server/scene_dependency_watcher.rb +79 -0
- data/lib/vizcore/server/websocket_handler.rb +60 -10
- data/lib/vizcore/server.rb +4 -0
- data/lib/vizcore/sync/osc_message.rb +103 -0
- data/lib/vizcore/sync/osc_receiver.rb +68 -0
- data/lib/vizcore/sync.rb +4 -0
- data/lib/vizcore/templates/midi_control_scene.rb +3 -1
- data/lib/vizcore/templates/plugin_layer.rb +20 -0
- data/lib/vizcore/templates/plugin_readme.md +23 -0
- data/lib/vizcore/templates/plugin_renderer.js +43 -0
- data/lib/vizcore/templates/plugin_scene.rb +14 -0
- data/lib/vizcore/templates/project_readme.md +7 -23
- data/lib/vizcore/templates/rubykaigi_scene.rb +30 -0
- data/lib/vizcore/version.rb +1 -1
- data/lib/vizcore.rb +27 -0
- data/scripts/browser_capture.mjs +75 -0
- data/sig/vizcore.rbs +362 -0
- metadata +83 -3
- data/docs/GETTING_STARTED.md +0 -105
|
@@ -20,6 +20,40 @@ void main() {
|
|
|
20
20
|
}
|
|
21
21
|
`;
|
|
22
22
|
|
|
23
|
+
export const resolveParticleForces = ({ x, y, index, time, forceField, turbulence, bassExplosion, audio }) => {
|
|
24
|
+
const bass = clampNumber(audio?.bands?.low, 0, 1);
|
|
25
|
+
const mid = clampNumber(audio?.bands?.mid, 0, 1);
|
|
26
|
+
const high = clampNumber(audio?.bands?.high, 0, 1);
|
|
27
|
+
const pulse = clampNumber(audio?.beat_pulse || (audio?.beat ? 1 : 0), 0, 1);
|
|
28
|
+
const field = String(forceField || "drift").toLowerCase();
|
|
29
|
+
|
|
30
|
+
let fx = 0;
|
|
31
|
+
let fy = 0;
|
|
32
|
+
|
|
33
|
+
const radius = Math.max(0.001, Math.hypot(x, y));
|
|
34
|
+
const nx = x / radius;
|
|
35
|
+
const ny = y / radius;
|
|
36
|
+
|
|
37
|
+
if (field === "vortex") {
|
|
38
|
+
const strength = 0.00018 + mid * 0.00065;
|
|
39
|
+
fx += -ny * strength;
|
|
40
|
+
fy += nx * strength;
|
|
41
|
+
} else if (field === "pulse") {
|
|
42
|
+
const strength = (bass * 0.0007 + pulse * 0.0012) * (1 + clampNumber(bassExplosion, 0, 3));
|
|
43
|
+
fx += nx * strength;
|
|
44
|
+
fy += ny * strength;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const turb = clampNumber(turbulence, 0, 3);
|
|
48
|
+
if (turb > 0) {
|
|
49
|
+
const noise = Math.sin(time * (1.7 + high * 3.0) + index * 12.9898);
|
|
50
|
+
fx += Math.cos(noise * 6.28318530718) * turb * 0.00018;
|
|
51
|
+
fy += Math.sin(noise * 6.28318530718) * turb * 0.00018;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return [fx, fy];
|
|
55
|
+
};
|
|
56
|
+
|
|
23
57
|
export class ParticleSystem {
|
|
24
58
|
constructor(gl, shaderManager) {
|
|
25
59
|
this.gl = gl;
|
|
@@ -39,11 +73,11 @@ export class ParticleSystem {
|
|
|
39
73
|
this.velocities = new Float32Array(0);
|
|
40
74
|
}
|
|
41
75
|
|
|
42
|
-
render({ count, speed, size, audio, time }) {
|
|
76
|
+
render({ count, speed, size, forceField, turbulence, bassExplosion, sparkle, color, audio, time }) {
|
|
43
77
|
const particleCount = clampInt(count, 200, 20_000);
|
|
44
78
|
this.ensureParticles(particleCount);
|
|
45
|
-
this.updateParticles(speed, audio, time);
|
|
46
|
-
this.draw(size, audio);
|
|
79
|
+
this.updateParticles({ speed, forceField, turbulence, bassExplosion, audio, time });
|
|
80
|
+
this.draw({ size, sparkle, color, audio });
|
|
47
81
|
}
|
|
48
82
|
|
|
49
83
|
ensureParticles(nextCount) {
|
|
@@ -67,9 +101,9 @@ export class ParticleSystem {
|
|
|
67
101
|
}
|
|
68
102
|
}
|
|
69
103
|
|
|
70
|
-
updateParticles(speed, audio, time) {
|
|
104
|
+
updateParticles({ speed, forceField, turbulence, bassExplosion, audio, time }) {
|
|
71
105
|
const motion = 0.4 + clampNumber(speed, 0, 4);
|
|
72
|
-
const beatBoost = audio?.beat ? 1.4 : 1.0;
|
|
106
|
+
const beatBoost = audio?.beat || audio?.beat_pulse ? 1.4 : 1.0;
|
|
73
107
|
const drift = 0.0008 + clampNumber(audio?.amplitude, 0, 1) * 0.0018;
|
|
74
108
|
|
|
75
109
|
for (let index = 0; index < this.count; index += 1) {
|
|
@@ -83,6 +117,12 @@ export class ParticleSystem {
|
|
|
83
117
|
vx += swirl * 0.01;
|
|
84
118
|
vy -= swirl * 0.01;
|
|
85
119
|
|
|
120
|
+
const [fx, fy] = resolveParticleForces({ x, y, index, time, forceField, turbulence, bassExplosion, audio });
|
|
121
|
+
vx += fx;
|
|
122
|
+
vy += fy;
|
|
123
|
+
vx = clampNumber(vx * 0.997, -0.035, 0.035);
|
|
124
|
+
vy = clampNumber(vy * 0.997, -0.035, 0.035);
|
|
125
|
+
|
|
86
126
|
x += vx * motion * beatBoost;
|
|
87
127
|
y += vy * motion * beatBoost;
|
|
88
128
|
|
|
@@ -102,12 +142,13 @@ export class ParticleSystem {
|
|
|
102
142
|
}
|
|
103
143
|
}
|
|
104
144
|
|
|
105
|
-
draw(size, audio) {
|
|
145
|
+
draw({ size, sparkle, color, audio }) {
|
|
106
146
|
const gl = this.gl;
|
|
107
|
-
const pointSize = 1.5 + clampNumber(size, 0, 32);
|
|
108
147
|
const amplitude = clampNumber(audio?.amplitude, 0, 1);
|
|
109
148
|
const bass = clampNumber(audio?.bands?.low, 0, 1);
|
|
110
149
|
const high = clampNumber(audio?.bands?.high, 0, 1);
|
|
150
|
+
const sparkleAmount = clampNumber(sparkle, 0, 3) * high;
|
|
151
|
+
const pointSize = 1.5 + clampNumber(size, 0, 32) + sparkleAmount * 1.5;
|
|
111
152
|
|
|
112
153
|
gl.useProgram(this.program);
|
|
113
154
|
gl.bindBuffer(gl.ARRAY_BUFFER, this.buffer);
|
|
@@ -116,11 +157,16 @@ export class ParticleSystem {
|
|
|
116
157
|
gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
117
158
|
|
|
118
159
|
gl.uniform1f(this.pointSizeLocation, pointSize);
|
|
160
|
+
const resolvedColor = Array.isArray(color) ? color : [
|
|
161
|
+
0.35 + bass * 0.45,
|
|
162
|
+
0.55 + high * 0.35 + sparkleAmount * 0.08,
|
|
163
|
+
0.95 + amplitude * 0.05 + sparkleAmount * 0.05
|
|
164
|
+
];
|
|
119
165
|
gl.uniform3f(
|
|
120
166
|
this.colorLocation,
|
|
121
|
-
0
|
|
122
|
-
|
|
123
|
-
0
|
|
167
|
+
clampNumber(resolvedColor[0], 0, 1),
|
|
168
|
+
clampNumber(resolvedColor[1], 0, 1),
|
|
169
|
+
clampNumber(resolvedColor[2], 0, 1)
|
|
124
170
|
);
|
|
125
171
|
gl.drawArrays(gl.POINTS, 0, this.count);
|
|
126
172
|
}
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
const SPECTROGRAM_VERTEX_SHADER = `#version 300 es
|
|
2
|
+
in vec2 a_position;
|
|
3
|
+
in vec2 a_uv;
|
|
4
|
+
out vec2 v_uv;
|
|
5
|
+
void main() {
|
|
6
|
+
v_uv = a_uv;
|
|
7
|
+
gl_Position = vec4(a_position, 0.0, 1.0);
|
|
8
|
+
}
|
|
9
|
+
`;
|
|
10
|
+
|
|
11
|
+
const SPECTROGRAM_FRAGMENT_SHADER = `#version 300 es
|
|
12
|
+
precision mediump float;
|
|
13
|
+
in vec2 v_uv;
|
|
14
|
+
uniform sampler2D u_texture;
|
|
15
|
+
uniform float u_opacity;
|
|
16
|
+
out vec4 outColor;
|
|
17
|
+
|
|
18
|
+
void main() {
|
|
19
|
+
vec4 texel = texture(u_texture, v_uv);
|
|
20
|
+
outColor = vec4(texel.rgb, texel.a * u_opacity);
|
|
21
|
+
}
|
|
22
|
+
`;
|
|
23
|
+
|
|
24
|
+
const QUAD_VERTICES = new Float32Array([
|
|
25
|
+
-1.0, -1.0, 0.0, 1.0,
|
|
26
|
+
1.0, -1.0, 1.0, 1.0,
|
|
27
|
+
-1.0, 1.0, 0.0, 0.0,
|
|
28
|
+
1.0, 1.0, 1.0, 0.0
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
export class SpectrogramRenderer {
|
|
32
|
+
constructor(gl, shaderManager) {
|
|
33
|
+
this.gl = gl;
|
|
34
|
+
this.shaderManager = shaderManager;
|
|
35
|
+
this.program = this.shaderManager.getProgram(
|
|
36
|
+
"spectrogram-renderer",
|
|
37
|
+
SPECTROGRAM_VERTEX_SHADER,
|
|
38
|
+
SPECTROGRAM_FRAGMENT_SHADER
|
|
39
|
+
);
|
|
40
|
+
this.positionLocation = this.gl.getAttribLocation(this.program, "a_position");
|
|
41
|
+
this.uvLocation = this.gl.getAttribLocation(this.program, "a_uv");
|
|
42
|
+
this.textureLocation = this.gl.getUniformLocation(this.program, "u_texture");
|
|
43
|
+
this.opacityLocation = this.gl.getUniformLocation(this.program, "u_opacity");
|
|
44
|
+
this.histories = new Map();
|
|
45
|
+
|
|
46
|
+
this.buffer = this.gl.createBuffer();
|
|
47
|
+
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.buffer);
|
|
48
|
+
this.gl.bufferData(this.gl.ARRAY_BUFFER, QUAD_VERTICES, this.gl.STATIC_DRAW);
|
|
49
|
+
|
|
50
|
+
this.canvas = document.createElement("canvas");
|
|
51
|
+
this.ctx = this.canvas.getContext("2d");
|
|
52
|
+
|
|
53
|
+
this.texture = this.gl.createTexture();
|
|
54
|
+
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
|
|
55
|
+
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
|
56
|
+
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
|
57
|
+
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
|
58
|
+
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
render({ key, audio, params = {} }) {
|
|
62
|
+
const bins = normalizeSpectrogramBins(params.bins);
|
|
63
|
+
const historySize = normalizeSpectrogramHistory(params.history);
|
|
64
|
+
const scroll = normalizeSpectrogramScroll(params.scroll);
|
|
65
|
+
const gain = normalizeSpectrogramGain(params.gain);
|
|
66
|
+
const spectrum = normalizeSpectrogramSpectrum(audio?.fft, bins, gain);
|
|
67
|
+
const history = this.updateHistory({ key, bins, historySize, spectrum });
|
|
68
|
+
const image = buildSpectrogramPixels({ history, bins, historySize, scroll });
|
|
69
|
+
|
|
70
|
+
this.drawImage(image);
|
|
71
|
+
this.uploadTexture();
|
|
72
|
+
this.drawQuad({ opacity: 1 });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
updateHistory({ key, bins, historySize, spectrum }) {
|
|
76
|
+
const cacheKey = String(key || "default");
|
|
77
|
+
const current = this.histories.get(cacheKey);
|
|
78
|
+
const history = current?.bins === bins && current?.historySize === historySize ? current.frames : [];
|
|
79
|
+
history.push(spectrum);
|
|
80
|
+
while (history.length > historySize) {
|
|
81
|
+
history.shift();
|
|
82
|
+
}
|
|
83
|
+
this.histories.set(cacheKey, { bins, historySize, frames: history });
|
|
84
|
+
return history;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
drawImage({ width, height, pixels }) {
|
|
88
|
+
if (this.canvas.width !== width) {
|
|
89
|
+
this.canvas.width = width;
|
|
90
|
+
}
|
|
91
|
+
if (this.canvas.height !== height) {
|
|
92
|
+
this.canvas.height = height;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const imageData = this.ctx.createImageData(width, height);
|
|
96
|
+
imageData.data.set(pixels);
|
|
97
|
+
this.ctx.putImageData(imageData, 0, 0);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
uploadTexture() {
|
|
101
|
+
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
|
|
102
|
+
this.gl.texImage2D(
|
|
103
|
+
this.gl.TEXTURE_2D,
|
|
104
|
+
0,
|
|
105
|
+
this.gl.RGBA,
|
|
106
|
+
this.gl.RGBA,
|
|
107
|
+
this.gl.UNSIGNED_BYTE,
|
|
108
|
+
this.canvas
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
drawQuad({ opacity }) {
|
|
113
|
+
const gl = this.gl;
|
|
114
|
+
gl.useProgram(this.program);
|
|
115
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.buffer);
|
|
116
|
+
gl.enableVertexAttribArray(this.positionLocation);
|
|
117
|
+
gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 16, 0);
|
|
118
|
+
gl.enableVertexAttribArray(this.uvLocation);
|
|
119
|
+
gl.vertexAttribPointer(this.uvLocation, 2, gl.FLOAT, false, 16, 8);
|
|
120
|
+
|
|
121
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
122
|
+
gl.bindTexture(gl.TEXTURE_2D, this.texture);
|
|
123
|
+
gl.uniform1i(this.textureLocation, 0);
|
|
124
|
+
gl.uniform1f(this.opacityLocation, clamp(Number(opacity || 1), 0, 1));
|
|
125
|
+
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export const normalizeSpectrogramScroll = (value) => {
|
|
130
|
+
const scroll = String(value || "vertical").trim().toLowerCase();
|
|
131
|
+
return scroll === "horizontal" ? "horizontal" : "vertical";
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
export const normalizeSpectrogramBins = (value) => {
|
|
135
|
+
return clampInt(value || 64, 16, 256);
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
export const normalizeSpectrogramHistory = (value) => {
|
|
139
|
+
return clampInt(value || 96, 16, 512);
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
export const normalizeSpectrogramGain = (value) => {
|
|
143
|
+
const gain = Number(value);
|
|
144
|
+
if (!Number.isFinite(gain)) return 1;
|
|
145
|
+
return clamp(gain, 0.1, 8);
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
export const normalizeSpectrogramSpectrum = (value, bins, gain = 1) => {
|
|
149
|
+
const input = Array.isArray(value) || ArrayBuffer.isView(value) ? Array.from(value) : [];
|
|
150
|
+
const output = [];
|
|
151
|
+
const safeBins = normalizeSpectrogramBins(bins);
|
|
152
|
+
const safeGain = normalizeSpectrogramGain(gain);
|
|
153
|
+
|
|
154
|
+
for (let index = 0; index < safeBins; index += 1) {
|
|
155
|
+
const progress = safeBins === 1 ? 0 : index / (safeBins - 1);
|
|
156
|
+
output.push(clamp(sampleSpectrum(input, progress) * safeGain, 0, 1));
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return output;
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
export const buildSpectrogramPixels = ({ history, bins, historySize, scroll }) => {
|
|
163
|
+
const safeBins = normalizeSpectrogramBins(bins);
|
|
164
|
+
const safeHistorySize = normalizeSpectrogramHistory(historySize);
|
|
165
|
+
const direction = normalizeSpectrogramScroll(scroll);
|
|
166
|
+
const width = direction === "vertical" ? safeBins : safeHistorySize;
|
|
167
|
+
const height = direction === "vertical" ? safeHistorySize : safeBins;
|
|
168
|
+
const pixels = new Uint8ClampedArray(width * height * 4);
|
|
169
|
+
const frames = Array.isArray(history) ? history.slice(-safeHistorySize) : [];
|
|
170
|
+
|
|
171
|
+
frames.forEach((frame, frameIndex) => {
|
|
172
|
+
const timeIndex = safeHistorySize - frames.length + frameIndex;
|
|
173
|
+
for (let bin = 0; bin < safeBins; bin += 1) {
|
|
174
|
+
const value = clamp(Number(frame?.[bin] || 0), 0, 1);
|
|
175
|
+
const x = direction === "vertical" ? bin : timeIndex;
|
|
176
|
+
const y = direction === "vertical" ? timeIndex : safeBins - 1 - bin;
|
|
177
|
+
writePixel(pixels, width, x, y, spectrogramColor(value));
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
return { width, height, pixels };
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
export const spectrogramColor = (value) => {
|
|
185
|
+
const energy = clamp(Number(value || 0), 0, 1);
|
|
186
|
+
const mid = 1 - Math.abs(energy * 2 - 1);
|
|
187
|
+
return [
|
|
188
|
+
Math.round(10 + energy * 245),
|
|
189
|
+
Math.round(18 + Math.max(0, energy - 0.18) * 250),
|
|
190
|
+
Math.round(36 + mid * 155 + energy * 36),
|
|
191
|
+
Math.round(energy * 255)
|
|
192
|
+
];
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
const writePixel = (pixels, width, x, y, color) => {
|
|
196
|
+
const offset = ((y * width) + x) * 4;
|
|
197
|
+
pixels[offset] = color[0];
|
|
198
|
+
pixels[offset + 1] = color[1];
|
|
199
|
+
pixels[offset + 2] = color[2];
|
|
200
|
+
pixels[offset + 3] = color[3];
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
const sampleSpectrum = (spectrum, progress) => {
|
|
204
|
+
if (!spectrum.length) return 0;
|
|
205
|
+
|
|
206
|
+
const position = progress * (spectrum.length - 1);
|
|
207
|
+
const left = Math.floor(position);
|
|
208
|
+
const right = Math.min(left + 1, spectrum.length - 1);
|
|
209
|
+
const mix = position - left;
|
|
210
|
+
const from = finiteNumber(spectrum[left], 0);
|
|
211
|
+
const to = finiteNumber(spectrum[right], 0);
|
|
212
|
+
return clamp(from + (to - from) * mix, 0, 1);
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
const finiteNumber = (value, fallback) => {
|
|
216
|
+
const numeric = Number(value);
|
|
217
|
+
return Number.isFinite(numeric) ? numeric : fallback;
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
const clampInt = (value, min, max) => {
|
|
221
|
+
const numeric = Number(value);
|
|
222
|
+
if (!Number.isFinite(numeric)) return min;
|
|
223
|
+
return Math.round(Math.min(Math.max(numeric, min), max));
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
const clamp = (value, min, max) => Math.min(Math.max(value, min), max);
|
|
@@ -55,9 +55,9 @@ export class TextRenderer {
|
|
|
55
55
|
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
render({ content, fontSize, audio, time, color, glowStrength }) {
|
|
59
|
-
const
|
|
60
|
-
if (!
|
|
58
|
+
render({ content, fontSize, audio, time, color, fontFamily, align, letterSpacing, strokeWidth, strokeColor, shadowColor, shadowBlur, glowStrength }) {
|
|
59
|
+
const lines = normalizeTextLines(content);
|
|
60
|
+
if (!lines.length) {
|
|
61
61
|
return;
|
|
62
62
|
}
|
|
63
63
|
|
|
@@ -65,15 +65,24 @@ export class TextRenderer {
|
|
|
65
65
|
|
|
66
66
|
const amp = clamp(Number(audio?.amplitude || 0), 0, 1);
|
|
67
67
|
const beatBoost = audio?.beat ? 1.0 : 0.0;
|
|
68
|
-
const
|
|
68
|
+
const singleLineMax = Math.floor(this.canvas.height * 0.22);
|
|
69
|
+
const multilineMax = Math.floor((this.canvas.height * 0.58) / Math.max(lines.length, 1));
|
|
70
|
+
const maxFontSize = Math.max(32, Math.min(singleLineMax, multilineMax));
|
|
69
71
|
const dynamicSize = Math.round(
|
|
70
72
|
clamp(Number(fontSize || 96), 18, maxFontSize) * (1 + amp * 0.08 + beatBoost * 0.04)
|
|
71
73
|
);
|
|
72
74
|
this.drawTextToCanvas({
|
|
73
|
-
|
|
75
|
+
lines,
|
|
74
76
|
fontSize: dynamicSize,
|
|
75
77
|
time,
|
|
76
78
|
color,
|
|
79
|
+
fontFamily,
|
|
80
|
+
align,
|
|
81
|
+
letterSpacing,
|
|
82
|
+
strokeWidth,
|
|
83
|
+
strokeColor,
|
|
84
|
+
shadowColor,
|
|
85
|
+
shadowBlur,
|
|
77
86
|
amplitude: amp,
|
|
78
87
|
glowStrength: Number(glowStrength ?? 0.15)
|
|
79
88
|
});
|
|
@@ -81,7 +90,7 @@ export class TextRenderer {
|
|
|
81
90
|
this.drawQuad({ intensity: 0.85 + amp * 0.15 });
|
|
82
91
|
}
|
|
83
92
|
|
|
84
|
-
drawTextToCanvas({
|
|
93
|
+
drawTextToCanvas({ lines, fontSize, time, color, fontFamily, align, letterSpacing, strokeWidth, strokeColor, shadowColor, shadowBlur, amplitude, glowStrength }) {
|
|
85
94
|
const ctx = this.ctx;
|
|
86
95
|
ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
|
87
96
|
|
|
@@ -91,14 +100,30 @@ export class TextRenderer {
|
|
|
91
100
|
const safeColor = typeof color === "string" && color.trim() ? color : "#e5f3ff";
|
|
92
101
|
const glow = clamp(Number(glowStrength || 0), 0, 1) * (1.5 + amplitude * 5.0);
|
|
93
102
|
const xShift = Math.sin(time * 2.0) * (2 + amplitude * 4);
|
|
103
|
+
const textAlign = normalizeTextAlign(align);
|
|
104
|
+
const spacing = normalizeLetterSpacing(letterSpacing);
|
|
105
|
+
const stroke = clamp(Number(strokeWidth || 0), 0, 24);
|
|
106
|
+
const shadow = shadowBlur === undefined ? glow : clamp(Number(shadowBlur || 0), 0, 80);
|
|
94
107
|
|
|
95
|
-
ctx.textAlign =
|
|
108
|
+
ctx.textAlign = textAlign;
|
|
96
109
|
ctx.textBaseline = "middle";
|
|
97
|
-
ctx.font = `700 ${fontSize}px
|
|
98
|
-
ctx.shadowColor = "rgba(110, 208, 255, 0.35)";
|
|
99
|
-
ctx.shadowBlur =
|
|
110
|
+
ctx.font = `700 ${fontSize}px ${normalizeFontFamily(fontFamily)}`;
|
|
111
|
+
ctx.shadowColor = normalizeTextColor(shadowColor, "rgba(110, 208, 255, 0.35)");
|
|
112
|
+
ctx.shadowBlur = shadow;
|
|
100
113
|
ctx.fillStyle = safeColor;
|
|
101
|
-
|
|
114
|
+
const x = resolveTextX(this.canvas.width, textAlign) + xShift;
|
|
115
|
+
const lineHeight = fontSize * 1.16;
|
|
116
|
+
const startY = this.canvas.height / 2 - ((lines.length - 1) * lineHeight) / 2;
|
|
117
|
+
lines.forEach((line, index) => {
|
|
118
|
+
const y = startY + index * lineHeight;
|
|
119
|
+
if (stroke > 0) {
|
|
120
|
+
ctx.lineJoin = "round";
|
|
121
|
+
ctx.lineWidth = stroke;
|
|
122
|
+
ctx.strokeStyle = normalizeTextColor(strokeColor, safeColor);
|
|
123
|
+
drawText(ctx, { text: line, x, y, align: textAlign, letterSpacing: spacing, method: "strokeText" });
|
|
124
|
+
}
|
|
125
|
+
drawText(ctx, { text: line, x, y, align: textAlign, letterSpacing: spacing, method: "fillText" });
|
|
126
|
+
});
|
|
102
127
|
}
|
|
103
128
|
|
|
104
129
|
syncCanvasSize() {
|
|
@@ -141,3 +166,79 @@ export class TextRenderer {
|
|
|
141
166
|
}
|
|
142
167
|
|
|
143
168
|
const clamp = (value, min, max) => Math.min(Math.max(value, min), max);
|
|
169
|
+
|
|
170
|
+
export const normalizeTextAlign = (value) => {
|
|
171
|
+
const align = String(value || "center").trim().toLowerCase();
|
|
172
|
+
if (align === "left" || align === "right" || align === "center") {
|
|
173
|
+
return align;
|
|
174
|
+
}
|
|
175
|
+
return "center";
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
export const resolveTextX = (width, align) => {
|
|
179
|
+
const canvasWidth = Number(width) || 0;
|
|
180
|
+
if (align === "left") return canvasWidth * 0.12;
|
|
181
|
+
if (align === "right") return canvasWidth * 0.88;
|
|
182
|
+
return canvasWidth * 0.5;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
export const normalizeTextLines = (value) => {
|
|
186
|
+
return String(value || "")
|
|
187
|
+
.split(/\r?\n/)
|
|
188
|
+
.map((line) => line.trim())
|
|
189
|
+
.filter((line) => line.length > 0)
|
|
190
|
+
.slice(0, 6);
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
export const normalizeLetterSpacing = (value) => {
|
|
194
|
+
const spacing = Number(value);
|
|
195
|
+
if (!Number.isFinite(spacing)) return 0;
|
|
196
|
+
return clamp(spacing, 0, 96);
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
export const measureLetterSpacedText = (ctx, text, letterSpacing = 0) => {
|
|
200
|
+
const chars = Array.from(String(text || ""));
|
|
201
|
+
if (!chars.length) return 0;
|
|
202
|
+
|
|
203
|
+
const spacing = normalizeLetterSpacing(letterSpacing);
|
|
204
|
+
const glyphWidth = chars.reduce((total, char) => {
|
|
205
|
+
return total + Number(ctx.measureText(char)?.width || 0);
|
|
206
|
+
}, 0);
|
|
207
|
+
return glyphWidth + spacing * (chars.length - 1);
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
export const resolveLetterSpacedStartX = (ctx, text, x, align, letterSpacing = 0) => {
|
|
211
|
+
const width = measureLetterSpacedText(ctx, text, letterSpacing);
|
|
212
|
+
if (align === "right") return x - width;
|
|
213
|
+
if (align === "center") return x - width / 2;
|
|
214
|
+
return x;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
export const normalizeFontFamily = (value) => {
|
|
218
|
+
const family = String(value || "").trim();
|
|
219
|
+
if (!family) return "\"IBM Plex Sans\", \"Noto Sans JP\", sans-serif";
|
|
220
|
+
if (family.includes(",")) return `${family}, "IBM Plex Sans", "Noto Sans JP", sans-serif`;
|
|
221
|
+
|
|
222
|
+
return `"${family.replaceAll("\"", "")}", "IBM Plex Sans", "Noto Sans JP", sans-serif`;
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
const normalizeTextColor = (value, fallback) => {
|
|
226
|
+
const color = String(value || "").trim();
|
|
227
|
+
return color || fallback;
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const drawText = (ctx, { text, x, y, align, letterSpacing, method }) => {
|
|
231
|
+
if (letterSpacing <= 0) {
|
|
232
|
+
ctx[method](text, x, y);
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const originalAlign = ctx.textAlign;
|
|
237
|
+
ctx.textAlign = "left";
|
|
238
|
+
let cursor = resolveLetterSpacedStartX(ctx, text, x, align, letterSpacing);
|
|
239
|
+
for (const char of Array.from(text)) {
|
|
240
|
+
ctx[method](char, cursor, y);
|
|
241
|
+
cursor += Number(ctx.measureText(char)?.width || 0) + letterSpacing;
|
|
242
|
+
}
|
|
243
|
+
ctx.textAlign = originalAlign;
|
|
244
|
+
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const RECONNECT_INTERVAL_MS = 1000;
|
|
2
2
|
const READY_STATE_CONNECTING = 0;
|
|
3
3
|
const READY_STATE_OPEN = 1;
|
|
4
|
+
export const PROTOCOL_VERSION = "vizcore.frame.v1";
|
|
4
5
|
|
|
5
6
|
export class WebSocketClient {
|
|
6
7
|
constructor(url, callbacks = {}) {
|
|
@@ -8,6 +9,7 @@ export class WebSocketClient {
|
|
|
8
9
|
this.onFrame = callbacks.onFrame || (() => {});
|
|
9
10
|
this.onSceneChange = callbacks.onSceneChange || (() => {});
|
|
10
11
|
this.onConfigUpdate = callbacks.onConfigUpdate || (() => {});
|
|
12
|
+
this.onLatencyProbe = callbacks.onLatencyProbe || (() => {});
|
|
11
13
|
this.onStatus = callbacks.onStatus || (() => {});
|
|
12
14
|
this.socket = null;
|
|
13
15
|
this.reconnectTimer = null;
|
|
@@ -97,6 +99,10 @@ export class WebSocketClient {
|
|
|
97
99
|
return;
|
|
98
100
|
}
|
|
99
101
|
|
|
102
|
+
if (message.protocol && message.protocol !== PROTOCOL_VERSION) {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
|
|
100
106
|
if (message.type === "audio_frame") {
|
|
101
107
|
this.onFrame(message.payload);
|
|
102
108
|
return;
|
|
@@ -109,6 +115,11 @@ export class WebSocketClient {
|
|
|
109
115
|
|
|
110
116
|
if (message.type === "config_update") {
|
|
111
117
|
this.onConfigUpdate(message.payload);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (message.type === "latency_probe") {
|
|
122
|
+
this.onLatencyProbe(message.payload);
|
|
112
123
|
}
|
|
113
124
|
}
|
|
114
125
|
|
|
@@ -122,7 +133,7 @@ export class WebSocketClient {
|
|
|
122
133
|
}
|
|
123
134
|
|
|
124
135
|
try {
|
|
125
|
-
this.socket.send(JSON.stringify({ type, payload }));
|
|
136
|
+
this.socket.send(JSON.stringify({ protocol: PROTOCOL_VERSION, type, payload }));
|
|
126
137
|
return true;
|
|
127
138
|
} catch {
|
|
128
139
|
return false;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Vizcore
|
|
4
|
+
module Analysis
|
|
5
|
+
# Scales audio features against a rolling amplitude peak for repeatable mappings.
|
|
6
|
+
class AdaptiveNormalizer
|
|
7
|
+
DEFAULT_WINDOW_SIZE = 128
|
|
8
|
+
DEFAULT_TARGET = 0.85
|
|
9
|
+
DEFAULT_FLOOR = 0.05
|
|
10
|
+
|
|
11
|
+
# @param window_size [Integer] number of recent active frames used to track the peak
|
|
12
|
+
# @param target [Numeric] desired level for the rolling peak
|
|
13
|
+
# @param floor [Numeric] minimum peak level used when calculating gain
|
|
14
|
+
def initialize(window_size: DEFAULT_WINDOW_SIZE, target: DEFAULT_TARGET, floor: DEFAULT_FLOOR)
|
|
15
|
+
@window_size = normalize_window_size(window_size)
|
|
16
|
+
@target = normalize_unit(target, DEFAULT_TARGET)
|
|
17
|
+
@floor = normalize_unit(floor, DEFAULT_FLOOR)
|
|
18
|
+
@history = []
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @param amplitude [Numeric] current RMS amplitude
|
|
22
|
+
# @param bands [Hash] current frequency band values
|
|
23
|
+
# @param fft [Array<Numeric>] current FFT preview values
|
|
24
|
+
# @return [Hash] normalized feature values plus the applied gain
|
|
25
|
+
def call(amplitude:, bands:, fft:)
|
|
26
|
+
current_amplitude = normalize_unit(amplitude, 0.0)
|
|
27
|
+
@history << current_amplitude
|
|
28
|
+
@history.shift while @history.length > @window_size
|
|
29
|
+
|
|
30
|
+
gain = @target / [@history.max.to_f, @floor].max
|
|
31
|
+
{
|
|
32
|
+
amplitude: scale_value(current_amplitude, gain),
|
|
33
|
+
bands: scale_hash(bands, gain),
|
|
34
|
+
fft: scale_array(fft, gain),
|
|
35
|
+
gain: gain
|
|
36
|
+
}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def normalize_window_size(value)
|
|
42
|
+
Integer(value).clamp(1, 10_000)
|
|
43
|
+
rescue ArgumentError, TypeError
|
|
44
|
+
DEFAULT_WINDOW_SIZE
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def normalize_unit(value, fallback)
|
|
48
|
+
Float(value).clamp(0.0, 1.0)
|
|
49
|
+
rescue ArgumentError, TypeError
|
|
50
|
+
fallback
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def scale_hash(values, gain)
|
|
54
|
+
Hash(values).transform_values { |value| scale_value(value, gain) }
|
|
55
|
+
rescue StandardError
|
|
56
|
+
{}
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def scale_array(values, gain)
|
|
60
|
+
Array(values).map { |value| scale_value(value, gain) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def scale_value(value, gain)
|
|
64
|
+
(Float(value) * gain).clamp(0.0, 1.0)
|
|
65
|
+
rescue ArgumentError, TypeError
|
|
66
|
+
0.0
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -10,11 +10,13 @@ module Vizcore
|
|
|
10
10
|
# @param sensitivity [Float] multiplier applied to moving average energy
|
|
11
11
|
# @param refractory_frames [Integer] minimum frames between beat events
|
|
12
12
|
# @param min_history [Integer] minimum history size before detecting beats
|
|
13
|
-
|
|
13
|
+
# @param min_energy [Float] absolute energy floor required for beat detection
|
|
14
|
+
def initialize(history_size: 43, sensitivity: 1.35, refractory_frames: 4, min_history: 8, min_energy: 1e-6)
|
|
14
15
|
@history_size = Integer(history_size)
|
|
15
16
|
@sensitivity = Float(sensitivity)
|
|
16
17
|
@refractory_frames = Integer(refractory_frames)
|
|
17
18
|
@min_history = Integer(min_history)
|
|
19
|
+
@min_energy = Float(min_energy)
|
|
18
20
|
@energy_history = []
|
|
19
21
|
@frame_index = 0
|
|
20
22
|
@last_beat_frame = -@refractory_frames
|
|
@@ -29,7 +31,7 @@ module Vizcore
|
|
|
29
31
|
threshold = average_energy * @sensitivity
|
|
30
32
|
enough_history = @energy_history.length >= @min_history
|
|
31
33
|
refractory_ok = (@frame_index - @last_beat_frame) > @refractory_frames
|
|
32
|
-
beat = enough_history && refractory_ok && instant_energy > threshold && instant_energy
|
|
34
|
+
beat = enough_history && refractory_ok && instant_energy > threshold && instant_energy >= @min_energy
|
|
33
35
|
|
|
34
36
|
if beat
|
|
35
37
|
@beat_count += 1
|