gsplat 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +202 -0
- data/README.md +236 -0
- data/docs/ACCEPTANCE.md +60 -0
- data/docs/BENCHMARKS.md +79 -0
- data/docs/DECISIONS.md +52 -0
- data/docs/MIGRATION.md +117 -0
- data/docs/PROFILE.md +75 -0
- data/docs/PROGRESS.md +199 -0
- data/docs/decisions/0000-template.md +17 -0
- data/docs/decisions/0001-pin-golden-reference-and-cpu-validation.md +27 -0
- data/docs/decisions/0002-retain-ruby-fallbacks-for-native-backend.md +26 -0
- data/docs/decisions/0003-share-projection-for-distorted-cameras.md +26 -0
- data/docs/decisions/0004-use-portable-world-space-reference-paths.md +26 -0
- data/docs/decisions/0005-reuse-ewa-core-for-2dgs.md +24 -0
- data/docs/decisions/0006-keep-eval3d-as-portable-reference.md +25 -0
- data/docs/decisions/0007-share-compositor-semantics-for-contribution-indices.md +25 -0
- data/examples/data/README.md +14 -0
- data/examples/data/colmap/images/view_000.png +0 -0
- data/examples/data/colmap/images/view_001.png +0 -0
- data/examples/data/colmap/images/view_002.png +0 -0
- data/examples/data/colmap/sparse/0/cameras.txt +2 -0
- data/examples/data/colmap/sparse/0/images.txt +6 -0
- data/examples/data/colmap/sparse/0/points3D.txt +17 -0
- data/examples/data/splats.ply +0 -0
- data/examples/fit_image.rb +35 -0
- data/examples/generate_sample_data.rb +147 -0
- data/examples/render_path.rb +117 -0
- data/examples/simple_trainer.rb +76 -0
- data/ext/gsplat_native/common.h +54 -0
- data/ext/gsplat_native/extconf.rb +39 -0
- data/ext/gsplat_native/gsplat_native.c +60 -0
- data/ext/gsplat_native/intersections.c +211 -0
- data/ext/gsplat_native/projection.c +199 -0
- data/ext/gsplat_native/raster_backward.c +129 -0
- data/ext/gsplat_native/raster_backward_bridge.c +69 -0
- data/ext/gsplat_native/raster_forward.c +150 -0
- data/ext/gsplat_native/rasterization.h +51 -0
- data/ext/gsplat_native/spherical_harmonics.c +129 -0
- data/gsplat.gemspec +36 -0
- data/lib/gsplat/autograd/context.rb +60 -0
- data/lib/gsplat/autograd/function.rb +68 -0
- data/lib/gsplat/autograd/variable.rb +159 -0
- data/lib/gsplat/backend/ruby/accumulate.rb +139 -0
- data/lib/gsplat/backend/ruby/accumulate_backward.rb +40 -0
- data/lib/gsplat/backend/ruby/eval3d_rasterizer.rb +175 -0
- data/lib/gsplat/backend/ruby/isect_tiles.rb +198 -0
- data/lib/gsplat/backend/ruby/projection.rb +251 -0
- data/lib/gsplat/backend/ruby/projection_backward.rb +190 -0
- data/lib/gsplat/backend/ruby/projection_covariance_vjp.rb +72 -0
- data/lib/gsplat/backend/ruby/projection_input_vjp.rb +252 -0
- data/lib/gsplat/backend/ruby/quat_scale_to_covar_preci.rb +139 -0
- data/lib/gsplat/backend/ruby/rasterize_to_indices_in_range.rb +121 -0
- data/lib/gsplat/backend/ruby/rasterize_to_pixels.rb +199 -0
- data/lib/gsplat/backend/ruby/rasterize_to_pixels_backward.rb +121 -0
- data/lib/gsplat/backend/ruby/spherical_harmonics.rb +135 -0
- data/lib/gsplat/backend/ruby/tile_compositor.rb +50 -0
- data/lib/gsplat/backend/ruby/tile_compositor_backward.rb +84 -0
- data/lib/gsplat/backend.rb +79 -0
- data/lib/gsplat/compression/grid_sort.rb +79 -0
- data/lib/gsplat/compression/kmeans.rb +121 -0
- data/lib/gsplat/compression/png.rb +147 -0
- data/lib/gsplat/compression/png_codec.rb +133 -0
- data/lib/gsplat/compression/quantizer.rb +79 -0
- data/lib/gsplat/io/checkpoint.rb +145 -0
- data/lib/gsplat/io/colmap.rb +175 -0
- data/lib/gsplat/io/colmap_binary.rb +98 -0
- data/lib/gsplat/io/colmap_text.rb +84 -0
- data/lib/gsplat/io/image.rb +63 -0
- data/lib/gsplat/io/image_backends.rb +81 -0
- data/lib/gsplat/io/npy.rb +189 -0
- data/lib/gsplat/io/ply.rb +185 -0
- data/lib/gsplat/io/ply_reader.rb +142 -0
- data/lib/gsplat/io/zip_archive.rb +183 -0
- data/lib/gsplat/math/camera_distortion.rb +123 -0
- data/lib/gsplat/math/camera_projection.rb +202 -0
- data/lib/gsplat/math/mat.rb +114 -0
- data/lib/gsplat/math/quaternion.rb +175 -0
- data/lib/gsplat/math/small_matrix_primitives.rb +112 -0
- data/lib/gsplat/math/spherical_harmonic_basis.rb +148 -0
- data/lib/gsplat/math/ssim.rb +153 -0
- data/lib/gsplat/native.rb +30 -0
- data/lib/gsplat/native_ops.rb +148 -0
- data/lib/gsplat/native_raster_ops.rb +76 -0
- data/lib/gsplat/ops/accumulate.rb +68 -0
- data/lib/gsplat/ops/eval3d_rasterize.rb +62 -0
- data/lib/gsplat/ops/isect_tiles.rb +38 -0
- data/lib/gsplat/ops/projection.rb +154 -0
- data/lib/gsplat/ops/quat_scale_to_covar_preci.rb +118 -0
- data/lib/gsplat/ops/rasterize_to_indices_in_range.rb +38 -0
- data/lib/gsplat/ops/rasterize_to_pixels.rb +105 -0
- data/lib/gsplat/ops/relocation.rb +94 -0
- data/lib/gsplat/ops/spherical_harmonics.rb +71 -0
- data/lib/gsplat/ops/tensor_shape_ops.rb +173 -0
- data/lib/gsplat/ops/tensor_value_ops.rb +150 -0
- data/lib/gsplat/optim/adam.rb +212 -0
- data/lib/gsplat/optim/lr_scheduler.rb +36 -0
- data/lib/gsplat/optim/selective_adam.rb +68 -0
- data/lib/gsplat/rasterization.rb +130 -0
- data/lib/gsplat/rasterization_2dgs.rb +140 -0
- data/lib/gsplat/rasterization_helpers.rb +162 -0
- data/lib/gsplat/rasterization_validation.rb +99 -0
- data/lib/gsplat/strategy/base.rb +49 -0
- data/lib/gsplat/strategy/default.rb +188 -0
- data/lib/gsplat/strategy/mcmc.rb +103 -0
- data/lib/gsplat/strategy/mcmc_ops.rb +143 -0
- data/lib/gsplat/strategy/ops.rb +165 -0
- data/lib/gsplat/training/config.rb +91 -0
- data/lib/gsplat/training/image_fitter.rb +158 -0
- data/lib/gsplat/training/losses.rb +191 -0
- data/lib/gsplat/training/scene.rb +116 -0
- data/lib/gsplat/training/trainer.rb +236 -0
- data/lib/gsplat/utils.rb +110 -0
- data/lib/gsplat/version.rb +6 -0
- data/lib/gsplat.rb +98 -0
- metadata +181 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gsplat
|
|
4
|
+
module Backend
|
|
5
|
+
# Brute-force Numo alpha compositor used as a rasterization reference.
|
|
6
|
+
module RubyAccumulate
|
|
7
|
+
ALPHA_CLAMP = 0.999
|
|
8
|
+
ALPHA_SKIP = 1.0 / 255.0
|
|
9
|
+
TRANSMITTANCE_STOP = 1e-4
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
14
|
+
def forward(means2d, conics, opacities, colors, backgrounds, width, height)
|
|
15
|
+
# rubocop:enable Metrics/ParameterLists
|
|
16
|
+
inputs = validate_inputs(means2d, conics, opacities, colors, backgrounds, width, height)
|
|
17
|
+
means2d, conics, opacities, colors, backgrounds = inputs
|
|
18
|
+
camera_count, gaussian_count = means2d.shape[0...2]
|
|
19
|
+
channel_count = colors.shape[-1]
|
|
20
|
+
pixel_count = width * height
|
|
21
|
+
pixels_x, pixels_y = pixel_coordinates(means2d.class, width, height)
|
|
22
|
+
render_colors = means2d.class.zeros(camera_count, pixel_count, channel_count)
|
|
23
|
+
render_alphas = means2d.class.zeros(camera_count, pixel_count)
|
|
24
|
+
last_ids = Numo::Int32.zeros(camera_count, pixel_count)
|
|
25
|
+
camera_count.times do |camera_index|
|
|
26
|
+
composited, transmittance, camera_last_ids = composite_camera(
|
|
27
|
+
means2d[camera_index, true, true],
|
|
28
|
+
conics[camera_index, true, true],
|
|
29
|
+
opacities[camera_index, true],
|
|
30
|
+
colors[camera_index, true, true],
|
|
31
|
+
pixels_x,
|
|
32
|
+
pixels_y,
|
|
33
|
+
gaussian_count
|
|
34
|
+
)
|
|
35
|
+
if backgrounds
|
|
36
|
+
composited += transmittance.reshape(pixel_count, 1) *
|
|
37
|
+
backgrounds[camera_index, true].reshape(1, channel_count)
|
|
38
|
+
end
|
|
39
|
+
render_colors[camera_index, true, true] = composited
|
|
40
|
+
render_alphas[camera_index, true] = 1 - transmittance
|
|
41
|
+
last_ids[camera_index, true] = camera_last_ids + (camera_index * gaussian_count)
|
|
42
|
+
end
|
|
43
|
+
[
|
|
44
|
+
render_colors.reshape(camera_count, height, width, channel_count),
|
|
45
|
+
render_alphas.reshape(camera_count, height, width, 1),
|
|
46
|
+
last_ids.reshape(camera_count, height, width)
|
|
47
|
+
]
|
|
48
|
+
end
|
|
49
|
+
# rubocop:enable Metrics/AbcSize
|
|
50
|
+
|
|
51
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
|
52
|
+
def validate_inputs(means2d, conics, opacities, colors, backgrounds, width, height)
|
|
53
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
|
54
|
+
unless means2d.is_a?(Numo::NArray) && [Numo::SFloat, Numo::DFloat].include?(means2d.class)
|
|
55
|
+
raise ArgumentError, "means2d must be Numo::SFloat or Numo::DFloat"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
valid_means = means2d.ndim == 3 && means2d.shape[-1] == 2
|
|
59
|
+
raise ShapeError, "expected means2d [C,N,2], got #{means2d.shape.inspect}" unless valid_means
|
|
60
|
+
|
|
61
|
+
leading_shape = means2d.shape[0...2]
|
|
62
|
+
conics = means2d.class.cast(conics)
|
|
63
|
+
opacities = means2d.class.cast(opacities)
|
|
64
|
+
colors = means2d.class.cast(colors)
|
|
65
|
+
valid = conics.shape == leading_shape + [3] &&
|
|
66
|
+
opacities.shape == leading_shape &&
|
|
67
|
+
colors.ndim == 3 && colors.shape[0...2] == leading_shape
|
|
68
|
+
unless valid
|
|
69
|
+
raise ShapeError,
|
|
70
|
+
"expected conics [C,N,3], opacities [C,N], and colors [C,N,D], " \
|
|
71
|
+
"got #{conics.shape.inspect}, #{opacities.shape.inspect}, and #{colors.shape.inspect}"
|
|
72
|
+
end
|
|
73
|
+
backgrounds = validate_backgrounds(means2d.class, backgrounds, means2d.shape[0], colors.shape[-1])
|
|
74
|
+
valid_size = width.is_a?(Integer) && width.positive? && height.is_a?(Integer) && height.positive?
|
|
75
|
+
raise ArgumentError, "width and height must be positive integers" unless valid_size
|
|
76
|
+
|
|
77
|
+
[means2d, conics, opacities, colors, backgrounds]
|
|
78
|
+
end
|
|
79
|
+
private_class_method :validate_inputs
|
|
80
|
+
|
|
81
|
+
def validate_backgrounds(type, backgrounds, camera_count, channel_count)
|
|
82
|
+
return nil unless backgrounds
|
|
83
|
+
|
|
84
|
+
value = type.cast(backgrounds)
|
|
85
|
+
expected = [camera_count, channel_count]
|
|
86
|
+
unless value.shape == expected
|
|
87
|
+
raise ShapeError, "expected backgrounds #{expected.inspect}, got #{value.shape.inspect}"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
value
|
|
91
|
+
end
|
|
92
|
+
private_class_method :validate_backgrounds
|
|
93
|
+
|
|
94
|
+
def pixel_coordinates(type, width, height)
|
|
95
|
+
x_values = Array.new(height) { (0...width).map { |column| column + 0.5 } }.flatten
|
|
96
|
+
y_values = (0...height).flat_map { |row| Array.new(width, row + 0.5) }
|
|
97
|
+
[type.cast(x_values), type.cast(y_values)]
|
|
98
|
+
end
|
|
99
|
+
private_class_method :pixel_coordinates
|
|
100
|
+
|
|
101
|
+
# rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
102
|
+
def composite_camera(means2d, conics, opacities, colors, pixels_x, pixels_y, gaussian_count)
|
|
103
|
+
# rubocop:enable Metrics/ParameterLists
|
|
104
|
+
pixel_count = pixels_x.size
|
|
105
|
+
channel_count = colors.shape[-1]
|
|
106
|
+
transmittance = means2d.class.ones(pixel_count)
|
|
107
|
+
composited = means2d.class.zeros(pixel_count, channel_count)
|
|
108
|
+
last_ids = Numo::Int32.zeros(pixel_count)
|
|
109
|
+
active = Numo::Bit.ones(pixel_count)
|
|
110
|
+
gaussian_count.times do |gaussian_index|
|
|
111
|
+
delta_x = means2d[gaussian_index, 0] - pixels_x
|
|
112
|
+
delta_y = means2d[gaussian_index, 1] - pixels_y
|
|
113
|
+
sigma = (0.5 * (
|
|
114
|
+
(conics[gaussian_index, 0] * (delta_x**2)) +
|
|
115
|
+
(conics[gaussian_index, 2] * (delta_y**2))
|
|
116
|
+
)) + (conics[gaussian_index, 1] * delta_x * delta_y)
|
|
117
|
+
alpha = opacities[gaussian_index] * Numo::NMath.exp(-sigma)
|
|
118
|
+
above_clamp = alpha.gt(ALPHA_CLAMP)
|
|
119
|
+
alpha[above_clamp] = ALPHA_CLAMP if above_clamp.any?
|
|
120
|
+
candidate = active & sigma.ge(0) & alpha.ge(ALPHA_SKIP)
|
|
121
|
+
next_transmittance = transmittance * (1 - alpha)
|
|
122
|
+
accepted = candidate & next_transmittance.gt(TRANSMITTANCE_STOP)
|
|
123
|
+
contribution = means2d.class.zeros(pixel_count)
|
|
124
|
+
contribution[accepted] = (alpha * transmittance)[accepted] if accepted.any?
|
|
125
|
+
composited += contribution.reshape(pixel_count, 1) *
|
|
126
|
+
colors[gaussian_index, true].reshape(1, channel_count)
|
|
127
|
+
transmittance[accepted] = next_transmittance[accepted] if accepted.any?
|
|
128
|
+
last_ids[accepted] = gaussian_index if accepted.any?
|
|
129
|
+
terminating = candidate & next_transmittance.le(TRANSMITTANCE_STOP)
|
|
130
|
+
active[terminating] = 0 if terminating.any?
|
|
131
|
+
break unless active.any?
|
|
132
|
+
end
|
|
133
|
+
[composited, transmittance, last_ids]
|
|
134
|
+
end
|
|
135
|
+
# rubocop:enable Metrics/AbcSize
|
|
136
|
+
private_class_method :composite_camera
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gsplat
|
|
4
|
+
module Backend
|
|
5
|
+
# Brute-force compositor VJP routed through the one-tile reverse kernel.
|
|
6
|
+
module RubyAccumulateBackward
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
# rubocop:disable Metrics/ParameterLists
|
|
10
|
+
def backward(means2d, conics, opacities, colors, backgrounds, width, height,
|
|
11
|
+
render_alphas, last_ids, grad_render_colors, grad_render_alphas)
|
|
12
|
+
# rubocop:enable Metrics/ParameterLists
|
|
13
|
+
camera_count, gaussian_count = means2d.shape[0...2]
|
|
14
|
+
offsets = Numo::Int32.zeros(camera_count, 1, 1)
|
|
15
|
+
camera_count.times { |camera_index| offsets[camera_index, 0, 0] = camera_index * gaussian_count }
|
|
16
|
+
flatten_ids = Numo::Int32.new(camera_count * gaussian_count).seq
|
|
17
|
+
gradients, = RubyRasterizeToPixelsBackward.backward(
|
|
18
|
+
means2d,
|
|
19
|
+
conics,
|
|
20
|
+
colors,
|
|
21
|
+
opacities,
|
|
22
|
+
backgrounds,
|
|
23
|
+
nil,
|
|
24
|
+
width,
|
|
25
|
+
height,
|
|
26
|
+
[width, height].max,
|
|
27
|
+
offsets,
|
|
28
|
+
flatten_ids,
|
|
29
|
+
render_alphas,
|
|
30
|
+
last_ids,
|
|
31
|
+
grad_render_colors,
|
|
32
|
+
grad_render_alphas,
|
|
33
|
+
absgrad: false
|
|
34
|
+
)
|
|
35
|
+
grad_means, grad_conics, grad_colors, grad_opacities, grad_backgrounds = gradients
|
|
36
|
+
[grad_means, grad_conics, grad_opacities, grad_colors, grad_backgrounds]
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gsplat
|
|
4
|
+
module Backend
|
|
5
|
+
# World-space Gaussian evaluator used by 3DGUT-style rendering.
|
|
6
|
+
module RubyEval3dRasterizer
|
|
7
|
+
ALPHA_CLAMP = 0.99
|
|
8
|
+
ALPHA_SKIP = 1.0 / 255
|
|
9
|
+
TRANSMITTANCE_STOP = 1e-4
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
14
|
+
def forward(means, quats, scales, colors, opacities, backgrounds, viewmats:, intrinsics:,
|
|
15
|
+
width:, height:, tile_size:, offsets:, flatten_ids:, camera_model:,
|
|
16
|
+
use_hit_distance: false, return_normals: false)
|
|
17
|
+
# rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
|
|
18
|
+
values = validate_inputs(
|
|
19
|
+
means, quats, scales, colors, opacities, backgrounds, viewmats, intrinsics,
|
|
20
|
+
width, height, offsets, flatten_ids, camera_model
|
|
21
|
+
)
|
|
22
|
+
means, quats, scales, colors, opacities, backgrounds, viewmats, intrinsics = values
|
|
23
|
+
camera_count, gaussian_count, channels = colors.shape
|
|
24
|
+
rendered = means.class.zeros(camera_count, height, width, channels)
|
|
25
|
+
alphas = means.class.zeros(camera_count, height, width, 1)
|
|
26
|
+
rendered_normals = means.class.zeros(camera_count, height, width, 3)
|
|
27
|
+
rotations = Math::Quaternion.to_rotmat(quats)
|
|
28
|
+
camera_count.times do |camera|
|
|
29
|
+
frame = camera_frame(viewmats[camera, true, true])
|
|
30
|
+
height.times do |row|
|
|
31
|
+
width.times do |column|
|
|
32
|
+
tile_x = column / tile_size
|
|
33
|
+
tile_y = row / tile_size
|
|
34
|
+
range = intersection_range(offsets, flatten_ids, camera, tile_x, tile_y)
|
|
35
|
+
ray = ray_for_pixel(frame, intrinsics[camera, true, true], column, row)
|
|
36
|
+
pixel, transmittance, pixel_normal = composite_pixel(
|
|
37
|
+
means, rotations, scales, colors, opacities, flatten_ids, range,
|
|
38
|
+
camera, gaussian_count, ray, use_hit_distance, return_normals
|
|
39
|
+
)
|
|
40
|
+
pixel += backgrounds[camera, true] * transmittance if backgrounds
|
|
41
|
+
rendered[camera, row, column, true] = pixel
|
|
42
|
+
alphas[camera, row, column, 0] = 1 - transmittance
|
|
43
|
+
rendered_normals[camera, row, column, true] = pixel_normal
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
[rendered, alphas, rendered_normals]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
|
51
|
+
def validate_inputs(means, quats, scales, colors, opacities, backgrounds, viewmats,
|
|
52
|
+
intrinsics, width, height, offsets, flatten_ids, camera_model)
|
|
53
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
|
54
|
+
unless camera_model == "pinhole"
|
|
55
|
+
raise ArgumentError, "hit-distance rendering currently requires pinhole camera_model"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
type = means.class
|
|
59
|
+
quats = type.cast(quats)
|
|
60
|
+
scales = type.cast(scales)
|
|
61
|
+
colors = type.cast(colors)
|
|
62
|
+
opacities = type.cast(opacities)
|
|
63
|
+
viewmats = type.cast(viewmats)
|
|
64
|
+
intrinsics = type.cast(intrinsics)
|
|
65
|
+
camera_count = viewmats.shape[0]
|
|
66
|
+
gaussian_count = means.shape[0]
|
|
67
|
+
valid = means.shape == [gaussian_count, 3] &&
|
|
68
|
+
quats.shape == [gaussian_count, 4] &&
|
|
69
|
+
scales.shape == [gaussian_count, 3] &&
|
|
70
|
+
colors.ndim == 3 && colors.shape[0...2] == [camera_count, gaussian_count] &&
|
|
71
|
+
opacities.shape == [camera_count, gaussian_count] &&
|
|
72
|
+
intrinsics.shape == [camera_count, 3, 3]
|
|
73
|
+
raise ShapeError, "invalid world-space rasterization inputs" unless valid
|
|
74
|
+
raise ArgumentError, "scales must be positive" unless scales.gt(0).all?
|
|
75
|
+
|
|
76
|
+
backgrounds = type.cast(backgrounds) if backgrounds
|
|
77
|
+
if backgrounds && backgrounds.shape != [camera_count, colors.shape[-1]]
|
|
78
|
+
raise ShapeError, "expected backgrounds [#{camera_count},#{colors.shape[-1]}]"
|
|
79
|
+
end
|
|
80
|
+
raise ArgumentError, "width and height must be positive" unless width.positive? && height.positive?
|
|
81
|
+
raise ShapeError, "expected offsets [C,TH,TW]" unless offsets.shape[0] == camera_count
|
|
82
|
+
raise ShapeError, "expected flatten_ids [M]" unless flatten_ids.ndim == 1
|
|
83
|
+
|
|
84
|
+
[means, quats, scales, colors, opacities, backgrounds, viewmats, intrinsics]
|
|
85
|
+
end
|
|
86
|
+
private_class_method :validate_inputs
|
|
87
|
+
|
|
88
|
+
def camera_frame(viewmat)
|
|
89
|
+
rotation = viewmat[0...3, 0...3]
|
|
90
|
+
translation = viewmat[0...3, 3]
|
|
91
|
+
[-translation.dot(rotation), rotation]
|
|
92
|
+
end
|
|
93
|
+
private_class_method :camera_frame
|
|
94
|
+
|
|
95
|
+
def ray_for_pixel(frame, intrinsics, column, row)
|
|
96
|
+
origin, rotation = frame
|
|
97
|
+
camera_direction = origin.class[
|
|
98
|
+
(column + 0.5 - intrinsics[0, 2]) / intrinsics[0, 0],
|
|
99
|
+
(row + 0.5 - intrinsics[1, 2]) / intrinsics[1, 1],
|
|
100
|
+
1.0
|
|
101
|
+
]
|
|
102
|
+
direction = camera_direction.dot(rotation)
|
|
103
|
+
[origin, direction / ::Math.sqrt((direction**2).sum)]
|
|
104
|
+
end
|
|
105
|
+
private_class_method :ray_for_pixel
|
|
106
|
+
|
|
107
|
+
def intersection_range(offsets, flatten_ids, camera, tile_x, tile_y)
|
|
108
|
+
tile_height, tile_width = offsets.shape[-2..]
|
|
109
|
+
flat = (((camera * tile_height) + tile_y) * tile_width) + tile_x
|
|
110
|
+
start_index = offsets[flat].to_i
|
|
111
|
+
end_index = flat + 1 < offsets.size ? offsets[flat + 1].to_i : flatten_ids.size
|
|
112
|
+
start_index...end_index
|
|
113
|
+
end
|
|
114
|
+
private_class_method :intersection_range
|
|
115
|
+
|
|
116
|
+
# rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
|
|
117
|
+
def composite_pixel(means, rotations, scales, colors, opacities, flatten_ids, range,
|
|
118
|
+
camera, gaussian_count, ray, use_hit_distance, return_normals)
|
|
119
|
+
# rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
|
|
120
|
+
pixel = means.class.zeros(colors.shape[-1])
|
|
121
|
+
pixel_normal = means.class.zeros(3)
|
|
122
|
+
transmittance = 1.0
|
|
123
|
+
range.each do |intersection|
|
|
124
|
+
flat_id = flatten_ids[intersection].to_i
|
|
125
|
+
gaussian = flat_id % gaussian_count
|
|
126
|
+
gray_distance, hit_distance = ray_gaussian(
|
|
127
|
+
ray, means[gaussian, true], rotations[gaussian, true, true], scales[gaussian, true]
|
|
128
|
+
)
|
|
129
|
+
next unless gray_distance
|
|
130
|
+
|
|
131
|
+
opacity = opacities[camera, gaussian].to_f
|
|
132
|
+
alpha = [opacity * ::Math.exp(-0.5 * gray_distance), ALPHA_CLAMP].min
|
|
133
|
+
next if alpha < ALPHA_SKIP
|
|
134
|
+
|
|
135
|
+
next_transmittance = transmittance * (1 - alpha)
|
|
136
|
+
break if next_transmittance <= TRANSMITTANCE_STOP
|
|
137
|
+
|
|
138
|
+
weight = alpha * transmittance
|
|
139
|
+
feature = colors[camera, gaussian, true].dup
|
|
140
|
+
feature[feature.size - 1] = hit_distance if use_hit_distance
|
|
141
|
+
pixel += feature * weight
|
|
142
|
+
if return_normals
|
|
143
|
+
normal = rotations[gaussian, true, 2].dup
|
|
144
|
+
normal *= -1 if (normal * ray[1]).sum.positive?
|
|
145
|
+
pixel_normal += normal * weight
|
|
146
|
+
end
|
|
147
|
+
transmittance = next_transmittance
|
|
148
|
+
end
|
|
149
|
+
[pixel, transmittance, pixel_normal]
|
|
150
|
+
end
|
|
151
|
+
private_class_method :composite_pixel
|
|
152
|
+
|
|
153
|
+
# rubocop:disable Metrics/AbcSize
|
|
154
|
+
def ray_gaussian(ray, mean, rotation, scale)
|
|
155
|
+
origin, direction = ray
|
|
156
|
+
local_origin = (origin - mean).dot(rotation) / scale
|
|
157
|
+
local_direction = direction.dot(rotation) / scale
|
|
158
|
+
local_direction /= ::Math.sqrt((local_direction**2).sum)
|
|
159
|
+
hit_t = -(local_direction * local_origin).sum
|
|
160
|
+
return [nil, nil] if hit_t.negative?
|
|
161
|
+
|
|
162
|
+
cross = local_direction.class[
|
|
163
|
+
(local_direction[1] * local_origin[2]) - (local_direction[2] * local_origin[1]),
|
|
164
|
+
(local_direction[2] * local_origin[0]) - (local_direction[0] * local_origin[2]),
|
|
165
|
+
(local_direction[0] * local_origin[1]) - (local_direction[1] * local_origin[0])
|
|
166
|
+
]
|
|
167
|
+
gray_distance = (cross**2).sum.to_f
|
|
168
|
+
hit_vector = scale * local_direction * hit_t
|
|
169
|
+
[gray_distance, ::Math.sqrt((hit_vector**2).sum)]
|
|
170
|
+
end
|
|
171
|
+
# rubocop:enable Metrics/AbcSize
|
|
172
|
+
private_class_method :ray_gaussian
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gsplat
|
|
4
|
+
module Backend
|
|
5
|
+
# Numo/Ruby tile intersection enumeration and offset encoding.
|
|
6
|
+
module RubyIsectTiles
|
|
7
|
+
TILE_KEY_LOW_BITS = 32
|
|
8
|
+
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# rubocop:disable Metrics/ParameterLists
|
|
12
|
+
def forward(means2d, radii, depths, tile_size, tile_width, tile_height, sort:)
|
|
13
|
+
# rubocop:enable Metrics/ParameterLists
|
|
14
|
+
means2d, radii, depths = validate_inputs(
|
|
15
|
+
means2d,
|
|
16
|
+
radii,
|
|
17
|
+
depths,
|
|
18
|
+
tile_size,
|
|
19
|
+
tile_width,
|
|
20
|
+
tile_height
|
|
21
|
+
)
|
|
22
|
+
camera_count, gaussian_count = means2d.shape[0...2]
|
|
23
|
+
bounds, tiles_per_gauss = tile_bounds(
|
|
24
|
+
means2d,
|
|
25
|
+
radii,
|
|
26
|
+
tile_size,
|
|
27
|
+
tile_width,
|
|
28
|
+
tile_height
|
|
29
|
+
)
|
|
30
|
+
intersection_count = tiles_per_gauss.sum.to_i
|
|
31
|
+
isect_ids = Numo::Int64.zeros(intersection_count)
|
|
32
|
+
flatten_ids = Numo::Int32.zeros(intersection_count)
|
|
33
|
+
tile_bits = tile_n_bits(tile_width, tile_height)
|
|
34
|
+
validate_key_width!(camera_count, tile_bits)
|
|
35
|
+
write_intersections!(
|
|
36
|
+
isect_ids,
|
|
37
|
+
flatten_ids,
|
|
38
|
+
bounds,
|
|
39
|
+
tiles_per_gauss,
|
|
40
|
+
depths,
|
|
41
|
+
gaussian_count,
|
|
42
|
+
tile_width,
|
|
43
|
+
tile_bits
|
|
44
|
+
)
|
|
45
|
+
if sort && intersection_count.positive?
|
|
46
|
+
keys = isect_ids.to_a
|
|
47
|
+
order = (0...intersection_count).sort_by { |index| [keys[index], index] }
|
|
48
|
+
isect_ids = isect_ids[order].dup
|
|
49
|
+
flatten_ids = flatten_ids[order].dup
|
|
50
|
+
end
|
|
51
|
+
[tiles_per_gauss, isect_ids, flatten_ids]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def offset_encode(isect_ids, camera_count, tile_width, tile_height)
|
|
55
|
+
validate_grid!(camera_count, tile_width, tile_height)
|
|
56
|
+
raise ShapeError, "expected isect_ids [M]" unless isect_ids.is_a?(Numo::NArray) && isect_ids.ndim == 1
|
|
57
|
+
|
|
58
|
+
keys = Numo::Int64.cast(isect_ids)
|
|
59
|
+
counts = Numo::Int64.zeros(camera_count, tile_height, tile_width)
|
|
60
|
+
tile_bits = tile_n_bits(tile_width, tile_height)
|
|
61
|
+
tile_mask = (1 << tile_bits) - 1
|
|
62
|
+
keys.each do |key|
|
|
63
|
+
upper = key >> TILE_KEY_LOW_BITS
|
|
64
|
+
camera_id = upper >> tile_bits
|
|
65
|
+
tile_id = upper & tile_mask
|
|
66
|
+
validate_decoded_key!(camera_id, tile_id, camera_count, tile_width, tile_height)
|
|
67
|
+
counts[camera_id, tile_id / tile_width, tile_id % tile_width] += 1
|
|
68
|
+
end
|
|
69
|
+
offsets = Numo::Int32.zeros(camera_count, tile_height, tile_width)
|
|
70
|
+
running = 0
|
|
71
|
+
counts.size.times do |index|
|
|
72
|
+
offsets[index] = running
|
|
73
|
+
running += counts[index]
|
|
74
|
+
end
|
|
75
|
+
offsets
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# rubocop:disable Metrics/ParameterLists
|
|
79
|
+
def validate_inputs(means2d, radii, depths, tile_size, tile_width, tile_height)
|
|
80
|
+
# rubocop:enable Metrics/ParameterLists
|
|
81
|
+
unless means2d.is_a?(Numo::NArray) && [Numo::SFloat, Numo::DFloat].include?(means2d.class)
|
|
82
|
+
raise ArgumentError, "means2d must be Numo::SFloat or Numo::DFloat"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
valid_means = means2d.ndim == 3 && means2d.shape[-1] == 2
|
|
86
|
+
raise ShapeError, "expected means2d [C,N,2], got #{means2d.shape.inspect}" unless valid_means
|
|
87
|
+
|
|
88
|
+
leading_shape = means2d.shape[0...2]
|
|
89
|
+
valid_radii = [leading_shape, leading_shape + [2]].include?(radii.shape)
|
|
90
|
+
unless valid_radii && depths.shape == leading_shape
|
|
91
|
+
raise ShapeError,
|
|
92
|
+
"expected radii [C,N] or [C,N,2] and depths #{leading_shape.inspect}, " \
|
|
93
|
+
"got #{radii.shape.inspect} and #{depths.shape.inspect}"
|
|
94
|
+
end
|
|
95
|
+
validate_grid!(means2d.shape[0], tile_width, tile_height)
|
|
96
|
+
valid_tile_size = tile_size.is_a?(Integer) && tile_size.positive?
|
|
97
|
+
raise ArgumentError, "tile_size must be a positive integer" unless valid_tile_size
|
|
98
|
+
|
|
99
|
+
[means2d, means2d.class.cast(radii), means2d.class.cast(depths)]
|
|
100
|
+
end
|
|
101
|
+
private_class_method :validate_inputs
|
|
102
|
+
|
|
103
|
+
def validate_grid!(camera_count, tile_width, tile_height)
|
|
104
|
+
values = [camera_count, tile_width, tile_height]
|
|
105
|
+
return if values.all? { |value| value.is_a?(Integer) && value.positive? }
|
|
106
|
+
|
|
107
|
+
raise ArgumentError, "camera count and tile dimensions must be positive integers"
|
|
108
|
+
end
|
|
109
|
+
private_class_method :validate_grid!
|
|
110
|
+
|
|
111
|
+
# rubocop:disable Metrics/AbcSize
|
|
112
|
+
def tile_bounds(means2d, radii, tile_size, tile_width, tile_height)
|
|
113
|
+
camera_count, gaussian_count = means2d.shape[0...2]
|
|
114
|
+
bounds = Numo::Int32.zeros(camera_count, gaussian_count, 4)
|
|
115
|
+
counts = Numo::Int32.zeros(camera_count, gaussian_count)
|
|
116
|
+
elliptical = radii.ndim == 3
|
|
117
|
+
camera_count.times do |camera_index|
|
|
118
|
+
gaussian_count.times do |gaussian_index|
|
|
119
|
+
if elliptical
|
|
120
|
+
radius_x = radii[camera_index, gaussian_index, 0].to_f
|
|
121
|
+
radius_y = radii[camera_index, gaussian_index, 1].to_f
|
|
122
|
+
else
|
|
123
|
+
radius_x = radii[camera_index, gaussian_index].to_f
|
|
124
|
+
radius_y = radius_x
|
|
125
|
+
end
|
|
126
|
+
next unless radius_x.positive? && radius_y.positive?
|
|
127
|
+
|
|
128
|
+
mean_x = means2d[camera_index, gaussian_index, 0].to_f
|
|
129
|
+
mean_y = means2d[camera_index, gaussian_index, 1].to_f
|
|
130
|
+
min_x = clamp_tile(((mean_x - radius_x) / tile_size).floor, tile_width)
|
|
131
|
+
max_x = clamp_tile(((mean_x + radius_x) / tile_size).ceil, tile_width)
|
|
132
|
+
min_y = clamp_tile(((mean_y - radius_y) / tile_size).floor, tile_height)
|
|
133
|
+
max_y = clamp_tile(((mean_y + radius_y) / tile_size).ceil, tile_height)
|
|
134
|
+
bounds[camera_index, gaussian_index, true] = [min_x, min_y, max_x, max_y]
|
|
135
|
+
counts[camera_index, gaussian_index] = (max_x - min_x) * (max_y - min_y)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
[bounds, counts]
|
|
139
|
+
end
|
|
140
|
+
# rubocop:enable Metrics/AbcSize
|
|
141
|
+
private_class_method :tile_bounds
|
|
142
|
+
|
|
143
|
+
def clamp_tile(value, maximum)
|
|
144
|
+
value.clamp(0, maximum)
|
|
145
|
+
end
|
|
146
|
+
private_class_method :clamp_tile
|
|
147
|
+
|
|
148
|
+
# rubocop:disable Metrics/ParameterLists
|
|
149
|
+
def write_intersections!(keys, flatten_ids, bounds, counts, depths, gaussian_count, tile_width, tile_bits)
|
|
150
|
+
# rubocop:enable Metrics/ParameterLists
|
|
151
|
+
write_index = 0
|
|
152
|
+
counts.shape[0].times do |camera_index|
|
|
153
|
+
gaussian_count.times do |gaussian_index|
|
|
154
|
+
next if counts[camera_index, gaussian_index].zero?
|
|
155
|
+
|
|
156
|
+
min_x, min_y, max_x, max_y = bounds[camera_index, gaussian_index, true].to_a
|
|
157
|
+
upper_camera = camera_index << tile_bits
|
|
158
|
+
depth_bits = float32_bits(depths[camera_index, gaussian_index])
|
|
159
|
+
(min_y...max_y).each do |tile_y|
|
|
160
|
+
(min_x...max_x).each do |tile_x|
|
|
161
|
+
tile_id = (tile_y * tile_width) + tile_x
|
|
162
|
+
keys[write_index] = ((upper_camera | tile_id) << TILE_KEY_LOW_BITS) | depth_bits
|
|
163
|
+
flatten_ids[write_index] = (camera_index * gaussian_count) + gaussian_index
|
|
164
|
+
write_index += 1
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
private_class_method :write_intersections!
|
|
171
|
+
|
|
172
|
+
def float32_bits(value)
|
|
173
|
+
[value.to_f].pack("g").unpack1("N")
|
|
174
|
+
end
|
|
175
|
+
private_class_method :float32_bits
|
|
176
|
+
|
|
177
|
+
def tile_n_bits(tile_width, tile_height)
|
|
178
|
+
tile_count = tile_width * tile_height
|
|
179
|
+
tile_count == 1 ? 0 : ::Math.log2(tile_count).ceil
|
|
180
|
+
end
|
|
181
|
+
private_class_method :tile_n_bits
|
|
182
|
+
|
|
183
|
+
def validate_key_width!(camera_count, tile_bits)
|
|
184
|
+
return if (camera_count - 1).bit_length + tile_bits <= TILE_KEY_LOW_BITS
|
|
185
|
+
|
|
186
|
+
raise ArgumentError, "camera and tile ids exceed the 64-bit intersection key layout"
|
|
187
|
+
end
|
|
188
|
+
private_class_method :validate_key_width!
|
|
189
|
+
|
|
190
|
+
def validate_decoded_key!(camera_id, tile_id, camera_count, tile_width, tile_height)
|
|
191
|
+
return if camera_id.between?(0, camera_count - 1) && tile_id.between?(0, (tile_width * tile_height) - 1)
|
|
192
|
+
|
|
193
|
+
raise Gsplat::Error, "intersection key contains an out-of-range camera or tile id"
|
|
194
|
+
end
|
|
195
|
+
private_class_method :validate_decoded_key!
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|