gsplat 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +202 -0
- data/README.md +236 -0
- data/docs/ACCEPTANCE.md +60 -0
- data/docs/BENCHMARKS.md +79 -0
- data/docs/DECISIONS.md +52 -0
- data/docs/MIGRATION.md +117 -0
- data/docs/PROFILE.md +75 -0
- data/docs/PROGRESS.md +199 -0
- data/docs/decisions/0000-template.md +17 -0
- data/docs/decisions/0001-pin-golden-reference-and-cpu-validation.md +27 -0
- data/docs/decisions/0002-retain-ruby-fallbacks-for-native-backend.md +26 -0
- data/docs/decisions/0003-share-projection-for-distorted-cameras.md +26 -0
- data/docs/decisions/0004-use-portable-world-space-reference-paths.md +26 -0
- data/docs/decisions/0005-reuse-ewa-core-for-2dgs.md +24 -0
- data/docs/decisions/0006-keep-eval3d-as-portable-reference.md +25 -0
- data/docs/decisions/0007-share-compositor-semantics-for-contribution-indices.md +25 -0
- data/examples/data/README.md +14 -0
- data/examples/data/colmap/images/view_000.png +0 -0
- data/examples/data/colmap/images/view_001.png +0 -0
- data/examples/data/colmap/images/view_002.png +0 -0
- data/examples/data/colmap/sparse/0/cameras.txt +2 -0
- data/examples/data/colmap/sparse/0/images.txt +6 -0
- data/examples/data/colmap/sparse/0/points3D.txt +17 -0
- data/examples/data/splats.ply +0 -0
- data/examples/fit_image.rb +35 -0
- data/examples/generate_sample_data.rb +147 -0
- data/examples/render_path.rb +117 -0
- data/examples/simple_trainer.rb +76 -0
- data/ext/gsplat_native/common.h +54 -0
- data/ext/gsplat_native/extconf.rb +39 -0
- data/ext/gsplat_native/gsplat_native.c +60 -0
- data/ext/gsplat_native/intersections.c +211 -0
- data/ext/gsplat_native/projection.c +199 -0
- data/ext/gsplat_native/raster_backward.c +129 -0
- data/ext/gsplat_native/raster_backward_bridge.c +69 -0
- data/ext/gsplat_native/raster_forward.c +150 -0
- data/ext/gsplat_native/rasterization.h +51 -0
- data/ext/gsplat_native/spherical_harmonics.c +129 -0
- data/gsplat.gemspec +36 -0
- data/lib/gsplat/autograd/context.rb +60 -0
- data/lib/gsplat/autograd/function.rb +68 -0
- data/lib/gsplat/autograd/variable.rb +159 -0
- data/lib/gsplat/backend/ruby/accumulate.rb +139 -0
- data/lib/gsplat/backend/ruby/accumulate_backward.rb +40 -0
- data/lib/gsplat/backend/ruby/eval3d_rasterizer.rb +175 -0
- data/lib/gsplat/backend/ruby/isect_tiles.rb +198 -0
- data/lib/gsplat/backend/ruby/projection.rb +251 -0
- data/lib/gsplat/backend/ruby/projection_backward.rb +190 -0
- data/lib/gsplat/backend/ruby/projection_covariance_vjp.rb +72 -0
- data/lib/gsplat/backend/ruby/projection_input_vjp.rb +252 -0
- data/lib/gsplat/backend/ruby/quat_scale_to_covar_preci.rb +139 -0
- data/lib/gsplat/backend/ruby/rasterize_to_indices_in_range.rb +121 -0
- data/lib/gsplat/backend/ruby/rasterize_to_pixels.rb +199 -0
- data/lib/gsplat/backend/ruby/rasterize_to_pixels_backward.rb +121 -0
- data/lib/gsplat/backend/ruby/spherical_harmonics.rb +135 -0
- data/lib/gsplat/backend/ruby/tile_compositor.rb +50 -0
- data/lib/gsplat/backend/ruby/tile_compositor_backward.rb +84 -0
- data/lib/gsplat/backend.rb +79 -0
- data/lib/gsplat/compression/grid_sort.rb +79 -0
- data/lib/gsplat/compression/kmeans.rb +121 -0
- data/lib/gsplat/compression/png.rb +147 -0
- data/lib/gsplat/compression/png_codec.rb +133 -0
- data/lib/gsplat/compression/quantizer.rb +79 -0
- data/lib/gsplat/io/checkpoint.rb +145 -0
- data/lib/gsplat/io/colmap.rb +175 -0
- data/lib/gsplat/io/colmap_binary.rb +98 -0
- data/lib/gsplat/io/colmap_text.rb +84 -0
- data/lib/gsplat/io/image.rb +63 -0
- data/lib/gsplat/io/image_backends.rb +81 -0
- data/lib/gsplat/io/npy.rb +189 -0
- data/lib/gsplat/io/ply.rb +185 -0
- data/lib/gsplat/io/ply_reader.rb +142 -0
- data/lib/gsplat/io/zip_archive.rb +183 -0
- data/lib/gsplat/math/camera_distortion.rb +123 -0
- data/lib/gsplat/math/camera_projection.rb +202 -0
- data/lib/gsplat/math/mat.rb +114 -0
- data/lib/gsplat/math/quaternion.rb +175 -0
- data/lib/gsplat/math/small_matrix_primitives.rb +112 -0
- data/lib/gsplat/math/spherical_harmonic_basis.rb +148 -0
- data/lib/gsplat/math/ssim.rb +153 -0
- data/lib/gsplat/native.rb +30 -0
- data/lib/gsplat/native_ops.rb +148 -0
- data/lib/gsplat/native_raster_ops.rb +76 -0
- data/lib/gsplat/ops/accumulate.rb +68 -0
- data/lib/gsplat/ops/eval3d_rasterize.rb +62 -0
- data/lib/gsplat/ops/isect_tiles.rb +38 -0
- data/lib/gsplat/ops/projection.rb +154 -0
- data/lib/gsplat/ops/quat_scale_to_covar_preci.rb +118 -0
- data/lib/gsplat/ops/rasterize_to_indices_in_range.rb +38 -0
- data/lib/gsplat/ops/rasterize_to_pixels.rb +105 -0
- data/lib/gsplat/ops/relocation.rb +94 -0
- data/lib/gsplat/ops/spherical_harmonics.rb +71 -0
- data/lib/gsplat/ops/tensor_shape_ops.rb +173 -0
- data/lib/gsplat/ops/tensor_value_ops.rb +150 -0
- data/lib/gsplat/optim/adam.rb +212 -0
- data/lib/gsplat/optim/lr_scheduler.rb +36 -0
- data/lib/gsplat/optim/selective_adam.rb +68 -0
- data/lib/gsplat/rasterization.rb +130 -0
- data/lib/gsplat/rasterization_2dgs.rb +140 -0
- data/lib/gsplat/rasterization_helpers.rb +162 -0
- data/lib/gsplat/rasterization_validation.rb +99 -0
- data/lib/gsplat/strategy/base.rb +49 -0
- data/lib/gsplat/strategy/default.rb +188 -0
- data/lib/gsplat/strategy/mcmc.rb +103 -0
- data/lib/gsplat/strategy/mcmc_ops.rb +143 -0
- data/lib/gsplat/strategy/ops.rb +165 -0
- data/lib/gsplat/training/config.rb +91 -0
- data/lib/gsplat/training/image_fitter.rb +158 -0
- data/lib/gsplat/training/losses.rb +191 -0
- data/lib/gsplat/training/scene.rb +116 -0
- data/lib/gsplat/training/trainer.rb +236 -0
- data/lib/gsplat/utils.rb +110 -0
- data/lib/gsplat/version.rb +6 -0
- data/lib/gsplat.rb +98 -0
- metadata +181 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/accumulate"
|
|
4
|
+
require_relative "../backend/ruby/accumulate_backward"
|
|
5
|
+
|
|
6
|
+
# Brute-force reference rasterization API.
|
|
7
|
+
module Gsplat
|
|
8
|
+
# Differentiable operation classes and low-level tensor adapters.
|
|
9
|
+
module Ops
|
|
10
|
+
# Differentiable reference alpha compositor.
|
|
11
|
+
class Accumulate < Autograd::Function
|
|
12
|
+
class << self
|
|
13
|
+
# rubocop:disable Metrics/ParameterLists
|
|
14
|
+
def forward(context, means2d, conics, opacities, colors, backgrounds, width, height)
|
|
15
|
+
# rubocop:enable Metrics/ParameterLists
|
|
16
|
+
render_colors, render_alphas, last_ids = Backend.dispatch(
|
|
17
|
+
:accumulate_forward,
|
|
18
|
+
means2d,
|
|
19
|
+
conics,
|
|
20
|
+
opacities,
|
|
21
|
+
colors,
|
|
22
|
+
backgrounds,
|
|
23
|
+
width,
|
|
24
|
+
height
|
|
25
|
+
)
|
|
26
|
+
context.save(
|
|
27
|
+
means2d, conics, opacities, colors, backgrounds, width, height,
|
|
28
|
+
render_alphas, last_ids
|
|
29
|
+
)
|
|
30
|
+
[render_colors, render_alphas]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Propagates color and alpha gradients through the reference compositor.
|
|
34
|
+
# @api private
|
|
35
|
+
def backward(context, grad_render_colors, grad_render_alphas)
|
|
36
|
+
saved = context.saved_values
|
|
37
|
+
gradients = Backend.dispatch(
|
|
38
|
+
:accumulate_backward,
|
|
39
|
+
*saved,
|
|
40
|
+
grad_render_colors,
|
|
41
|
+
grad_render_alphas
|
|
42
|
+
)
|
|
43
|
+
[*gradients, nil, nil]
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
class << self
|
|
50
|
+
# Composites every Gaussian over every pixel without tile acceleration.
|
|
51
|
+
# rubocop:disable Metrics/ParameterLists
|
|
52
|
+
def accumulate(means2d, conics, opacities, colors, width:, height:, backgrounds: nil)
|
|
53
|
+
# rubocop:enable Metrics/ParameterLists
|
|
54
|
+
inputs = [means2d, conics, opacities, colors, backgrounds]
|
|
55
|
+
return Ops::Accumulate.apply(*inputs, width, height) if inputs.any?(Autograd::Variable)
|
|
56
|
+
|
|
57
|
+
Backend.dispatch(
|
|
58
|
+
:accumulate_forward,
|
|
59
|
+
*inputs,
|
|
60
|
+
width,
|
|
61
|
+
height
|
|
62
|
+
).first(2)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
Backend.register(:accumulate_forward, :ruby, Backend::RubyAccumulate.method(:forward))
|
|
67
|
+
Backend.register(:accumulate_backward, :ruby, Backend::RubyAccumulateBackward.method(:backward))
|
|
68
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/eval3d_rasterizer"
|
|
4
|
+
|
|
5
|
+
module Gsplat
|
|
6
|
+
module Ops
|
|
7
|
+
# Differentiable world-space reference rasterizer with optional normals.
|
|
8
|
+
class Eval3dRasterize < Autograd::Function
|
|
9
|
+
class << self
|
|
10
|
+
# rubocop:disable Metrics/ParameterLists
|
|
11
|
+
def forward(context, means, quats, scales, colors, opacities, backgrounds, **options)
|
|
12
|
+
# rubocop:enable Metrics/ParameterLists
|
|
13
|
+
context.save(means, quats, scales, colors, opacities, backgrounds, options)
|
|
14
|
+
Backend::RubyEval3dRasterizer.forward(
|
|
15
|
+
means, quats, scales, colors, opacities, backgrounds, **options
|
|
16
|
+
)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Computes a central-difference VJP for the portable eval3d path.
|
|
20
|
+
# @api private
|
|
21
|
+
def backward(context, grad_rendered, grad_alphas, grad_normals)
|
|
22
|
+
*inputs, options = context.saved_values
|
|
23
|
+
gradient_outputs = [grad_rendered, grad_alphas, grad_normals]
|
|
24
|
+
context.needs_input_grad.each_with_index.map do |needed, input_index|
|
|
25
|
+
next unless needed
|
|
26
|
+
|
|
27
|
+
numerical_vjp(inputs, input_index, gradient_outputs, options)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def numerical_vjp(inputs, input_index, gradient_outputs, options)
|
|
34
|
+
input = inputs[input_index]
|
|
35
|
+
epsilon = input.is_a?(Numo::DFloat) ? 1e-5 : 1e-3
|
|
36
|
+
gradient = input.class.zeros(*input.shape)
|
|
37
|
+
input.size.times do |element|
|
|
38
|
+
positive_inputs = inputs.dup
|
|
39
|
+
negative_inputs = inputs.dup
|
|
40
|
+
positive_inputs[input_index] = perturb(input, element, epsilon)
|
|
41
|
+
negative_inputs[input_index] = perturb(input, element, -epsilon)
|
|
42
|
+
positive = objective(positive_inputs, gradient_outputs, options)
|
|
43
|
+
negative = objective(negative_inputs, gradient_outputs, options)
|
|
44
|
+
gradient[element] = (positive - negative) / (2 * epsilon)
|
|
45
|
+
end
|
|
46
|
+
gradient
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def perturb(input, index, amount)
|
|
50
|
+
flat = input.flatten.dup
|
|
51
|
+
flat[index] += amount
|
|
52
|
+
flat.reshape(*input.shape)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def objective(inputs, gradient_outputs, options)
|
|
56
|
+
outputs = Backend::RubyEval3dRasterizer.forward(*inputs, **options)
|
|
57
|
+
outputs.zip(gradient_outputs).sum { |output, gradient| (output * gradient).sum }.to_f
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/isect_tiles"
|
|
4
|
+
|
|
5
|
+
# Tile intersection APIs.
|
|
6
|
+
module Gsplat
|
|
7
|
+
class << self
|
|
8
|
+
# Enumerates Gaussian/tile intersections.
|
|
9
|
+
# rubocop:disable Metrics/ParameterLists
|
|
10
|
+
def isect_tiles(means2d, radii, depths, tile_size, tile_width, tile_height, sort: true)
|
|
11
|
+
# rubocop:enable Metrics/ParameterLists
|
|
12
|
+
Backend.dispatch(
|
|
13
|
+
:isect_tiles,
|
|
14
|
+
means2d,
|
|
15
|
+
radii,
|
|
16
|
+
depths,
|
|
17
|
+
tile_size,
|
|
18
|
+
tile_width,
|
|
19
|
+
tile_height,
|
|
20
|
+
sort: sort
|
|
21
|
+
)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Encodes the starting intersection index for every camera tile.
|
|
25
|
+
def isect_offset_encode(isect_ids, camera_count, tile_width, tile_height)
|
|
26
|
+
Backend.dispatch(
|
|
27
|
+
:isect_offset_encode,
|
|
28
|
+
isect_ids,
|
|
29
|
+
camera_count,
|
|
30
|
+
tile_width,
|
|
31
|
+
tile_height
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
Backend.register(:isect_tiles, :ruby, Backend::RubyIsectTiles.method(:forward))
|
|
37
|
+
Backend.register(:isect_offset_encode, :ruby, Backend::RubyIsectTiles.method(:offset_encode))
|
|
38
|
+
end
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/projection"
|
|
4
|
+
require_relative "../backend/ruby/projection_backward"
|
|
5
|
+
require_relative "../backend/ruby/projection_covariance_vjp"
|
|
6
|
+
require_relative "../backend/ruby/projection_input_vjp"
|
|
7
|
+
|
|
8
|
+
# Differentiable camera projection operations.
|
|
9
|
+
module Gsplat
|
|
10
|
+
module Ops
|
|
11
|
+
# Fused world-to-camera transform, covariance projection, and culling.
|
|
12
|
+
class FullyFusedProjection < Autograd::Function
|
|
13
|
+
class << self
|
|
14
|
+
# Adds the optional compensation placeholder used by the public tuple.
|
|
15
|
+
# @api private
|
|
16
|
+
def apply(*inputs, calc_compensations: false, **options)
|
|
17
|
+
outputs = super
|
|
18
|
+
calc_compensations ? outputs : [*outputs, nil]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# rubocop:disable Metrics/ParameterLists
|
|
22
|
+
def forward(context, means, covars, quaternions, scales, viewmats, intrinsics, width, height, **options)
|
|
23
|
+
# rubocop:enable Metrics/ParameterLists
|
|
24
|
+
context.save(means, covars, quaternions, scales, viewmats, intrinsics, width, height, options)
|
|
25
|
+
outputs = Backend.dispatch(
|
|
26
|
+
:fully_fused_projection_forward,
|
|
27
|
+
means,
|
|
28
|
+
covars,
|
|
29
|
+
quaternions,
|
|
30
|
+
scales,
|
|
31
|
+
viewmats,
|
|
32
|
+
intrinsics,
|
|
33
|
+
width,
|
|
34
|
+
height,
|
|
35
|
+
**options
|
|
36
|
+
)
|
|
37
|
+
options.fetch(:calc_compensations) ? outputs : outputs.first(4)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# rubocop:disable Metrics/ParameterLists
|
|
41
|
+
def backward(context, _grad_radii, grad_means2d, grad_depths, grad_conics, grad_compensations = nil)
|
|
42
|
+
# rubocop:enable Metrics/ParameterLists
|
|
43
|
+
means, covars, quaternions, scales, viewmats, intrinsics, width, height, options = context.saved_values
|
|
44
|
+
input_gradients = Backend.dispatch(
|
|
45
|
+
:fully_fused_projection_backward,
|
|
46
|
+
means,
|
|
47
|
+
covars,
|
|
48
|
+
quaternions,
|
|
49
|
+
scales,
|
|
50
|
+
viewmats,
|
|
51
|
+
intrinsics,
|
|
52
|
+
width,
|
|
53
|
+
height,
|
|
54
|
+
grad_means2d,
|
|
55
|
+
grad_depths,
|
|
56
|
+
grad_conics,
|
|
57
|
+
grad_compensations,
|
|
58
|
+
**options
|
|
59
|
+
)
|
|
60
|
+
[*input_gradients, nil, nil, nil, nil]
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
class << self
|
|
67
|
+
# Transforms world-space means/covariances into camera space.
|
|
68
|
+
#
|
|
69
|
+
# @param means [Numo::NArray] [N,3]
|
|
70
|
+
# @param covars [Numo::NArray] [N,3,3] or packed [N,6]
|
|
71
|
+
# @param viewmats [Numo::NArray] [C,4,4] world-to-camera transforms
|
|
72
|
+
# @return [Array<Numo::NArray>] camera means `[C,N,3]` and covariances `[C,N,3,3]`
|
|
73
|
+
def world_to_cam(means, covars, viewmats)
|
|
74
|
+
Math::CameraProjection.world_to_cam(means, covars, viewmats)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Projects camera-space means/covariances with a pinhole model.
|
|
78
|
+
#
|
|
79
|
+
# @param means [Numo::NArray] [C,N,3]
|
|
80
|
+
# @param covars [Numo::NArray] [C,N,3,3]
|
|
81
|
+
# @param intrinsics [Numo::NArray] [C,3,3]
|
|
82
|
+
# @return [Array<Numo::NArray>] projected means `[C,N,2]` and covariances `[C,N,2,2]`
|
|
83
|
+
def persp_proj(means, covars, intrinsics, width, height)
|
|
84
|
+
Math::CameraProjection.persp_proj(means, covars, intrinsics, width, height)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Projects camera-space means/covariances with an orthographic model.
|
|
88
|
+
#
|
|
89
|
+
# @param means [Numo::NArray] [C,N,3]
|
|
90
|
+
# @param covars [Numo::NArray] [C,N,3,3]
|
|
91
|
+
# @param intrinsics [Numo::NArray] [C,3,3]
|
|
92
|
+
# @return [Array<Numo::NArray>] projected means `[C,N,2]` and covariances `[C,N,2,2]`
|
|
93
|
+
def ortho_proj(means, covars, intrinsics, width, height)
|
|
94
|
+
Math::CameraProjection.ortho_proj(means, covars, intrinsics, width, height)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Projects and culls a dense camera batch.
|
|
98
|
+
#
|
|
99
|
+
# Inputs use float32/float64 Numo arrays or {Autograd::Variable}; geometry is
|
|
100
|
+
# `[N,3]`, views `[C,4,4]`, intrinsics `[C,3,3]`, and outputs are
|
|
101
|
+
# radii `[C,N,2]`, means `[C,N,2]`, depths `[C,N]`, conics `[C,N,3]`,
|
|
102
|
+
# plus optional compensations `[C,N]`.
|
|
103
|
+
#
|
|
104
|
+
# @return [Array<(Numo::NArray, Autograd::Variable, nil)>]
|
|
105
|
+
# rubocop:disable Metrics/ParameterLists, Naming/MethodParameterName
|
|
106
|
+
def fully_fused_projection(means, viewmats:, ks:, width:, height:, covars: nil, quats: nil, scales: nil,
|
|
107
|
+
eps2d: 0.3, near_plane: 0.01, far_plane: 1e10, radius_clip: 0.0,
|
|
108
|
+
calc_compensations: false, camera_model: "pinhole",
|
|
109
|
+
radial_coeffs: nil, tangential_coeffs: nil, thin_prism_coeffs: nil,
|
|
110
|
+
global_z_order: true)
|
|
111
|
+
# rubocop:enable Metrics/ParameterLists, Naming/MethodParameterName
|
|
112
|
+
inputs = [means, covars, quats, scales, viewmats, ks]
|
|
113
|
+
options = {
|
|
114
|
+
eps2d: eps2d,
|
|
115
|
+
near_plane: near_plane,
|
|
116
|
+
far_plane: far_plane,
|
|
117
|
+
radius_clip: radius_clip,
|
|
118
|
+
calc_compensations: calc_compensations,
|
|
119
|
+
camera_model: camera_model,
|
|
120
|
+
radial_coeffs: radial_coeffs,
|
|
121
|
+
tangential_coeffs: tangential_coeffs,
|
|
122
|
+
thin_prism_coeffs: thin_prism_coeffs,
|
|
123
|
+
global_z_order: global_z_order
|
|
124
|
+
}
|
|
125
|
+
if inputs.any?(Autograd::Variable)
|
|
126
|
+
return Ops::FullyFusedProjection.apply(
|
|
127
|
+
*inputs,
|
|
128
|
+
width,
|
|
129
|
+
height,
|
|
130
|
+
**options
|
|
131
|
+
)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
Backend.dispatch(
|
|
135
|
+
:fully_fused_projection_forward,
|
|
136
|
+
*inputs,
|
|
137
|
+
width,
|
|
138
|
+
height,
|
|
139
|
+
**options
|
|
140
|
+
)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
Backend.register(
|
|
145
|
+
:fully_fused_projection_forward,
|
|
146
|
+
:ruby,
|
|
147
|
+
Backend::RubyProjection.method(:forward)
|
|
148
|
+
)
|
|
149
|
+
Backend.register(
|
|
150
|
+
:fully_fused_projection_backward,
|
|
151
|
+
:ruby,
|
|
152
|
+
Backend::RubyProjectionBackward.method(:backward)
|
|
153
|
+
)
|
|
154
|
+
end
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/quat_scale_to_covar_preci"
|
|
4
|
+
|
|
5
|
+
# Differentiable quaternion/scale covariance operation.
|
|
6
|
+
module Gsplat
|
|
7
|
+
module Ops
|
|
8
|
+
# Differentiable conversion from wxyz quaternions/scales to covariance and precision.
|
|
9
|
+
class QuatScaleToCovarPreci < Autograd::Function
|
|
10
|
+
class << self
|
|
11
|
+
# Applies the operation while retaining Python gsplat's two tuple positions.
|
|
12
|
+
#
|
|
13
|
+
# @param quaternions [Autograd::Variable, Numo::NArray] [...,4]
|
|
14
|
+
# @param scales [Autograd::Variable, Numo::NArray] [...,3]
|
|
15
|
+
# @param compute_covar [Boolean]
|
|
16
|
+
# @param compute_preci [Boolean]
|
|
17
|
+
# @param triu [Boolean] return [...,6] upper triangles when true
|
|
18
|
+
# @return [Array<(Autograd::Variable, nil)>]
|
|
19
|
+
def apply(quaternions, scales, compute_covar: true, compute_preci: true, triu: false)
|
|
20
|
+
validate_selection!(compute_covar, compute_preci)
|
|
21
|
+
outputs = super
|
|
22
|
+
return outputs if compute_covar && compute_preci
|
|
23
|
+
return [outputs, nil] if compute_covar
|
|
24
|
+
|
|
25
|
+
[nil, outputs]
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Evaluates covariance/precision tensors and records inputs for VJP.
|
|
29
|
+
# @api private
|
|
30
|
+
def forward(context, quaternions, scales, **options)
|
|
31
|
+
compute_covar = options.fetch(:compute_covar)
|
|
32
|
+
compute_preci = options.fetch(:compute_preci)
|
|
33
|
+
triu = options.fetch(:triu)
|
|
34
|
+
context.save(quaternions, scales, compute_covar, compute_preci, triu)
|
|
35
|
+
covariance, precision = Backend.dispatch(
|
|
36
|
+
:quat_scale_to_covar_preci_forward,
|
|
37
|
+
quaternions,
|
|
38
|
+
scales,
|
|
39
|
+
compute_covar: compute_covar,
|
|
40
|
+
compute_preci: compute_preci,
|
|
41
|
+
triu: triu
|
|
42
|
+
)
|
|
43
|
+
return [covariance, precision] if compute_covar && compute_preci
|
|
44
|
+
|
|
45
|
+
compute_covar ? covariance : precision
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Propagates matrix gradients to quaternions and scales.
|
|
49
|
+
# @api private
|
|
50
|
+
def backward(context, *grad_outputs)
|
|
51
|
+
quaternions, scales, compute_covar, compute_preci, triu = context.saved_values
|
|
52
|
+
grad_covar = compute_covar ? grad_outputs.shift : nil
|
|
53
|
+
grad_preci = compute_preci ? grad_outputs.shift : nil
|
|
54
|
+
Backend.dispatch(
|
|
55
|
+
:quat_scale_to_covar_preci_backward,
|
|
56
|
+
quaternions,
|
|
57
|
+
scales,
|
|
58
|
+
grad_covar,
|
|
59
|
+
grad_preci,
|
|
60
|
+
triu: triu
|
|
61
|
+
)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def validate_selection!(compute_covar, compute_preci)
|
|
67
|
+
return if compute_covar || compute_preci
|
|
68
|
+
|
|
69
|
+
raise ArgumentError, "at least one of compute_covar or compute_preci must be true"
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
class << self
|
|
76
|
+
# Converts wxyz quaternions and scales to covariance and/or precision matrices.
|
|
77
|
+
#
|
|
78
|
+
# @param quaternions [Autograd::Variable, Numo::NArray] [...,4]
|
|
79
|
+
# @param scales [Autograd::Variable, Numo::NArray] [...,3]
|
|
80
|
+
# @param compute_covar [Boolean]
|
|
81
|
+
# @param compute_preci [Boolean]
|
|
82
|
+
# @param triu [Boolean] return [...,6] upper triangles when true
|
|
83
|
+
# @return [Array<(Autograd::Variable, Numo::NArray, nil)>]
|
|
84
|
+
def quat_scale_to_covar_preci(quaternions, scales, compute_covar: true, compute_preci: true, triu: false)
|
|
85
|
+
if [quaternions, scales].any?(Autograd::Variable)
|
|
86
|
+
return Ops::QuatScaleToCovarPreci.apply(
|
|
87
|
+
quaternions,
|
|
88
|
+
scales,
|
|
89
|
+
compute_covar: compute_covar,
|
|
90
|
+
compute_preci: compute_preci,
|
|
91
|
+
triu: triu
|
|
92
|
+
)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
raise ArgumentError, "at least one output must be requested" unless compute_covar || compute_preci
|
|
96
|
+
|
|
97
|
+
Backend.dispatch(
|
|
98
|
+
:quat_scale_to_covar_preci_forward,
|
|
99
|
+
quaternions,
|
|
100
|
+
scales,
|
|
101
|
+
compute_covar: compute_covar,
|
|
102
|
+
compute_preci: compute_preci,
|
|
103
|
+
triu: triu
|
|
104
|
+
)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
Backend.register(
|
|
109
|
+
:quat_scale_to_covar_preci_forward,
|
|
110
|
+
:ruby,
|
|
111
|
+
Backend::RubyQuatScaleToCovarPreci.method(:forward)
|
|
112
|
+
)
|
|
113
|
+
Backend.register(
|
|
114
|
+
:quat_scale_to_covar_preci_backward,
|
|
115
|
+
:ruby,
|
|
116
|
+
Backend::RubyQuatScaleToCovarPreci.method(:backward)
|
|
117
|
+
)
|
|
118
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/accumulate"
|
|
4
|
+
require_relative "../backend/ruby/rasterize_to_pixels"
|
|
5
|
+
require_relative "../backend/ruby/rasterize_to_indices_in_range"
|
|
6
|
+
|
|
7
|
+
# Differentiable Gaussian rasterization primitives.
|
|
8
|
+
module Gsplat
|
|
9
|
+
class << self
|
|
10
|
+
# Enumerates Gaussian contributions for tile-list batches in depth order.
|
|
11
|
+
#
|
|
12
|
+
# @return [Array<Numo::Int64>] Gaussian, pixel, and image IDs
|
|
13
|
+
# rubocop:disable Metrics/ParameterLists
|
|
14
|
+
def rasterize_to_indices_in_range(range_start, range_end, transmittances, means2d, conics, opacities,
|
|
15
|
+
width, height, tile_size, isect_offsets, flatten_ids)
|
|
16
|
+
# rubocop:enable Metrics/ParameterLists
|
|
17
|
+
tensors = [transmittances, means2d, conics, opacities, isect_offsets, flatten_ids].map do |value|
|
|
18
|
+
Ops::TensorOps.data(value)
|
|
19
|
+
end
|
|
20
|
+
Backend.dispatch(
|
|
21
|
+
:rasterize_to_indices_in_range,
|
|
22
|
+
range_start,
|
|
23
|
+
range_end,
|
|
24
|
+
*tensors.first(4),
|
|
25
|
+
width,
|
|
26
|
+
height,
|
|
27
|
+
tile_size,
|
|
28
|
+
*tensors.last(2)
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
Backend.register(
|
|
34
|
+
:rasterize_to_indices_in_range,
|
|
35
|
+
:ruby,
|
|
36
|
+
Backend::RubyRasterizeToIndicesInRange.method(:forward)
|
|
37
|
+
)
|
|
38
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../backend/ruby/tile_compositor"
|
|
4
|
+
require_relative "../backend/ruby/rasterize_to_pixels"
|
|
5
|
+
require_relative "../backend/ruby/tile_compositor_backward"
|
|
6
|
+
require_relative "../backend/ruby/rasterize_to_pixels_backward"
|
|
7
|
+
|
|
8
|
+
# Differentiable tile rasterization API.
|
|
9
|
+
module Gsplat
|
|
10
|
+
module Ops
|
|
11
|
+
# Tile-accelerated alpha compositor.
|
|
12
|
+
class RasterizeToPixels < Autograd::Function
|
|
13
|
+
class << self
|
|
14
|
+
# rubocop:disable Metrics/ParameterLists
|
|
15
|
+
def forward(context, means2d, conics, colors, opacities, backgrounds, masks, width, height,
|
|
16
|
+
tile_size, isect_offsets, flatten_ids, absgrad:)
|
|
17
|
+
# rubocop:enable Metrics/ParameterLists
|
|
18
|
+
render_colors, render_alphas, last_ids = Backend.dispatch(
|
|
19
|
+
:rasterize_to_pixels_forward,
|
|
20
|
+
means2d,
|
|
21
|
+
conics,
|
|
22
|
+
colors,
|
|
23
|
+
opacities,
|
|
24
|
+
backgrounds,
|
|
25
|
+
masks,
|
|
26
|
+
width,
|
|
27
|
+
height,
|
|
28
|
+
tile_size,
|
|
29
|
+
isect_offsets,
|
|
30
|
+
flatten_ids
|
|
31
|
+
)
|
|
32
|
+
context.save(
|
|
33
|
+
means2d, conics, colors, opacities, backgrounds, masks, width, height,
|
|
34
|
+
tile_size, isect_offsets, flatten_ids, render_alphas, last_ids, absgrad
|
|
35
|
+
)
|
|
36
|
+
[render_colors, render_alphas]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Propagates pixel color and alpha gradients to projected attributes.
|
|
40
|
+
# @api private
|
|
41
|
+
def backward(context, grad_render_colors, grad_render_alphas)
|
|
42
|
+
saved = context.saved_values
|
|
43
|
+
means2d, conics, colors, opacities, backgrounds, masks, width, height,
|
|
44
|
+
tile_size, offsets, flatten_ids, render_alphas, last_ids, absgrad = saved
|
|
45
|
+
gradients, means2d_absgrad = Backend.dispatch(
|
|
46
|
+
:rasterize_to_pixels_backward,
|
|
47
|
+
means2d,
|
|
48
|
+
conics,
|
|
49
|
+
colors,
|
|
50
|
+
opacities,
|
|
51
|
+
backgrounds,
|
|
52
|
+
masks,
|
|
53
|
+
width,
|
|
54
|
+
height,
|
|
55
|
+
tile_size,
|
|
56
|
+
offsets,
|
|
57
|
+
flatten_ids,
|
|
58
|
+
render_alphas,
|
|
59
|
+
last_ids,
|
|
60
|
+
grad_render_colors,
|
|
61
|
+
grad_render_alphas,
|
|
62
|
+
absgrad: absgrad
|
|
63
|
+
)
|
|
64
|
+
if absgrad && context.inputs[0].is_a?(Autograd::Variable)
|
|
65
|
+
context.inputs[0].accumulate_absgrad(means2d_absgrad)
|
|
66
|
+
end
|
|
67
|
+
[*gradients, nil, nil, nil, nil, nil, nil]
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
class << self
|
|
74
|
+
# Alpha-composites sorted tile intersections.
|
|
75
|
+
#
|
|
76
|
+
# Projected inputs have shapes `means2d [C,N,2]`, `conics [C,N,3]`,
|
|
77
|
+
# `colors [C,N,D]`, and `opacities [C,N]`. Outputs are color
|
|
78
|
+
# `[C,H,W,D]` and alpha `[C,H,W,1]`.
|
|
79
|
+
#
|
|
80
|
+
# @return [Array<(Numo::NArray, Autograd::Variable)>]
|
|
81
|
+
# rubocop:disable Metrics/ParameterLists
|
|
82
|
+
def rasterize_to_pixels(means2d, conics, colors, opacities, width, height, tile_size,
|
|
83
|
+
isect_offsets, flatten_ids, backgrounds: nil, masks: nil, absgrad: false)
|
|
84
|
+
# rubocop:enable Metrics/ParameterLists
|
|
85
|
+
inputs = [
|
|
86
|
+
means2d, conics, colors, opacities, backgrounds, masks, width, height,
|
|
87
|
+
tile_size, isect_offsets, flatten_ids
|
|
88
|
+
]
|
|
89
|
+
return Ops::RasterizeToPixels.apply(*inputs, absgrad: absgrad) if inputs.any?(Autograd::Variable)
|
|
90
|
+
|
|
91
|
+
Backend.dispatch(:rasterize_to_pixels_forward, *inputs).first(2)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
Backend.register(
|
|
96
|
+
:rasterize_to_pixels_forward,
|
|
97
|
+
:ruby,
|
|
98
|
+
Backend::RubyRasterizeToPixels.method(:forward)
|
|
99
|
+
)
|
|
100
|
+
Backend.register(
|
|
101
|
+
:rasterize_to_pixels_backward,
|
|
102
|
+
:ruby,
|
|
103
|
+
Backend::RubyRasterizeToPixelsBackward.method(:backward)
|
|
104
|
+
)
|
|
105
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Closed-form relocation primitive and public wrapper.
|
|
4
|
+
module Gsplat
|
|
5
|
+
module Ops
|
|
6
|
+
# Closed-form opacity and scale update from 3DGS-MCMC equation 9.
|
|
7
|
+
module Relocation
|
|
8
|
+
# Default maximum split ratio represented by {.binomial_table}.
|
|
9
|
+
DEFAULT_N_MAX = 51
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# Builds the binomial coefficient lookup used by relocation.
|
|
14
|
+
#
|
|
15
|
+
# @param n_max [Integer] maximum split ratio, inclusive
|
|
16
|
+
# @param dtype [Class] Numo floating-point class
|
|
17
|
+
# @return [Numo::NArray] [n_max,n_max]
|
|
18
|
+
def binomial_table(n_max: DEFAULT_N_MAX, dtype: Numo::SFloat)
|
|
19
|
+
raise ArgumentError, "n_max must be a positive integer" unless n_max.is_a?(Integer) && n_max.positive?
|
|
20
|
+
|
|
21
|
+
output = dtype.zeros(n_max, n_max)
|
|
22
|
+
n_max.times do |n_value|
|
|
23
|
+
(n_value + 1).times { |k_value| output[n_value, k_value] = binomial(n_value, k_value) }
|
|
24
|
+
end
|
|
25
|
+
output
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Recalculates activated opacities and scales for a requested split ratio.
|
|
29
|
+
#
|
|
30
|
+
# @param opacities [Numo::NArray] [N], activated values in [0,1]
|
|
31
|
+
# @param scales [Numo::NArray] [N,3], activated positive scales
|
|
32
|
+
# @param ratios [Numo::NArray] [N], clamped to 1..n_max
|
|
33
|
+
# @param binoms [Numo::NArray] [n_max,n_max]
|
|
34
|
+
# @return [Array<Numo::NArray>] new opacities and scales
|
|
35
|
+
def compute(opacities, scales, ratios, binoms: binomial_table(dtype: opacities.class))
|
|
36
|
+
validate_inputs!(opacities, scales, ratios, binoms)
|
|
37
|
+
output_opacities = opacities.class.zeros(opacities.shape[0])
|
|
38
|
+
output_scales = scales.class.zeros(*scales.shape)
|
|
39
|
+
opacities.shape[0].times do |index|
|
|
40
|
+
ratio = ratios[index].to_i.clamp(1, binoms.shape[0])
|
|
41
|
+
opacity = opacities[index].to_f
|
|
42
|
+
new_opacity = 1.0 - ((1.0 - opacity)**(1.0 / ratio))
|
|
43
|
+
coefficient = opacity / denominator(new_opacity, ratio, binoms)
|
|
44
|
+
output_opacities[index] = new_opacity
|
|
45
|
+
output_scales[index, true] = scales[index, true] * coefficient
|
|
46
|
+
end
|
|
47
|
+
[output_opacities, output_scales]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def binomial(n_value, k_value)
|
|
51
|
+
return 1 if k_value.zero? || k_value == n_value
|
|
52
|
+
|
|
53
|
+
k_value = [k_value, n_value - k_value].min
|
|
54
|
+
(1..k_value).reduce(1) { |value, index| (value * (n_value - k_value + index)) / index }
|
|
55
|
+
end
|
|
56
|
+
private_class_method :binomial
|
|
57
|
+
|
|
58
|
+
def denominator(opacity, ratio, binoms)
|
|
59
|
+
(1..ratio).sum do |i_value|
|
|
60
|
+
(0...i_value).sum do |k_value|
|
|
61
|
+
sign = k_value.even? ? 1.0 : -1.0
|
|
62
|
+
binoms[i_value - 1, k_value].to_f * sign * (opacity**(k_value + 1)) / ::Math.sqrt(k_value + 1)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
private_class_method :denominator
|
|
67
|
+
|
|
68
|
+
def validate_inputs!(opacities, scales, ratios, binoms)
|
|
69
|
+
unless opacities.is_a?(Numo::NArray) && [Numo::SFloat, Numo::DFloat].include?(opacities.class)
|
|
70
|
+
raise ArgumentError, "opacities must be Numo::SFloat or Numo::DFloat"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
count = opacities.shape[0] if opacities.ndim == 1
|
|
74
|
+
valid = count && scales.shape == [count, 3] && ratios.shape == [count] &&
|
|
75
|
+
binoms.ndim == 2 && binoms.shape[0] == binoms.shape[1]
|
|
76
|
+
return if valid
|
|
77
|
+
|
|
78
|
+
raise ShapeError,
|
|
79
|
+
"expected opacities [N], scales [N,3], ratios [N], binoms [M,M]; " \
|
|
80
|
+
"got #{opacities.shape.inspect}, #{scales.shape.inspect}, " \
|
|
81
|
+
"#{ratios.shape.inspect}, #{binoms.shape.inspect}"
|
|
82
|
+
end
|
|
83
|
+
private_class_method :validate_inputs!
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
class << self
|
|
88
|
+
# Computes the deterministic 3DGS-MCMC relocation update.
|
|
89
|
+
def relocation(opacities, scales, ratios, binoms: nil)
|
|
90
|
+
options = binoms ? { binoms: binoms } : {}
|
|
91
|
+
Ops::Relocation.compute(opacities, scales, ratios, **options)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|