gsplat 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +202 -0
  3. data/README.md +236 -0
  4. data/docs/ACCEPTANCE.md +60 -0
  5. data/docs/BENCHMARKS.md +79 -0
  6. data/docs/DECISIONS.md +52 -0
  7. data/docs/MIGRATION.md +117 -0
  8. data/docs/PROFILE.md +75 -0
  9. data/docs/PROGRESS.md +199 -0
  10. data/docs/decisions/0000-template.md +17 -0
  11. data/docs/decisions/0001-pin-golden-reference-and-cpu-validation.md +27 -0
  12. data/docs/decisions/0002-retain-ruby-fallbacks-for-native-backend.md +26 -0
  13. data/docs/decisions/0003-share-projection-for-distorted-cameras.md +26 -0
  14. data/docs/decisions/0004-use-portable-world-space-reference-paths.md +26 -0
  15. data/docs/decisions/0005-reuse-ewa-core-for-2dgs.md +24 -0
  16. data/docs/decisions/0006-keep-eval3d-as-portable-reference.md +25 -0
  17. data/docs/decisions/0007-share-compositor-semantics-for-contribution-indices.md +25 -0
  18. data/examples/data/README.md +14 -0
  19. data/examples/data/colmap/images/view_000.png +0 -0
  20. data/examples/data/colmap/images/view_001.png +0 -0
  21. data/examples/data/colmap/images/view_002.png +0 -0
  22. data/examples/data/colmap/sparse/0/cameras.txt +2 -0
  23. data/examples/data/colmap/sparse/0/images.txt +6 -0
  24. data/examples/data/colmap/sparse/0/points3D.txt +17 -0
  25. data/examples/data/splats.ply +0 -0
  26. data/examples/fit_image.rb +35 -0
  27. data/examples/generate_sample_data.rb +147 -0
  28. data/examples/render_path.rb +117 -0
  29. data/examples/simple_trainer.rb +76 -0
  30. data/ext/gsplat_native/common.h +54 -0
  31. data/ext/gsplat_native/extconf.rb +39 -0
  32. data/ext/gsplat_native/gsplat_native.c +60 -0
  33. data/ext/gsplat_native/intersections.c +211 -0
  34. data/ext/gsplat_native/projection.c +199 -0
  35. data/ext/gsplat_native/raster_backward.c +129 -0
  36. data/ext/gsplat_native/raster_backward_bridge.c +69 -0
  37. data/ext/gsplat_native/raster_forward.c +150 -0
  38. data/ext/gsplat_native/rasterization.h +51 -0
  39. data/ext/gsplat_native/spherical_harmonics.c +129 -0
  40. data/gsplat.gemspec +36 -0
  41. data/lib/gsplat/autograd/context.rb +60 -0
  42. data/lib/gsplat/autograd/function.rb +68 -0
  43. data/lib/gsplat/autograd/variable.rb +159 -0
  44. data/lib/gsplat/backend/ruby/accumulate.rb +139 -0
  45. data/lib/gsplat/backend/ruby/accumulate_backward.rb +40 -0
  46. data/lib/gsplat/backend/ruby/eval3d_rasterizer.rb +175 -0
  47. data/lib/gsplat/backend/ruby/isect_tiles.rb +198 -0
  48. data/lib/gsplat/backend/ruby/projection.rb +251 -0
  49. data/lib/gsplat/backend/ruby/projection_backward.rb +190 -0
  50. data/lib/gsplat/backend/ruby/projection_covariance_vjp.rb +72 -0
  51. data/lib/gsplat/backend/ruby/projection_input_vjp.rb +252 -0
  52. data/lib/gsplat/backend/ruby/quat_scale_to_covar_preci.rb +139 -0
  53. data/lib/gsplat/backend/ruby/rasterize_to_indices_in_range.rb +121 -0
  54. data/lib/gsplat/backend/ruby/rasterize_to_pixels.rb +199 -0
  55. data/lib/gsplat/backend/ruby/rasterize_to_pixels_backward.rb +121 -0
  56. data/lib/gsplat/backend/ruby/spherical_harmonics.rb +135 -0
  57. data/lib/gsplat/backend/ruby/tile_compositor.rb +50 -0
  58. data/lib/gsplat/backend/ruby/tile_compositor_backward.rb +84 -0
  59. data/lib/gsplat/backend.rb +79 -0
  60. data/lib/gsplat/compression/grid_sort.rb +79 -0
  61. data/lib/gsplat/compression/kmeans.rb +121 -0
  62. data/lib/gsplat/compression/png.rb +147 -0
  63. data/lib/gsplat/compression/png_codec.rb +133 -0
  64. data/lib/gsplat/compression/quantizer.rb +79 -0
  65. data/lib/gsplat/io/checkpoint.rb +145 -0
  66. data/lib/gsplat/io/colmap.rb +175 -0
  67. data/lib/gsplat/io/colmap_binary.rb +98 -0
  68. data/lib/gsplat/io/colmap_text.rb +84 -0
  69. data/lib/gsplat/io/image.rb +63 -0
  70. data/lib/gsplat/io/image_backends.rb +81 -0
  71. data/lib/gsplat/io/npy.rb +189 -0
  72. data/lib/gsplat/io/ply.rb +185 -0
  73. data/lib/gsplat/io/ply_reader.rb +142 -0
  74. data/lib/gsplat/io/zip_archive.rb +183 -0
  75. data/lib/gsplat/math/camera_distortion.rb +123 -0
  76. data/lib/gsplat/math/camera_projection.rb +202 -0
  77. data/lib/gsplat/math/mat.rb +114 -0
  78. data/lib/gsplat/math/quaternion.rb +175 -0
  79. data/lib/gsplat/math/small_matrix_primitives.rb +112 -0
  80. data/lib/gsplat/math/spherical_harmonic_basis.rb +148 -0
  81. data/lib/gsplat/math/ssim.rb +153 -0
  82. data/lib/gsplat/native.rb +30 -0
  83. data/lib/gsplat/native_ops.rb +148 -0
  84. data/lib/gsplat/native_raster_ops.rb +76 -0
  85. data/lib/gsplat/ops/accumulate.rb +68 -0
  86. data/lib/gsplat/ops/eval3d_rasterize.rb +62 -0
  87. data/lib/gsplat/ops/isect_tiles.rb +38 -0
  88. data/lib/gsplat/ops/projection.rb +154 -0
  89. data/lib/gsplat/ops/quat_scale_to_covar_preci.rb +118 -0
  90. data/lib/gsplat/ops/rasterize_to_indices_in_range.rb +38 -0
  91. data/lib/gsplat/ops/rasterize_to_pixels.rb +105 -0
  92. data/lib/gsplat/ops/relocation.rb +94 -0
  93. data/lib/gsplat/ops/spherical_harmonics.rb +71 -0
  94. data/lib/gsplat/ops/tensor_shape_ops.rb +173 -0
  95. data/lib/gsplat/ops/tensor_value_ops.rb +150 -0
  96. data/lib/gsplat/optim/adam.rb +212 -0
  97. data/lib/gsplat/optim/lr_scheduler.rb +36 -0
  98. data/lib/gsplat/optim/selective_adam.rb +68 -0
  99. data/lib/gsplat/rasterization.rb +130 -0
  100. data/lib/gsplat/rasterization_2dgs.rb +140 -0
  101. data/lib/gsplat/rasterization_helpers.rb +162 -0
  102. data/lib/gsplat/rasterization_validation.rb +99 -0
  103. data/lib/gsplat/strategy/base.rb +49 -0
  104. data/lib/gsplat/strategy/default.rb +188 -0
  105. data/lib/gsplat/strategy/mcmc.rb +103 -0
  106. data/lib/gsplat/strategy/mcmc_ops.rb +143 -0
  107. data/lib/gsplat/strategy/ops.rb +165 -0
  108. data/lib/gsplat/training/config.rb +91 -0
  109. data/lib/gsplat/training/image_fitter.rb +158 -0
  110. data/lib/gsplat/training/losses.rb +191 -0
  111. data/lib/gsplat/training/scene.rb +116 -0
  112. data/lib/gsplat/training/trainer.rb +236 -0
  113. data/lib/gsplat/utils.rb +110 -0
  114. data/lib/gsplat/version.rb +6 -0
  115. data/lib/gsplat.rb +98 -0
  116. metadata +181 -0
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/accumulate"
4
+ require_relative "../backend/ruby/accumulate_backward"
5
+
6
+ # Brute-force reference rasterization API.
7
+ module Gsplat
8
+ # Differentiable operation classes and low-level tensor adapters.
9
+ module Ops
10
+ # Differentiable reference alpha compositor.
11
+ class Accumulate < Autograd::Function
12
+ class << self
13
+ # rubocop:disable Metrics/ParameterLists
14
+ def forward(context, means2d, conics, opacities, colors, backgrounds, width, height)
15
+ # rubocop:enable Metrics/ParameterLists
16
+ render_colors, render_alphas, last_ids = Backend.dispatch(
17
+ :accumulate_forward,
18
+ means2d,
19
+ conics,
20
+ opacities,
21
+ colors,
22
+ backgrounds,
23
+ width,
24
+ height
25
+ )
26
+ context.save(
27
+ means2d, conics, opacities, colors, backgrounds, width, height,
28
+ render_alphas, last_ids
29
+ )
30
+ [render_colors, render_alphas]
31
+ end
32
+
33
+ # Propagates color and alpha gradients through the reference compositor.
34
+ # @api private
35
+ def backward(context, grad_render_colors, grad_render_alphas)
36
+ saved = context.saved_values
37
+ gradients = Backend.dispatch(
38
+ :accumulate_backward,
39
+ *saved,
40
+ grad_render_colors,
41
+ grad_render_alphas
42
+ )
43
+ [*gradients, nil, nil]
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ class << self
50
+ # Composites every Gaussian over every pixel without tile acceleration.
51
+ # rubocop:disable Metrics/ParameterLists
52
+ def accumulate(means2d, conics, opacities, colors, width:, height:, backgrounds: nil)
53
+ # rubocop:enable Metrics/ParameterLists
54
+ inputs = [means2d, conics, opacities, colors, backgrounds]
55
+ return Ops::Accumulate.apply(*inputs, width, height) if inputs.any?(Autograd::Variable)
56
+
57
+ Backend.dispatch(
58
+ :accumulate_forward,
59
+ *inputs,
60
+ width,
61
+ height
62
+ ).first(2)
63
+ end
64
+ end
65
+
66
+ Backend.register(:accumulate_forward, :ruby, Backend::RubyAccumulate.method(:forward))
67
+ Backend.register(:accumulate_backward, :ruby, Backend::RubyAccumulateBackward.method(:backward))
68
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/eval3d_rasterizer"
4
+
5
+ module Gsplat
6
+ module Ops
7
+ # Differentiable world-space reference rasterizer with optional normals.
8
+ class Eval3dRasterize < Autograd::Function
9
+ class << self
10
+ # rubocop:disable Metrics/ParameterLists
11
+ def forward(context, means, quats, scales, colors, opacities, backgrounds, **options)
12
+ # rubocop:enable Metrics/ParameterLists
13
+ context.save(means, quats, scales, colors, opacities, backgrounds, options)
14
+ Backend::RubyEval3dRasterizer.forward(
15
+ means, quats, scales, colors, opacities, backgrounds, **options
16
+ )
17
+ end
18
+
19
+ # Computes a central-difference VJP for the portable eval3d path.
20
+ # @api private
21
+ def backward(context, grad_rendered, grad_alphas, grad_normals)
22
+ *inputs, options = context.saved_values
23
+ gradient_outputs = [grad_rendered, grad_alphas, grad_normals]
24
+ context.needs_input_grad.each_with_index.map do |needed, input_index|
25
+ next unless needed
26
+
27
+ numerical_vjp(inputs, input_index, gradient_outputs, options)
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def numerical_vjp(inputs, input_index, gradient_outputs, options)
34
+ input = inputs[input_index]
35
+ epsilon = input.is_a?(Numo::DFloat) ? 1e-5 : 1e-3
36
+ gradient = input.class.zeros(*input.shape)
37
+ input.size.times do |element|
38
+ positive_inputs = inputs.dup
39
+ negative_inputs = inputs.dup
40
+ positive_inputs[input_index] = perturb(input, element, epsilon)
41
+ negative_inputs[input_index] = perturb(input, element, -epsilon)
42
+ positive = objective(positive_inputs, gradient_outputs, options)
43
+ negative = objective(negative_inputs, gradient_outputs, options)
44
+ gradient[element] = (positive - negative) / (2 * epsilon)
45
+ end
46
+ gradient
47
+ end
48
+
49
+ def perturb(input, index, amount)
50
+ flat = input.flatten.dup
51
+ flat[index] += amount
52
+ flat.reshape(*input.shape)
53
+ end
54
+
55
+ def objective(inputs, gradient_outputs, options)
56
+ outputs = Backend::RubyEval3dRasterizer.forward(*inputs, **options)
57
+ outputs.zip(gradient_outputs).sum { |output, gradient| (output * gradient).sum }.to_f
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/isect_tiles"
4
+
5
+ # Tile intersection APIs.
6
+ module Gsplat
7
+ class << self
8
+ # Enumerates Gaussian/tile intersections.
9
+ # rubocop:disable Metrics/ParameterLists
10
+ def isect_tiles(means2d, radii, depths, tile_size, tile_width, tile_height, sort: true)
11
+ # rubocop:enable Metrics/ParameterLists
12
+ Backend.dispatch(
13
+ :isect_tiles,
14
+ means2d,
15
+ radii,
16
+ depths,
17
+ tile_size,
18
+ tile_width,
19
+ tile_height,
20
+ sort: sort
21
+ )
22
+ end
23
+
24
+ # Encodes the starting intersection index for every camera tile.
25
+ def isect_offset_encode(isect_ids, camera_count, tile_width, tile_height)
26
+ Backend.dispatch(
27
+ :isect_offset_encode,
28
+ isect_ids,
29
+ camera_count,
30
+ tile_width,
31
+ tile_height
32
+ )
33
+ end
34
+ end
35
+
36
+ Backend.register(:isect_tiles, :ruby, Backend::RubyIsectTiles.method(:forward))
37
+ Backend.register(:isect_offset_encode, :ruby, Backend::RubyIsectTiles.method(:offset_encode))
38
+ end
@@ -0,0 +1,154 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/projection"
4
+ require_relative "../backend/ruby/projection_backward"
5
+ require_relative "../backend/ruby/projection_covariance_vjp"
6
+ require_relative "../backend/ruby/projection_input_vjp"
7
+
8
+ # Differentiable camera projection operations.
9
+ module Gsplat
10
+ module Ops
11
+ # Fused world-to-camera transform, covariance projection, and culling.
12
+ class FullyFusedProjection < Autograd::Function
13
+ class << self
14
+ # Adds the optional compensation placeholder used by the public tuple.
15
+ # @api private
16
+ def apply(*inputs, calc_compensations: false, **options)
17
+ outputs = super
18
+ calc_compensations ? outputs : [*outputs, nil]
19
+ end
20
+
21
+ # rubocop:disable Metrics/ParameterLists
22
+ def forward(context, means, covars, quaternions, scales, viewmats, intrinsics, width, height, **options)
23
+ # rubocop:enable Metrics/ParameterLists
24
+ context.save(means, covars, quaternions, scales, viewmats, intrinsics, width, height, options)
25
+ outputs = Backend.dispatch(
26
+ :fully_fused_projection_forward,
27
+ means,
28
+ covars,
29
+ quaternions,
30
+ scales,
31
+ viewmats,
32
+ intrinsics,
33
+ width,
34
+ height,
35
+ **options
36
+ )
37
+ options.fetch(:calc_compensations) ? outputs : outputs.first(4)
38
+ end
39
+
40
+ # rubocop:disable Metrics/ParameterLists
41
+ def backward(context, _grad_radii, grad_means2d, grad_depths, grad_conics, grad_compensations = nil)
42
+ # rubocop:enable Metrics/ParameterLists
43
+ means, covars, quaternions, scales, viewmats, intrinsics, width, height, options = context.saved_values
44
+ input_gradients = Backend.dispatch(
45
+ :fully_fused_projection_backward,
46
+ means,
47
+ covars,
48
+ quaternions,
49
+ scales,
50
+ viewmats,
51
+ intrinsics,
52
+ width,
53
+ height,
54
+ grad_means2d,
55
+ grad_depths,
56
+ grad_conics,
57
+ grad_compensations,
58
+ **options
59
+ )
60
+ [*input_gradients, nil, nil, nil, nil]
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ class << self
67
+ # Transforms world-space means/covariances into camera space.
68
+ #
69
+ # @param means [Numo::NArray] [N,3]
70
+ # @param covars [Numo::NArray] [N,3,3] or packed [N,6]
71
+ # @param viewmats [Numo::NArray] [C,4,4] world-to-camera transforms
72
+ # @return [Array<Numo::NArray>] camera means `[C,N,3]` and covariances `[C,N,3,3]`
73
+ def world_to_cam(means, covars, viewmats)
74
+ Math::CameraProjection.world_to_cam(means, covars, viewmats)
75
+ end
76
+
77
+ # Projects camera-space means/covariances with a pinhole model.
78
+ #
79
+ # @param means [Numo::NArray] [C,N,3]
80
+ # @param covars [Numo::NArray] [C,N,3,3]
81
+ # @param intrinsics [Numo::NArray] [C,3,3]
82
+ # @return [Array<Numo::NArray>] projected means `[C,N,2]` and covariances `[C,N,2,2]`
83
+ def persp_proj(means, covars, intrinsics, width, height)
84
+ Math::CameraProjection.persp_proj(means, covars, intrinsics, width, height)
85
+ end
86
+
87
+ # Projects camera-space means/covariances with an orthographic model.
88
+ #
89
+ # @param means [Numo::NArray] [C,N,3]
90
+ # @param covars [Numo::NArray] [C,N,3,3]
91
+ # @param intrinsics [Numo::NArray] [C,3,3]
92
+ # @return [Array<Numo::NArray>] projected means `[C,N,2]` and covariances `[C,N,2,2]`
93
+ def ortho_proj(means, covars, intrinsics, width, height)
94
+ Math::CameraProjection.ortho_proj(means, covars, intrinsics, width, height)
95
+ end
96
+
97
+ # Projects and culls a dense camera batch.
98
+ #
99
+ # Inputs use float32/float64 Numo arrays or {Autograd::Variable}; geometry is
100
+ # `[N,3]`, views `[C,4,4]`, intrinsics `[C,3,3]`, and outputs are
101
+ # radii `[C,N,2]`, means `[C,N,2]`, depths `[C,N]`, conics `[C,N,3]`,
102
+ # plus optional compensations `[C,N]`.
103
+ #
104
+ # @return [Array<(Numo::NArray, Autograd::Variable, nil)>]
105
+ # rubocop:disable Metrics/ParameterLists, Naming/MethodParameterName
106
+ def fully_fused_projection(means, viewmats:, ks:, width:, height:, covars: nil, quats: nil, scales: nil,
107
+ eps2d: 0.3, near_plane: 0.01, far_plane: 1e10, radius_clip: 0.0,
108
+ calc_compensations: false, camera_model: "pinhole",
109
+ radial_coeffs: nil, tangential_coeffs: nil, thin_prism_coeffs: nil,
110
+ global_z_order: true)
111
+ # rubocop:enable Metrics/ParameterLists, Naming/MethodParameterName
112
+ inputs = [means, covars, quats, scales, viewmats, ks]
113
+ options = {
114
+ eps2d: eps2d,
115
+ near_plane: near_plane,
116
+ far_plane: far_plane,
117
+ radius_clip: radius_clip,
118
+ calc_compensations: calc_compensations,
119
+ camera_model: camera_model,
120
+ radial_coeffs: radial_coeffs,
121
+ tangential_coeffs: tangential_coeffs,
122
+ thin_prism_coeffs: thin_prism_coeffs,
123
+ global_z_order: global_z_order
124
+ }
125
+ if inputs.any?(Autograd::Variable)
126
+ return Ops::FullyFusedProjection.apply(
127
+ *inputs,
128
+ width,
129
+ height,
130
+ **options
131
+ )
132
+ end
133
+
134
+ Backend.dispatch(
135
+ :fully_fused_projection_forward,
136
+ *inputs,
137
+ width,
138
+ height,
139
+ **options
140
+ )
141
+ end
142
+ end
143
+
144
+ Backend.register(
145
+ :fully_fused_projection_forward,
146
+ :ruby,
147
+ Backend::RubyProjection.method(:forward)
148
+ )
149
+ Backend.register(
150
+ :fully_fused_projection_backward,
151
+ :ruby,
152
+ Backend::RubyProjectionBackward.method(:backward)
153
+ )
154
+ end
@@ -0,0 +1,118 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/quat_scale_to_covar_preci"
4
+
5
+ # Differentiable quaternion/scale covariance operation.
6
+ module Gsplat
7
+ module Ops
8
+ # Differentiable conversion from wxyz quaternions/scales to covariance and precision.
9
+ class QuatScaleToCovarPreci < Autograd::Function
10
+ class << self
11
+ # Applies the operation while retaining Python gsplat's two tuple positions.
12
+ #
13
+ # @param quaternions [Autograd::Variable, Numo::NArray] [...,4]
14
+ # @param scales [Autograd::Variable, Numo::NArray] [...,3]
15
+ # @param compute_covar [Boolean]
16
+ # @param compute_preci [Boolean]
17
+ # @param triu [Boolean] return [...,6] upper triangles when true
18
+ # @return [Array<(Autograd::Variable, nil)>]
19
+ def apply(quaternions, scales, compute_covar: true, compute_preci: true, triu: false)
20
+ validate_selection!(compute_covar, compute_preci)
21
+ outputs = super
22
+ return outputs if compute_covar && compute_preci
23
+ return [outputs, nil] if compute_covar
24
+
25
+ [nil, outputs]
26
+ end
27
+
28
+ # Evaluates covariance/precision tensors and records inputs for VJP.
29
+ # @api private
30
+ def forward(context, quaternions, scales, **options)
31
+ compute_covar = options.fetch(:compute_covar)
32
+ compute_preci = options.fetch(:compute_preci)
33
+ triu = options.fetch(:triu)
34
+ context.save(quaternions, scales, compute_covar, compute_preci, triu)
35
+ covariance, precision = Backend.dispatch(
36
+ :quat_scale_to_covar_preci_forward,
37
+ quaternions,
38
+ scales,
39
+ compute_covar: compute_covar,
40
+ compute_preci: compute_preci,
41
+ triu: triu
42
+ )
43
+ return [covariance, precision] if compute_covar && compute_preci
44
+
45
+ compute_covar ? covariance : precision
46
+ end
47
+
48
+ # Propagates matrix gradients to quaternions and scales.
49
+ # @api private
50
+ def backward(context, *grad_outputs)
51
+ quaternions, scales, compute_covar, compute_preci, triu = context.saved_values
52
+ grad_covar = compute_covar ? grad_outputs.shift : nil
53
+ grad_preci = compute_preci ? grad_outputs.shift : nil
54
+ Backend.dispatch(
55
+ :quat_scale_to_covar_preci_backward,
56
+ quaternions,
57
+ scales,
58
+ grad_covar,
59
+ grad_preci,
60
+ triu: triu
61
+ )
62
+ end
63
+
64
+ private
65
+
66
+ def validate_selection!(compute_covar, compute_preci)
67
+ return if compute_covar || compute_preci
68
+
69
+ raise ArgumentError, "at least one of compute_covar or compute_preci must be true"
70
+ end
71
+ end
72
+ end
73
+ end
74
+
75
+ class << self
76
+ # Converts wxyz quaternions and scales to covariance and/or precision matrices.
77
+ #
78
+ # @param quaternions [Autograd::Variable, Numo::NArray] [...,4]
79
+ # @param scales [Autograd::Variable, Numo::NArray] [...,3]
80
+ # @param compute_covar [Boolean]
81
+ # @param compute_preci [Boolean]
82
+ # @param triu [Boolean] return [...,6] upper triangles when true
83
+ # @return [Array<(Autograd::Variable, Numo::NArray, nil)>]
84
+ def quat_scale_to_covar_preci(quaternions, scales, compute_covar: true, compute_preci: true, triu: false)
85
+ if [quaternions, scales].any?(Autograd::Variable)
86
+ return Ops::QuatScaleToCovarPreci.apply(
87
+ quaternions,
88
+ scales,
89
+ compute_covar: compute_covar,
90
+ compute_preci: compute_preci,
91
+ triu: triu
92
+ )
93
+ end
94
+
95
+ raise ArgumentError, "at least one output must be requested" unless compute_covar || compute_preci
96
+
97
+ Backend.dispatch(
98
+ :quat_scale_to_covar_preci_forward,
99
+ quaternions,
100
+ scales,
101
+ compute_covar: compute_covar,
102
+ compute_preci: compute_preci,
103
+ triu: triu
104
+ )
105
+ end
106
+ end
107
+
108
+ Backend.register(
109
+ :quat_scale_to_covar_preci_forward,
110
+ :ruby,
111
+ Backend::RubyQuatScaleToCovarPreci.method(:forward)
112
+ )
113
+ Backend.register(
114
+ :quat_scale_to_covar_preci_backward,
115
+ :ruby,
116
+ Backend::RubyQuatScaleToCovarPreci.method(:backward)
117
+ )
118
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/accumulate"
4
+ require_relative "../backend/ruby/rasterize_to_pixels"
5
+ require_relative "../backend/ruby/rasterize_to_indices_in_range"
6
+
7
+ # Differentiable Gaussian rasterization primitives.
8
+ module Gsplat
9
+ class << self
10
+ # Enumerates Gaussian contributions for tile-list batches in depth order.
11
+ #
12
+ # @return [Array<Numo::Int64>] Gaussian, pixel, and image IDs
13
+ # rubocop:disable Metrics/ParameterLists
14
+ def rasterize_to_indices_in_range(range_start, range_end, transmittances, means2d, conics, opacities,
15
+ width, height, tile_size, isect_offsets, flatten_ids)
16
+ # rubocop:enable Metrics/ParameterLists
17
+ tensors = [transmittances, means2d, conics, opacities, isect_offsets, flatten_ids].map do |value|
18
+ Ops::TensorOps.data(value)
19
+ end
20
+ Backend.dispatch(
21
+ :rasterize_to_indices_in_range,
22
+ range_start,
23
+ range_end,
24
+ *tensors.first(4),
25
+ width,
26
+ height,
27
+ tile_size,
28
+ *tensors.last(2)
29
+ )
30
+ end
31
+ end
32
+
33
+ Backend.register(
34
+ :rasterize_to_indices_in_range,
35
+ :ruby,
36
+ Backend::RubyRasterizeToIndicesInRange.method(:forward)
37
+ )
38
+ end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../backend/ruby/tile_compositor"
4
+ require_relative "../backend/ruby/rasterize_to_pixels"
5
+ require_relative "../backend/ruby/tile_compositor_backward"
6
+ require_relative "../backend/ruby/rasterize_to_pixels_backward"
7
+
8
+ # Differentiable tile rasterization API.
9
+ module Gsplat
10
+ module Ops
11
+ # Tile-accelerated alpha compositor.
12
+ class RasterizeToPixels < Autograd::Function
13
+ class << self
14
+ # rubocop:disable Metrics/ParameterLists
15
+ def forward(context, means2d, conics, colors, opacities, backgrounds, masks, width, height,
16
+ tile_size, isect_offsets, flatten_ids, absgrad:)
17
+ # rubocop:enable Metrics/ParameterLists
18
+ render_colors, render_alphas, last_ids = Backend.dispatch(
19
+ :rasterize_to_pixels_forward,
20
+ means2d,
21
+ conics,
22
+ colors,
23
+ opacities,
24
+ backgrounds,
25
+ masks,
26
+ width,
27
+ height,
28
+ tile_size,
29
+ isect_offsets,
30
+ flatten_ids
31
+ )
32
+ context.save(
33
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
34
+ tile_size, isect_offsets, flatten_ids, render_alphas, last_ids, absgrad
35
+ )
36
+ [render_colors, render_alphas]
37
+ end
38
+
39
+ # Propagates pixel color and alpha gradients to projected attributes.
40
+ # @api private
41
+ def backward(context, grad_render_colors, grad_render_alphas)
42
+ saved = context.saved_values
43
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
44
+ tile_size, offsets, flatten_ids, render_alphas, last_ids, absgrad = saved
45
+ gradients, means2d_absgrad = Backend.dispatch(
46
+ :rasterize_to_pixels_backward,
47
+ means2d,
48
+ conics,
49
+ colors,
50
+ opacities,
51
+ backgrounds,
52
+ masks,
53
+ width,
54
+ height,
55
+ tile_size,
56
+ offsets,
57
+ flatten_ids,
58
+ render_alphas,
59
+ last_ids,
60
+ grad_render_colors,
61
+ grad_render_alphas,
62
+ absgrad: absgrad
63
+ )
64
+ if absgrad && context.inputs[0].is_a?(Autograd::Variable)
65
+ context.inputs[0].accumulate_absgrad(means2d_absgrad)
66
+ end
67
+ [*gradients, nil, nil, nil, nil, nil, nil]
68
+ end
69
+ end
70
+ end
71
+ end
72
+
73
+ class << self
74
+ # Alpha-composites sorted tile intersections.
75
+ #
76
+ # Projected inputs have shapes `means2d [C,N,2]`, `conics [C,N,3]`,
77
+ # `colors [C,N,D]`, and `opacities [C,N]`. Outputs are color
78
+ # `[C,H,W,D]` and alpha `[C,H,W,1]`.
79
+ #
80
+ # @return [Array<(Numo::NArray, Autograd::Variable)>]
81
+ # rubocop:disable Metrics/ParameterLists
82
+ def rasterize_to_pixels(means2d, conics, colors, opacities, width, height, tile_size,
83
+ isect_offsets, flatten_ids, backgrounds: nil, masks: nil, absgrad: false)
84
+ # rubocop:enable Metrics/ParameterLists
85
+ inputs = [
86
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
87
+ tile_size, isect_offsets, flatten_ids
88
+ ]
89
+ return Ops::RasterizeToPixels.apply(*inputs, absgrad: absgrad) if inputs.any?(Autograd::Variable)
90
+
91
+ Backend.dispatch(:rasterize_to_pixels_forward, *inputs).first(2)
92
+ end
93
+ end
94
+
95
+ Backend.register(
96
+ :rasterize_to_pixels_forward,
97
+ :ruby,
98
+ Backend::RubyRasterizeToPixels.method(:forward)
99
+ )
100
+ Backend.register(
101
+ :rasterize_to_pixels_backward,
102
+ :ruby,
103
+ Backend::RubyRasterizeToPixelsBackward.method(:backward)
104
+ )
105
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Closed-form relocation primitive and public wrapper.
4
+ module Gsplat
5
+ module Ops
6
+ # Closed-form opacity and scale update from 3DGS-MCMC equation 9.
7
+ module Relocation
8
+ # Default maximum split ratio represented by {.binomial_table}.
9
+ DEFAULT_N_MAX = 51
10
+
11
+ module_function
12
+
13
+ # Builds the binomial coefficient lookup used by relocation.
14
+ #
15
+ # @param n_max [Integer] maximum split ratio, inclusive
16
+ # @param dtype [Class] Numo floating-point class
17
+ # @return [Numo::NArray] [n_max,n_max]
18
+ def binomial_table(n_max: DEFAULT_N_MAX, dtype: Numo::SFloat)
19
+ raise ArgumentError, "n_max must be a positive integer" unless n_max.is_a?(Integer) && n_max.positive?
20
+
21
+ output = dtype.zeros(n_max, n_max)
22
+ n_max.times do |n_value|
23
+ (n_value + 1).times { |k_value| output[n_value, k_value] = binomial(n_value, k_value) }
24
+ end
25
+ output
26
+ end
27
+
28
+ # Recalculates activated opacities and scales for a requested split ratio.
29
+ #
30
+ # @param opacities [Numo::NArray] [N], activated values in [0,1]
31
+ # @param scales [Numo::NArray] [N,3], activated positive scales
32
+ # @param ratios [Numo::NArray] [N], clamped to 1..n_max
33
+ # @param binoms [Numo::NArray] [n_max,n_max]
34
+ # @return [Array<Numo::NArray>] new opacities and scales
35
+ def compute(opacities, scales, ratios, binoms: binomial_table(dtype: opacities.class))
36
+ validate_inputs!(opacities, scales, ratios, binoms)
37
+ output_opacities = opacities.class.zeros(opacities.shape[0])
38
+ output_scales = scales.class.zeros(*scales.shape)
39
+ opacities.shape[0].times do |index|
40
+ ratio = ratios[index].to_i.clamp(1, binoms.shape[0])
41
+ opacity = opacities[index].to_f
42
+ new_opacity = 1.0 - ((1.0 - opacity)**(1.0 / ratio))
43
+ coefficient = opacity / denominator(new_opacity, ratio, binoms)
44
+ output_opacities[index] = new_opacity
45
+ output_scales[index, true] = scales[index, true] * coefficient
46
+ end
47
+ [output_opacities, output_scales]
48
+ end
49
+
50
+ def binomial(n_value, k_value)
51
+ return 1 if k_value.zero? || k_value == n_value
52
+
53
+ k_value = [k_value, n_value - k_value].min
54
+ (1..k_value).reduce(1) { |value, index| (value * (n_value - k_value + index)) / index }
55
+ end
56
+ private_class_method :binomial
57
+
58
+ def denominator(opacity, ratio, binoms)
59
+ (1..ratio).sum do |i_value|
60
+ (0...i_value).sum do |k_value|
61
+ sign = k_value.even? ? 1.0 : -1.0
62
+ binoms[i_value - 1, k_value].to_f * sign * (opacity**(k_value + 1)) / ::Math.sqrt(k_value + 1)
63
+ end
64
+ end
65
+ end
66
+ private_class_method :denominator
67
+
68
+ def validate_inputs!(opacities, scales, ratios, binoms)
69
+ unless opacities.is_a?(Numo::NArray) && [Numo::SFloat, Numo::DFloat].include?(opacities.class)
70
+ raise ArgumentError, "opacities must be Numo::SFloat or Numo::DFloat"
71
+ end
72
+
73
+ count = opacities.shape[0] if opacities.ndim == 1
74
+ valid = count && scales.shape == [count, 3] && ratios.shape == [count] &&
75
+ binoms.ndim == 2 && binoms.shape[0] == binoms.shape[1]
76
+ return if valid
77
+
78
+ raise ShapeError,
79
+ "expected opacities [N], scales [N,3], ratios [N], binoms [M,M]; " \
80
+ "got #{opacities.shape.inspect}, #{scales.shape.inspect}, " \
81
+ "#{ratios.shape.inspect}, #{binoms.shape.inspect}"
82
+ end
83
+ private_class_method :validate_inputs!
84
+ end
85
+ end
86
+
87
+ class << self
88
+ # Computes the deterministic 3DGS-MCMC relocation update.
89
+ def relocation(opacities, scales, ratios, binoms: nil)
90
+ options = binoms ? { binoms: binoms } : {}
91
+ Ops::Relocation.compute(opacities, scales, ratios, **options)
92
+ end
93
+ end
94
+ end