gsplat 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +202 -0
  3. data/README.md +236 -0
  4. data/docs/ACCEPTANCE.md +60 -0
  5. data/docs/BENCHMARKS.md +79 -0
  6. data/docs/DECISIONS.md +52 -0
  7. data/docs/MIGRATION.md +117 -0
  8. data/docs/PROFILE.md +75 -0
  9. data/docs/PROGRESS.md +199 -0
  10. data/docs/decisions/0000-template.md +17 -0
  11. data/docs/decisions/0001-pin-golden-reference-and-cpu-validation.md +27 -0
  12. data/docs/decisions/0002-retain-ruby-fallbacks-for-native-backend.md +26 -0
  13. data/docs/decisions/0003-share-projection-for-distorted-cameras.md +26 -0
  14. data/docs/decisions/0004-use-portable-world-space-reference-paths.md +26 -0
  15. data/docs/decisions/0005-reuse-ewa-core-for-2dgs.md +24 -0
  16. data/docs/decisions/0006-keep-eval3d-as-portable-reference.md +25 -0
  17. data/docs/decisions/0007-share-compositor-semantics-for-contribution-indices.md +25 -0
  18. data/examples/data/README.md +14 -0
  19. data/examples/data/colmap/images/view_000.png +0 -0
  20. data/examples/data/colmap/images/view_001.png +0 -0
  21. data/examples/data/colmap/images/view_002.png +0 -0
  22. data/examples/data/colmap/sparse/0/cameras.txt +2 -0
  23. data/examples/data/colmap/sparse/0/images.txt +6 -0
  24. data/examples/data/colmap/sparse/0/points3D.txt +17 -0
  25. data/examples/data/splats.ply +0 -0
  26. data/examples/fit_image.rb +35 -0
  27. data/examples/generate_sample_data.rb +147 -0
  28. data/examples/render_path.rb +117 -0
  29. data/examples/simple_trainer.rb +76 -0
  30. data/ext/gsplat_native/common.h +54 -0
  31. data/ext/gsplat_native/extconf.rb +39 -0
  32. data/ext/gsplat_native/gsplat_native.c +60 -0
  33. data/ext/gsplat_native/intersections.c +211 -0
  34. data/ext/gsplat_native/projection.c +199 -0
  35. data/ext/gsplat_native/raster_backward.c +129 -0
  36. data/ext/gsplat_native/raster_backward_bridge.c +69 -0
  37. data/ext/gsplat_native/raster_forward.c +150 -0
  38. data/ext/gsplat_native/rasterization.h +51 -0
  39. data/ext/gsplat_native/spherical_harmonics.c +129 -0
  40. data/gsplat.gemspec +36 -0
  41. data/lib/gsplat/autograd/context.rb +60 -0
  42. data/lib/gsplat/autograd/function.rb +68 -0
  43. data/lib/gsplat/autograd/variable.rb +159 -0
  44. data/lib/gsplat/backend/ruby/accumulate.rb +139 -0
  45. data/lib/gsplat/backend/ruby/accumulate_backward.rb +40 -0
  46. data/lib/gsplat/backend/ruby/eval3d_rasterizer.rb +175 -0
  47. data/lib/gsplat/backend/ruby/isect_tiles.rb +198 -0
  48. data/lib/gsplat/backend/ruby/projection.rb +251 -0
  49. data/lib/gsplat/backend/ruby/projection_backward.rb +190 -0
  50. data/lib/gsplat/backend/ruby/projection_covariance_vjp.rb +72 -0
  51. data/lib/gsplat/backend/ruby/projection_input_vjp.rb +252 -0
  52. data/lib/gsplat/backend/ruby/quat_scale_to_covar_preci.rb +139 -0
  53. data/lib/gsplat/backend/ruby/rasterize_to_indices_in_range.rb +121 -0
  54. data/lib/gsplat/backend/ruby/rasterize_to_pixels.rb +199 -0
  55. data/lib/gsplat/backend/ruby/rasterize_to_pixels_backward.rb +121 -0
  56. data/lib/gsplat/backend/ruby/spherical_harmonics.rb +135 -0
  57. data/lib/gsplat/backend/ruby/tile_compositor.rb +50 -0
  58. data/lib/gsplat/backend/ruby/tile_compositor_backward.rb +84 -0
  59. data/lib/gsplat/backend.rb +79 -0
  60. data/lib/gsplat/compression/grid_sort.rb +79 -0
  61. data/lib/gsplat/compression/kmeans.rb +121 -0
  62. data/lib/gsplat/compression/png.rb +147 -0
  63. data/lib/gsplat/compression/png_codec.rb +133 -0
  64. data/lib/gsplat/compression/quantizer.rb +79 -0
  65. data/lib/gsplat/io/checkpoint.rb +145 -0
  66. data/lib/gsplat/io/colmap.rb +175 -0
  67. data/lib/gsplat/io/colmap_binary.rb +98 -0
  68. data/lib/gsplat/io/colmap_text.rb +84 -0
  69. data/lib/gsplat/io/image.rb +63 -0
  70. data/lib/gsplat/io/image_backends.rb +81 -0
  71. data/lib/gsplat/io/npy.rb +189 -0
  72. data/lib/gsplat/io/ply.rb +185 -0
  73. data/lib/gsplat/io/ply_reader.rb +142 -0
  74. data/lib/gsplat/io/zip_archive.rb +183 -0
  75. data/lib/gsplat/math/camera_distortion.rb +123 -0
  76. data/lib/gsplat/math/camera_projection.rb +202 -0
  77. data/lib/gsplat/math/mat.rb +114 -0
  78. data/lib/gsplat/math/quaternion.rb +175 -0
  79. data/lib/gsplat/math/small_matrix_primitives.rb +112 -0
  80. data/lib/gsplat/math/spherical_harmonic_basis.rb +148 -0
  81. data/lib/gsplat/math/ssim.rb +153 -0
  82. data/lib/gsplat/native.rb +30 -0
  83. data/lib/gsplat/native_ops.rb +148 -0
  84. data/lib/gsplat/native_raster_ops.rb +76 -0
  85. data/lib/gsplat/ops/accumulate.rb +68 -0
  86. data/lib/gsplat/ops/eval3d_rasterize.rb +62 -0
  87. data/lib/gsplat/ops/isect_tiles.rb +38 -0
  88. data/lib/gsplat/ops/projection.rb +154 -0
  89. data/lib/gsplat/ops/quat_scale_to_covar_preci.rb +118 -0
  90. data/lib/gsplat/ops/rasterize_to_indices_in_range.rb +38 -0
  91. data/lib/gsplat/ops/rasterize_to_pixels.rb +105 -0
  92. data/lib/gsplat/ops/relocation.rb +94 -0
  93. data/lib/gsplat/ops/spherical_harmonics.rb +71 -0
  94. data/lib/gsplat/ops/tensor_shape_ops.rb +173 -0
  95. data/lib/gsplat/ops/tensor_value_ops.rb +150 -0
  96. data/lib/gsplat/optim/adam.rb +212 -0
  97. data/lib/gsplat/optim/lr_scheduler.rb +36 -0
  98. data/lib/gsplat/optim/selective_adam.rb +68 -0
  99. data/lib/gsplat/rasterization.rb +130 -0
  100. data/lib/gsplat/rasterization_2dgs.rb +140 -0
  101. data/lib/gsplat/rasterization_helpers.rb +162 -0
  102. data/lib/gsplat/rasterization_validation.rb +99 -0
  103. data/lib/gsplat/strategy/base.rb +49 -0
  104. data/lib/gsplat/strategy/default.rb +188 -0
  105. data/lib/gsplat/strategy/mcmc.rb +103 -0
  106. data/lib/gsplat/strategy/mcmc_ops.rb +143 -0
  107. data/lib/gsplat/strategy/ops.rb +165 -0
  108. data/lib/gsplat/training/config.rb +91 -0
  109. data/lib/gsplat/training/image_fitter.rb +158 -0
  110. data/lib/gsplat/training/losses.rb +191 -0
  111. data/lib/gsplat/training/scene.rb +116 -0
  112. data/lib/gsplat/training/trainer.rb +236 -0
  113. data/lib/gsplat/utils.rb +110 -0
  114. data/lib/gsplat/version.rb +6 -0
  115. data/lib/gsplat.rb +98 -0
  116. metadata +181 -0
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Backend
5
+ # Brute-force Numo alpha compositor used as a rasterization reference.
6
+ module RubyAccumulate
7
+ ALPHA_CLAMP = 0.999
8
+ ALPHA_SKIP = 1.0 / 255.0
9
+ TRANSMITTANCE_STOP = 1e-4
10
+
11
+ module_function
12
+
13
+ # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
14
+ def forward(means2d, conics, opacities, colors, backgrounds, width, height)
15
+ # rubocop:enable Metrics/ParameterLists
16
+ inputs = validate_inputs(means2d, conics, opacities, colors, backgrounds, width, height)
17
+ means2d, conics, opacities, colors, backgrounds = inputs
18
+ camera_count, gaussian_count = means2d.shape[0...2]
19
+ channel_count = colors.shape[-1]
20
+ pixel_count = width * height
21
+ pixels_x, pixels_y = pixel_coordinates(means2d.class, width, height)
22
+ render_colors = means2d.class.zeros(camera_count, pixel_count, channel_count)
23
+ render_alphas = means2d.class.zeros(camera_count, pixel_count)
24
+ last_ids = Numo::Int32.zeros(camera_count, pixel_count)
25
+ camera_count.times do |camera_index|
26
+ composited, transmittance, camera_last_ids = composite_camera(
27
+ means2d[camera_index, true, true],
28
+ conics[camera_index, true, true],
29
+ opacities[camera_index, true],
30
+ colors[camera_index, true, true],
31
+ pixels_x,
32
+ pixels_y,
33
+ gaussian_count
34
+ )
35
+ if backgrounds
36
+ composited += transmittance.reshape(pixel_count, 1) *
37
+ backgrounds[camera_index, true].reshape(1, channel_count)
38
+ end
39
+ render_colors[camera_index, true, true] = composited
40
+ render_alphas[camera_index, true] = 1 - transmittance
41
+ last_ids[camera_index, true] = camera_last_ids + (camera_index * gaussian_count)
42
+ end
43
+ [
44
+ render_colors.reshape(camera_count, height, width, channel_count),
45
+ render_alphas.reshape(camera_count, height, width, 1),
46
+ last_ids.reshape(camera_count, height, width)
47
+ ]
48
+ end
49
+ # rubocop:enable Metrics/AbcSize
50
+
51
+ # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
52
+ def validate_inputs(means2d, conics, opacities, colors, backgrounds, width, height)
53
+ # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
54
+ unless means2d.is_a?(Numo::NArray) && [Numo::SFloat, Numo::DFloat].include?(means2d.class)
55
+ raise ArgumentError, "means2d must be Numo::SFloat or Numo::DFloat"
56
+ end
57
+
58
+ valid_means = means2d.ndim == 3 && means2d.shape[-1] == 2
59
+ raise ShapeError, "expected means2d [C,N,2], got #{means2d.shape.inspect}" unless valid_means
60
+
61
+ leading_shape = means2d.shape[0...2]
62
+ conics = means2d.class.cast(conics)
63
+ opacities = means2d.class.cast(opacities)
64
+ colors = means2d.class.cast(colors)
65
+ valid = conics.shape == leading_shape + [3] &&
66
+ opacities.shape == leading_shape &&
67
+ colors.ndim == 3 && colors.shape[0...2] == leading_shape
68
+ unless valid
69
+ raise ShapeError,
70
+ "expected conics [C,N,3], opacities [C,N], and colors [C,N,D], " \
71
+ "got #{conics.shape.inspect}, #{opacities.shape.inspect}, and #{colors.shape.inspect}"
72
+ end
73
+ backgrounds = validate_backgrounds(means2d.class, backgrounds, means2d.shape[0], colors.shape[-1])
74
+ valid_size = width.is_a?(Integer) && width.positive? && height.is_a?(Integer) && height.positive?
75
+ raise ArgumentError, "width and height must be positive integers" unless valid_size
76
+
77
+ [means2d, conics, opacities, colors, backgrounds]
78
+ end
79
+ private_class_method :validate_inputs
80
+
81
+ def validate_backgrounds(type, backgrounds, camera_count, channel_count)
82
+ return nil unless backgrounds
83
+
84
+ value = type.cast(backgrounds)
85
+ expected = [camera_count, channel_count]
86
+ unless value.shape == expected
87
+ raise ShapeError, "expected backgrounds #{expected.inspect}, got #{value.shape.inspect}"
88
+ end
89
+
90
+ value
91
+ end
92
+ private_class_method :validate_backgrounds
93
+
94
+ def pixel_coordinates(type, width, height)
95
+ x_values = Array.new(height) { (0...width).map { |column| column + 0.5 } }.flatten
96
+ y_values = (0...height).flat_map { |row| Array.new(width, row + 0.5) }
97
+ [type.cast(x_values), type.cast(y_values)]
98
+ end
99
+ private_class_method :pixel_coordinates
100
+
101
+ # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
102
+ def composite_camera(means2d, conics, opacities, colors, pixels_x, pixels_y, gaussian_count)
103
+ # rubocop:enable Metrics/ParameterLists
104
+ pixel_count = pixels_x.size
105
+ channel_count = colors.shape[-1]
106
+ transmittance = means2d.class.ones(pixel_count)
107
+ composited = means2d.class.zeros(pixel_count, channel_count)
108
+ last_ids = Numo::Int32.zeros(pixel_count)
109
+ active = Numo::Bit.ones(pixel_count)
110
+ gaussian_count.times do |gaussian_index|
111
+ delta_x = means2d[gaussian_index, 0] - pixels_x
112
+ delta_y = means2d[gaussian_index, 1] - pixels_y
113
+ sigma = (0.5 * (
114
+ (conics[gaussian_index, 0] * (delta_x**2)) +
115
+ (conics[gaussian_index, 2] * (delta_y**2))
116
+ )) + (conics[gaussian_index, 1] * delta_x * delta_y)
117
+ alpha = opacities[gaussian_index] * Numo::NMath.exp(-sigma)
118
+ above_clamp = alpha.gt(ALPHA_CLAMP)
119
+ alpha[above_clamp] = ALPHA_CLAMP if above_clamp.any?
120
+ candidate = active & sigma.ge(0) & alpha.ge(ALPHA_SKIP)
121
+ next_transmittance = transmittance * (1 - alpha)
122
+ accepted = candidate & next_transmittance.gt(TRANSMITTANCE_STOP)
123
+ contribution = means2d.class.zeros(pixel_count)
124
+ contribution[accepted] = (alpha * transmittance)[accepted] if accepted.any?
125
+ composited += contribution.reshape(pixel_count, 1) *
126
+ colors[gaussian_index, true].reshape(1, channel_count)
127
+ transmittance[accepted] = next_transmittance[accepted] if accepted.any?
128
+ last_ids[accepted] = gaussian_index if accepted.any?
129
+ terminating = candidate & next_transmittance.le(TRANSMITTANCE_STOP)
130
+ active[terminating] = 0 if terminating.any?
131
+ break unless active.any?
132
+ end
133
+ [composited, transmittance, last_ids]
134
+ end
135
+ # rubocop:enable Metrics/AbcSize
136
+ private_class_method :composite_camera
137
+ end
138
+ end
139
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Backend
5
+ # Brute-force compositor VJP routed through the one-tile reverse kernel.
6
+ module RubyAccumulateBackward
7
+ module_function
8
+
9
+ # rubocop:disable Metrics/ParameterLists
10
+ def backward(means2d, conics, opacities, colors, backgrounds, width, height,
11
+ render_alphas, last_ids, grad_render_colors, grad_render_alphas)
12
+ # rubocop:enable Metrics/ParameterLists
13
+ camera_count, gaussian_count = means2d.shape[0...2]
14
+ offsets = Numo::Int32.zeros(camera_count, 1, 1)
15
+ camera_count.times { |camera_index| offsets[camera_index, 0, 0] = camera_index * gaussian_count }
16
+ flatten_ids = Numo::Int32.new(camera_count * gaussian_count).seq
17
+ gradients, = RubyRasterizeToPixelsBackward.backward(
18
+ means2d,
19
+ conics,
20
+ colors,
21
+ opacities,
22
+ backgrounds,
23
+ nil,
24
+ width,
25
+ height,
26
+ [width, height].max,
27
+ offsets,
28
+ flatten_ids,
29
+ render_alphas,
30
+ last_ids,
31
+ grad_render_colors,
32
+ grad_render_alphas,
33
+ absgrad: false
34
+ )
35
+ grad_means, grad_conics, grad_colors, grad_opacities, grad_backgrounds = gradients
36
+ [grad_means, grad_conics, grad_opacities, grad_colors, grad_backgrounds]
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,175 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Backend
5
+ # World-space Gaussian evaluator used by 3DGUT-style rendering.
6
+ module RubyEval3dRasterizer
7
+ ALPHA_CLAMP = 0.99
8
+ ALPHA_SKIP = 1.0 / 255
9
+ TRANSMITTANCE_STOP = 1e-4
10
+
11
+ module_function
12
+
13
+ # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
14
+ def forward(means, quats, scales, colors, opacities, backgrounds, viewmats:, intrinsics:,
15
+ width:, height:, tile_size:, offsets:, flatten_ids:, camera_model:,
16
+ use_hit_distance: false, return_normals: false)
17
+ # rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
18
+ values = validate_inputs(
19
+ means, quats, scales, colors, opacities, backgrounds, viewmats, intrinsics,
20
+ width, height, offsets, flatten_ids, camera_model
21
+ )
22
+ means, quats, scales, colors, opacities, backgrounds, viewmats, intrinsics = values
23
+ camera_count, gaussian_count, channels = colors.shape
24
+ rendered = means.class.zeros(camera_count, height, width, channels)
25
+ alphas = means.class.zeros(camera_count, height, width, 1)
26
+ rendered_normals = means.class.zeros(camera_count, height, width, 3)
27
+ rotations = Math::Quaternion.to_rotmat(quats)
28
+ camera_count.times do |camera|
29
+ frame = camera_frame(viewmats[camera, true, true])
30
+ height.times do |row|
31
+ width.times do |column|
32
+ tile_x = column / tile_size
33
+ tile_y = row / tile_size
34
+ range = intersection_range(offsets, flatten_ids, camera, tile_x, tile_y)
35
+ ray = ray_for_pixel(frame, intrinsics[camera, true, true], column, row)
36
+ pixel, transmittance, pixel_normal = composite_pixel(
37
+ means, rotations, scales, colors, opacities, flatten_ids, range,
38
+ camera, gaussian_count, ray, use_hit_distance, return_normals
39
+ )
40
+ pixel += backgrounds[camera, true] * transmittance if backgrounds
41
+ rendered[camera, row, column, true] = pixel
42
+ alphas[camera, row, column, 0] = 1 - transmittance
43
+ rendered_normals[camera, row, column, true] = pixel_normal
44
+ end
45
+ end
46
+ end
47
+ [rendered, alphas, rendered_normals]
48
+ end
49
+
50
+ # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
51
+ def validate_inputs(means, quats, scales, colors, opacities, backgrounds, viewmats,
52
+ intrinsics, width, height, offsets, flatten_ids, camera_model)
53
+ # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
54
+ unless camera_model == "pinhole"
55
+ raise ArgumentError, "hit-distance rendering currently requires pinhole camera_model"
56
+ end
57
+
58
+ type = means.class
59
+ quats = type.cast(quats)
60
+ scales = type.cast(scales)
61
+ colors = type.cast(colors)
62
+ opacities = type.cast(opacities)
63
+ viewmats = type.cast(viewmats)
64
+ intrinsics = type.cast(intrinsics)
65
+ camera_count = viewmats.shape[0]
66
+ gaussian_count = means.shape[0]
67
+ valid = means.shape == [gaussian_count, 3] &&
68
+ quats.shape == [gaussian_count, 4] &&
69
+ scales.shape == [gaussian_count, 3] &&
70
+ colors.ndim == 3 && colors.shape[0...2] == [camera_count, gaussian_count] &&
71
+ opacities.shape == [camera_count, gaussian_count] &&
72
+ intrinsics.shape == [camera_count, 3, 3]
73
+ raise ShapeError, "invalid world-space rasterization inputs" unless valid
74
+ raise ArgumentError, "scales must be positive" unless scales.gt(0).all?
75
+
76
+ backgrounds = type.cast(backgrounds) if backgrounds
77
+ if backgrounds && backgrounds.shape != [camera_count, colors.shape[-1]]
78
+ raise ShapeError, "expected backgrounds [#{camera_count},#{colors.shape[-1]}]"
79
+ end
80
+ raise ArgumentError, "width and height must be positive" unless width.positive? && height.positive?
81
+ raise ShapeError, "expected offsets [C,TH,TW]" unless offsets.shape[0] == camera_count
82
+ raise ShapeError, "expected flatten_ids [M]" unless flatten_ids.ndim == 1
83
+
84
+ [means, quats, scales, colors, opacities, backgrounds, viewmats, intrinsics]
85
+ end
86
+ private_class_method :validate_inputs
87
+
88
+ def camera_frame(viewmat)
89
+ rotation = viewmat[0...3, 0...3]
90
+ translation = viewmat[0...3, 3]
91
+ [-translation.dot(rotation), rotation]
92
+ end
93
+ private_class_method :camera_frame
94
+
95
+ def ray_for_pixel(frame, intrinsics, column, row)
96
+ origin, rotation = frame
97
+ camera_direction = origin.class[
98
+ (column + 0.5 - intrinsics[0, 2]) / intrinsics[0, 0],
99
+ (row + 0.5 - intrinsics[1, 2]) / intrinsics[1, 1],
100
+ 1.0
101
+ ]
102
+ direction = camera_direction.dot(rotation)
103
+ [origin, direction / ::Math.sqrt((direction**2).sum)]
104
+ end
105
+ private_class_method :ray_for_pixel
106
+
107
+ def intersection_range(offsets, flatten_ids, camera, tile_x, tile_y)
108
+ tile_height, tile_width = offsets.shape[-2..]
109
+ flat = (((camera * tile_height) + tile_y) * tile_width) + tile_x
110
+ start_index = offsets[flat].to_i
111
+ end_index = flat + 1 < offsets.size ? offsets[flat + 1].to_i : flatten_ids.size
112
+ start_index...end_index
113
+ end
114
+ private_class_method :intersection_range
115
+
116
+ # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
117
+ def composite_pixel(means, rotations, scales, colors, opacities, flatten_ids, range,
118
+ camera, gaussian_count, ray, use_hit_distance, return_normals)
119
+ # rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
120
+ pixel = means.class.zeros(colors.shape[-1])
121
+ pixel_normal = means.class.zeros(3)
122
+ transmittance = 1.0
123
+ range.each do |intersection|
124
+ flat_id = flatten_ids[intersection].to_i
125
+ gaussian = flat_id % gaussian_count
126
+ gray_distance, hit_distance = ray_gaussian(
127
+ ray, means[gaussian, true], rotations[gaussian, true, true], scales[gaussian, true]
128
+ )
129
+ next unless gray_distance
130
+
131
+ opacity = opacities[camera, gaussian].to_f
132
+ alpha = [opacity * ::Math.exp(-0.5 * gray_distance), ALPHA_CLAMP].min
133
+ next if alpha < ALPHA_SKIP
134
+
135
+ next_transmittance = transmittance * (1 - alpha)
136
+ break if next_transmittance <= TRANSMITTANCE_STOP
137
+
138
+ weight = alpha * transmittance
139
+ feature = colors[camera, gaussian, true].dup
140
+ feature[feature.size - 1] = hit_distance if use_hit_distance
141
+ pixel += feature * weight
142
+ if return_normals
143
+ normal = rotations[gaussian, true, 2].dup
144
+ normal *= -1 if (normal * ray[1]).sum.positive?
145
+ pixel_normal += normal * weight
146
+ end
147
+ transmittance = next_transmittance
148
+ end
149
+ [pixel, transmittance, pixel_normal]
150
+ end
151
+ private_class_method :composite_pixel
152
+
153
+ # rubocop:disable Metrics/AbcSize
154
+ def ray_gaussian(ray, mean, rotation, scale)
155
+ origin, direction = ray
156
+ local_origin = (origin - mean).dot(rotation) / scale
157
+ local_direction = direction.dot(rotation) / scale
158
+ local_direction /= ::Math.sqrt((local_direction**2).sum)
159
+ hit_t = -(local_direction * local_origin).sum
160
+ return [nil, nil] if hit_t.negative?
161
+
162
+ cross = local_direction.class[
163
+ (local_direction[1] * local_origin[2]) - (local_direction[2] * local_origin[1]),
164
+ (local_direction[2] * local_origin[0]) - (local_direction[0] * local_origin[2]),
165
+ (local_direction[0] * local_origin[1]) - (local_direction[1] * local_origin[0])
166
+ ]
167
+ gray_distance = (cross**2).sum.to_f
168
+ hit_vector = scale * local_direction * hit_t
169
+ [gray_distance, ::Math.sqrt((hit_vector**2).sum)]
170
+ end
171
+ # rubocop:enable Metrics/AbcSize
172
+ private_class_method :ray_gaussian
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,198 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Backend
5
+ # Numo/Ruby tile intersection enumeration and offset encoding.
6
+ module RubyIsectTiles
7
+ TILE_KEY_LOW_BITS = 32
8
+
9
+ module_function
10
+
11
+ # rubocop:disable Metrics/ParameterLists
12
+ def forward(means2d, radii, depths, tile_size, tile_width, tile_height, sort:)
13
+ # rubocop:enable Metrics/ParameterLists
14
+ means2d, radii, depths = validate_inputs(
15
+ means2d,
16
+ radii,
17
+ depths,
18
+ tile_size,
19
+ tile_width,
20
+ tile_height
21
+ )
22
+ camera_count, gaussian_count = means2d.shape[0...2]
23
+ bounds, tiles_per_gauss = tile_bounds(
24
+ means2d,
25
+ radii,
26
+ tile_size,
27
+ tile_width,
28
+ tile_height
29
+ )
30
+ intersection_count = tiles_per_gauss.sum.to_i
31
+ isect_ids = Numo::Int64.zeros(intersection_count)
32
+ flatten_ids = Numo::Int32.zeros(intersection_count)
33
+ tile_bits = tile_n_bits(tile_width, tile_height)
34
+ validate_key_width!(camera_count, tile_bits)
35
+ write_intersections!(
36
+ isect_ids,
37
+ flatten_ids,
38
+ bounds,
39
+ tiles_per_gauss,
40
+ depths,
41
+ gaussian_count,
42
+ tile_width,
43
+ tile_bits
44
+ )
45
+ if sort && intersection_count.positive?
46
+ keys = isect_ids.to_a
47
+ order = (0...intersection_count).sort_by { |index| [keys[index], index] }
48
+ isect_ids = isect_ids[order].dup
49
+ flatten_ids = flatten_ids[order].dup
50
+ end
51
+ [tiles_per_gauss, isect_ids, flatten_ids]
52
+ end
53
+
54
+ def offset_encode(isect_ids, camera_count, tile_width, tile_height)
55
+ validate_grid!(camera_count, tile_width, tile_height)
56
+ raise ShapeError, "expected isect_ids [M]" unless isect_ids.is_a?(Numo::NArray) && isect_ids.ndim == 1
57
+
58
+ keys = Numo::Int64.cast(isect_ids)
59
+ counts = Numo::Int64.zeros(camera_count, tile_height, tile_width)
60
+ tile_bits = tile_n_bits(tile_width, tile_height)
61
+ tile_mask = (1 << tile_bits) - 1
62
+ keys.each do |key|
63
+ upper = key >> TILE_KEY_LOW_BITS
64
+ camera_id = upper >> tile_bits
65
+ tile_id = upper & tile_mask
66
+ validate_decoded_key!(camera_id, tile_id, camera_count, tile_width, tile_height)
67
+ counts[camera_id, tile_id / tile_width, tile_id % tile_width] += 1
68
+ end
69
+ offsets = Numo::Int32.zeros(camera_count, tile_height, tile_width)
70
+ running = 0
71
+ counts.size.times do |index|
72
+ offsets[index] = running
73
+ running += counts[index]
74
+ end
75
+ offsets
76
+ end
77
+
78
+ # rubocop:disable Metrics/ParameterLists
79
+ def validate_inputs(means2d, radii, depths, tile_size, tile_width, tile_height)
80
+ # rubocop:enable Metrics/ParameterLists
81
+ unless means2d.is_a?(Numo::NArray) && [Numo::SFloat, Numo::DFloat].include?(means2d.class)
82
+ raise ArgumentError, "means2d must be Numo::SFloat or Numo::DFloat"
83
+ end
84
+
85
+ valid_means = means2d.ndim == 3 && means2d.shape[-1] == 2
86
+ raise ShapeError, "expected means2d [C,N,2], got #{means2d.shape.inspect}" unless valid_means
87
+
88
+ leading_shape = means2d.shape[0...2]
89
+ valid_radii = [leading_shape, leading_shape + [2]].include?(radii.shape)
90
+ unless valid_radii && depths.shape == leading_shape
91
+ raise ShapeError,
92
+ "expected radii [C,N] or [C,N,2] and depths #{leading_shape.inspect}, " \
93
+ "got #{radii.shape.inspect} and #{depths.shape.inspect}"
94
+ end
95
+ validate_grid!(means2d.shape[0], tile_width, tile_height)
96
+ valid_tile_size = tile_size.is_a?(Integer) && tile_size.positive?
97
+ raise ArgumentError, "tile_size must be a positive integer" unless valid_tile_size
98
+
99
+ [means2d, means2d.class.cast(radii), means2d.class.cast(depths)]
100
+ end
101
+ private_class_method :validate_inputs
102
+
103
+ def validate_grid!(camera_count, tile_width, tile_height)
104
+ values = [camera_count, tile_width, tile_height]
105
+ return if values.all? { |value| value.is_a?(Integer) && value.positive? }
106
+
107
+ raise ArgumentError, "camera count and tile dimensions must be positive integers"
108
+ end
109
+ private_class_method :validate_grid!
110
+
111
+ # rubocop:disable Metrics/AbcSize
112
+ def tile_bounds(means2d, radii, tile_size, tile_width, tile_height)
113
+ camera_count, gaussian_count = means2d.shape[0...2]
114
+ bounds = Numo::Int32.zeros(camera_count, gaussian_count, 4)
115
+ counts = Numo::Int32.zeros(camera_count, gaussian_count)
116
+ elliptical = radii.ndim == 3
117
+ camera_count.times do |camera_index|
118
+ gaussian_count.times do |gaussian_index|
119
+ if elliptical
120
+ radius_x = radii[camera_index, gaussian_index, 0].to_f
121
+ radius_y = radii[camera_index, gaussian_index, 1].to_f
122
+ else
123
+ radius_x = radii[camera_index, gaussian_index].to_f
124
+ radius_y = radius_x
125
+ end
126
+ next unless radius_x.positive? && radius_y.positive?
127
+
128
+ mean_x = means2d[camera_index, gaussian_index, 0].to_f
129
+ mean_y = means2d[camera_index, gaussian_index, 1].to_f
130
+ min_x = clamp_tile(((mean_x - radius_x) / tile_size).floor, tile_width)
131
+ max_x = clamp_tile(((mean_x + radius_x) / tile_size).ceil, tile_width)
132
+ min_y = clamp_tile(((mean_y - radius_y) / tile_size).floor, tile_height)
133
+ max_y = clamp_tile(((mean_y + radius_y) / tile_size).ceil, tile_height)
134
+ bounds[camera_index, gaussian_index, true] = [min_x, min_y, max_x, max_y]
135
+ counts[camera_index, gaussian_index] = (max_x - min_x) * (max_y - min_y)
136
+ end
137
+ end
138
+ [bounds, counts]
139
+ end
140
+ # rubocop:enable Metrics/AbcSize
141
+ private_class_method :tile_bounds
142
+
143
+ def clamp_tile(value, maximum)
144
+ value.clamp(0, maximum)
145
+ end
146
+ private_class_method :clamp_tile
147
+
148
+ # rubocop:disable Metrics/ParameterLists
149
+ def write_intersections!(keys, flatten_ids, bounds, counts, depths, gaussian_count, tile_width, tile_bits)
150
+ # rubocop:enable Metrics/ParameterLists
151
+ write_index = 0
152
+ counts.shape[0].times do |camera_index|
153
+ gaussian_count.times do |gaussian_index|
154
+ next if counts[camera_index, gaussian_index].zero?
155
+
156
+ min_x, min_y, max_x, max_y = bounds[camera_index, gaussian_index, true].to_a
157
+ upper_camera = camera_index << tile_bits
158
+ depth_bits = float32_bits(depths[camera_index, gaussian_index])
159
+ (min_y...max_y).each do |tile_y|
160
+ (min_x...max_x).each do |tile_x|
161
+ tile_id = (tile_y * tile_width) + tile_x
162
+ keys[write_index] = ((upper_camera | tile_id) << TILE_KEY_LOW_BITS) | depth_bits
163
+ flatten_ids[write_index] = (camera_index * gaussian_count) + gaussian_index
164
+ write_index += 1
165
+ end
166
+ end
167
+ end
168
+ end
169
+ end
170
+ private_class_method :write_intersections!
171
+
172
+ def float32_bits(value)
173
+ [value.to_f].pack("g").unpack1("N")
174
+ end
175
+ private_class_method :float32_bits
176
+
177
+ def tile_n_bits(tile_width, tile_height)
178
+ tile_count = tile_width * tile_height
179
+ tile_count == 1 ? 0 : ::Math.log2(tile_count).ceil
180
+ end
181
+ private_class_method :tile_n_bits
182
+
183
+ def validate_key_width!(camera_count, tile_bits)
184
+ return if (camera_count - 1).bit_length + tile_bits <= TILE_KEY_LOW_BITS
185
+
186
+ raise ArgumentError, "camera and tile ids exceed the 64-bit intersection key layout"
187
+ end
188
+ private_class_method :validate_key_width!
189
+
190
+ def validate_decoded_key!(camera_id, tile_id, camera_count, tile_width, tile_height)
191
+ return if camera_id.between?(0, camera_count - 1) && tile_id.between?(0, (tile_width * tile_height) - 1)
192
+
193
+ raise Gsplat::Error, "intersection key contains an out-of-range camera or tile id"
194
+ end
195
+ private_class_method :validate_decoded_key!
196
+ end
197
+ end
198
+ end