gsplat 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +202 -0
  3. data/README.md +236 -0
  4. data/docs/ACCEPTANCE.md +60 -0
  5. data/docs/BENCHMARKS.md +79 -0
  6. data/docs/DECISIONS.md +52 -0
  7. data/docs/MIGRATION.md +117 -0
  8. data/docs/PROFILE.md +75 -0
  9. data/docs/PROGRESS.md +199 -0
  10. data/docs/decisions/0000-template.md +17 -0
  11. data/docs/decisions/0001-pin-golden-reference-and-cpu-validation.md +27 -0
  12. data/docs/decisions/0002-retain-ruby-fallbacks-for-native-backend.md +26 -0
  13. data/docs/decisions/0003-share-projection-for-distorted-cameras.md +26 -0
  14. data/docs/decisions/0004-use-portable-world-space-reference-paths.md +26 -0
  15. data/docs/decisions/0005-reuse-ewa-core-for-2dgs.md +24 -0
  16. data/docs/decisions/0006-keep-eval3d-as-portable-reference.md +25 -0
  17. data/docs/decisions/0007-share-compositor-semantics-for-contribution-indices.md +25 -0
  18. data/examples/data/README.md +14 -0
  19. data/examples/data/colmap/images/view_000.png +0 -0
  20. data/examples/data/colmap/images/view_001.png +0 -0
  21. data/examples/data/colmap/images/view_002.png +0 -0
  22. data/examples/data/colmap/sparse/0/cameras.txt +2 -0
  23. data/examples/data/colmap/sparse/0/images.txt +6 -0
  24. data/examples/data/colmap/sparse/0/points3D.txt +17 -0
  25. data/examples/data/splats.ply +0 -0
  26. data/examples/fit_image.rb +35 -0
  27. data/examples/generate_sample_data.rb +147 -0
  28. data/examples/render_path.rb +117 -0
  29. data/examples/simple_trainer.rb +76 -0
  30. data/ext/gsplat_native/common.h +54 -0
  31. data/ext/gsplat_native/extconf.rb +39 -0
  32. data/ext/gsplat_native/gsplat_native.c +60 -0
  33. data/ext/gsplat_native/intersections.c +211 -0
  34. data/ext/gsplat_native/projection.c +199 -0
  35. data/ext/gsplat_native/raster_backward.c +129 -0
  36. data/ext/gsplat_native/raster_backward_bridge.c +69 -0
  37. data/ext/gsplat_native/raster_forward.c +150 -0
  38. data/ext/gsplat_native/rasterization.h +51 -0
  39. data/ext/gsplat_native/spherical_harmonics.c +129 -0
  40. data/gsplat.gemspec +36 -0
  41. data/lib/gsplat/autograd/context.rb +60 -0
  42. data/lib/gsplat/autograd/function.rb +68 -0
  43. data/lib/gsplat/autograd/variable.rb +159 -0
  44. data/lib/gsplat/backend/ruby/accumulate.rb +139 -0
  45. data/lib/gsplat/backend/ruby/accumulate_backward.rb +40 -0
  46. data/lib/gsplat/backend/ruby/eval3d_rasterizer.rb +175 -0
  47. data/lib/gsplat/backend/ruby/isect_tiles.rb +198 -0
  48. data/lib/gsplat/backend/ruby/projection.rb +251 -0
  49. data/lib/gsplat/backend/ruby/projection_backward.rb +190 -0
  50. data/lib/gsplat/backend/ruby/projection_covariance_vjp.rb +72 -0
  51. data/lib/gsplat/backend/ruby/projection_input_vjp.rb +252 -0
  52. data/lib/gsplat/backend/ruby/quat_scale_to_covar_preci.rb +139 -0
  53. data/lib/gsplat/backend/ruby/rasterize_to_indices_in_range.rb +121 -0
  54. data/lib/gsplat/backend/ruby/rasterize_to_pixels.rb +199 -0
  55. data/lib/gsplat/backend/ruby/rasterize_to_pixels_backward.rb +121 -0
  56. data/lib/gsplat/backend/ruby/spherical_harmonics.rb +135 -0
  57. data/lib/gsplat/backend/ruby/tile_compositor.rb +50 -0
  58. data/lib/gsplat/backend/ruby/tile_compositor_backward.rb +84 -0
  59. data/lib/gsplat/backend.rb +79 -0
  60. data/lib/gsplat/compression/grid_sort.rb +79 -0
  61. data/lib/gsplat/compression/kmeans.rb +121 -0
  62. data/lib/gsplat/compression/png.rb +147 -0
  63. data/lib/gsplat/compression/png_codec.rb +133 -0
  64. data/lib/gsplat/compression/quantizer.rb +79 -0
  65. data/lib/gsplat/io/checkpoint.rb +145 -0
  66. data/lib/gsplat/io/colmap.rb +175 -0
  67. data/lib/gsplat/io/colmap_binary.rb +98 -0
  68. data/lib/gsplat/io/colmap_text.rb +84 -0
  69. data/lib/gsplat/io/image.rb +63 -0
  70. data/lib/gsplat/io/image_backends.rb +81 -0
  71. data/lib/gsplat/io/npy.rb +189 -0
  72. data/lib/gsplat/io/ply.rb +185 -0
  73. data/lib/gsplat/io/ply_reader.rb +142 -0
  74. data/lib/gsplat/io/zip_archive.rb +183 -0
  75. data/lib/gsplat/math/camera_distortion.rb +123 -0
  76. data/lib/gsplat/math/camera_projection.rb +202 -0
  77. data/lib/gsplat/math/mat.rb +114 -0
  78. data/lib/gsplat/math/quaternion.rb +175 -0
  79. data/lib/gsplat/math/small_matrix_primitives.rb +112 -0
  80. data/lib/gsplat/math/spherical_harmonic_basis.rb +148 -0
  81. data/lib/gsplat/math/ssim.rb +153 -0
  82. data/lib/gsplat/native.rb +30 -0
  83. data/lib/gsplat/native_ops.rb +148 -0
  84. data/lib/gsplat/native_raster_ops.rb +76 -0
  85. data/lib/gsplat/ops/accumulate.rb +68 -0
  86. data/lib/gsplat/ops/eval3d_rasterize.rb +62 -0
  87. data/lib/gsplat/ops/isect_tiles.rb +38 -0
  88. data/lib/gsplat/ops/projection.rb +154 -0
  89. data/lib/gsplat/ops/quat_scale_to_covar_preci.rb +118 -0
  90. data/lib/gsplat/ops/rasterize_to_indices_in_range.rb +38 -0
  91. data/lib/gsplat/ops/rasterize_to_pixels.rb +105 -0
  92. data/lib/gsplat/ops/relocation.rb +94 -0
  93. data/lib/gsplat/ops/spherical_harmonics.rb +71 -0
  94. data/lib/gsplat/ops/tensor_shape_ops.rb +173 -0
  95. data/lib/gsplat/ops/tensor_value_ops.rb +150 -0
  96. data/lib/gsplat/optim/adam.rb +212 -0
  97. data/lib/gsplat/optim/lr_scheduler.rb +36 -0
  98. data/lib/gsplat/optim/selective_adam.rb +68 -0
  99. data/lib/gsplat/rasterization.rb +130 -0
  100. data/lib/gsplat/rasterization_2dgs.rb +140 -0
  101. data/lib/gsplat/rasterization_helpers.rb +162 -0
  102. data/lib/gsplat/rasterization_validation.rb +99 -0
  103. data/lib/gsplat/strategy/base.rb +49 -0
  104. data/lib/gsplat/strategy/default.rb +188 -0
  105. data/lib/gsplat/strategy/mcmc.rb +103 -0
  106. data/lib/gsplat/strategy/mcmc_ops.rb +143 -0
  107. data/lib/gsplat/strategy/ops.rb +165 -0
  108. data/lib/gsplat/training/config.rb +91 -0
  109. data/lib/gsplat/training/image_fitter.rb +158 -0
  110. data/lib/gsplat/training/losses.rb +191 -0
  111. data/lib/gsplat/training/scene.rb +116 -0
  112. data/lib/gsplat/training/trainer.rb +236 -0
  113. data/lib/gsplat/utils.rb +110 -0
  114. data/lib/gsplat/version.rb +6 -0
  115. data/lib/gsplat.rb +98 -0
  116. metadata +181 -0
@@ -0,0 +1,112 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Math
5
+ # Internal shape, casting, and closed-form primitives for Mat.
6
+ module SmallMatrixPrimitives
7
+ private
8
+
9
+ def prepare_matrix(matrices, size, dtype)
10
+ typed = cast_float(matrices, dtype)
11
+ validate_rank!(typed, 2)
12
+ unless typed.shape[-2, 2] == [size, size]
13
+ raise ShapeError, "expected [...,#{size},#{size}], got #{typed.shape.inspect}"
14
+ end
15
+
16
+ leading_shape = typed.shape[0...-2]
17
+ batch_size = leading_shape.empty? ? 1 : leading_shape.inject(:*)
18
+ [typed, leading_shape, typed.reshape(batch_size, size, size)]
19
+ end
20
+
21
+ def cast_float(array, dtype)
22
+ raise ArgumentError, "expected a Numo::NArray" unless array.is_a?(Numo::NArray)
23
+
24
+ type = case dtype
25
+ when nil then array.class
26
+ when :float32 then Numo::SFloat
27
+ when :float64 then Numo::DFloat
28
+ else dtype
29
+ end
30
+ unless [Numo::SFloat, Numo::DFloat].include?(type)
31
+ raise ArgumentError, "dtype must be Numo::SFloat or Numo::DFloat"
32
+ end
33
+
34
+ type.cast(array)
35
+ end
36
+
37
+ def validate_rank!(array, trailing_dimensions)
38
+ return if array.ndim >= trailing_dimensions
39
+
40
+ raise ShapeError, "expected at least #{trailing_dimensions} dimensions, got #{array.shape.inspect}"
41
+ end
42
+
43
+ def reshape_vector(vector, leading_shape)
44
+ return vector.class.cast(vector[0]) if leading_shape.empty?
45
+
46
+ vector.reshape(*leading_shape)
47
+ end
48
+
49
+ def reshape_matrix(matrix, leading_shape, *trailing_shape)
50
+ matrix.reshape(*(leading_shape + trailing_shape))
51
+ end
52
+
53
+ def ensure_invertible!(determinant, epsilon)
54
+ return unless determinant.abs.le(epsilon).any?
55
+
56
+ raise Gsplat::Error, "matrix is singular within epsilon #{epsilon}"
57
+ end
58
+
59
+ def default_epsilon(type)
60
+ type == Numo::DFloat ? 1e-12 : 1e-6
61
+ end
62
+
63
+ def determinant3(matrix)
64
+ a = matrix[true, 0, 0]
65
+ b = matrix[true, 0, 1]
66
+ c = matrix[true, 0, 2]
67
+ d = matrix[true, 1, 0]
68
+ e = matrix[true, 1, 1]
69
+ f = matrix[true, 1, 2]
70
+ g = matrix[true, 2, 0]
71
+ h = matrix[true, 2, 1]
72
+ i = matrix[true, 2, 2]
73
+ (a * ((e * i) - (f * h))) - (b * ((d * i) - (f * g))) + (c * ((d * h) - (e * g)))
74
+ end
75
+
76
+ # rubocop:disable Metrics/AbcSize
77
+ def fill_inverse3!(output, matrix, determinant)
78
+ a = matrix[true, 0, 0]
79
+ b = matrix[true, 0, 1]
80
+ c = matrix[true, 0, 2]
81
+ d = matrix[true, 1, 0]
82
+ e = matrix[true, 1, 1]
83
+ f = matrix[true, 1, 2]
84
+ g = matrix[true, 2, 0]
85
+ h = matrix[true, 2, 1]
86
+ i = matrix[true, 2, 2]
87
+ output[true, 0, 0] = ((e * i) - (f * h)) / determinant
88
+ output[true, 0, 1] = ((c * h) - (b * i)) / determinant
89
+ output[true, 0, 2] = ((b * f) - (c * e)) / determinant
90
+ output[true, 1, 0] = ((f * g) - (d * i)) / determinant
91
+ output[true, 1, 1] = ((a * i) - (c * g)) / determinant
92
+ output[true, 1, 2] = ((c * d) - (a * f)) / determinant
93
+ output[true, 2, 0] = ((d * h) - (e * g)) / determinant
94
+ output[true, 2, 1] = ((b * g) - (a * h)) / determinant
95
+ output[true, 2, 2] = ((a * e) - (b * d)) / determinant
96
+ end
97
+ # rubocop:enable Metrics/AbcSize
98
+
99
+ def multiply_flat(left, right, rows, shared, columns)
100
+ output = left.class.zeros(left.shape[0], rows, columns)
101
+ rows.times do |row|
102
+ columns.times do |column|
103
+ shared.times do |inner|
104
+ output[true, row, column] += left[true, row, inner] * right[true, inner, column]
105
+ end
106
+ end
107
+ end
108
+ output
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,148 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Math
5
+ # Real SH bases and normalized-direction derivatives in gsplat/Inria order.
6
+ module SphericalHarmonicBasis
7
+ C0 = 0.2820947917738781
8
+ C1 = 0.48860251190292
9
+ C2 = [1.092548430592079, 0.3153915652525201, 0.5462742152960395].freeze
10
+ C3 = [0.5900435899266435, 2.890611442640554, 0.4570457994644658,
11
+ 0.3731763325901154, 1.445305721320277].freeze
12
+ C4 = [2.5033429417967046, -1.770130769779931, 0.9461746957575601,
13
+ -0.6690465435572892, 0.1057855469152043, 0.47308734787878,
14
+ 0.6258357354491763].freeze
15
+
16
+ module_function
17
+
18
+ # Evaluates bases and derivatives with respect to already-normalized directions.
19
+ #
20
+ # Constants and ordering match gsplat 1.5.3 cuda/_torch_impl.py.
21
+ #
22
+ # @param directions [Numo::NArray] [N,3] normalized directions
23
+ # @param degree [Integer] 0..4
24
+ # @param basis_count [Integer] coefficient basis width
25
+ # @return [Array<Numo::NArray>] bases [N,K], derivatives [N,K,3]
26
+ def evaluate(directions, degree, basis_count)
27
+ count = directions.shape[0]
28
+ bases = directions.class.zeros(count, basis_count)
29
+ derivatives = directions.class.zeros(count, basis_count, 3)
30
+ x_coord = directions[true, 0]
31
+ y_coord = directions[true, 1]
32
+ z_coord = directions[true, 2]
33
+ bases[true, 0] = C0
34
+ fill_degree_one!(bases, derivatives, x_coord, y_coord, z_coord) if degree >= 1
35
+ fill_degree_two!(bases, derivatives, x_coord, y_coord, z_coord) if degree >= 2
36
+ fill_degree_three!(bases, derivatives, x_coord, y_coord, z_coord) if degree >= 3
37
+ fill_degree_four!(bases, derivatives, x_coord, y_coord, z_coord) if degree >= 4
38
+ [bases, derivatives]
39
+ end
40
+
41
+ def fill_degree_one!(bases, derivatives, x_coord, y_coord, z_coord)
42
+ bases[true, 1] = -C1 * y_coord
43
+ bases[true, 2] = C1 * z_coord
44
+ bases[true, 3] = -C1 * x_coord
45
+ derivatives[true, 1, 1] = -C1
46
+ derivatives[true, 2, 2] = C1
47
+ derivatives[true, 3, 0] = -C1
48
+ end
49
+ private_class_method :fill_degree_one!
50
+
51
+ # rubocop:disable Metrics/AbcSize
52
+ def fill_degree_two!(bases, derivatives, x_coord, y_coord, z_coord)
53
+ bases[true, 4] = C2[0] * x_coord * y_coord
54
+ bases[true, 5] = -C2[0] * y_coord * z_coord
55
+ bases[true, 6] = C2[1] * ((3 * (z_coord**2)) - 1)
56
+ bases[true, 7] = -C2[0] * x_coord * z_coord
57
+ bases[true, 8] = C2[2] * ((x_coord**2) - (y_coord**2))
58
+ derivatives[true, 4, 0] = C2[0] * y_coord
59
+ derivatives[true, 4, 1] = C2[0] * x_coord
60
+ derivatives[true, 5, 1] = -C2[0] * z_coord
61
+ derivatives[true, 5, 2] = -C2[0] * y_coord
62
+ derivatives[true, 6, 2] = 6 * C2[1] * z_coord
63
+ derivatives[true, 7, 0] = -C2[0] * z_coord
64
+ derivatives[true, 7, 2] = -C2[0] * x_coord
65
+ derivatives[true, 8, 0] = 2 * C2[2] * x_coord
66
+ derivatives[true, 8, 1] = -2 * C2[2] * y_coord
67
+ end
68
+ private_class_method :fill_degree_two!
69
+
70
+ def fill_degree_three!(bases, derivatives, x_coord, y_coord, z_coord)
71
+ bases[true, 9] = -C3[0] * y_coord * ((3 * (x_coord**2)) - (y_coord**2))
72
+ bases[true, 10] = C3[1] * x_coord * y_coord * z_coord
73
+ bases[true, 11] = -C3[2] * y_coord * ((5 * (z_coord**2)) - 1)
74
+ bases[true, 12] = C3[3] * z_coord * ((5 * (z_coord**2)) - 3)
75
+ bases[true, 13] = -C3[2] * x_coord * ((5 * (z_coord**2)) - 1)
76
+ bases[true, 14] = C3[4] * z_coord * ((x_coord**2) - (y_coord**2))
77
+ bases[true, 15] = -C3[0] * x_coord * ((x_coord**2) - (3 * (y_coord**2)))
78
+ fill_degree_three_derivatives!(derivatives, x_coord, y_coord, z_coord)
79
+ end
80
+ private_class_method :fill_degree_three!
81
+
82
+ def fill_degree_three_derivatives!(derivatives, x_coord, y_coord, z_coord)
83
+ derivatives[true, 9, 0] = -6 * C3[0] * x_coord * y_coord
84
+ derivatives[true, 9, 1] = -3 * C3[0] * ((x_coord**2) - (y_coord**2))
85
+ derivatives[true, 10, 0] = C3[1] * y_coord * z_coord
86
+ derivatives[true, 10, 1] = C3[1] * x_coord * z_coord
87
+ derivatives[true, 10, 2] = C3[1] * x_coord * y_coord
88
+ derivatives[true, 11, 1] = -C3[2] * ((5 * (z_coord**2)) - 1)
89
+ derivatives[true, 11, 2] = -10 * C3[2] * y_coord * z_coord
90
+ derivatives[true, 12, 2] = C3[3] * ((15 * (z_coord**2)) - 3)
91
+ derivatives[true, 13, 0] = -C3[2] * ((5 * (z_coord**2)) - 1)
92
+ derivatives[true, 13, 2] = -10 * C3[2] * x_coord * z_coord
93
+ derivatives[true, 14, 0] = 2 * C3[4] * x_coord * z_coord
94
+ derivatives[true, 14, 1] = -2 * C3[4] * y_coord * z_coord
95
+ derivatives[true, 14, 2] = C3[4] * ((x_coord**2) - (y_coord**2))
96
+ derivatives[true, 15, 0] = -3 * C3[0] * ((x_coord**2) - (y_coord**2))
97
+ derivatives[true, 15, 1] = 6 * C3[0] * x_coord * y_coord
98
+ end
99
+ private_class_method :fill_degree_three_derivatives!
100
+
101
+ def fill_degree_four!(bases, derivatives, x_coord, y_coord, z_coord)
102
+ x2 = x_coord**2
103
+ y2 = y_coord**2
104
+ z2 = z_coord**2
105
+ bases[true, 16] = C4[0] * x_coord * y_coord * (x2 - y2)
106
+ bases[true, 17] = C4[1] * y_coord * z_coord * ((3 * x2) - y2)
107
+ bases[true, 18] = C4[2] * x_coord * y_coord * ((7 * z2) - 1)
108
+ bases[true, 19] = C4[3] * y_coord * z_coord * ((7 * z2) - 3)
109
+ bases[true, 20] = C4[4] * ((z2 * ((35 * z2) - 30)) + 3)
110
+ bases[true, 21] = C4[3] * x_coord * z_coord * ((7 * z2) - 3)
111
+ bases[true, 22] = C4[5] * (x2 - y2) * ((7 * z2) - 1)
112
+ bases[true, 23] = C4[1] * x_coord * z_coord * (x2 - (3 * y2))
113
+ bases[true, 24] = C4[6] * ((x2**2) - (6 * x2 * y2) + (y2**2))
114
+ fill_degree_four_derivatives!(derivatives, x_coord, y_coord, z_coord)
115
+ end
116
+ private_class_method :fill_degree_four!
117
+
118
+ def fill_degree_four_derivatives!(derivatives, x_coord, y_coord, z_coord)
119
+ x_squared = x_coord**2
120
+ y_squared = y_coord**2
121
+ z_squared = z_coord**2
122
+ derivatives[true, 16, 0] = C4[0] * y_coord * ((3 * x_squared) - y_squared)
123
+ derivatives[true, 16, 1] = C4[0] * x_coord * (x_squared - (3 * y_squared))
124
+ derivatives[true, 17, 0] = 6 * C4[1] * x_coord * y_coord * z_coord
125
+ derivatives[true, 17, 1] = 3 * C4[1] * z_coord * (x_squared - y_squared)
126
+ derivatives[true, 17, 2] = C4[1] * y_coord * ((3 * x_squared) - y_squared)
127
+ derivatives[true, 18, 0] = C4[2] * y_coord * ((7 * z_squared) - 1)
128
+ derivatives[true, 18, 1] = C4[2] * x_coord * ((7 * z_squared) - 1)
129
+ derivatives[true, 18, 2] = 14 * C4[2] * x_coord * y_coord * z_coord
130
+ derivatives[true, 19, 1] = C4[3] * z_coord * ((7 * z_squared) - 3)
131
+ derivatives[true, 19, 2] = C4[3] * y_coord * ((21 * z_squared) - 3)
132
+ derivatives[true, 20, 2] = C4[4] * ((140 * (z_coord**3)) - (60 * z_coord))
133
+ derivatives[true, 21, 0] = C4[3] * z_coord * ((7 * z_squared) - 3)
134
+ derivatives[true, 21, 2] = C4[3] * x_coord * ((21 * z_squared) - 3)
135
+ derivatives[true, 22, 0] = 2 * C4[5] * x_coord * ((7 * z_squared) - 1)
136
+ derivatives[true, 22, 1] = -2 * C4[5] * y_coord * ((7 * z_squared) - 1)
137
+ derivatives[true, 22, 2] = 14 * C4[5] * z_coord * (x_squared - y_squared)
138
+ derivatives[true, 23, 0] = 3 * C4[1] * z_coord * (x_squared - y_squared)
139
+ derivatives[true, 23, 1] = -6 * C4[1] * x_coord * y_coord * z_coord
140
+ derivatives[true, 23, 2] = C4[1] * x_coord * (x_squared - (3 * y_squared))
141
+ derivatives[true, 24, 0] = 4 * C4[6] * x_coord * (x_squared - (3 * y_squared))
142
+ derivatives[true, 24, 1] = 4 * C4[6] * y_coord * (y_squared - (3 * x_squared))
143
+ end
144
+ # rubocop:enable Metrics/AbcSize
145
+ private_class_method :fill_degree_four_derivatives!
146
+ end
147
+ end
148
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ module Math
5
+ # SSIM forward and analytic VJP with a grouped Gaussian convolution.
6
+ module Ssim
7
+ # Intermediate tensors retained by {.backward}.
8
+ Cache = Data.define(
9
+ :image_a, :image_b, :mu_a, :mu_b, :ssim_map,
10
+ :numerator_mean, :numerator_variance,
11
+ :denominator_mean, :denominator_variance, :kernel, :layout
12
+ )
13
+
14
+ module_function
15
+
16
+ # rubocop:disable Metrics/AbcSize
17
+ def forward(image_a, image_b, layout:)
18
+ first, resolved_layout = to_nchw(image_a, layout)
19
+ second, second_layout = to_nchw(image_b, layout)
20
+ validate_inputs!(first, second, resolved_layout, second_layout)
21
+ kernel = gaussian_kernel(first.class)
22
+ mu_a = convolve(first, kernel)
23
+ mu_b = convolve(second, kernel)
24
+ variance_a = convolve(first**2, kernel) - (mu_a**2)
25
+ variance_b = convolve(second**2, kernel) - (mu_b**2)
26
+ covariance = convolve(first * second, kernel) - (mu_a * mu_b)
27
+ numerator_mean = (2 * mu_a * mu_b) + (0.01**2)
28
+ numerator_variance = (2 * covariance) + (0.03**2)
29
+ denominator_mean = (mu_a**2) + (mu_b**2) + (0.01**2)
30
+ denominator_variance = variance_a + variance_b + (0.03**2)
31
+ ssim_map = (numerator_mean * numerator_variance) /
32
+ (denominator_mean * denominator_variance)
33
+ cache = Cache.new(
34
+ image_a: first, image_b: second, mu_a: mu_a, mu_b: mu_b, ssim_map: ssim_map,
35
+ numerator_mean: numerator_mean, numerator_variance: numerator_variance,
36
+ denominator_mean: denominator_mean, denominator_variance: denominator_variance,
37
+ kernel: kernel, layout: resolved_layout
38
+ )
39
+ [ssim_map.mean.to_f, cache]
40
+ end
41
+ # rubocop:enable Metrics/AbcSize
42
+
43
+ # rubocop:disable Metrics/AbcSize
44
+ def backward(cache, grad_output)
45
+ factor = grad_output.to_f / cache.ssim_map.size
46
+ denominator = cache.denominator_mean * cache.denominator_variance
47
+ grad_num_mean = factor * cache.numerator_variance / denominator
48
+ grad_num_variance = factor * cache.numerator_mean / denominator
49
+ grad_den_mean = -factor * cache.ssim_map / cache.denominator_mean
50
+ grad_den_variance = -factor * cache.ssim_map / cache.denominator_variance
51
+ grad_mu_a = (2 * cache.mu_b * grad_num_mean) -
52
+ (2 * cache.mu_b * grad_num_variance) +
53
+ (2 * cache.mu_a * grad_den_mean) -
54
+ (2 * cache.mu_a * grad_den_variance)
55
+ grad_mu_b = (2 * cache.mu_a * grad_num_mean) -
56
+ (2 * cache.mu_a * grad_num_variance) +
57
+ (2 * cache.mu_b * grad_den_mean) -
58
+ (2 * cache.mu_b * grad_den_variance)
59
+ grad_cross = 2 * grad_num_variance
60
+ filtered_variance = convolve(grad_den_variance, cache.kernel)
61
+ filtered_cross = convolve(grad_cross, cache.kernel)
62
+ grad_a = convolve(grad_mu_a, cache.kernel) +
63
+ (2 * cache.image_a * filtered_variance) +
64
+ (cache.image_b * filtered_cross)
65
+ grad_b = convolve(grad_mu_b, cache.kernel) +
66
+ (2 * cache.image_b * filtered_variance) +
67
+ (cache.image_a * filtered_cross)
68
+ [from_nchw(grad_a, cache.layout), from_nchw(grad_b, cache.layout)]
69
+ end
70
+ # rubocop:enable Metrics/AbcSize
71
+
72
+ def gaussian_kernel(type)
73
+ coordinates = (-5..5).map { |value| ::Math.exp(-(value**2) / (2 * (1.5**2))) }
74
+ total = coordinates.sum
75
+ vector = coordinates.map { |value| value / total }
76
+ type.cast(vector.product(vector).map { |left, right| left * right }).reshape(11, 11)
77
+ end
78
+ private_class_method :gaussian_kernel
79
+
80
+ def convolve(input, kernel)
81
+ output = input.class.zeros(*input.shape)
82
+ height = input.shape[2]
83
+ width = input.shape[3]
84
+ 11.times do |kernel_y|
85
+ next if (kernel_y - 5).abs >= height
86
+
87
+ destination_y, source_y = shifted_ranges(height, kernel_y - 5)
88
+ 11.times do |kernel_x|
89
+ next if (kernel_x - 5).abs >= width
90
+
91
+ destination_x, source_x = shifted_ranges(width, kernel_x - 5)
92
+ destination = [true, true, destination_y, destination_x]
93
+ source = [true, true, source_y, source_x]
94
+ output[*destination] = output[*destination] + (input[*source] * kernel[kernel_y, kernel_x])
95
+ end
96
+ end
97
+ output
98
+ end
99
+ private_class_method :convolve
100
+
101
+ def shifted_ranges(length, offset)
102
+ if offset >= 0
103
+ [0...(length - offset), offset...length]
104
+ else
105
+ [(-offset)...length, 0...(length + offset)]
106
+ end
107
+ end
108
+ private_class_method :shifted_ranges
109
+
110
+ def to_nchw(image, layout)
111
+ unless [Numo::SFloat, Numo::DFloat].include?(image.class)
112
+ raise ArgumentError, "images must be Numo floating-point arrays"
113
+ end
114
+
115
+ resolved = layout == :auto ? infer_layout(image) : layout
116
+ converted = case resolved
117
+ when :nchw then image
118
+ when :nhwc then image.transpose(0, 3, 1, 2)
119
+ when :chw then image.reshape(1, *image.shape)
120
+ when :hwc then image.transpose(2, 0, 1).reshape(1, image.shape[2], image.shape[0], image.shape[1])
121
+ else raise ArgumentError, "layout must be :auto, :nchw, :nhwc, :chw, or :hwc"
122
+ end
123
+ [converted, resolved]
124
+ end
125
+ private_class_method :to_nchw
126
+
127
+ def from_nchw(image, layout)
128
+ case layout
129
+ when :nchw then image
130
+ when :nhwc then image.transpose(0, 2, 3, 1)
131
+ when :chw then image[0, true, true, true].dup
132
+ when :hwc then image[0, true, true, true].transpose(1, 2, 0)
133
+ end
134
+ end
135
+ private_class_method :from_nchw
136
+
137
+ def infer_layout(image)
138
+ return image.shape[-1] <= 4 && image.shape[1] > 4 ? :nhwc : :nchw if image.ndim == 4
139
+ return image.shape[-1] <= 4 && image.shape[0] > 4 ? :hwc : :chw if image.ndim == 3
140
+
141
+ raise ShapeError, "expected a 3D or 4D image, got #{image.shape.inspect}"
142
+ end
143
+ private_class_method :infer_layout
144
+
145
+ def validate_inputs!(first, second, first_layout, second_layout)
146
+ return if first.shape == second.shape && first_layout == second_layout && first.ndim == 4
147
+
148
+ raise ShapeError, "SSIM image shapes/layouts differ: #{first.shape.inspect} and #{second.shape.inspect}"
149
+ end
150
+ private_class_method :validate_inputs!
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ # Optional C implementation entrypoint.
5
+ module Native
6
+ class << self
7
+ attr_reader :load_error
8
+
9
+ def available?
10
+ @available == true
11
+ end
12
+
13
+ def mark_available!
14
+ @available = true
15
+ end
16
+
17
+ def mark_unavailable!(error)
18
+ @available = false
19
+ @load_error = error
20
+ end
21
+ end
22
+ end
23
+ end
24
+
25
+ begin
26
+ require_relative "gsplat_native"
27
+ Gsplat::Native.mark_available!
28
+ rescue LoadError => e
29
+ Gsplat::Native.mark_unavailable!(e)
30
+ end
@@ -0,0 +1,148 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "native_raster_ops"
4
+
5
+ # Native operation registration and validated Ruby fallbacks.
6
+ module Gsplat
7
+ # Ruby-facing validation and fallback wrappers for native operations.
8
+ module NativeOps
9
+ module_function
10
+
11
+ def spherical_harmonics_forward(degree, directions, coefficients, masks: nil)
12
+ prepared = Backend::RubySphericalHarmonics.send(
13
+ :validate_inputs, degree, directions, coefficients, masks
14
+ )
15
+ typed_directions, typed_coefficients, typed_masks, = prepared
16
+ return Backend::RubySphericalHarmonics.forward(degree, directions, coefficients, masks: masks) if typed_masks
17
+ unless typed_directions.is_a?(Numo::SFloat)
18
+ return Backend::RubySphericalHarmonics.forward(degree, directions, coefficients, masks: masks)
19
+ end
20
+
21
+ Native.spherical_harmonics_forward_sfloat(
22
+ degree,
23
+ typed_directions.dup,
24
+ typed_coefficients.dup
25
+ )
26
+ end
27
+
28
+ # rubocop:disable Metrics/ParameterLists
29
+ def projection_forward(means, covars, quaternions, scales, viewmats, intrinsics, width, height,
30
+ eps2d:, near_plane:, far_plane:, radius_clip:, calc_compensations:, camera_model:,
31
+ radial_coeffs: nil, tangential_coeffs: nil, thin_prism_coeffs: nil,
32
+ global_z_order: true)
33
+ # rubocop:enable Metrics/ParameterLists
34
+ extended = camera_model.to_s == "fisheye" ||
35
+ [radial_coeffs, tangential_coeffs, thin_prism_coeffs].any?
36
+ unless means.is_a?(Numo::SFloat) && !extended && global_z_order
37
+ return Backend::RubyProjection.forward(
38
+ means, covars, quaternions, scales, viewmats, intrinsics, width, height,
39
+ eps2d: eps2d, near_plane: near_plane, far_plane: far_plane,
40
+ radius_clip: radius_clip, calc_compensations: calc_compensations,
41
+ camera_model: camera_model, radial_coeffs: radial_coeffs,
42
+ tangential_coeffs: tangential_coeffs, thin_prism_coeffs: thin_prism_coeffs,
43
+ global_z_order: global_z_order
44
+ )
45
+ end
46
+ prepared = Backend::RubyProjection.send(
47
+ :prepare_inputs,
48
+ means, covars, quaternions, scales, viewmats, intrinsics, width, height,
49
+ eps2d, near_plane, far_plane, radius_clip, camera_model
50
+ )
51
+ typed_means, typed_covars, typed_views, typed_intrinsics = prepared
52
+ Native.projection_forward_sfloat(
53
+ typed_means.dup,
54
+ typed_covars.dup,
55
+ typed_views.dup,
56
+ typed_intrinsics.dup,
57
+ width,
58
+ height,
59
+ eps2d,
60
+ near_plane,
61
+ far_plane,
62
+ radius_clip,
63
+ calc_compensations,
64
+ camera_model.to_s == "ortho"
65
+ )
66
+ end
67
+
68
+ # rubocop:disable Metrics/ParameterLists
69
+ def isect_tiles(means2d, radii, depths, tile_size, tile_width, tile_height, sort:)
70
+ # rubocop:enable Metrics/ParameterLists
71
+ prepared = Backend::RubyIsectTiles.send(
72
+ :validate_inputs, means2d, radii, depths, tile_size, tile_width, tile_height
73
+ )
74
+ typed_means, typed_radii, typed_depths = prepared
75
+ unless typed_means.is_a?(Numo::SFloat)
76
+ return Backend::RubyIsectTiles.forward(
77
+ means2d, radii, depths, tile_size, tile_width, tile_height, sort: sort
78
+ )
79
+ end
80
+ tile_bits = Backend::RubyIsectTiles.send(:tile_n_bits, tile_width, tile_height)
81
+ Backend::RubyIsectTiles.send(:validate_key_width!, typed_means.shape[0], tile_bits)
82
+
83
+ Native.isect_tiles_sfloat(
84
+ typed_means.dup,
85
+ typed_radii.dup,
86
+ typed_depths.dup,
87
+ tile_size,
88
+ tile_width,
89
+ tile_height,
90
+ sort
91
+ )
92
+ end
93
+
94
+ def isect_offset_encode(keys, camera_count, tile_width, tile_height)
95
+ Backend::RubyIsectTiles.send(:validate_grid!, camera_count, tile_width, tile_height)
96
+ unless keys.is_a?(Numo::Int64)
97
+ return Backend::RubyIsectTiles.offset_encode(keys, camera_count, tile_width, tile_height)
98
+ end
99
+ raise ShapeError, "expected isect_ids [M]" unless keys.ndim == 1
100
+
101
+ Native.isect_offset_encode_int64(keys.dup, camera_count, tile_width, tile_height)
102
+ end
103
+ end
104
+
105
+ if Native.available?
106
+ Backend.register(
107
+ :spherical_harmonics_forward,
108
+ :native,
109
+ NativeOps.method(:spherical_harmonics_forward)
110
+ )
111
+ Backend.register(
112
+ :spherical_harmonics_backward,
113
+ :native,
114
+ Backend::RubySphericalHarmonics.method(:backward)
115
+ )
116
+ Backend.register(
117
+ :fully_fused_projection_forward,
118
+ :native,
119
+ NativeOps.method(:projection_forward)
120
+ )
121
+ Backend.register(
122
+ :fully_fused_projection_backward,
123
+ :native,
124
+ Backend::RubyProjectionBackward.method(:backward)
125
+ )
126
+ Backend.register(:isect_tiles, :native, NativeOps.method(:isect_tiles))
127
+ Backend.register(
128
+ :isect_offset_encode,
129
+ :native,
130
+ NativeOps.method(:isect_offset_encode)
131
+ )
132
+ Backend.register(
133
+ :rasterize_to_pixels_forward,
134
+ :native,
135
+ NativeRasterOps.method(:forward)
136
+ )
137
+ Backend.register(
138
+ :rasterize_to_pixels_backward,
139
+ :native,
140
+ NativeRasterOps.method(:backward)
141
+ )
142
+ Backend.register(
143
+ :rasterize_to_indices_in_range,
144
+ :native,
145
+ Backend::RubyRasterizeToIndicesInRange.method(:forward)
146
+ )
147
+ end
148
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gsplat
4
+ # Validated float32 bridge for native tile rasterization.
5
+ module NativeRasterOps
6
+ module_function
7
+
8
+ # rubocop:disable Metrics/ParameterLists
9
+ def forward(means2d, conics, colors, opacities, backgrounds, masks, width, height,
10
+ tile_size, isect_offsets, flatten_ids)
11
+ # rubocop:enable Metrics/ParameterLists
12
+ prepared = Backend::RubyRasterizeToPixels.send(
13
+ :validate_inputs,
14
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
15
+ tile_size, isect_offsets, flatten_ids
16
+ )
17
+ unless means2d.is_a?(Numo::SFloat)
18
+ return Backend::RubyRasterizeToPixels.forward(
19
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
20
+ tile_size, isect_offsets, flatten_ids
21
+ )
22
+ end
23
+
24
+ typed_means, typed_conics, typed_colors, typed_opacities,
25
+ typed_backgrounds, typed_masks, typed_offsets, typed_ids = prepared
26
+ Native.rasterize_forward_sfloat(
27
+ typed_means.dup, typed_conics.dup, typed_colors.dup, typed_opacities.dup,
28
+ typed_backgrounds&.dup, native_masks(typed_masks), width, height, tile_size,
29
+ typed_offsets.dup, typed_ids.dup
30
+ )
31
+ end
32
+
33
+ # rubocop:disable Metrics/ParameterLists
34
+ def backward(means2d, conics, colors, opacities, backgrounds, masks, width, height,
35
+ tile_size, isect_offsets, flatten_ids, render_alphas, last_ids,
36
+ grad_render_colors, grad_render_alphas, absgrad:)
37
+ # rubocop:enable Metrics/ParameterLists
38
+ unless means2d.is_a?(Numo::SFloat)
39
+ return Backend::RubyRasterizeToPixelsBackward.backward(
40
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
41
+ tile_size, isect_offsets, flatten_ids, render_alphas, last_ids,
42
+ grad_render_colors, grad_render_alphas, absgrad: absgrad
43
+ )
44
+ end
45
+ prepared = Backend::RubyRasterizeToPixels.send(
46
+ :validate_inputs,
47
+ means2d, conics, colors, opacities, backgrounds, masks, width, height,
48
+ tile_size, isect_offsets, flatten_ids
49
+ )
50
+ typed_means, typed_conics, typed_colors, typed_opacities,
51
+ typed_backgrounds, typed_masks, typed_offsets, typed_ids = prepared
52
+ color_shape = [typed_means.shape[0], height, width, typed_colors.shape[-1]]
53
+ alpha_shape = [typed_means.shape[0], height, width, 1]
54
+ typed_color_grad = validate_gradient(grad_render_colors, color_shape)
55
+ typed_alpha_grad = validate_gradient(grad_render_alphas, alpha_shape)
56
+ Native.rasterize_backward_sfloat(
57
+ typed_means.dup, typed_conics.dup, typed_colors.dup, typed_opacities.dup,
58
+ typed_backgrounds&.dup, native_masks(typed_masks), width, height, tile_size,
59
+ typed_offsets.dup, typed_ids.dup, Numo::SFloat.cast(render_alphas).dup,
60
+ Numo::Int32.cast(last_ids).dup, typed_color_grad.dup, typed_alpha_grad.dup, absgrad
61
+ )
62
+ end
63
+
64
+ def native_masks(masks)
65
+ masks && Numo::Int32.cast(masks).dup
66
+ end
67
+ private_class_method :native_masks
68
+
69
+ def validate_gradient(gradient, shape)
70
+ Backend::RubyRasterizeToPixelsBackward.send(
71
+ :validate_gradient, Numo::SFloat, gradient, shape
72
+ )
73
+ end
74
+ private_class_method :validate_gradient
75
+ end
76
+ end