contrek 1.4.1 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/Gemfile.lock +4 -1
  4. data/README.md +218 -38
  5. data/contrek.gemspec +1 -0
  6. data/ext/cpp_polygon_finder/PolygonFinder/src/ContrekApi.h +2 -2
  7. data/ext/cpp_polygon_finder/PolygonFinder/src/Tests.cpp +52 -148
  8. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/RawBitmap.cpp +12 -4
  9. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/RawBitmap.h +1 -0
  10. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/AsciiSource.cpp +46 -0
  11. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/AsciiSource.h +29 -0
  12. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/GeoLocalization.h +32 -0
  13. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/PngSource.cpp +103 -0
  14. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/PngSource.h +40 -0
  15. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/RasterSource.h +22 -0
  16. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/RasterStreamer.h +92 -0
  17. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/TiffSource.cpp +301 -0
  18. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/bitmaps/streaming/TiffSource.h +51 -0
  19. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/FinderUtils.cpp +6 -8
  20. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/PolygonFinder.cpp +7 -2
  21. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/PolygonFinder.h +3 -1
  22. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/concurrent/Finder.cpp +6 -0
  23. data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/concurrent/GeoJsonStreamingMerger.h +51 -6
  24. data/ext/cpp_polygon_finder/cpp_polygon_finder.cpp +83 -0
  25. data/ext/cpp_polygon_finder/extconf.rb +9 -0
  26. data/lib/contrek/bitmaps/bitmap.rb +2 -2
  27. data/lib/contrek/bitmaps/chunky_bitmap.rb +23 -8
  28. data/lib/contrek/bitmaps/png_bitmap.rb +4 -0
  29. data/lib/contrek/bitmaps/rgba_bitmap.rb +50 -0
  30. data/lib/contrek/bitmaps/streaming/raster_streamer.rb +66 -0
  31. data/lib/contrek/bitmaps/streaming/sources/ascii.rb +33 -0
  32. data/lib/contrek/bitmaps/streaming/sources/base.rb +27 -0
  33. data/lib/contrek/bitmaps/streaming/sources/png.rb +105 -0
  34. data/lib/contrek/bitmaps/streaming/sources/tiff.rb +349 -0
  35. data/lib/contrek/finder/concurrent/finder.rb +3 -2
  36. data/lib/contrek/finder/concurrent/geo_json_streaming_merger.rb +19 -2
  37. data/lib/contrek/finder/polygon_finder.rb +6 -3
  38. data/lib/contrek/version.rb +1 -1
  39. data/lib/contrek.rb +6 -0
  40. metadata +30 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eada79f09d6eede09420a2f933496d8839152f741d9c90e7d6b63fc7537bec29
4
- data.tar.gz: 5652df8b513dfb26db08a1640cf59e2642a0feba7df2f9ce62eec0fe4e6dc445
3
+ metadata.gz: 802afd86666e1705cc50240c6f7d79abfd9c7e93c315b1222199e79836fdd9e2
4
+ data.tar.gz: e9f593c3ef9a661720f2587b44d65406839b25c2bc1690b8c4bfae477f967de6
5
5
  SHA512:
6
- metadata.gz: 8dc280b616db1de56688ca94f5c1bcef1bf02cb81ff820638ebdf0bed821ba37745799a579a2660da6dc0934afa46a94aae51affdcb3b6fd987710a84f5f85d9
7
- data.tar.gz: c815702b9abafa8b5e6de4e1f2e05ab6b03ed91248efe0786807307db6ebff19a7e2c9d002fadf94a12e2c9f41a8c07fe0ca05c01f5a838722090a737c888983
6
+ metadata.gz: 8e4d8940a0f6cb93fc0ee685808c8889110bedbfd604fb8de3e3aca6b0fb9caa69d57bec518cc50ce15314a6ffa45e525fc947b60d52d71403510b6c77b7302f
7
+ data.tar.gz: e12d7f30b9dec7ee474ab5c400f86f8331cc8eaa8e1571e6c8b895ddea4a4f0633806d2ccb0b4fdd12380d77caa714c4b6c85b9f347e3bf6ff70f39b0a1cb9d9
data/CHANGELOG.md CHANGED
@@ -156,3 +156,9 @@ All notable changes to this project will be documented in this file.
156
156
 
157
157
  ## [1.4.1] - 2026-08-23
158
158
  - The number_of_threads effective value is now returned inside metadata output struct too.
159
+
160
+ ## [1.4.2] - 2026-09-06
161
+ - Contrek takes care of the streaming process, providing an API that greatly simplifies the implementation of streaming raster processing.
162
+
163
+ ## [1.4.3] - 2026-09-09
164
+ - Added TIFF/GeoTIFF streaming support with geolocation metadata and direct WGS84 GeoJSON output.
data/Gemfile.lock CHANGED
@@ -1,9 +1,10 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- contrek (1.4.1)
4
+ contrek (1.4.3)
5
5
  chunky_png (~> 1.4)
6
6
  concurrent-ruby (~> 1.3.5)
7
+ ffi (~> 1.17)
7
8
  rice (= 4.5.0)
8
9
 
9
10
  GEM
@@ -14,6 +15,8 @@ GEM
14
15
  chunky_png (1.4.0)
15
16
  concurrent-ruby (1.3.5)
16
17
  diff-lcs (1.6.1)
18
+ ffi (1.17.4)
19
+ ffi (1.17.4-x86_64-linux-gnu)
17
20
  json (2.11.3)
18
21
  language_server-protocol (3.17.0.4)
19
22
  lint_roller (1.1.0)
data/README.md CHANGED
@@ -2,12 +2,14 @@
2
2
 
3
3
  **Contrek** is a standalone **C++17 contour tracing and polygonization library** for raster images.
4
4
 
5
- It was originally developed to solve a practical problem: extracting polygons from very large raster images without loading the entire image into memory. Over time the implementation evolved into a reusable library that can process PNG images as well as raw memory buffers, while preserving polygon topology during the entire tracing process.
5
+ It was originally developed to solve a practical problem: extracting polygons from very large raster images without loading the entire image into memory. Over time the implementation evolved into a reusable library that can process PNG and TIFF/GeoTIFF images as well as raw memory buffers, while preserving polygon topology during the entire tracing process.
6
6
 
7
- The engine is based on programmable pixel matchers, allowing the caller to decide which pixels belong to the regions being traced. Polygon coordinates can be streamed as they are produced, making the library suitable for datasets that would otherwise require a large amount of RAM.
7
+ The engine is based on programmable pixel matchers, allowing the caller to decide which pixels belong to the regions being traced. Polygon coordinates can be streamed as they are produced, making the library suitable for datasets that would otherwise require a large amount of RAM. GeoTIFF geolocation can also be preserved when streaming directly to GeoJSON (EPSG:4326).
8
8
 
9
9
  Although the tracing engine is written in C++, Contrek is also distributed as a Ruby gem exposing almost the complete native API through an idiomatic Ruby interface.
10
10
 
11
+ A **Python** binding is available 👉 **[here](https://github.com/runout77/contrek-python)**
12
+
11
13
  ## About Contrek
12
14
 
13
15
  **Contrek** (**CON**tour **TREK**king) at its simplest, the library scans a bitmap and returns the contours of all regions matching a user-defined criterion. Each region is represented by one outer polygon and, when necessary, one or more inner polygons describing holes.
@@ -25,58 +27,227 @@ The image below shows a simple example. Every non-white pixel is considered part
25
27
 
26
28
  Most contour tracing libraries process an image sequentially. That approach is perfectly adequate for many workloads, but it becomes less practical once images become very large or when memory usage starts to matter.
27
29
 
28
- Contrek follows a different strategy.
30
+ Contrek supports monothread extraction but also different strategies.
29
31
 
30
- Instead of processing the whole image at once, it can split it into independent vertical stripes. Each stripe is traced separately and the resulting polygons are merged afterwards. Most of the implementation complexity lies in this merge phase, whose purpose is to reconstruct polygons crossing stripe boundaries without breaking their topology.
32
+ ### Mode 1: Single-threaded processing
31
33
 
32
- The same design makes it possible to use the library in two different ways:
34
+ <table>
35
+ <tr>
36
+ <td width="40%" valign="top">
37
+ <img src="docs/images/modes/mode1.jpg" width="100%" alt="Mode 1">
38
+ </td>
39
+ <td width="60%" valign="top">
40
+ The entire image is processed using a single core.
33
41
 
34
- - **Parallel processing**, where several stripes are traced simultaneously on different CPU cores.
35
- - **Streaming processing**, where only a small portion of the image is kept in memory at any given time.
42
+ **Profile:** low speed; low memory efficiency.
43
+ <br><br>
44
+ <center><img src="docs/images/modes/mode1_panel.jpg" width="80%" alt="Mode 1"></center>
45
+ </td></tr></table>
36
46
 
37
- The tracing algorithm itself is identical in both cases; only the execution strategy changes.
47
+ Trace polygons from a PNG file (`CPPPngBitMap`).
38
48
 
39
- ## Parallel execution
49
+ ```ruby
50
+ png_bitmap = CPPPngBitMap.new("spec/files/images/labyrinth3.png")
51
+ color = Contrek::Bitmaps::RgbCppColor.new(r: 255, g: 255, b: 255, a: 255)
52
+ rgb_matcher = CPPRGBNotMatcher.new(color.raw)
53
+ polygonfinder = CPPPolygonFinder.new(
54
+ png_bitmap,
55
+ rgb_matcher,
56
+ nil,
57
+ { versus: :a,
58
+ compress: {
59
+ visvalingam: true,
60
+ visvalingam_tolerance: 1.5
61
+ }
62
+ }
63
+ )
64
+ result = polygonfinder.process_info
65
+ puts result.metadata[:number_of_threads] # => 0
66
+ ```
40
67
 
41
- When multiple CPU cores are available, independent stripes can be processed concurrently. The actual speedup depends on the image content and on how much work is required during the merge stage, but large datasets generally benefit from additional cores.
68
+ `versus` accepts `:a`,`:o` for "anticlockwise"/"clockwise".
42
69
 
43
- This approach allows Contrek to make effective use of modern multicore processors without changing the resulting geometry.
44
70
 
45
- ## Streaming large datasets
71
+ ### Mode 2: Parallel processing
46
72
 
47
- Loading an entire gigapixel image into memory is often unnecessary.
73
+ <table>
74
+ <tr>
75
+ <td width="40%" valign="top">
76
+ <img src="docs/images/modes/mode2.jpg" width="100%" alt="Mode 1">
77
+ </td>
78
+ <td width="60%" valign="top">
79
+ The entire image is loaded first, then split into tiles and processed across multiple CPU cores. Partial results are progressively and dynamically merged: there is no predefined merge order, and adjacent pairs are processed as soon as they become available.
80
+ This mode prioritizes performance, using parallelism both for tile processing and for merging partial results.
48
81
 
49
- Contrek can instead process the image incrementally by reading one stripe at a time. Only the current working buffer needs to be allocated, making memory consumption predictable regardless of the final image height.
82
+ **Profile:** maximum speed, with processing time decreasing as more cores become available; low memory efficiency.
83
+ <br><br>
84
+ <center><img src="docs/images/modes/mode2_panel.jpg" width="80%" alt="Mode 1"></center>
85
+ </td></tr></table>
50
86
 
51
- This execution mode is particularly useful when processing images that are too large to fit comfortably into RAM.
87
+ Use 8 threads and 8 tiles
52
88
 
53
- ## Merging polygons
89
+ ```ruby
90
+ png_bitmap = CPPPngBitMap.new("spec/files/images/sample_10240x10240.png")
91
+ color = Contrek::Bitmaps::RgbCppColor.new(r: 255, g: 255, b: 255, a: 255)
92
+ rgb_matcher = CPPRGBNotMatcher.new(color.raw)
93
+ polygonfinder = Contrek::Cpp::CPPConcurrentFinder.new(
94
+ number_of_threads: 8,
95
+ bitmap: png_bitmap,
96
+ matcher: rgb_matcher,
97
+ options: {number_of_tiles: 8, versus: :o, compress: {uniq: true}}
98
+ )
99
+ result = polygonfinder.process_info
100
+ puts result.metadata[:benchmarks].inspect
101
+ # => {"compress"=>5.08, "init"=>230.79, "inner"=>6.54, "outer"=>84.11, "total"=>235.87}
102
+ puts result.metadata[:number_of_threads] # => 8
103
+ ```
104
+
105
+ ### Mode 3: Input streaming
54
106
 
55
- Whenever a polygon crosses the boundary between two adjacent stripes, the library reconstructs it during the merge phase.
107
+ <table>
108
+ <tr>
109
+ <td width="40%" valign="top">
110
+ <img src="docs/images/modes/mode3.jpg" width="100%" alt="Mode 1">
111
+ </td>
112
+ <td width="60%" valign="top">
113
+ The image does not need to be loaded entirely into memory. Instead, it can be read progressively using a fixed-size buffer. For example, with a PNG source, this can be done using libspng's progressive decoding.
114
+ Adjacent tiles share an overlapping scanline to preserve geometry continuity across tile boundaries. Once all tiles have been added, the merge is performed (optionally using multiple threads) to reconstruct the complete geometry.
115
+ This mode provides a trade-off between performance and memory usage: the entire raster does not need to be kept in RAM, while the vector state required to build the final result is retained.
116
+
117
+ **Profile:** medium speed; medium-high memory efficiency.
118
+ <br><br>
119
+ <center><img src="docs/images/modes/mode3_panel.jpg" width="80%" alt="Mode 1"></center>
120
+ </td></tr></table>
121
+
122
+ Trace polygons from two in-memory pattern strings (`CPPBitMap`, useful for synthetic tiles or tests).
123
+ Up is 6 rows height, down is 5 rows. Total after merging: 10 rows, because one row is the shared scanline
56
124
 
57
- This reconstruction preserves polygon connectivity across stripe boundaries, so the final output is equivalent to tracing the entire image in a single pass.
125
+ ```ruby
126
+ up = " 00000000000000 " \
127
+ " 00000000000000 " \
128
+ " 00 00 " \
129
+ " 00 00 " \
130
+ " 00 00 " \
131
+ " 00 00 "
132
+
133
+ down = " 00 00 " \
134
+ " 00 00 " \
135
+ " 00 00 " \
136
+ " 00000000000000 " \
137
+ " 00000000000000 "
138
+ matcher = CPPValueNotMatcher.new(" ")
139
+ result_up = CPPPolygonFinder.new(CPPBitMap.new(up, 30),
140
+ matcher,
141
+ nil,
142
+ {versus: :a, bounds: true}).process_info
143
+ result_down = CPPPolygonFinder.new(CPPBitMap.new(down, 30),
144
+ matcher,
145
+ nil,
146
+ {versus: :a, bounds: true}).process_info
147
+
148
+ step_finder = Contrek::Cpp::CPPConcurrentVerticalMerger.new(options: {compress: {linear: true}})
149
+ step_finder.add_tile(result_up)
150
+ step_finder.add_tile(result_down)
151
+ result = step_finder.process_info
152
+
153
+ expect(result.metadata[:groups]).to eq(1)
154
+ expect(result.metadata[:width]).to eq(30)
155
+ expect(result.metadata[:height]).to eq(10)
156
+ ```
157
+ See an other example of **[input streaming graphical result](docs/images/stripes/merging_polygons.md)**.
158
+
159
+ The same operation using Contrek's built-in streaming API and progressive PNG decoding via libspng:
160
+
161
+ ```ruby
162
+ # PNG image to stream from
163
+ source = CPPPngSource.new("./spec/files/images/labyrinth2.png")
164
+ # streamer with a 20-row window
165
+ streamer = CPPRasterStreamer.new(source, stripe_height: 20)
166
+ white = Contrek::Bitmaps::RgbCppColor.new(r: 255, g: 255, b: 255, a: 255)
167
+ matcher = CPPRGBNotMatcher.new(white.raw)
168
+ # reusable bitmap where libspng decodes rows directly
169
+ buffer_bitmap = CPPRawBitMap.new(source.width, streamer.stripe_height)
170
+ finder = Contrek::Cpp::CPPConcurrentVerticalMerger.new(options: {compress: {uniq: true, linear: true}})
171
+
172
+ # streaming loop, 20 rows each
173
+ streamer.each(buffer_bitmap) do |bitmap, buffer_rows, buffer_size|
174
+ tile = CPPPolygonFinder.new(
175
+ bitmap,
176
+ matcher,
177
+ nil,
178
+ {processing_height: buffer_rows, versus: :o, bounds: true}
179
+ ).process_info
180
+ finder.add_tile(tile)
181
+ end
182
+ result = finder.process_info
183
+ puts result.metadata[:width] # => 130
184
+ puts result.metadata[:height] # => 130
185
+ puts result.points[0][:outer].first.inspect # => {:x=>10, :y=>4} # first point
186
+ puts result.points[0][:outer].size # => 1171 points
187
+ ```
188
+
189
+ ### Mode 4: End-to-end streaming
58
190
 
59
191
  <table>
60
- <tr>
61
- <td width="50%" style="padding: 0; background-color: white;">
62
- <img src="docs/images/stripes/whole_0.png" width="100%"><br>
63
- <img src="docs/images/stripes/whole_256.png" width="100%"><br>
64
- <img src="docs/images/stripes/whole_512.png" width="100%"><br>
65
- <img src="docs/images/stripes/whole_768.png" width="100%">
66
- </td>
67
- <td width="50%" align="center" style="vertical-align: middle; background-color: white;">
68
- <strong>Full Topological Reconstruction</strong><br><br>
69
- <img src="docs/images/stripes/whole.png" width="90%">
70
- </td>
71
- </tr>
72
- <tr>
73
- <td colspan="2" align="center" style="background-color: white;">
74
- <em><b>Left:</b> Image split into 4 independent memory buffers (stripes).</em><br>
75
- <em><b>Right:</b> Contrek ensures <b>perfect topological continuity</b> during merging.</em><br>
76
- 🔴 <b>Red:</b> Outer contours &nbsp;&nbsp; | &nbsp;&nbsp; 🟢 <b>Green:</b> Inner zones
77
- </td>
78
- </tr>
79
- </table>
192
+ <tr>
193
+ <td width="40%" valign="top">
194
+ <img src="docs/images/modes/mode4.jpg" width="100%" alt="Mode 1">
195
+ </td>
196
+ <td width="60%" valign="top">
197
+ This mode extends the incremental processing used in Mode 3 by streaming the output as well.
198
+ The image is read and processed one tile at a time. Each new tile is immediately merged with the current state. As processing moves forward, whenever a geometry is complete and can no longer be affected by subsequent tiles, it is finalized and written directly to the SVG file.
199
+ This limits both the amount of raster data kept in memory and the accumulation of generated vector geometries. It is particularly well suited to very large datasets or cases where the vector output itself can become significant in size.
200
+
201
+ **Profile:** low speed; maximum memory efficiency.
202
+ <br><br>
203
+ <center><img src="docs/images/modes/mode4_panel.jpg" width="80%" alt="Mode 1"></center>
204
+ </td></tr></table>
205
+
206
+ Trace polygons from 2 in-memory pattern strings.
207
+
208
+ ```ruby
209
+ stripe1 = " 000 000" \
210
+ " 00 00 00 0" \
211
+ " 00 00 00 0" \
212
+ " 00 00 00 0" \
213
+ " 000 00 0" \
214
+ " 00 0"
215
+
216
+ stripe2 = " 00 0" \
217
+ " 000 00 0" \
218
+ "00 00 00 0" \
219
+ " 000 00 0" \
220
+ " 00 0" \
221
+ " 0000000000000"
222
+ # streaming to svg file pattern
223
+ stripes = [stripe1, stripe2]
224
+ shared_stream = Contrek::Cpp::CPPTempfile.new("output.svg")
225
+ step_finder = Contrek::Cpp::CPPSvgConcurrentStreamingMerger.new(
226
+ options: {bounds: true, compress: {uniq: true, linear: true, douglas_peucker: true}},
227
+ stream_to: shared_stream,
228
+ total_width: 18,
229
+ total_height: 11
230
+ )
231
+ matcher = CPPValueNotMatcher.new(" ")
232
+ stripes.each do |stripe|
233
+ stripe_result = CPPPolygonFinder.new(CPPBitMap.new(stripe, 18),
234
+ matcher,
235
+ nil,
236
+ {versus: :a, bounds: true}).process_info
237
+ last = stripes.last == stripe
238
+ step_finder.add_tile(stripe_result, last)
239
+ end
240
+ result = step_finder.process_info
241
+ expect(result.metadata[:groups]).to eq(3)
242
+ expect(result.metadata[:width]).to eq(18)
243
+ expect(result.metadata[:height]).to eq(11)
244
+ shared_stream.rewind
245
+ puts shared_stream.read.inspect
246
+ # => "<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"18\" height=\"11\">
247
+ # <polygon points=\"3,3 5,5 8,5 10,3 8,0 5,0\" class=\"out\"/>
248
+ # <polygon points=\"7,1 8,3 6,4 5,2\" class=\"in\"/>
249
+ # <polygon points=\"5,11 18,11 18,0 15,0\" class=\"out\"/> ...
250
+ ```
80
251
 
81
252
  ## Benchmarking
82
253
 
@@ -134,6 +305,15 @@ During installation the native C++ extension is compiled automatically.
134
305
 
135
306
  Once installed, the C++ implementation is used by default. A pure Ruby implementation is also included and can be selected explicitly whenever native extensions are not desired.
136
307
 
308
+ ### TIFF / GeoTIFF support
309
+
310
+ TIFF and GeoTIFF streaming support requires the system `libtiff` and `libgeotiff` libraries.
311
+ Ubuntu / Debian:
312
+
313
+ ```bash
314
+ sudo apt install libtiff-dev libgeotiff-dev
315
+ ```
316
+
137
317
  # Usage
138
318
 
139
319
  The simplest way to use Contrek is through the `Contrek.contour!` helper.
data/contrek.gemspec CHANGED
@@ -32,6 +32,7 @@ Gem::Specification.new do |s|
32
32
 
33
33
  s.add_dependency "chunky_png", "~> 1.4"
34
34
  s.add_dependency "concurrent-ruby", "~> 1.3.5"
35
+ s.add_dependency "ffi", "~> 1.17"
35
36
  s.add_dependency "rice", "= 4.5.0"
36
37
 
37
38
  s.required_ruby_version = ">= 3.0.0"
@@ -38,8 +38,8 @@ enum class Connectivity {
38
38
 
39
39
 
40
40
  struct Config {
41
- int threads = 4;
42
- int tiles = 2;
41
+ int threads = 0;
42
+ int tiles = 1;
43
43
  Versus versus = Versus::A;
44
44
  bool compress_unique = false;
45
45
  bool compress_linear = false;
@@ -28,6 +28,8 @@
28
28
  #include "polygon/bitmaps/RawBitmap.h"
29
29
  #include "polygon/bitmaps/RemoteFastPngBitmap.h"
30
30
  #include "polygon/bitmaps/spng.h"
31
+ #include "polygon/bitmaps/streaming/RasterStreamer.h"
32
+ #include "polygon/bitmaps/streaming/PngSource.h"
31
33
  #include "polygon/matchers/Matcher.h"
32
34
  #include "polygon/matchers/RGBMatcher.h"
33
35
  #include "polygon/matchers/RGBNotMatcher.h"
@@ -100,7 +102,7 @@ void Tests::test_b()
100
102
 
101
103
  Options options = {
102
104
  {"versus", Identifier{"o"}},
103
- {"number_of_tiles",2},
105
+ {"number_of_tiles", 2},
104
106
  {"compress", Options{
105
107
  {"uniq", true},
106
108
  {"linear", true},
@@ -202,7 +204,7 @@ void Tests::test_e()
202
204
 
203
205
  Options options = {
204
206
  {"versus", Identifier{"a"}},
205
- {"number_of_tiles",2},
207
+ {"number_of_tiles", 2},
206
208
  {"compress", Options{
207
209
  {"uniq", true},
208
210
  }},
@@ -243,7 +245,7 @@ void Tests::test_g()
243
245
 
244
246
  Options options = {
245
247
  {"versus", Identifier{"a"}},
246
- {"number_of_tiles",2},
248
+ {"number_of_tiles", 2},
247
249
  {"compress", Options{
248
250
  {"uniq", true},
249
251
  }},
@@ -321,70 +323,29 @@ double get_peak_rss() {
321
323
  This approach allows for processing large PNG files on systems where memory
322
324
  would otherwise be insufficient.
323
325
  */
326
+
324
327
  void stream_png_image(const std::string& filepath, uint32_t stripe_height, bool generate_svg = false, bool generate_png = false) {
325
328
  std::vector<ProcessResult*> result_clones;
326
329
  Options options = {};
327
330
  VerticalMerger vmerger(0, options);
328
331
 
329
- // opens image to stream
330
- FILE* fp = fopen(filepath.c_str(), "rb");
331
- if (!fp) {
332
- std::cerr << "Unable open file: " << filepath << std::endl;
333
- return;
334
- }
335
-
336
- // exams image
337
- spng_ctx *ctx = spng_ctx_new(0);
338
- spng_set_png_file(ctx, fp);
339
- struct spng_ihdr ihdr;
340
- if (spng_get_ihdr(ctx, &ihdr)) {
341
- fclose(fp);
342
- spng_ctx_free(ctx);
343
- return;
344
- }
345
- uint32_t total_width = ihdr.width;
346
- uint32_t total_height = ihdr.height;
332
+ PngSource source(filepath);
333
+ RasterStreamer streamer(source, stripe_height);
347
334
 
348
335
  // allocates stripe buffer
349
- RawBitmap stripe_bitmap;
350
- std::cout << total_width << " " << stripe_height << std::endl;
351
- stripe_bitmap.define(total_width, stripe_height, 4, true);
336
+ RawBitmap stripe_bitmap(source.width(), stripe_height);
352
337
  RGBNotMatcher not_matcher(-1);
353
338
 
354
- if (spng_decode_image(ctx, NULL, 0, SPNG_FMT_RGBA8, SPNG_DECODE_PROGRESSIVE)) {
355
- fclose(fp);
356
- spng_ctx_free(ctx);
357
- return;
358
- }
359
-
360
- size_t row_size = static_cast<size_t>(total_width) * 4;
361
339
  int stripe_count = 0;
340
+
362
341
  // main stripes loop
363
- for (uint32_t current_y_offset = 0; current_y_offset < total_height; current_y_offset += stripe_height) {
364
- uint32_t uncovered_height = total_height - current_y_offset;
365
-
366
- // copy previous last line to the next new one (each contigue stripe must share one pixel scanline)
367
- if (current_y_offset > 0) {
368
- unsigned char* last_row_prev = const_cast<unsigned char*>(stripe_bitmap.get_row_ptr(stripe_height - 1));
369
- unsigned char* first_row_curr = const_cast<unsigned char*>(stripe_bitmap.get_row_ptr(0));
370
- std::memcpy(first_row_curr, last_row_prev, row_size);
371
- }
372
- // clears the part of the stripe wont be overwritten by png data
373
- if (uncovered_height < stripe_height)
374
- { stripe_bitmap.draw_filled_rectangle(0, 1, total_width, stripe_height - 1, 255, 255, 255);
375
- }
376
- // decoding data directly in the stripe buffer
377
- uint32_t lines_to_read = std::min(stripe_height, total_height - current_y_offset);
378
- for (uint32_t y = (current_y_offset == 0 ? 0 : 1); y < lines_to_read; y++) {
379
- unsigned char* row_ptr = const_cast<unsigned char*>(stripe_bitmap.get_row_ptr(y));
380
- int ret = spng_decode_row(ctx, row_ptr, row_size);
381
- if (ret != 0 && ret != SPNG_EOI) break;
382
- }
342
+ streamer.each(stripe_bitmap, [&](Bitmap& bitmap, uint32_t buffer_rows, std::size_t, std::size_t) {
383
343
  // stripe contour tracing
384
344
  Options options = {
345
+ {"processing_height", static_cast<int>(buffer_rows)},
385
346
  {"versus", Identifier{"a"}},
386
347
  };
387
- PolygonFinder polygon_finder(&stripe_bitmap, &not_matcher, nullptr, options);
348
+ PolygonFinder polygon_finder(&bitmap, &not_matcher, nullptr, options);
388
349
  ProcessResult *result = polygon_finder.process_info();
389
350
  if (result) {
390
351
  std::cout << "stripe " << stripe_count << ": found polygons " << result->groups << std::endl;
@@ -392,7 +353,7 @@ void stream_png_image(const std::string& filepath, uint32_t stripe_height, bool
392
353
  vmerger.add_tile(*result);
393
354
  }
394
355
  stripe_count++;
395
- }
356
+ });
396
357
 
397
358
  std::cout << "Merging polygons ..." << std::endl;
398
359
  ProcessResult *merged_result = vmerger.process_info();
@@ -400,8 +361,7 @@ void stream_png_image(const std::string& filepath, uint32_t stripe_height, bool
400
361
  if (merged_result) {
401
362
  std::cout << "Founds total polygons: " << merged_result->groups << std::endl;
402
363
  if (generate_png) {
403
- RawBitmap full_bitmap;
404
- full_bitmap.define(total_width, total_height, 4, true);
364
+ RawBitmap full_bitmap(source.width(), source.height());
405
365
  full_bitmap.fill(255, 255, 255);
406
366
  merged_result->draw_on_bitmap(full_bitmap);
407
367
  std::cout << "Saving whole png ..." << std::endl;
@@ -413,18 +373,18 @@ void stream_png_image(const std::string& filepath, uint32_t stripe_height, bool
413
373
  merged_result->save_svg("whole.svg");
414
374
  std::cout << "Svg saved!" << std::endl;
415
375
  }
376
+ delete merged_result;
416
377
  }
417
- delete merged_result;
378
+
418
379
  // frees memory
419
380
  for (auto c : result_clones) {
420
381
  delete c;
421
382
  }
422
- spng_ctx_free(ctx);
423
- fclose(fp);
424
383
  }
425
384
 
426
385
  void Tests::test_i() {
427
386
  stream_png_image("../images/graphs_1024x1024.png", 300, true);
387
+ // stream_png_image("../images/test_20480x20480.png", 2000, true);
428
388
  std::cout << "Memory usage peak: " << get_peak_rss() << " MB" << std::endl;
429
389
  }
430
390
 
@@ -440,103 +400,47 @@ double get_current_rss_mb() {
440
400
  }
441
401
 
442
402
  void stream_progressive_png_image(const std::string& filepath, uint32_t stripe_height) {
443
- std::vector<ProcessResult*> result_clones;
444
- Options voptions = {
445
- {"bounds", true},
446
- };
447
- // opens image to stream
448
- FILE* fp = fopen(filepath.c_str(), "rb");
449
- if (!fp) {
450
- std::cerr << "Unable open file: " << filepath << std::endl;
451
- return;
452
- }
453
-
454
- // exams image
455
- spng_ctx *ctx = spng_ctx_new(0);
456
- spng_set_png_file(ctx, fp);
457
- struct spng_ihdr ihdr;
458
- if (spng_get_ihdr(ctx, &ihdr)) {
459
- fclose(fp);
460
- spng_ctx_free(ctx);
461
- return;
462
- }
463
- uint32_t total_width = ihdr.width;
464
- uint32_t total_height = ihdr.height;
465
-
466
- // allocates stripe buffer
467
- RawBitmap stripe_bitmap;
468
- stripe_bitmap.define(total_width, stripe_height, 4, true);
469
- RGBNotMatcher not_matcher(-1);
470
- if (spng_decode_image(ctx, NULL, 0, SPNG_FMT_RGBA8, SPNG_DECODE_PROGRESSIVE)) {
471
- fclose(fp);
472
- spng_ctx_free(ctx);
473
- return;
474
- }
475
-
476
- // allocates streaming svg buffer
477
- std::string output_path = "streaming_buffer.svg";
478
- std::ofstream shared_stream(output_path, std::ios::out | std::ios::binary);
479
- if (!shared_stream) {
480
- std::cerr << "Error: Unable creating output streaming file!" << std::endl;
481
- }
482
- std::vector<char> buffer(4 * 1024 * 1024); // Buffer (4MB)
483
- shared_stream.rdbuf()->pubsetbuf(buffer.data(), buffer.size());
484
-
485
- SvgStreamingMerger vmerger(0, voptions, &shared_stream, total_width, total_height);
486
- // GeoJsonStreamingMerger vmerger(0, &varguments, &shared_stream, 4294901760);
487
- try {
488
- size_t row_size = static_cast<size_t>(total_width) * 4;
489
- int stripe_count = 0;
490
- // main stripes loop
491
- for (uint32_t current_y_offset = 0; current_y_offset < total_height; current_y_offset += stripe_height) {
492
- uint32_t uncovered_height = total_height - current_y_offset;
493
-
494
- // copy previous last line to the next new one (each contigue stripe must share one pixel scanline)
495
- if (current_y_offset > 0) {
496
- unsigned char* last_row_prev = const_cast<unsigned char*>(stripe_bitmap.get_row_ptr(stripe_height - 1));
497
- unsigned char* first_row_curr = const_cast<unsigned char*>(stripe_bitmap.get_row_ptr(0));
498
- std::memcpy(first_row_curr, last_row_prev, row_size);
499
- }
500
- // clears the part of the stripe wont be overwritten by png data
501
- if (uncovered_height < stripe_height)
502
- { stripe_bitmap.draw_filled_rectangle(0, 1, total_width, stripe_height - 1, 255, 255, 255);
503
- }
504
- // decoding data directly in the stripe buffer
505
- uint32_t lines_to_read = std::min(stripe_height, total_height - current_y_offset);
506
- for (uint32_t y = (current_y_offset == 0 ? 0 : 1); y < lines_to_read; y++) {
507
- unsigned char* row_ptr = const_cast<unsigned char*>(stripe_bitmap.get_row_ptr(y));
508
- int ret = spng_decode_row(ctx, row_ptr, row_size);
509
- if (ret != 0 && ret != SPNG_EOI) break;
510
- }
511
- // stripe contour tracing
512
- Options options = {
403
+ PngSource source(filepath);
404
+ RasterStreamer streamer(source, stripe_height);
405
+ RawBitmap bitmap(source.width(), stripe_height);
406
+ RGBNotMatcher not_matcher(-1);
407
+
408
+ std::ofstream stream("streaming_buffer.svg");
409
+
410
+ SvgStreamingMerger merger(
411
+ 0,
412
+ {{"bounds", true}},
413
+ &stream,
414
+ source.width(),
415
+ source.height());
416
+
417
+ uint32_t processed_rows = 0;
418
+ bool first = true;
419
+ int stripe_count = 0;
420
+ streamer.each(
421
+ bitmap,
422
+ [&](Bitmap& bitmap, uint32_t buffer_rows, std::size_t, std::size_t) {
423
+ PolygonFinder finder(&bitmap, &not_matcher, nullptr,
424
+ {
425
+ {"processing_height", static_cast<int>(buffer_rows)},
513
426
  {"versus", Identifier{"a"}},
514
427
  {"bounds", true},
515
428
  {"compress", Options{
516
429
  {"uniq", true},
517
430
  {"linear", true},
518
431
  }},
519
- };
520
-
521
- PolygonFinder polygon_finder(&stripe_bitmap, &not_matcher, nullptr, options);
522
- ProcessResult *result = polygon_finder.process_info();
523
- if (result) {
524
- std::cout << "stripe " << stripe_count << ": found polygons " << result->groups << std::endl;
525
- vmerger.add_tile(*result, !(current_y_offset + stripe_height < total_height));
526
- delete result;
527
- std::cout << "-> RSS before merge: " << get_current_rss_mb() << " MB" << std::endl;
528
- }
529
- stripe_count++;
432
+ });
433
+ ProcessResult* result = finder.process_info();
434
+ if (result) {
435
+ std::cout << "stripe " << stripe_count++ << ": found polygons " << result->groups << std::endl;
436
+ processed_rows += buffer_rows - (first ? 0 : RasterStreamer::OVERLAP);
437
+ merger.add_tile(*result, processed_rows == source.height());
438
+ delete result;
530
439
  }
531
- ProcessResult *merged_result = vmerger.process_info();
532
- std::cout << "total found polygons " << merged_result->groups << std::endl;
533
- delete merged_result;
534
- } catch (const std::exception& e) {
535
- std::cerr << "\n[ERROR] Processing exception: " << e.what() << std::endl;
536
- if (shared_stream.is_open()) shared_stream.close();
537
- }
538
- spng_ctx_free(ctx);
539
- fclose(fp);
440
+ first = false;
441
+ });
442
+
443
+ delete merger.process_info();
540
444
  }
541
445
 
542
446
  void Tests::test_l() {