contrek 1.4.0 → 1.4.2

data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/FinderUtils.cpp

@@ -17,18 +17,17 @@ void FinderUtils::sanitize_options(pf_Options& options, const Options& incoming_
   const Identifier versus = incoming_options.get<Identifier>("versus", options.versus == Node::A ? "a" : "o");
   options.versus = versus == "a" ? Node::A : Node::O;
   options.number_of_tiles = std::max(1, static_cast<int>(incoming_options.get<std::int64_t>(
-    "number_of_tiles",
-    options.number_of_tiles)));
+    "number_of_tiles", options.number_of_tiles)));
   options.connectivity_offset = incoming_options.get<std::int64_t>(
-    "connectivity",
-    options.connectivity_offset == 1 ? 8 : 4) == 8;
+    "connectivity", options.connectivity_offset == 1 ? 8 : 4) == 8;
   options.treemap = incoming_options.get<bool>("treemap", options.treemap);
   options.named_sequences = incoming_options.get<bool>(
-    "named_sequences",
-    options.named_sequences);
+    "named_sequences", options.named_sequences);
   options.bounds = incoming_options.get<bool>("bounds", options.bounds);
   options.unsafe_mode = incoming_options.get<bool>("unsafe_mode", options.unsafe_mode);
   options.deterministic = incoming_options.get<bool>("deterministic", options.deterministic);
+  options.processing_height = static_cast<int>(incoming_options.get<std::int64_t>(
+    "processing_height", options.processing_height));
 
   if (const Options* compress = incoming_options.get_options("compress")) {
     options.compress_uniq = compress->get<bool>("uniq", options.compress_uniq);
@@ -39,8 +38,7 @@ void FinderUtils::sanitize_options(pf_Options& options, const Options& incoming_
     if (compress->contains("visvalingam_tolerance")) {
       options.compress_visvalingam = true;
       options.compress_visvalingam_tolerance = static_cast<float>(compress->get<double>(
-        "visvalingam_tolerance",
-        options.compress_visvalingam_tolerance));
+        "visvalingam_tolerance", options.compress_visvalingam_tolerance));
     }
   }
 

data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/PolygonFinder.cpp

@@ -11,6 +11,7 @@
 #include <list>
 #include <map>
 #include <ctime>
+#include <cmath>
 #include <typeinfo>
 #include <string>
 #include <vector>
@@ -40,7 +41,11 @@ PolygonFinder::PolygonFinder(Bitmap *bitmap,
 { this->rgb_matcher = dynamic_cast<RGBMatcher*>(matcher);
   FinderUtils::sanitize_options(this->options, options);
 
-  this->node_cluster = new NodeCluster(source_bitmap->h(), source_bitmap->w(), &this->options);
+  this->processing_height = std::abs(this->options.processing_height == 0 ? source_bitmap->h() : this->options.processing_height);
+  if (this->processing_height > source_bitmap->h()) {
+    throw std::invalid_argument("Option processing_height must be less or equal than image height!");
+  }
+  this->node_cluster = new NodeCluster(this->processing_height, source_bitmap->w(), &this->options);
 
   //= SCAN ==============//
   cpu_timer.start();
@@ -106,10 +111,11 @@ ProcessResult* PolygonFinder::process_info() {
   pr->benchmarks = std::move(this->reports);
   pr->treemap = this->node_cluster->treemap;
   pr->width = this->source_bitmap->w();
-  pr->height = this->source_bitmap->h();
+  pr->height = this->processing_height;
   pr->has_bounds = this->node_cluster->options->bounds;
   pr->versus = this->options.versus;
   pr->options = this->incoming_options_;
+  pr->number_of_threads = 0;
 
   if (this->node_cluster->options->named_sequences && typeid(*this->source_bitmap) == typeid(Bitmap))
   { std::string sequence;

data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/PolygonFinder.h

@@ -54,6 +54,7 @@ struct pf_Options {
   bool unsafe_mode = false;
   bool bounds = false;
   bool deterministic = false;
+  int processing_height = 0;
   int connectivity_offset = 0;
   float compress_visvalingam_tolerance = 10.0;
   int number_of_tiles = 1;
@@ -69,6 +70,7 @@ struct ProcessResult {
   int groups;
   int width, height;
   int versus;
+  int number_of_threads;
   bool has_bounds = false;
   std::map<std::string, double> benchmarks;
   std::list<Polygon> polygons;
@@ -158,6 +160,7 @@ class PolygonFinder {
  protected:
   int start_x;
   int end_x;
+  int processing_height;
   Options incoming_options_;
 
  private:
@@ -172,7 +175,7 @@ class PolygonFinder {
 
   template <typename M, typename F>
   void run_loop(M* specific_matcher, F&& fetch_color, int offset) {
-    int img_h = this->source_bitmap->h();
+    int img_h = this->processing_height;
     int bpp = this->source_bitmap->get_bytes_per_pixel();
 
     for (int y = 0; y < img_h; y++) {

data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/concurrent/Finder.cpp

@@ -32,6 +32,9 @@ Finder::Finder(int number_of_threads, Bitmap *bitmap, Matcher *matcher, const Op
 { cpu_timer.start();
   FinderUtils::sanitize_options(this->options_, options);
 
+  if (this->options_.processing_height != 0) {
+    this->height = std::abs(this->options_.processing_height);
+  }
   double cw = static_cast<double>(this->maximum_width_) / this->options_.number_of_tiles;
   if (cw < 1.0) {
     throw std::runtime_error("One pixel tile width minimum!");
@@ -54,6 +57,9 @@ Finder::Finder(int number_of_threads, Bitmap *bitmap, Matcher *matcher, const Op
       if (this->options_.treemap) {
         base_arguments["treemap"] = true;
       }
+      if (this->options_.processing_height != 0) {
+        base_arguments["processing_height"] = this->options_.processing_height;
+      }
 
       CpuTimer t;
       t.start();
@@ -166,6 +172,7 @@ ProcessResult* Finder::process_info() {
   pr->has_bounds = this->options_.bounds;
   pr->versus = this->options_.versus;
   pr->options = this->input_options;
+  pr->number_of_threads = this->number_of_threads();
   FakeCluster fake_cluster(pr->polygons, this->options_);
   cpu_timer.start();
   fake_cluster.compress_coords(pr->polygons, this->options_);

data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/concurrent/Poolable.h

@@ -38,6 +38,7 @@ class Poolable {
     }
   }
   void wait();
+  int number_of_threads() const { return number_of_threads_; }
 
  private:
   std::vector<std::thread> workers;

data/ext/cpp_polygon_finder/PolygonFinder/src/polygon/finder/concurrent/Tile.cpp

@@ -24,10 +24,10 @@ Tile::Tile(Finder *finder, int start_x, int end_x, std::string name, int index,
 : finder(finder),
   start_x_(start_x),
   end_x_(end_x),
-  name_(name),
   index_(index),
-  benchmarks(b),
-  order_(order) {
+  order_(order),
+  name_(name),
+  benchmarks(b) {
   this->shapes_pool = new ShapePool();
   this->shapes_pool->set_owner(this);
   this->shapes_pools.push_back(this->shapes_pool);

data/ext/cpp_polygon_finder/cpp_polygon_finder.cpp

@@ -37,6 +37,12 @@
 #include "PolygonFinder/src/polygon/bitmaps/FastPngBitmap.cpp"
 #include "PolygonFinder/src/polygon/bitmaps/RemoteFastPngBitmap.h"
 #include "PolygonFinder/src/polygon/bitmaps/RemoteFastPngBitmap.cpp"
+#include "PolygonFinder/src/polygon/bitmaps/streaming/RasterSource.h"
+#include "PolygonFinder/src/polygon/bitmaps/streaming/AsciiSource.h"
+#include "PolygonFinder/src/polygon/bitmaps/streaming/AsciiSource.cpp"
+#include "PolygonFinder/src/polygon/bitmaps/streaming/PngSource.h"
+#include "PolygonFinder/src/polygon/bitmaps/streaming/PngSource.cpp"
+#include "PolygonFinder/src/polygon/bitmaps/streaming/RasterStreamer.h"
 #include "PolygonFinder/src/polygon/matchers/Matcher.h"
 #include "PolygonFinder/src/polygon/matchers/Matcher.cpp"
 #include "PolygonFinder/src/polygon/matchers/ValueNotMatcher.h"
@@ -232,6 +238,7 @@ class To_Ruby<ProcessResult*>
     return_me[Symbol("height")] = pr->height;
     return_me[Symbol("versus")] = Symbol(pr->versus == Node::O ? "o" : "a");
     return_me[Symbol("options")] = options_to_ruby(pr->options);
+    return_me[Symbol("number_of_threads")] = pr->number_of_threads;
 
     Rice::Array out;
     for (Polygon& x : pr->polygons)
@@ -394,6 +401,7 @@ void Init_cpp_polygon_finder() {
     .define_method("value_at", &Bitmap::value_at)
     .define_method("w", &Bitmap::w)
     .define_method("h", &Bitmap::h)
+    .define_method("get_bytes_per_pixel", &Bitmap::get_bytes_per_pixel)
     .define_method("error", &Bitmap::error)
     .define_method("clear", &Bitmap::clear)
     .define_method("print", &Bitmap::print);
@@ -401,6 +409,7 @@ void Init_cpp_polygon_finder() {
   Data_Type<RawBitmap> rb_cRawBitmap =
     define_class<RawBitmap, Bitmap>("CPPRawBitMap")
     .define_constructor(Constructor<RawBitmap>())
+    .define_constructor(Constructor<RawBitmap, uint, uint>())
     .define_method("rgb_value_at", &RawBitmap::rgb_value_at)
     .define_method("w", &RawBitmap::w)
     .define_method("h", &RawBitmap::h)
@@ -429,6 +438,44 @@ void Init_cpp_polygon_finder() {
     .define_method("error", &FastPngBitmap::error)
     .define_method("print", &FastPngBitmap::print);
 
+  Data_Type<RasterSource> rb_cRasterSource =
+    define_class<RasterSource>("CPPRasterSource");
+
+  Data_Type<AsciiSource> rb_cAsciiSource =
+    define_class<AsciiSource, RasterSource>("CPPAsciiSource")
+    .define_constructor(Constructor<AsciiSource, Bitmap&>(),Arg("bitmap"))
+    .define_method("width",&AsciiSource::width)
+    .define_method("height",&AsciiSource::height)
+    .define_method("get_bytes_per_pixel",&AsciiSource::get_bytes_per_pixel);
+
+  Data_Type<PngSource> rb_cPngSource =
+    define_class<PngSource, RasterSource>("CPPPngSource")
+    .define_constructor(Constructor<PngSource, std::string>(),Arg("filepath"))
+    .define_method("width",&PngSource::width)
+    .define_method("height",&PngSource::height)
+    .define_method("get_bytes_per_pixel",&PngSource::get_bytes_per_pixel);
+
+  Data_Type<RasterStreamer> rb_cRasterStreamer =
+    define_class<RasterStreamer>("CPPRasterStreamer")
+    .define_constructor(Constructor<RasterStreamer, RasterSource&, uint32_t>(),Arg("source"),Arg("stripe_height"))
+    .define_method("stripe_height",&RasterStreamer::stripe_height)
+    .define_method("each",[](RasterStreamer& self, Bitmap& buffer) {
+        VALUE block = rb_block_proc();
+        self.each(
+          buffer,
+          [block](Bitmap& bitmap,uint32_t buffer_rows,std::size_t buffer_size) {
+            Rice::Data_Object<Bitmap> rb_bitmap(&bitmap);
+            Rice::Object(block).call(
+              "call",
+              rb_bitmap,
+              buffer_rows,
+              buffer_size
+            );
+          }
+        );
+      }
+    );
+
   Data_Type<Matcher> rb_cMatcher =
     define_class<Matcher>("CPPMatcher")
     .define_constructor(Constructor<Matcher, char>())

data/lib/contrek/bitmaps/bitmap.rb

@@ -5,7 +5,7 @@ module Contrek
     class Bitmap
       include Painting
 
-      def scan(start_x: 0, end_x: w)
+      def scan(start_x: 0, end_x: w, end_y: h)
         x = start_x
         y = 0
         loop do
@@ -14,7 +14,7 @@ module Contrek
           if x == end_x
             x = start_x
             y += 1
-            break if y == h
+            break if y == end_y
           end
         end
       end

data/lib/contrek/bitmaps/chunky_bitmap.rb

@@ -1,15 +1,19 @@
 # frozen_string_literal: true
 
+require "ffi"
+
 module Contrek
   module Bitmaps
     class ChunkyBitmap < Bitmap
       def initialize(data, mod)
-        @raw = data.dup
         @module = mod
+        @height = data.bytesize / @module
+        @raw = FFI::MemoryPointer.new(:uint8, data.bytesize)
+        @raw.put_bytes(0, data)
       end
 
       def clear(val = "0")
-        @raw = val * @module * h
+        @raw.put_bytes(0, val * @module * h)
       end
 
       def w
@@ -17,22 +21,30 @@ module Contrek
       end
 
       def h
-        @raw.size / @module
+        @height
+      end
+
+      def get_bytes_per_pixel
+        1
+      end
+
+      def get_row_ptr(y)
+        @raw + (@module * y)
       end
 
       def value_at(x, y)
-        @raw[y * @module + x]
+        @raw.get_uint8(y * @module + x).chr
       end
 
       def value_set(x, y, value)
         return if y >= h
         return if x >= w
 
-        @raw[y * @module + x] = value
+        @raw.put_uint8(y * @module + x, value.ord)
       end
 
       def dup!
-        ChunkyBitmap.new(@raw, @module)
+        ChunkyBitmap.new(@raw.read_bytes(@module * h), @module)
       end
 
       def draw_rect(x:, y:, width:, height:, color: "o", filled: true)
@@ -80,7 +92,7 @@ module Contrek
         puts label if label
         puts "  " + (0...@module).map { |i| (i % 10).to_s }.join
         n = 0
-        @raw.scan(/.{1,#{@module}}/).each do |line|
+        @raw.read_bytes(@module * h).scan(/.{1,#{@module}}/).each do |line|
           colored_line = line.chars.map { |c| colorize_char(c) }.join
           puts "#{n} #{colored_line}"
           n += 1
@@ -96,8 +108,11 @@ module Contrek
             transposed += value_at(x, y)
           end
         end
-        @raw = transposed
+        old_width = w
         @module = h
+        @height = old_width
+        @raw = FFI::MemoryPointer.new(:uint8, transposed.bytesize)
+        @raw.put_bytes(0, transposed)
       end
 
       private

data/lib/contrek/bitmaps/png_bitmap.rb

@@ -20,6 +20,10 @@ module Contrek
         @image.dimension.height
       end
 
+      def get_bytes_per_pixel
+        4
+      end
+
       def draw_line(start_x, start_y, end_x, end_y, value)
         @image.line_xiaolin_wu(start_x, start_y, end_x, end_y, value)
       end

data/lib/contrek/bitmaps/rgba_bitmap.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "ffi"
+
+module Contrek
+  module Bitmaps
+    class RgbaBitmap < Bitmap
+      BYTES_PER_PIXEL = 4
+
+      def initialize(width, height)
+        @width = width
+        @height = height
+        @raw = FFI::MemoryPointer.new(:uint8, @width * @height * BYTES_PER_PIXEL)
+      end
+
+      def w
+        @width
+      end
+
+      def h
+        @height
+      end
+
+      def get_bytes_per_pixel
+        BYTES_PER_PIXEL
+      end
+
+      def value_at(x, y)
+        rgb_value_at(x, y)
+      end
+
+      def get_row_ptr(y)
+        @raw + (y * @width * BYTES_PER_PIXEL)
+      end
+
+      def rgb_value_at(x, y)
+        index = (y * @width + x) * BYTES_PER_PIXEL
+        r = @raw.get_uint8(index)
+        g = @raw.get_uint8(index + 1)
+        b = @raw.get_uint8(index + 2)
+        a = @raw.get_uint8(index + 3)
+        r | (g << 8) | (b << 16) | (a << 24)
+      end
+
+      def read_bytes(size)
+        @raw.read_bytes(size)
+      end
+    end
+  end
+end

data/lib/contrek/bitmaps/streaming/raster_streamer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "ffi"
+
+module Contrek
+  module Bitmaps
+    module Streaming
+      class RasterStreamer
+        OVERLAP = 1
+
+        attr_reader :stripe_height
+
+        def initialize(source, stripe_height:)
+          @source = source
+          @stripe_height = stripe_height
+
+          raise ArgumentError, "stripe_height must be greater than 1" if @stripe_height <= OVERLAP
+        end
+
+        def each(buffer)
+          return enum_for(:each, buffer) unless block_given?
+
+          width = @source.width
+          height = @source.height
+          bytes_per_pixel = @source.get_bytes_per_pixel
+
+          row_size = width * bytes_per_pixel
+
+          source_row = 0
+          buffer_rows = 0
+          first_stripe = true
+
+          while source_row < height
+            unless first_stripe
+              previous_last_row = buffer.get_row_ptr(buffer_rows - 1)
+              current_first_row = buffer.get_row_ptr(0)
+              current_first_row.put_bytes(0, previous_last_row.read_bytes(row_size))
+            end
+
+            overlap_rows = first_stripe ? 0 : OVERLAP
+            available_rows = @stripe_height - overlap_rows
+            remaining_rows = height - source_row
+            rows_to_read = [available_rows, remaining_rows].min
+
+            rows_read = 0
+            rows_to_read.times do |i|
+              buffer_y = overlap_rows + i
+              row_ptr = buffer.get_row_ptr(buffer_y)
+              break unless @source.read_next_row(row_ptr, row_size)
+
+              source_row += 1
+              rows_read += 1
+            end
+
+            buffer_rows = overlap_rows + rows_read
+            break if buffer_rows == 0
+
+            yield buffer, buffer_rows, buffer_rows * row_size
+
+            first_stripe = false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrek/bitmaps/streaming/sources/ascii.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Contrek
+  module Bitmaps
+    module Streaming
+      module Sources
+        class Ascii < Base
+          extend Forwardable
+
+          def_delegator :@bitmap, :w, :width
+          def_delegator :@bitmap, :h, :height
+          def_delegators :@bitmap, :get_bytes_per_pixel
+
+          def initialize(bitmap)
+            @current_row = 0
+            @bitmap = bitmap
+          end
+
+          def read_next_row(destination, row_size)
+            return false if @current_row >= height
+
+            row = @bitmap.get_row_ptr(@current_row)
+            destination.put_bytes(0, row.read_bytes(row_size))
+            @current_row += 1
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrek/bitmaps/streaming/sources/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Contrek
+  module Bitmaps
+    module Streaming
+      module Sources
+        class Base
+          def width
+            raise NotImplementedError
+          end
+
+          def height
+            raise NotImplementedError
+          end
+
+          def get_bytes_per_pixel
+            raise NotImplementedError
+          end
+
+          def read_next_row(destination, row_size)
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrek/bitmaps/streaming/sources/png.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "cpp_polygon_finder"
+
+module Contrek
+  module Bitmaps
+    module Streaming
+      module Sources
+        class Png
+          attr_reader :width, :height
+
+          module LibSpng
+            extend FFI::Library
+
+            ffi_lib FFI::Library::CURRENT_PROCESS
+
+            SPNG_FMT_RGBA8 = 1
+            SPNG_DECODE_PROGRESSIVE = 256
+            SPNG_EOI = 75
+
+            class Ihdr < FFI::Struct
+              layout(
+                :width, :uint32,
+                :height, :uint32,
+                :bit_depth, :uint8,
+                :color_type, :uint8,
+                :compression_method, :uint8,
+                :filter_method, :uint8,
+                :interlace_method, :uint8
+              )
+            end
+
+            attach_function :spng_ctx_new, [:int], :pointer
+            attach_function :spng_ctx_free, [:pointer], :void
+            attach_function :spng_set_png_file, [:pointer, :pointer], :int
+            attach_function :spng_get_ihdr, [:pointer, :pointer], :int
+            attach_function :spng_decode_image, [:pointer, :pointer, :size_t, :int, :int], :int
+            attach_function :spng_decode_row, [:pointer, :pointer, :size_t], :int
+          end
+
+          module LibC
+            extend FFI::Library
+
+            ffi_lib FFI::Library::LIBC
+
+            attach_function :fopen, [:string, :string], :pointer
+            attach_function :fclose, [:pointer], :int
+          end
+
+          def initialize(filepath)
+            @fp = LibC.fopen(filepath, "rb")
+            raise "Unable to open PNG: #{filepath}" if @fp.null?
+
+            @ctx = LibSpng.spng_ctx_new(0)
+            raise "Unable to create spng context" if @ctx.null?
+
+            ret = LibSpng.spng_set_png_file(@ctx, @fp)
+            raise "spng_set_png_file failed: #{ret}" unless ret == 0
+
+            ihdr = LibSpng::Ihdr.new
+
+            ret = LibSpng.spng_get_ihdr(@ctx, ihdr)
+            raise "spng_get_ihdr failed: #{ret}" unless ret == 0
+
+            @width = ihdr[:width]
+            @height = ihdr[:height]
+
+            ret = LibSpng.spng_decode_image(
+              @ctx,
+              nil,
+              0,
+              LibSpng::SPNG_FMT_RGBA8,
+              LibSpng::SPNG_DECODE_PROGRESSIVE
+            )
+            raise "Unable to initialize progressive decoding: #{ret}" unless ret == 0
+          end
+
+          def get_bytes_per_pixel
+            4
+          end
+
+          def read_next_row(destination, row_size)
+            ret = LibSpng.spng_decode_row(@ctx, destination, row_size)
+            unless ret == 0 || ret == LibSpng::SPNG_EOI
+              raise "spng_decode_row failed: #{ret}"
+            end
+            true
+          end
+
+          def close
+            unless @ctx.nil? || @ctx.null?
+              LibSpng.spng_ctx_free(@ctx)
+              @ctx = nil
+            end
+            unless @fp.nil? || @fp.null?
+              LibC.fclose(@fp)
+              @fp = nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrek/finder/concurrent/finder.rb

@@ -30,7 +30,7 @@ module Contrek
           @options = options
           @clusters = []
           @maximum_width = bitmap.w
-          @height = bitmap.h
+          @height = options[:processing_height] || bitmap.h
           @number_of_tiles = options[:number_of_tiles] || (raise "number_of_tiles params is needed!")
           @number_of_tiles = 1 if @number_of_tiles <= 0
 
@@ -51,6 +51,7 @@ module Contrek
                 bitmap: bitmap,
                 matcher: matcher,
                 options: {
+                  processing_height: @options[:processing_height],
                   versus: current_versus,
                   bounds: true,
                   treemap: @options[:treemap],
@@ -71,7 +72,7 @@ module Contrek
 
             x = tile_end_x - 1
           end
-          process_tiles!(bitmap, height: bitmap.h, deterministic: @options[:deterministic])
+          process_tiles!(bitmap, height: @height, deterministic: @options[:deterministic])
         end.real
       end
 
@@ -95,7 +96,8 @@ module Contrek
           width: @maximum_width,
           height: @height,
           versus: options[:versus],
-          options: options
+          options: options,
+          number_of_threads: @number_of_threads
         }
         metadata[:treemap] = @whole_tile.compute_treemap if options[:treemap]
         Contrek::Finder::Result.new(raw_polygons, metadata)