circe 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 683391e9186e4a20233cb25ce28f37b7bf56c88586b1ac72f1312081e4e48c7d
-  data.tar.gz: 680a92f2318df871cdd2f6f98732f85adab5098e067fe7c4fa986832e5f32caf
+  metadata.gz: 046e41e59ccb0202ad32729da191ba2f003a9e3a0370f4454996fdd544f47103
+  data.tar.gz: 1d3d18f2bcacfc6ae6e8ff816d166b43f53686841e510fbf722a5a2c85f80080
 SHA512:
-  metadata.gz: 84422ff365869d1615c4cf17bbc035426e7a4cbfc5bd501104d5e1b2a5ea12535150de13e192b0a2849102fb0a49f7e174b2f9b804027614fc50f68858d139de
-  data.tar.gz: d679d1b61f155a5170426d007e8d06e73b5e01c12fd4dc12ab70ceddb29fc1c80a35ab178b83986c32f0f05e1b508d9e6cf56ce8fd4840bacbf1d67f96dc6312
+  metadata.gz: 17a645694b3ac252fe68048a58e63cbfe4d580a45ed8b3bdfda6cc0ff94e0ab821536a4bed63aff1a42b7ceba3637c889b0a767c9e51ab3e2f5bf4d4c193db53
+  data.tar.gz: 5fcffb7041bcd9f53f379576847748a5579d77b681b71e0eae810f3ce090aca5d83644c6bba442bfc06f3fd4e8bfb6c5550dd6ae172f9636bdf263822a94446e

data/ext/circe.cpp

@@ -119,6 +119,9 @@ static ID id_class;
 static ID id_png;
 static ID id_jpg;
 
+static ID id_label;
+static ID id_thickness;
+static ID id_color;
 
 static Yolo  *yolo;
 static YuNet *yunet;
@@ -143,18 +146,88 @@ draw_label(cv::Mat& img, string label, Point& origin,
 }
 
 static void
-draw_labelbox(cv::Mat& img, string label, Rect& box,
-	      Scalar& framecolor = BLUE, Scalar& textcolor = BLACK,
-	      int thickness = 1) {
+draw_box(cv::Mat& img, Rect& box,
+	 Scalar& framecolor = BLUE, int thickness = 1) {
 
     Point a = { box.x,             box.y              };
     Point b = { box.x + box.width, box.y + box.height };
 
     cv::rectangle(img, a, b, framecolor, thickness);
-    draw_label(img, label, a, textcolor, framecolor);    
 }
 
+static void
+draw_labelbox(cv::Mat& img, string label, Rect& box,
+	      Scalar& framecolor = BLUE, Scalar& textcolor = BLACK,
+	      int thickness = 1) {
+
+    Point o = { box.x, box.y };
+
+    draw_box(img, box, framecolor, thickness);
+    draw_label(img, label, o, textcolor, framecolor);    
+}
+
+
+VALUE
+circe_annotate(Mat& img, Rect& box, VALUE v_annotation, int *state) {
+    if (img.empty() || NIL_P(v_annotation))
+        return Qnil;
+
+    VALUE v_label      = Qnil;
+    VALUE v_color      = ULONG2NUM(0x0000ff);
+    VALUE v_thickness  = INT2NUM(1);
+  
+    VALUE s_label      = rb_id2sym(id_label);
+    VALUE s_color      = rb_id2sym(id_color);
+    VALUE s_thickness  = rb_id2sym(id_thickness);
+
+    switch (TYPE(v_annotation)) {
+    case T_NIL:
+        break;
+    case T_HASH:
+        v_thickness = rb_hash_aref(v_annotation, s_thickness);
+	v_color     = rb_hash_aref(v_annotation, s_color);
+	v_label     = rb_hash_aref(v_annotation, s_label);
+	break;
+    case T_ARRAY:
+        switch(RARRAY_LENINT(v_annotation)) {
+	default:
+	case 3: v_thickness = RARRAY_AREF(v_annotation, 2);
+	case 2: v_color     = RARRAY_AREF(v_annotation, 1);
+	case 1: v_label     = RARRAY_AREF(v_annotation, 0);
+	case 0: break;
+	}
+	break;
+    case T_STRING:
+        v_label = v_annotation;
+	break;
+    }
+  
+    // No color, no rendering
+    if (NIL_P(v_color))
+        return Qnil;
+  
+    long   rgb   = NUM2ULONG(v_color);
+    Scalar color = cv::Scalar((rgb >>  0) & 0xFF,
+			      (rgb >>  8) & 0xFF,
+			      (rgb >> 16) & 0xFF);
+  
+    if (! NIL_P(v_thickness)) {
+        int thickness = NUM2INT(v_thickness);
+	draw_box(img, box, color, thickness);
+    }
+    if (! NIL_P(v_label)) {
+        string label  = StringValueCStr(v_label);
+	Point  o      = { box.x, box.y };
+	draw_label(img, label, o, BLACK, color);
+    }
 
+    // Return normalized parameters
+    VALUE r = rb_hash_new();
+    rb_hash_aset(r, s_label,     v_label    );
+    rb_hash_aset(r, s_color,     v_color    );
+    rb_hash_aset(r, s_thickness, v_thickness);
+    return r;
+}
 
 
 
@@ -186,40 +259,11 @@ yunet_process_features(vector<YuNet::Face>& faces,
 	rb_ary_push(v_features, v_feature);
 	
 	if (!img.empty() && rb_block_given_p()) {
-	    VALUE v_annotation= rb_yield_splat(v_feature);
-	    VALUE v_label     = Qnil;
-	    VALUE v_color     = ULONG2NUM(0x0000ff);
-	    VALUE v_thickness = INT2NUM(1);
-
-	    switch (TYPE(v_annotation)) {
-	    case T_NIL:
-		break;
-	    case T_HASH:
-		break;
-	    case T_ARRAY:
-		switch(RARRAY_LENINT(v_annotation)) {
-		default:
-		case 3: v_thickness = RARRAY_AREF(v_annotation, 2);
-		case 2: v_color     = RARRAY_AREF(v_annotation, 1);
-		case 1: v_label     = RARRAY_AREF(v_annotation, 0);
-		case 0: break;
-		}
-		break;
-	    case T_STRING:
-		v_label = v_annotation;
-		break;
-	    }
-	    
-	    if (! NIL_P(v_label)) {
-		string label     = StringValueCStr(v_label);
-		long   rgb       = NUM2ULONG(v_color);
-		int    thickness = NUM2INT(v_thickness);
-		Scalar color     = cv::Scalar((rgb >>  0) & 0xFF,
-					      (rgb >>  8) & 0xFF,
-					      (rgb >> 16) & 0xFF);
-		draw_labelbox(img, label, box, color, BLACK, thickness);
-		
-		for (const auto& p : lmark) {
+	    VALUE v_annotation = rb_yield_splat(v_feature);
+	    VALUE cfg = circe_annotate(img, box, v_annotation, state);
+
+	    if (! NIL_P(cfg)) {
+	        for (const auto& p : lmark) {
 		    cv::circle(img, p, 3, cv::Scalar(255, 0, 0), 2);
 		}
 	    }
@@ -248,41 +292,9 @@ yolo_process_features(vector<Yolo::Item>& items,
 						     v_name, v_confidence);
 	rb_ary_push(v_features, v_feature);
 
-	if (rb_block_given_p()) {
+	if (!img.empty() && rb_block_given_p()) {
 	    VALUE v_annotation = rb_yield_splat(v_feature);
-	    
-	    VALUE v_label      = Qnil;
-	    VALUE v_color      = ULONG2NUM(0x0000ff);
-	    VALUE v_thickness  = INT2NUM(1);
-	    
-	    switch (TYPE(v_annotation)) {
-	    case T_NIL:
-		break;
-	    case T_HASH:
-		break;
-	    case T_ARRAY:
-		switch(RARRAY_LENINT(v_annotation)) {
-		default:
-		case 3: v_thickness = RARRAY_AREF(v_annotation, 2);
-		case 2: v_color     = RARRAY_AREF(v_annotation, 1);
-		case 1: v_label     = RARRAY_AREF(v_annotation, 0);
-		case 0: break;
-		}
-		break;
-	    case T_STRING:
-		v_label = v_annotation;
-		break;
-	    }
-	    
-	    if (! NIL_P(v_label)) {
-		string label     = StringValueCStr(v_label);
-		long   rgb       = NUM2ULONG(v_color);
-		int    thickness = NUM2INT(v_thickness);
-		Scalar color     = cv::Scalar((rgb >>  0) & 0xFF,
-					      (rgb >>  8) & 0xFF,
-					      (rgb >> 16) & 0xFF);
-		draw_labelbox(img, label, box, color, BLACK, thickness);
-	    }
+	    circe_annotate(img, box, v_annotation, state);
 	}
     }
 }
@@ -364,6 +376,10 @@ circe_m_analyze(int argc, VALUE* argv, VALUE self) {
 	cv::imencode(format, o_img, buf);
 	v_image = rb_str_new(reinterpret_cast<char*>(buf.data()), buf.size());
     }
+
+    i_img.release();
+    o_img.release();
+
     return rb_ary_new_from_args(2, v_features, v_image);
 
  exception:
@@ -382,22 +398,34 @@ void Init_core(void) {
     eCirceError = rb_define_class_under(cCirce, "Error", rb_eStandardError);
     // myclass = rb_const_get(mymodule, sym_myclass);
 
-    VALUE v_onnx_yolo  = rb_const_get(cCirce, rb_intern("ONNX_YOLO"));
-    VALUE v_onnx_yunet = rb_const_get(cCirce, rb_intern("ONNX_YUNET"));
+    VALUE v_onnx_yolo   = rb_const_get(cCirce, rb_intern("ONNX_YOLO"));
+    VALUE v_yolo_path   = RARRAY_AREF(v_onnx_yolo, 0);
+    VALUE v_yolo_height = RARRAY_AREF(v_onnx_yolo, 1);
+    VALUE v_yolo_width  = RARRAY_AREF(v_onnx_yolo, 2);
+
+    VALUE v_onnx_yunet  = rb_const_get(cCirce, rb_intern("ONNX_YUNET"));
+    VALUE v_yunet_path  = RARRAY_AREF(v_onnx_yunet, 0);
 
-    static Yolo  _yolo  = { StringValueCStr(v_onnx_yolo ) };
-    static YuNet _yunet = { StringValueCStr(v_onnx_yunet) };
+
+    
+    static Yolo  _yolo  = { StringValueCStr(v_yolo_path ),
+			    { NUM2INT(v_yolo_width),
+			      NUM2INT(v_yolo_height) }};
+    static YuNet _yunet = { StringValueCStr(v_yunet_path) };
 
     yolo  = &_yolo;
     yunet = &_yunet;
 
     
-    id_debug       = rb_intern_const("debug"   );
-    id_face        = rb_intern_const("face"    );
-    id_classify    = rb_intern_const("classify");
-    id_class       = rb_intern_const("class"   );
-    id_png         = rb_intern_const("png"     );
-    id_jpg         = rb_intern_const("jpg"     );
+    id_debug       = rb_intern_const("debug"    );
+    id_face        = rb_intern_const("face"     );
+    id_classify    = rb_intern_const("classify" );
+    id_class       = rb_intern_const("class"    );
+    id_png         = rb_intern_const("png"      );
+    id_jpg         = rb_intern_const("jpg"      );
+    id_label       = rb_intern_const("label"    );
+    id_thickness   = rb_intern_const("thickness");
+    id_color       = rb_intern_const("color"    );
     
     
     rb_define_method(cCirce, "analyze", circe_m_analyze, -1);

data/ext/yolo.cpp

@@ -1,3 +1,14 @@
+/*
+ * https://github.com/ultralytics/ultralytics/blob/main/examples/YOLOv8-CPP-Inference/inference.cpp
+ * https://github.com/opencv/opencv/blob/4.x/samples/dnn/yolo_detector.cpp
+ *
+ * yolov5 has an output of shape:
+ *  (batchSize, 25200, 85) (Num classes + box[x,y,w,h] + confidence[c])
+ * yolov8 has an output of shape:
+ *  (batchSize, 84,  8400) (Num classes + box[x,y,w,h])
+ *
+ * yolo export model=yolov8s.pt imgsz=480,640 format=onnx opset=12
+ */ 
 #include <tuple>
 #include <string>
 #include <opencv2/imgproc.hpp>
@@ -6,89 +17,144 @@
 #include "yolo.h"
 
 
-Yolo::Yolo(const std::string& model) {
-    net = cv::dnn::readNetFromONNX(model);
+Yolo::Yolo(const std::string& model, cv::Size size) {
+    this->net  = cv::dnn::readNetFromONNX(model);
+    this->size = size;
 }
 
 
 void Yolo::process(cv::Mat &img, std::vector<Yolo::Item> &items) {
+    int version = 5;
 
+    cv::Mat input = img;
+    if (letterBoxForSquare && size.width == size.height)
+        input = formatToSquare(input);
+    
     // Pre-process
     cv::Mat blob;
-    std::vector<cv::Mat> outputs;
-
-    cv::dnn::blobFromImage(img, blob, 1./255.,
-			   cv::Size(INPUT_WIDTH, INPUT_HEIGHT),
-			   cv::Scalar(), true, false);
+    cv::dnn::blobFromImage(input, blob, 1./255.,
+			   this->size, cv::Scalar(), true, false);
 
     // Process
+    std::vector<cv::Mat> outputs;
     net.setInput(blob);
     net.forward(outputs, net.getUnconnectedOutLayersNames());
 
-    // Post-process
-    std::vector<int>   class_ids;
-    std::vector<float> confidences;
-    std::vector<cv::Rect>  boxes;
+    // Output
+    int    rows       = outputs[0].size[1];
+    int    dimensions = outputs[0].size[2];
+
+    // Infer yolo version
+    //  -> Check if the shape[2] is more than shape[1] (yolov8)
+    if (dimensions > rows) {
+        version = 8;
+    }
+
+    // Adjust according to version
+    if (version == 8) {
+	//  cv::transposeND(outs[0], {0, 2, 1}, outs[0]);
+        rows       = outputs[0].size[2];
+        dimensions = outputs[0].size[1];
+        outputs[0] = outputs[0].reshape(1, dimensions);
+        cv::transpose(outputs[0], outputs[0]);
+    }
+
+    // Output
+    float *data    = (float *)outputs[0].data;
 
     // Resizing factor.
-    float x_factor = img.cols / INPUT_WIDTH;
-    float y_factor = img.rows / INPUT_HEIGHT;
+    float x_factor = input.cols / size.width;
+    float y_factor = input.rows / size.height;
 
-    float *data = (float *)outputs[0].data;
-    const int dimensions = 85;
+    // Post-process
+    std::vector<int>       class_ids;
+    std::vector<float>     confidences;
+    std::vector<cv::Rect>  boxes;
     
-    // 25200 for default size 640.
-    const int rows = 25200;
-    // Iterate through 25200 detections.
-    for (int i = 0; i < rows; ++i) {
-	float confidence = data[4];
-
-	// Discard bad detections and continue.
-	if (confidence >= CONFIDENCE_THRESHOLD) {
-            float *classes_scores = data + 5;
-
-            // Create a 1x85 Mat and store class scores of 80 classes.
-	    cv::Mat scores(1, names.size(), CV_32FC1, classes_scores);
-
-            // Perform minMaxLoc and acquire the index of best class score.
-	    cv::Point class_id;
-            double max_class_score;
-            minMaxLoc(scores, 0, &max_class_score, 0, &class_id);
-
-            // Continue if the class score is above the threshold.
-            if (max_class_score > SCORE_THRESHOLD) {
-                // Store class ID and confidence in the pre-defined
-		// respective vectors.
-                float cx = data[0];  // Center
-                float cy = data[1];
-                float w  = data[2];  // Box dimension
-                float h  = data[3];
-		
-                // Bounding box coordinates.
-                int left   = int((cx - 0.5 * w) * x_factor);
-                int top    = int((cy - 0.5 * h) * y_factor);
-                int width  = int(w              * x_factor);
-                int height = int(h              * y_factor);
-
-                // Store good detections in the boxes vector.
-                confidences.push_back(confidence);
+
+    if        (version == 5) {
+	for (int i = 0; i < rows; ++i) {
+            float confidence = data[4];
+
+            if (confidence >= CONFIDENCE_THRESHOLD) {
+                float *classes_scores = data + 5;
+
+                cv::Mat scores(1, classes.size(), CV_32FC1, classes_scores);
+                cv::Point class_id;
+                double max_class_score;
+
+                minMaxLoc(scores, 0, &max_class_score, 0, &class_id);
+
+                if (max_class_score > SCORE_THRESHOLD) {
+                    confidences.push_back(confidence);
+                    class_ids.push_back(class_id.x);
+
+                    float x = data[0];
+                    float y = data[1];
+                    float w = data[2];
+                    float h = data[3];
+
+                    int left   = int((x - 0.5 * w) * x_factor);
+                    int top    = int((y - 0.5 * h) * y_factor);
+                    int width  = int(w * x_factor);
+                    int height = int(h * y_factor);
+
+                    boxes.push_back(cv::Rect(left, top, width, height));
+                }
+            }
+	    data += dimensions;
+	}
+    } else if (version == 8) {
+	for (int i = 0; i < rows; ++i) {
+	    float *classes_scores = data + 4;
+	    
+            cv::Mat scores(1, classes.size(), CV_32FC1, classes_scores);
+            cv::Point class_id;
+            double maxClassScore;
+
+            minMaxLoc(scores, 0, &maxClassScore, 0, &class_id);
+
+            if (maxClassScore > SCORE_THRESHOLD) {
+                confidences.push_back(maxClassScore);
                 class_ids.push_back(class_id.x);
+
+                float x = data[0];
+                float y = data[1];
+                float w = data[2];
+                float h = data[3];
+
+                int left   = int((x - 0.5 * w) * x_factor);
+                int top    = int((y - 0.5 * h) * y_factor);
+                int width  = int(w * x_factor);
+                int height = int(h * y_factor);
+
                 boxes.push_back(cv::Rect(left, top, width, height));
             }
-        }
-        // Jump to the next row.
-        data += 85;
+	    data += dimensions;
+	}
     }
+   
     
     // Perform Non-Maximum Suppression and draw predictions.
-    std::vector<int> indices;
+    std::vector<int> nms_result;
     cv::dnn::NMSBoxes(boxes, confidences,
-		      SCORE_THRESHOLD, NMS_THRESHOLD, indices);
+		      SCORE_THRESHOLD, NMS_THRESHOLD, nms_result);
 
     items.clear();
 
-    for (int i = 0; i < indices.size(); i++) {
-        int  idx = indices[i];
-	items.push_back({ names[class_ids[idx]],confidences[idx],boxes[idx] });
+    for (int i = 0; i < nms_result.size(); i++) {
+        int  idx = nms_result[i];
+	items.push_back({ classes[class_ids[idx]],confidences[idx],boxes[idx] });
     }
 }
+
+
+cv::Mat Yolo::formatToSquare(const cv::Mat &source)
+{
+    int col = source.cols;
+    int row = source.rows;
+    int _max = MAX(col, row);
+    cv::Mat result = cv::Mat::zeros(_max, _max, CV_8UC3);
+    source.copyTo(result(cv::Rect(0, 0, col, row)));
+    return result;
+}

data/ext/yolo.h

@@ -16,12 +16,14 @@ public:
 private:
     static constexpr float INPUT_WIDTH          = 640.0;
     static constexpr float INPUT_HEIGHT         = 640.0;
-    static constexpr float SCORE_THRESHOLD      =   0.5;
-    static constexpr float NMS_THRESHOLD        =   0.45;
-    static constexpr float CONFIDENCE_THRESHOLD =   0.45;
+    static constexpr float CONFIDENCE_THRESHOLD =   0.25;
+    static constexpr float SCORE_THRESHOLD      =   0.45;
+    static constexpr float NMS_THRESHOLD        =   0.50;
 
+    bool letterBoxForSquare = true;
+    
  public:
-    const std::vector<std::string> names = {
+    const std::vector<std::string> classes = {
 	"person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train",
 	"truck", "boat", "traffic light", "fire hydrant", "stop sign",
 	"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
@@ -38,11 +40,13 @@ private:
     };
 
 public:
-    Yolo(const std::string& model);
+    Yolo(const std::string& model, cv::Size size);
     void process(cv::Mat &img, std::vector<Item> &items);
     
 private:
     cv::dnn::Net net;
+    cv::Size     size;
+    cv::Mat formatToSquare(const cv::Mat &source);
 };
 
 #endif

data/lib/circe/version.rb

@@ -1,3 +1,3 @@
 class Circe
-    VERSION = '0.1.0'
+    VERSION = '0.1.1'
 end

data/lib/circe.rb

@@ -4,8 +4,8 @@ class Circe
     
     # Don't know how to do it inside the c extension
     DATA_DIR   = File.join(__dir__, '..', 'data').freeze
-    ONNX_YOLO  = File.join(DATA_DIR, 'yolov5s.onnx')
-    ONNX_YUNET = File.join(DATA_DIR, 'face_detection_yunet_2022mar.onnx')
+    ONNX_YOLO  = [ File.join(DATA_DIR, 'yolov5s.onnx'), 640, 640 ]
+    ONNX_YUNET = [ File.join(DATA_DIR, 'face_detection_yunet_2022mar.onnx') ]
     
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: circe
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Stéphane D'Alu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-03 00:00:00.000000000 Z
+date: 2024-04-30 00:00:00.000000000 Z
 dependencies: []
 description: |2+
 
@@ -52,7 +52,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.2
+rubygems_version: 3.4.20
 signing_key:
 specification_version: 4
 summary: Face and object recognition