mork 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b09396164e58d22ccd0551b167333469bc754d0
-  data.tar.gz: 6bda803a414aceb17b8011fb1decadc2f5fd9848
+  metadata.gz: 3e2f106fe0351208351e34893145ccd9061de06f
+  data.tar.gz: ffffe95407f3fd7d09eb027ca9d8879f63bedfa9
 SHA512:
-  metadata.gz: a5cd7f9419791b146fd90b14b2ea518be9fd8c09ac11214b699d8308dbe7ed5e0dadc1fba7f68414a730422b959538ef77a8b15ad914088afd83037cde1fddf8
-  data.tar.gz: 54a38a8b8c6681731045d7bef9f98afb08af93715a896c98eba40c75581843d4d5ed39985fcf6ae005f202c0e671dd5c48c8358e812f115340ccff6b7f685eaa
+  metadata.gz: c9c8b692a4f540a214893169f94e0da6893e672479d62275b0cc8c900a6dc6a276ec9f222d2bbea705205c58b901035531a2da36f5eb7603dc4ae6b737abdd9f
+  data.tar.gz: dbf89308c0c80aa5694bcb63459872824301cccadfd429bf8c971fd83615afd4b1e5e90565992cc8878c453b9b85f66437b766aab8473ef1a9a0d54216dfc1ff

data/.gitignore

@@ -8,4 +8,6 @@ tmp/*
 .yardoc
 .ruby-version
 .ruby-gemset
-.rspec
+.rspec
+.byebug_history
+doc

data/README.md

@@ -12,18 +12,19 @@ Mork is a low-level library, and very much work in progress. It is not, and will
 - the PDF files generated by Mork are intended to be printed on regular printer paper
 - the entire response sheet must fit into a single page
 - after collecting the responses, a filled-out form should be acquired as a JPEG, PNG, or PDF image by a normal optical scanner or camera (i.e., no specialized equipment is necessary)
+- independent of how the sheet is printed and the image is acquired, all internal processing is done on a grayscale version of the bitmap
 - the response sheet contains the following items:
   - registration marks at each page corner
   - a bar code along the bottom margin to uniquely identify the sheet
   - a header area to print arbitrary information
   - a response area containing a list of numbered items (questions)
   - each item contains an arbitrary number of choice “cells”, each marked with a capital letter (A, B, C, ...)
-  - the far right side of the response area is reserved for a column of calibration cells that must remain blank
-- it is entirely up to the user to provide parameters that produce the desired response sheet layout; in particular, making sure that the header elements and choice cells fit in the available space, as Mork does not provide any type of "sanity" check at this time.
+  - the right edge of the response area is reserved for a column of calibration cells that must remain blank
+- it is entirely up to the user to provide parameters that produce the desired response sheet layout; in particular, making sure that the header elements and choice cells fit in the available space, as Mork does not provide any type of "sanity" check at this time
 
 ## Getting started
 
-First, make sure that ImageMagick is installed in your system. On the Mac, `brew` is the preferred package manager:
+First, make sure that ImageMagick is installed in your system. In OS X, `brew` is the preferred package manager:
 
     brew install imagemagick
 
@@ -90,37 +91,42 @@ page_size:                # all measurements in mm
   height:         297     # height of the paper sheet
 reg_marks:         
   margin:          10     # distance from each page border to registration mark center
-  radius:           2.5   # registration mark radius
-  crop:            12     # size of the registration mark search area (*)
+  radius:           3     # registration mark radius
+  crop:            20     # size of the registration mark search area (*)
   offset:           2     # distance between the search area and each page border (*)
-  blur:             2     # size of a gaussian blur filter to smooth overly pixelated registration marks (0 to skip)
-  dilate:           5     # size of a “dilate” filter to get rid of stray noise (0 to skip)
+  blur:             2     # size of a gaussian blur filter to smooth overly pixelated registration marks (0 to skip) (*)
+  dilate:           5     # size of a “dilate” filter to get rid of stray noise (0 to skip) (*)
+  contrast:        20     # minimum contrast between registration mark circles and the surrounding white paper
 header:           
   name:                   # ‘name’ is just a label; you can add arbitrary header elements
     top:            5     # margin relative to registration frame top side
-    left:           7.5   # margin relative to registration frame left side
-    width:         170    # text will be fitted to this width
-    size:           14    # font size
+    left:          15     # margin relative to registration frame left side
+    width:        160     # text will be fitted to this width
+    height:         7     # ...and this height
+    size:          14     # font size
   title:
-    top:            15
-    left:           7.5
-    width:         180
-    size:           12
+    top:           15
+    left:          15
+    width:        160
+    height:        12
+    size:          12
   code:
-    top:             5
-    left:          165
-    width:          20
-    size:           14
+    top:           35
+    left:         130
+    width:         57
+    height:        10
+    size:          14
   signature:
     top:            30
-    left:            7.5
+    left:           15
     width:         120
     height:         15
     size:            7
     box:          true    # header element will be enclosed in a box
 items:
-  top:              55.5  # response area margin, relative to reg frame
-  left:             10.5  # response area margin, relative to reg frame
+  threshold:         0.75 # mark detection threshold
+  top:              55    # response area margin, relative to reg frame
+  left:             11    # response area margin, relative to reg frame
   rows:             30    # number of items per column
   columns:           4    # number of columns
   column_width:     44    # 
@@ -136,7 +142,7 @@ barcode:
   bits:             40    # the maximum sheet identifier is 2 to the power or bits
   left:             15    # distance between registration frame side and the first barcode bit
   width:             3    # width of each barcode bit
-  height:            2.5  # height of each barcode bit from the registration frame bottom side
+  height:            3    # height of each barcode bit from the registration frame bottom side
   spacing:           4    # horizontal distance between adjacent barcode bit centers
 ```
 
@@ -144,7 +150,7 @@ Assuming that the above text is written in a file named `layout.yml`, here's how
 
 ```ruby
 s = SheetPDF.new content, 'layout.yml'
-s.write 'sheet.pdf'
+s.save 'sheet.pdf'
 system 'open sheet.pdf' # this works in OSX
 ```
 
@@ -156,28 +162,94 @@ If the `layout` argument is omitted, Mork will search for a file named `layout.y
 
 Assuming that a person has filled out a response sheet by darkening with a pen the selected choices, and that the sheet has been acquired as an image file, response scoring is performed by the `Mork::SheetOMR` class. Three pieces of information must be provided to the object constructor:
 
-- **image**: the path/filename of the bitmap image (currently accepts JPG, JPEG, PNG, PDF extensions; a resolution of 150-200 dpi is usually more than sufficient to obtain accurate readings)
-- **choices**: equivalent to the `choices` array of integers passed to the `SheetPDF` constructor as part of the `content` parameter (see above)
-- **layout**: same as for the `SheetPDF` class
+- **path**: mandatory path/filename of the bitmap image (accepts JPG, JPEG, PNG, PDF extensions; a resolution of 150-200 dpi is usually more than sufficient to obtain accurate readings)
+- **choices**: a named argument (ruby-2 style) equivalent to the `choices` array of integers passed to the `SheetPDF` constructor as part of the `content` parameter (see above). If omitted, the `choices` parameter is inferred from the layout
+- **layout_file**: same as for the `SheetPDF` class
 
 The following code shows how to create and analyze a SheetOMR based on a bitmap file named `image.jpg`:
 
 ```ruby
 # instantiating the object
-s = SheetOMR.new 'image.jpg', [5]*100, 'layout.yml'
+s = SheetOMR.new 'image.jpg', choices: [5]*100, layout_file: 'layout.yml'
 # detecting darkened choice cells for the 100 items
-chosen = s.mark_array
+chosen = s.marked_choices
 ```
 
-If all goes well, the `chosen` array will contain 100 sub-arrays, each containing the list of darkened choices for that item, where the first cell is indicated by a 0, the second by a 1, etc. It is also possible to show the scoring graphically:
+If all goes well, the `chosen` array will contain 100 sub-arrays, each containing the list of marked choices for that item, where the first cell is indicated by a 0, the second by a 1, etc. It is also possible to show the scoring graphically by applying an overlay on top of the scanned image.
 
 ```ruby
-s = SheetOMR.new 'image.jpg', 'layout.yml'
-s.highlight
-s.write 'highlights.jpg'
-system 'open highlights.jpg' # OSX only
+s = SheetOMR.new 'image.jpg', choices: [5]*100, layout_file: 'layout.yml'
+s.overlay :check, :marked
+s.save 'marked_choices.jpg'
+system 'open marked_choices.jpg' # this works in OSX
 ```
 
-#### Wait, why Mork?
+More than one overlay can be applied on a sheet. For example, in addition to checking the marked choice cells, the expected choices can be outlined to show correct vs. wrong responses:
 
-Because I used to have a crush on Mindy
+```ruby
+...
+correct = [[3], [0], [2], [1], [2]] # and so on...
+s.overlay :check, :marked
+s.overlay :outline, correct
+...
+```
+
+
+Scoring can only be performed if the sheet gets properly registered, which in turn depends on the quality of the scanned image.
+
+### Improving sheet registration and marking
+
+For Mork to be able to register the response sheet, the acquired image should be of _sufficient_ resolution and contrast, and it should be straight, with some white margin around the 4 registration circles.
+
+Mork tries to be tolerant of variations in the above parameters, but you should experiment with your own layout, actual printout, and actual scans.
+
+Check for object “validity” to make sure that registration succeeded:
+
+```ruby
+s = SheetOMR.new 'image.jpg', choices: [5]*100, layout_file: 'layout.yml'
+s.valid?
+```
+
+When registration fails, it is possible to get some information by displaying the status and by applying a dedicated overlay on the original image:
+
+```ruby
+s = SheetOMR.new 'image.jpg', choices: [5]*100, layout_file: 'layout.yml'
+unless s.valid?
+  s.save_registration 'unregistered.jpg'
+end
+```
+
+Experiment with the following layout settings to improve sheet registration:
+
+```yaml
+reg_marks:         
+  radius:    3 # a bigger registration mark can sometimes help
+  crop:     20 # make sure the registration mark lies comfortably
+               # inside the crop area
+  offset:    2 # increase the offset if a black background is
+               # visible along the page edges
+  blur:      2 # increase gaussian blur filter size if images are
+               # noisy
+  dilate:    5 # higher values can remove more specks and dust from
+               # the crop areas, but may also “eat out” too much of
+               # the registration mark
+  contrast: 20 # decrease this value if the contrast between the
+               # registration marks’ black and the surrounding white
+               # paper is low
+```
+
+In addition, you can fine tune the threshold for accepting a choice as marked:
+
+```yaml
+items:
+  threshold: 0.75 # higher values will result in fewer false positives, but
+                  # lightly darkened choices may be missed; conversely, lower
+                  # values will reduce misses but may increase false positives
+```
+
+If Mork fails to register/mark scoring sheets that you believe **should** be valid, please open an issue and attach a sample image.
+
+## API Reference
+The online rubygems docs are [here](http://www.rubydoc.info/gems/mork)
+
+_(Wait, why Mork? Because I used to have a crush on Mindy)_

data/lib/mork/coord.rb

@@ -1,4 +1,5 @@
 module Mork
+  # @private
   # The Coord class takes coordinates in the standard unit (e.g. mm)
   # and provides pixel-based coordinates useful for image manipulation
   class Coord
@@ -13,7 +14,6 @@ module Mork
 
     def to_hash
       { w: @w, h: @h, x: @x, y: @y }
-      5
     end
 
     def print

data/lib/mork/extensions.rb

@@ -1,3 +1,4 @@
+# @private
 class Array
   def mean
     @the_sample_mean ||= inject(:+)/length.to_f
@@ -13,12 +14,14 @@ class Array
   end
 end
 
+# @private
 class Fixnum
   def mm
     self * 2.83464566929134
   end
 end
 
+# @private
 class Float
   def mm
     self * 2.83464566929134

data/lib/mork/grid.rb

@@ -2,6 +2,7 @@ require 'yaml'
 require 'mork/grid_const'
 
 module Mork
+  # @private
   # The Grid is a set of expectations on what the response sheet should look like
   # It knows nothing about the actual scanned image.
   # All returned values are in the arbitrary units given in the configuration file
@@ -9,17 +10,18 @@ module Mork
     # Calling Grid.new without arguments creates the default boilerplate Grid
     def initialize(options=nil)
       @params = DGRID
+      if File.exists?('layout.yml')
+        @params.deeper_merge! symbolize YAML.load_file('layout.yml')
+      end
       case options
       when NilClass
-        if File.exists?('layout.yml')
-          @params.merge! symbolize YAML.load_file('layout.yml')
-        end
+        # do nothing
       when Hash
-        @params.merge! symbolize options
+        @params.deeper_merge! symbolize options
       when String
-        @params.merge! symbolize YAML.load_file(options)
+        @params.deeper_merge! symbolize YAML.load_file(options)
       else
-        raise "Invalid parameter in the Grid constructor: #{options.class.inspect}"
+        raise ArgumentError, "Invalid parameter in the Grid constructor: #{options.class.inspect}"
       end
     end
 
@@ -55,6 +57,10 @@ module Mork
       @params[:reg_marks][:blur].to_i
     end
 
+    def choice_threshold
+      @params[:items][:threshold].to_f
+    end
+
     #====================#
     private
     #====================#
@@ -118,6 +124,7 @@ module Mork
     def reg_off()          @params[:reg_marks][:offset].to_f   end
     def reg_frame_width()  page_width  - reg_margin * 2        end
     def reg_frame_height() page_height - reg_margin * 2        end
+    def reg_min_contrast() @params[:reg_marks][:contrast]      end
     def page_width()       @params[:page_size][:width].to_f    end
     def page_height()      @params[:page_size][:height].to_f   end
     def reg_margin()       @params[:reg_marks][:margin].to_f   end

data/lib/mork/grid_const.rb

@@ -12,10 +12,11 @@ module Mork
     reg_marks: {
       margin:      10,
       radius:       3,
-      offset:       2,
+      offset:       2, # distance between page edge and registraton mark search area
       crop:        20, # size of square where the regmark should be located
-      dilate:       0, # set to >0 to apply a dilate IM operation
-      blur:         0  # set to >0 to apply a blur IM operation
+      dilate:       5, # set to >0 to apply a dilate IM operation
+      blur:         2, # set to >0 to apply a blur IM operation
+      contrast:    20  # minimum contrast between registration mark circles and the white paper
     },
     header: {
       name: {
@@ -33,9 +34,9 @@ module Mork
         size:      12
       },
       code: {
-        top:        30,
-        left:      155,
-        width:      35,
+        top:        35,
+        left:      130,
+        width:      57,
         height:     10,
         size:       14,
         align:   :right
@@ -50,13 +51,14 @@ module Mork
       }
     }, # header end
     items: {
+      threshold:     0.75,
       columns:       4,
       column_width: 44,
       rows:         30,
       # from the top-left registration mark
       # to the center of the first choice cell
-      left:         10.5,
-      top:          55.5,
+      left:         11,
+      top:          55,
       # between choices
       x_spacing:     7,
       # between rows
@@ -71,7 +73,7 @@ module Mork
       # distance between right side of q num and left side of first choice cell
       number_width:  8,
       # width of question number text box
-      number_margin: 2,
+      number_margin: 2
     }, # items end
     barcode: {
       bits:         40,

data/lib/mork/grid_omr.rb

@@ -1,7 +1,9 @@
 require 'mork/grid'
 require 'mork/coord'
+require 'deep_merge/rails_compat'
 
 module Mork
+  # @private
   class GridOMR < Grid
     def initialize(options=nil)
       super options
@@ -13,12 +15,12 @@ module Mork
       self
     end
 
-    def barcode_bit_areas(bitstring = '1' * barcode_bits)
-      areas = []
-      bitstring.reverse.each_char.with_index do |c, i|
-        areas << barcode_bit_area(i+1) if c=='1'
+    def barcode_areas(bits)
+      [].tap do |areas|
+        bits.each_with_index do |b, i|
+          areas << barcode_bit_area(i+1) if b
+        end
       end
-      areas
     end
 
     # ===========================================
@@ -117,3 +119,11 @@ end
 #   (d[:w]-d[:h]).abs
 # end
 
+# # GET RID OF THIS!
+# def barcode_bit_areas(bitstring = '1' * barcode_bits)
+#   areas = []
+#   bitstring.reverse.each_char.with_index do |c, i|
+#     areas << barcode_bit_area(i+1) if c=='1'
+#   end
+#   areas
+# end

data/lib/mork/grid_pdf.rb

@@ -1,6 +1,7 @@
 require 'mork/grid'
 
 module Mork
+  # @private
   # GridPDF gets coordinates and measurements from a Grid and
   # provides SheetPDF with the properly computed values
   class GridPDF < Grid

data/lib/mork/magicko.rb

@@ -1,6 +1,7 @@
 require 'mini_magick'
 
 module Mork
+  # @private
   # Magicko: image management, done in two ways: 1) direct system calls to
   # imagemagick tools; 2) via the MiniMagick gem
   class Magicko
@@ -32,26 +33,28 @@ module Mork
       read_bytes "-crop #{c.cropper}#{b}#{d}"
     end
 
-    # MiniMagick stuff
+    ##################################
+    # Constructing MiniMagick commands
+    ##################################
 
-    def highlight_cells(coords)
+    def write_registration(fname)
+
+    end
+
+    def highlight(coords, rounded)
       @cmd << [:stroke, 'none']
       @cmd << [:fill, 'rgba(255, 255, 0, 0.3)']
-      coords.each do |c|
-        @cmd << [:draw, "roundrectangle #{c.choice_cell}"]
-      end
+      coords.each { |c| @cmd << [:draw, shape(c, rounded)] }
     end
 
-    def outline(coords)
+    def outline(coords, rounded)
       @cmd << [:stroke, 'green']
       @cmd << [:strokewidth, '2']
       @cmd << [:fill, 'none']
-      coords.each do |c|
-        @cmd << [:draw, "roundrectangle #{c.choice_cell}"]
-      end
+      coords.each { |c| @cmd << [:draw, shape(c, rounded)] }
     end
 
-    def cross(coords)
+    def check(coords, rounded)
       @cmd << [:stroke, 'red']
       @cmd << [:strokewidth, '3']
       coords.each do |c|
@@ -99,23 +102,21 @@ module Mork
       @cmd << [:draw, "polygon #{pts}"]
     end
 
-    def write(fname, reg)
-      if fname
-        MiniMagick::Tool::Convert.new(whiny: false) do |img|
-          img << @path
-          exec_mm_cmd img, reg
-          img << fname
-        end
-      else
-        MiniMagick::Tool::Mogrify.new(whiny: false) do |img|
-          img << @path
-          exec_mm_cmd img, reg
-        end
+    def save(fname, reg)
+      MiniMagick::Tool::Convert.new(whiny: false) do |img|
+        img << @path
+        img.distort(:perspective, pps(reg)) if reg
+        @cmd.each { |cmd| img.send(*cmd) }
+        img << fname
       end
     end
 
     private
 
+    def shape(c, rounded)
+      rounded ? "roundrectangle #{c.choice_cell}" : "rectangle #{c.rect_points}"
+    end
+
     # calling imagemagick and capturing the converted image
     # into an array of bytes
     def read_bytes(params=nil)
@@ -123,12 +124,6 @@ module Mork
       IO.read(s).unpack 'C*'
     end
 
-    def exec_mm_cmd(c, pp)
-      c.distort(:perspective, pps(pp)) if pp
-      @cmd.each { |cmd| c.send(*cmd) }
-    end
-
-
     # perspective points: brings the found registration area centers to the
     # original image boundaries; the result is that the registered image is
     # somewhat stretched, which should be okay
@@ -160,3 +155,26 @@ end
 # def raw_patch
 #   @raw_pixels ||= patch
 # end
+
+# def exec_mm_cmd(c, pp)
+#   c.distort(:perspective, pps(pp)) if pp
+#   @cmd.each { |cmd| c.send(*cmd) }
+# end
+
+# def highlight_cells(coords)
+#   @cmd << [:stroke, 'none']
+#   @cmd << [:fill, 'rgba(255, 255, 0, 0.3)']
+#   coords.each do |c|
+#     @cmd << [:draw, "roundrectangle #{c.choice_cell}"]
+#   end
+# end
+
+# def outline(coords)
+#   @cmd << [:stroke, 'green']
+#   @cmd << [:strokewidth, '2']
+#   @cmd << [:fill, 'none']
+#   coords.each do |c|
+#     @cmd << [:draw, "roundrectangle #{c.choice_cell}"]
+#   end
+# end
+