dsr 0.0.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e94dd71eedb7e3989dde0e7a35c73db47c97891211ab3620ef668bf6fb163dcb
-  data.tar.gz: af6324245ad6c157dc3a700f00121860c75bb828dc096ffcd4a4c1e15ae0dd96
+  metadata.gz: 621fa913c6e55e6c86613100c173069d3dded944de48c3d930b0852ee70733e8
+  data.tar.gz: a5086a089ea4da28aa6b5547d55718a87c292e388357b6d1df2943f885ef1347
 SHA512:
-  metadata.gz: 1a607b1cfaaec5321348face51201ec20baa16958775485f3a86fe67a1eb703be6a58bcf555045dbb181397644eb42240fbe28a38b4dd79f361de4a287e23897
-  data.tar.gz: 66fefe0318e13f2b2d60f60dd125af1577313ac038d1fc91215016fac21066f91abbb58ce91a6f1eabfcb2b39513c800f563cac567f3539e162b1ec05cfddf10
+  metadata.gz: 11af653e502854581722067ac9070e4234f20f77969f4ca30aa364e8b717b7aea1a82b5aee9105985ebf7090c8516167e32d147a058f73dab50c1f2373af7c64
+  data.tar.gz: 428a9d20a0df7c3310ca70c91fe88283888a5c9460fbed434d6f1ab90fd3f6dbc8e9de1e9327ed3ed4103e665cf78e8c83b24402ff6d0231adaaa42ba987196e

data/dsr.gemspec

@@ -1,15 +1,14 @@
 Gem::Specification.new do |spec|
   spec.name         = "dsr"
-  spec.version      = "0.0.0"
-  spec.summary      = "[WIP] Document Structure Recognizer -- currently a collection of common routines I use to build A.I.s"
+  spec.version      = "1.0.0"
+  spec.summary      = "Document Structure Recognizer -- a collection of common routines to build AIs"
 
   spec.author       = "Victor Maslov aka Nakilon"
   spec.email        = "nakilon@gmail.com"
   spec.license      = "MIT"
   spec.metadata     = {"source_code_uri" => "https://github.com/nakilon/dsr"}
 
-  spec.add_dependency "nakischema"
-  spec.add_dependency "hexapdf"
+  spec.add_dependency "pcbr"
 
-  spec.files        = %w{ LICENSE dsr.gemspec lib/dsr.rb }
+  spec.files        = %w{ LICENSE dsr.gemspec lib/dsr.rb lib/dsr/recognizer.rb }
 end

data/lib/dsr/recognizer.rb

@@ -0,0 +1,147 @@
+using(
+  ::Module.new do
+    AssertOneError = ::Class.new ::RuntimeError
+    refine ::Array do
+      def mean &block
+        sum(&block).fdiv size
+      end
+      def assert_one
+        return at 0 if 1 == size
+        raise ::Common::AssertOneError, "size: #{size}"
+      end
+    end
+  end
+)
+
+module DSR
+  module Recognizer
+
+    # joins two arrays of objects into a Hash of parents and children, sorted by passed block
+    # it is possible for parent to have no children
+    # examples:
+    #   Discord search results HTML page -- based on `node.rect.y`, channel names become keys, messages become values
+    # @param keys [Array] these become keys
+    # @param values [Array] these become values
+    # @return [Hash{Object => Array<Object>}
+    def self.detect_many_after_many keys, values, &block
+      fail if keys.any?(&:nil?)   # for 'if key' in the algorithm loop
+      # fail if keys.dup.uniq!
+      # fail if values.dup.uniq!
+      fail if (keys + values).map(&block).uniq!
+      key = nil
+      h = {}
+      vs = []
+      for e, from in (
+        keys.map{ |_| [_, :from_keys] } +
+        values.map{ |_| [_, :from_values] }
+      ).sort_by{ |e,| block.call e }
+        if :from_keys == from
+          h[key] = vs if key
+          key, vs = e, []
+        else
+          fail unless key
+          vs.push e
+        end
+      end
+      h[key] = vs if key
+      h
+    end
+
+    # sorts an array by a block and splits in chunks
+    # gauranteed to split into at least two chunks
+    # if you pass `ratio` then recursively splits if there is another `gap_size > prev_gap_size * ratio`
+    # if you pass `goal` then tries ratios from 0.0 to 1.0 to find the goal number of chunks
+    # if you pass nothing then it uses PCBR to chose the ratio
+    # for both `goal` and nothing variants it checks the result to be the same for all fitting ratios
+    # `ratio: 0` is effectively the same as just Ruby stdlib sort and chunk by block
+    # @param _array [Array<Object>]
+    # @param ratio [Float]
+    # @param goal [Integer]
+    # @yieldparam item [Object] an item of `_array`
+    # @yieldreturn [Numeric] something that can subtract from each other to measure a gap
+    def self.detect_distant_groups _array, goal: nil, ratio: nil
+
+      array = ::Struct.new(:item, :value, :i, :dist_left).then do |struct|
+        _array.map{ |e| struct.new e, yield(e) }.sort_by(&:value)
+      end
+      array.each_with_index{ |e, i| e.i = i }
+      array.each_cons(2){ |e1, e2| e2.dist_left = e2.value - e1.value }
+      f = lambda do |from, to, r: ratio, threshold: 0|
+        # next [] if from > to
+        next [[array[from]]] if from == to
+        max = array[from + 1..to].max_by(&:dist_left)
+        # next [] unless max
+        if threshold * r < max.dist_left
+          [*f[from, max.i - 1, r: r, threshold: max.dist_left], *f[max.i, to, r: r, threshold: max.dist_left]]
+        else
+          [array[from..to]]
+        end
+      end
+
+      if ratio
+        f[0, array.size - 1].map{ |_| _.map &:item }
+      elsif goal
+        0.step(1, 0.01).map do |r|
+          f[0, array.size - 1, r: r].map{ |_| _.map &:item }
+        end.tap do |ratios|
+          fail unless ratios.first.size == array.map(&:value).uniq.size
+          ratios.each_cons(2){ |a, b| fail if a.size < b.size }
+        end.select{ |_| _.size == goal }.tap do |splits|
+          fail if 1 < splits.uniq.size
+        end.first
+      else
+        require "pcbr"
+        pcbr = ::PCBR.new
+        0.step(1, 0.01).map.with_index do |r, i|
+          [i, f[0, array.size - 1, r: r].map{ |_| _.map &:item }]
+        end.tap do |ratios|
+          fail unless ratios.first.last.size == array.map(&:value).uniq.size
+          ratios.each_cons(2){ |(_, a), (_, b)| fail if a.size < b.size }
+        end.chunk{ |_, split| split.size }.each do |_, splits|
+          # v = [splits.first.first, splits.size]
+          v = [
+            splits.first.first,
+            -splits.map(&:last).mean do |split|
+              split.map(&:size).then do |sizes|
+                sizes.map do |size|
+                  (sizes.max - sizes.min).then do |_|
+                    # otherwise it makes Infinity, then NaN
+                    _.zero? ? size : size.fdiv(_)
+                  end
+                end.then do |sizes|
+                  # σ²
+                  sizes.mean{ |_| (_ - sizes.mean) ** 2 }
+                end
+              end
+            end
+          ]
+          pcbr.store splits.map(&:last), v
+        end
+        (best, _, score) = pcbr.table.reverse.max_by(&:last)
+        fail unless 0 < score
+        fail if 1 < best.uniq.size
+        best.first
+      end
+
+    end
+
+    # https://en.wikipedia.org/wiki/Injective_function
+    # picks the closest item from `codomain` for each `domain` element
+    # raises exception if picks intersect
+    # @param domain [Array]
+    # @param codomain [Array]
+    # @return [Array<Array(Object, Object)>]
+    # @yieldparam domain_item, codomain_item [Object]
+    # @yieldreturn [Numeric] distance
+    def self.injective_mapping domain, codomain
+      domain.map do |a|
+        [a, codomain.each_index.group_by{ |i| yield a, codomain[i] }.min.last.assert_one]
+      end.tap do |_|
+        fail if _.map(&:last).uniq!
+      end.map do |a, b|
+        [a, codomain[b]]
+      end
+    end
+
+  end
+end

data/lib/dsr.rb

@@ -1,11 +1,13 @@
 module DSR
 
-  Struct = ::Struct.new :text, :left, :bottom, :right, :top, :width, :height
-  private_constant :Struct
+  StructLinkable = ::Struct.new :ref, :left, :bottom, :right, :top
+  private_constant :StructLinkable
+  StructWithText = ::Struct.new :text, *StructLinkable.members.drop(1), :width, :height
+  private_constant :StructWithText
 
   class Texts < Array
-    def find_all_by_text text
-      self.class.new select{ |_| text == _.text }
+    def find_all text_or_regex
+      self.class.new select{ |_| text_or_regex === _.text }
     end
     def select_intersecting_vertically_with item
       self.class.new (self-[item]).select{ |_| _.bottom >= item.top && _.top <= item.bottom }
@@ -41,7 +43,7 @@ module DSR
         }
       }
     end["textAnnotations"].map do |text|
-      Struct.new text["description"],
+      StructWithText.new text["description"],
                  text["boundingPoly"]["vertices"].map{ |_| _["x"] }.min,
                  text["boundingPoly"]["vertices"].map{ |_| _["y"] }.max,
                  text["boundingPoly"]["vertices"].map{ |_| _["x"] }.max,
@@ -51,6 +53,7 @@ module DSR
     end )
   end
 
+  # TODO: refactor or deprecate?
   def self.link headers, array, direction, alignment, *priority
     l, r = case direction
     when :horizontal ; %i{ left right }
@@ -59,8 +62,8 @@ module DSR
     end
     headers = headers.sort_by(&l).map(&:dup)
     headers.each_cons(2){ |a, b| a[r], b[l] = [a[r], b[l]].max, [a[r], b[l]].min }
-    headers.first[l] = -Float::INFINITY
-    headers.last[r] = +Float::INFINITY
+    headers.first[l] = -::Float::INFINITY
+    headers.last[r] = +::Float::INFINITY
     headers.unshift headers.delete_at headers.index{ |_| priority.include? _.text } unless priority.empty?   # TODO: document/explain this
     array.sort_by(&l).each_with_object([]) do |cell, a|
       i = headers.public_send(alignment){ |_| (_[l].._[r]).include?((cell[l]+cell[r])/2) }
@@ -68,6 +71,7 @@ module DSR
       a[i] << cell
     end
   end
+
   def self.pdf2struct object
     require "hexapdf"
     processor = Class.new HexaPDF::Content::Processor do
@@ -78,7 +82,7 @@ module DSR
       end
       def show_text str
         boxes = decode_text_with_positioning str
-        @texts.push Struct.new boxes.string,
+        @texts.push StructWithText.new boxes.string,
           boxes.lower_left[0], -boxes.lower_left[1],
           boxes.upper_right[0], -boxes.upper_right[1],
           boxes.upper_right[0] - boxes.lower_left[0],
@@ -103,4 +107,30 @@ module DSR
     end.reject &:empty?
   end
 
+  def self.nodes2struct nodes
+    nodes.map do |node|
+      StructLinkable.new(node, *::JSON.load(node.page.evaluate(<<~HEREDOC, node)))
+        ( function(node) {
+          var x = scrollX, y = scrollY;
+          var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
+          rect.top += scrollY;
+          rect.left += scrollX;
+          var t = JSON.stringify( [rect.left, rect.bottom, rect.right, rect.top] );
+          return t;
+        } )(arguments[0])
+      HEREDOC
+    end
+  end
+
+  def self.capybara2struct nodes
+    nodes.map do |node|
+      rect = node.rect
+      StructLinkable.new node,
+        rect.x,
+        rect.y + rect.height,
+        rect.x + rect.width,
+        rect.y
+    end
+  end
+
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: dsr
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-16 00:00:00.000000000 Z
+date: 2026-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nakischema
+  name: pcbr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -24,21 +24,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: hexapdf
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
+description:
 email: nakilon@gmail.com
 executables: []
 extensions: []
@@ -47,12 +33,13 @@ files:
 - LICENSE
 - dsr.gemspec
 - lib/dsr.rb
-homepage: 
+- lib/dsr/recognizer.rb
+homepage:
 licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/nakilon/dsr
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -67,9 +54,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.25
-signing_key: 
+rubygems_version: 3.2.0
+signing_key:
 specification_version: 4
-summary: "[WIP] Document Structure Recognizer -- currently a collection of common
-  routines I use to build A.I.s"
+summary: Document Structure Recognizer -- a collection of common routines to build
+  AIs
 test_files: []