stupidedi 1.1.0

data/lib/stupidedi/builder/constraint_table.rb

@@ -0,0 +1,418 @@
+module Stupidedi
+  module Builder
+
+    #
+    # The {ConstraintTable} is a data structure that contains one or more
+    # {Instruction} values for the same segment identifier. Each concrete
+    # subclass implements different strategies for narrowing down the
+    # {Instruction} list.
+    #
+    # Reducing the number of valid {Instruction} values is important because
+    # executing more than one {Instruction} creates a non-deterministic state --
+    # more than one valid parse tree exists -- which slows the parser. Most
+    # often there is only one valid {Instruction} but the parser cannot
+    # (efficiently or at all) narrow the tree down without evaluating the
+    # constraints declared by each {Instruction}'s {Schema::SegmentUse}, which
+    # is done here.
+    #
+    class ConstraintTable
+
+      # @return [Array<Instruction>]
+      abstract :matches, :args => %w(segment_tok)
+
+      #
+      # Performs no filtering of the {Instruction} list. This is used when there
+      # already is a single {Instruction} or when a {Reader::SegmentTok} doesn't
+      # provide any more information to filter the list.
+      #
+      class Stub < ConstraintTable
+        def initialize(instructions)
+          @instructions = instructions
+        end
+
+        # @return [Array<Instruction>]
+        def matches(segment_tok, strict)
+          @instructions
+        end
+      end
+
+      #
+      # Chooses the {Instruction} that pops the fewest number of states. For
+      # example, in the X222 837P an HL segment signals the start of a new
+      # 2000 loop, but may or may not begin a new Table 2 -- the specifications
+      # aren't actually clear. This rule will avoid creating a new Table 2 if
+      # possible, and instead create a new 2000 loop under the current Table 2.
+      #
+      class DepthBased < ConstraintTable
+        def initialize(instructions)
+          @instructions = instructions
+        end
+
+        # @return [Array<Instruction>]
+        def matches(segment_tok, strict)
+          @__matches ||= begin
+            deepest = @instructions.head
+
+            @instructions.tail.each do |i|
+              if i.pop_count < deepest.pop_count
+                deepest = i
+              end
+            end
+
+            deepest.cons
+          end
+        end
+      end
+
+      #
+      # Chooses the subset of {Instruction} values based on the distinguishing
+      # values allowed by each {Schema::SegmentUse}. For instance, there are
+      # often several loops that begin with `NM1`, which are distinguished by
+      # the qualifier in element `NM101`.
+      #
+      class ValueBased < ConstraintTable
+        def initialize(instructions)
+          @instructions = instructions
+        end
+
+        # @return [Array<Instruction>]
+        def matches(segment_tok, strict)
+          invalid = true  # Were all possibly distinguishing elements invalid?
+          present = false # Were any possibly distinguishing elements present?
+
+          @__basis ||= basis(deepest(@instructions))
+          @__basis.head.each do |(n, m), map|
+            value = deconstruct(segment_tok.element_toks, n, m)
+
+            case value
+            when nil, :not_used, :default
+              # ignore
+            else
+              singleton = map.at(value)
+              present   = true
+
+              unless singleton.nil?
+                return singleton
+              else
+                if strict
+                  designator = "#{segment_tok.id}#{'%02d' % (n + 1)}"
+                  designator << "-%02d" % m unless m.nil?
+
+                  raise ArgumentError,
+                    "#{value.inspect} is not allowed in #{designator}"
+                end
+              end
+            end
+          end
+
+          # If we reach this line, none of the present elements could, on its
+          # own, narrow the search space to a single Instruction. We now test
+          # the combination of elements to iteratively narrow the search space
+          space = @instructions
+
+          # @todo: These filters should be ordered by probable effectiveness,
+          # so we narrow the search space by the largest amount in the fewest
+          # number of steps.
+          @__basis.last.each do |(n, m), map|
+            value = deconstruct(segment_tok.element_toks, n, m)
+
+            unless value.nil?
+              subset  = map.at(value)
+              present = true
+
+              unless subset.blank?
+                invalid = false
+                space  &= subset
+
+                if space.length <= 1
+                  return space
+                end
+              else
+                if strict
+                  designator = "#{segment_tok.id}#{'%02d' % n}"
+                  designator << "-%02d" % m unless m.nil?
+
+                  raise ArgumentError,
+                    "#{value.inspect} is not allowed in #{designator}"
+                end
+              end
+            end
+          end
+
+          if invalid and present
+            []
+          else
+            space
+          end
+        end
+
+      private
+
+        # Resolve conflicts between instructions that have identical SegmentUse
+        # values. For each SegmentUse, this chooses the Instruction that pops
+        # the fewest number of states.
+        #
+        # @return [Array<Instruction>]
+        def deepest(instructions)
+          deepest = Hash.new
+
+          instructions.each do |i|
+            key = i.segment_use.object_id
+
+            if deepest.defined_at?(key)
+              if deepest.at(key).pop_count > i.pop_count
+                deepest[key] = i
+              end
+            else
+              deepest[key] = i
+            end
+          end
+
+          deepest.values
+        end
+
+        # @return [Array(Array<(Integer, Integer, Map)>, Array<(Integer, Integer, Map)>)]
+        def basis(instructions)
+          disjoint_elements = []
+          distinct_elements = []
+
+          # The first SegmentUse is used to represent the structure that must
+          # be shared by the others: number of elements and type of elements
+          element_uses = instructions.head.segment_use.definition.element_uses
+
+          # Iterate over each element across all SegmentUses (think columns)
+          #   NM1*[IL]*[  ]*..*..*..*..*..*[  ]*..*..*{..}*..
+          #   NM1*[40]*[  ]*..*..*..*..*..*[  ]*..*..*{..}*..
+          element_uses.length.times do |n|
+            if element_uses.at(n).composite?
+              ms = 0 .. element_uses.at(n).definition.component_uses.length - 1
+            else
+              ms = [nil]
+            end
+
+            # If this is a composite element, we iterate over each component.
+            # Otherwise this loop iterates once with the index {m} set to nil.
+            ms.each do |m|
+              last  = nil        # the last subset we examined
+              total = Sets.empty # the union of all examined subsets
+
+              distinct = false
+              disjoint = true
+
+              instructions.each do |i|
+                element_use = i.segment_use.definition.element_uses.at(n)
+
+                unless m.nil?
+                  element_use = element_use.definition.component_uses.at(m)
+                end
+
+                allowed_vals = element_use.allowed_values
+
+                # We want to know if every instruction's set of allowed values
+                # is disjoint (with one another). Instead of comparing each set
+                # with every other set, which takes (N-1)! comparisons, we can
+                # do it in N steps.
+                disjoint &&= allowed_vals.disjoint?(total)
+
+                # We also want to know if one instruction's set of allowed vals
+                # contains elements that aren't present in at least one other
+                # set. The opposite condition is easy to test: all sets contain
+                # the same elements (are equal). So we can similarly, check this
+                # condition in N steps rather than (N-1)!
+                distinct ||= allowed_vals != last unless last.nil?
+
+                total = allowed_vals.union(total)
+                last  = allowed_vals
+              end
+
+            # puts "#{n}.#{m}: disjoint(#{disjoint}) distinct(#{distinct})"
+
+              if disjoint
+                # Since each instruction's set of allowed values is disjoint, we
+                # can build a function/hash that returns the single instruction,
+                # given one of the values. When given a value outside the set of
+                # all (combined) values, it returns nil.
+                disjoint_elements << [[n, m], build_disjoint(total, n, m, instructions)]
+              elsif distinct
+                # Not all instructions have the same set of allowed values. So
+                # we can build a function/hash that accepts one of the values
+                # and returns the subset of the instructions where that value
+                # can occur. This might be some, none, or all of the original
+                # instructions, so clearly this provides less information than
+                # if each allowed value set was disjoint.
+
+                # Currently disabled (and untested) because it doesn't look like
+                # any of the HIPAA schemas would use this -- so testing it would
+                # be a pain.
+                #
+                distinct_elements << [[n, m], build_distinct(total, n, m, instructions)]
+              end
+            end
+          end
+
+          [disjoint_elements, distinct_elements]
+        end
+
+        # @return [Hash<String, Array<Instruction>>]
+        def build_disjoint(total, n, m, instructions)
+          if total.finite?
+            # The sum of all allowed value sets is finite, so we know that each
+            # individual allowed value set is finite (we can iterate over it).
+            map = Hash.new
+
+            instructions.each do |i|
+              element_use = i.segment_use.definition.element_uses.at(n)
+
+              unless m.nil?
+                element_use = element_use.definition.component_uses.at(m)
+              end
+
+              allowed_vals = element_use.allowed_values
+              allowed_vals.each{|v| map[v] = i.cons }
+            end
+
+            map
+          else
+            # At least one of allowed value sets is infinite. This happens when
+            # it is RelativeComplement, which declares the values that are *not*
+            # allowed in the set.
+            map = Hash.new{|h,k| h[k] = instructions }
+
+            instructions.each do |i|
+              element_use = i.segment_use.definition.element_uses.at(n)
+              unless m.nil?
+                element_use = element_use.definition.component_uses.at(m)
+              end
+
+              allowed_vals = element_use.allowed_values
+
+              unless allowed_vals.finite?
+                allowed_vals.complement.each{|v| map[v] -= i }
+              end
+            end
+
+            # Clear the default_proc so accesses don't change the Hash
+            map.default = instructions
+            map
+          end
+        end
+
+        # @return [Hash<String, Array<Instruction>>]
+        def build_distinct(total, n, m, instructions)
+          if total.finite?
+            # The sum of all allowed value sets is finite, so we know that each
+            # individual allowed value set is finite (we can iterate over it).
+            map = Hash.new{|h,k| h[k] = [] }
+
+            instructions.each do |i|
+              element_use  = i.segment_use.definition.element_uses.at(n)
+
+              unless m.nil?
+                element_use = element_use.definition.component_uses.at(m)
+              end
+
+              allowed_vals = element_use.allowed_values
+              allowed_vals.each{|v| map[v] << i }
+            end
+
+            # Clear the default_proc so accesses don't change the Hash
+            map.default = []
+            map
+          else
+            # At least one of allowed value sets is infinite. This happens when
+            # it is RelativeComplement, which declares the values that are *not*
+            # allowed in the set.
+            map = Hash.new{|h,k| h[k] = instructions }
+
+            instructions.each do |i|
+              element_use  = i.segment_use.definition.element_uses.at(n)
+
+              unless m.nil?
+                element_use = element_use.definition.component_uses.at(m)
+              end
+
+              allowed_vals = element_use.allowed_values
+
+              unless allowed_vals.finite?
+                allowed_vals.complement.each{|v| map[v] -= i }
+              end
+            end
+
+            # Clear the default_proc so accesses don't change the Hash
+            map.default = instructions
+            map
+          end
+        end
+
+        # Return the value of the `m`-th elemnt, or if `n` is not nil, return
+        # the value of the `n`-th component from the `n`-th element. When the
+        # value is blank, the function returns `nil`.
+        #
+        # @param [Array<Reader::SimpleElementTok, Reader::CompositeElementTok>] element_toks
+        # @param [Integer] m
+        # @param [Integer, nil] n
+        #
+        # @return [String, nil]
+        def deconstruct(element_toks, m, n)
+          element_tok = element_toks.at(m)
+          element_tok = element_tok.element_toks.at(0) if element_tok.try(:repeated?)
+
+          if element_tok.blank?
+            nil
+          elsif n.nil?
+            element_tok.value
+          else
+            element_tok = element_tok.component_toks.at(n)
+
+            if element_tok.blank?
+              nil
+            else
+              element_tok.value
+            end
+          end
+        end
+      end
+
+    end
+
+    class << ConstraintTable
+      # @group Constructors
+      #########################################################################
+
+      # Given a list of {Instruction} values for the same segment identifier,
+      # this method constructs the appropriate concrete subclass of
+      # {ConstraintTable}.
+      #
+      # @param [Array<Instruction>] instructions
+      # @return [ConstraintTable]
+      def build(instructions)
+        if instructions.length <= 1
+          ConstraintTable::Stub.new(instructions)
+        elsif instructions.any?{|i| i.segment_use.nil? } and
+          not instructions.all?{|i| i.segment_use.nil? }
+          # When one of the instructions has a nil segment_use, it means
+          # the SegmentUse is determined when pushing the new state. There
+          # isn't a way to know the segment constraints from here.
+          ConstraintTable::Stub.new(instructions)
+        else
+          segment_uses = instructions.map{|i| i.segment_use }
+
+          if segment_uses.map{|u| u.object_id }.uniq.length <= 1
+            # The same SegmentUse may appear more than once, because the
+            # segment can be placed at different levels in the tree. If
+            # all the instructions have the same SegmentUse, they also have
+            # the same element constraints so we can't use them to narrow
+            # down the instruction list.
+            ConstraintTable::DepthBased.new(instructions)
+          else
+            ConstraintTable::ValueBased.new(instructions)
+          end
+        end
+      end
+
+      # @endgroup
+      #########################################################################
+    end
+
+  end
+end

data/lib/stupidedi/builder/generation.rb

@@ -0,0 +1,112 @@
+module Stupidedi
+  module Builder
+
+    module Generation
+
+      # @return [(StateMachine, Either<Reader::TokenReader>)]
+      def read(reader)
+        machine = self
+        reader  = reader.read_segment
+
+        while reader.defined?
+          reader = reader.flatmap do |segment_tok, reader|
+            machine, reader =
+              machine.insert(segment_tok, reader)
+
+            reader.read_segment
+          end
+        end
+
+        return machine, reader
+      end
+
+      # @return [(StateMachine, Reader::TokenReader)]
+      def insert(segment_tok, reader)
+        active = []
+
+        @active.each do |zipper|
+          state        = zipper.node
+          instructions = state.instructions.matches(segment_tok)
+
+          if instructions.empty?
+            active << zipper.append(FailureState.mksegment(segment_tok, state))
+            next
+          end
+
+          instructions.each do |op|
+            if op.push.nil?
+              table = zipper.node.instructions
+              value = zipper.node.zipper
+              state = zipper
+
+              op.pop_count.times do
+                value = value.up
+                state = state.up
+              end
+
+              # Create a new AbstractState node that has a new InstructionTable
+              # and also points to a new AbstractVal tree (with the new segment)
+              segment   = AbstractState.mksegment(segment_tok, op.segment_use)
+              successor = state.append(state.node.copy(
+                :zipper       => value.append(segment),
+                :instructions => table.pop(op.pop_count).drop(op.drop_count)))
+
+              unless op.pop_count.zero? or reader.stream?
+                # More general than checking if segment_tok is an ISE/GE segment
+                unless reader.separators.eql?(successor.node.separators) \
+                  and reader.segment_dict.eql?(successor.node.segment_dict)
+                  reader = reader.copy \
+                    :separators   => successor.node.separators,
+                    :segment_dict => successor.node.segment_dict
+                end
+              end
+            else
+              table = zipper.node.instructions
+              value = zipper.node.zipper
+              state = zipper
+
+              op.pop_count.times do
+                value = value.up
+                state = state.up
+              end
+
+              parent = state.node.copy \
+                :zipper       => value,
+                :children     => [],
+                :separators   => reader.separators,
+                :segment_dict => reader.segment_dict,
+                :instructions => table.pop(op.pop_count).drop(op.drop_count)
+
+              state = state.append(parent) unless state.root?
+
+              successor = op.push.push(state, parent, segment_tok, op.segment_use, @config)
+
+              # More general than checking if segment_tok is an ISA/GS segment
+              unless reader.separators.eql?(successor.node.separators) \
+                and reader.segment_dict.eql?(successor.node.segment_dict)
+                reader = reader.copy \
+                  :separators   => successor.node.separators,
+                  :segment_dict => successor.node.segment_dict
+              end
+            end
+
+            active << successor
+          end
+        end
+
+        return StateMachine.new(@config, active), reader
+      end
+
+      # @return [StateMachine]
+      def replace(segment_tok, reader)
+        # @todo
+      end
+
+      # @return [StateMachine]
+      def remove
+        # @todo
+      end
+
+    end
+  end
+end

data/lib/stupidedi/builder/instruction.rb

@@ -0,0 +1,102 @@
+module Stupidedi
+  module Builder
+
+    class Instruction
+      include Inspect
+
+      # The segment identifier to which this {Instruction} applies
+      #
+      # @return [Symbol]
+      attr_reader :segment_id
+
+      # The segment use contains helpful information about the context of the
+      # segment within a definition tree (eg the parent structure's definition).
+      # It also enumerates the allowed values for segment qualifiers, which is
+      # used to minimize non-determinism.
+      #
+      # @return [Schema::SegmentUse]
+      attr_reader :segment_use
+
+      # This indicates the number of levels to ascend and terminate within the
+      # tree before storing the segment.
+      #
+      # @return [Integer]
+      attr_reader :pop_count
+
+      # This controls the allowed order in which structures may occur, by
+      # indicating the number of instructions to remove from the beginning of
+      # the instruction table.
+      #
+      # Repeatable structures have a value that ensures that their instruction
+      # remains in the successor table.
+      #
+      # Sibling structures that have the same position (as defined by their
+      # {Schema::SegmentUse}) will have equal drop_count values such that all of
+      # the sibling instructions remain in the successor table when any one of
+      # them is executed.
+      #
+      # @return [Integer]
+      attr_reader :drop_count
+
+      # This indicates that a child node should be added to the tree, which
+      # will then contain the segment.
+      #
+      # When a segment indicates the start of a child structure, the class
+      # indicated by this attribute is expected to respond to `push` by
+      # creating a new {AbstractState}.
+      #
+      # @return [Zipper::AbstractCursor]
+      attr_reader :push
+
+      def initialize(segment_id, segment_use, pop, drop, push)
+        @segment_id, @segment_use, @pop_count, @drop_count, @push =
+          segment_id || segment_use.definition.id, segment_use, pop, drop, push
+      end
+
+      # @return [Instruction]
+      def copy(changes = {})
+        Instruction.new \
+          changes.fetch(:segment_id, @segment_id),
+          changes.fetch(:segment_use, @segment_use),
+          changes.fetch(:pop_count, @pop_count),
+          changes.fetch(:drop_count, @drop_count),
+          changes.fetch(:push, @push)
+      end
+
+      # @return [void]
+      def pretty_print(q)
+        id = '% 3s' % @segment_id.to_s
+
+        unless @segment_use.nil?
+          width = 18
+          name  = @segment_use.definition.name
+
+          # Truncate the segment name to `width` characters
+          if name.length > width - 2
+            id << ": #{name.slice(0, width - 2)}.."
+          else
+            id << ": #{name.ljust(width)}"
+          end
+        end
+
+        q.text "Instruction[#{'% 3s' % id}]"
+
+        q.group(6, "(", ")") do
+          q.breakable ""
+
+          q.text "pop: #{@pop_count},"
+          q.breakable
+
+          q.text "drop: #{@drop_count}"
+
+          unless @push.nil?
+            q.text ","
+            q.breakable
+            q.text "push: #{@push.try{|c| c.name.split('::').last}}"
+          end
+        end
+      end
+    end
+
+  end
+end