gov_codes 0.1.1 → 0.1.2

data/lib/gov_codes/dafecd/ri_sdi/index_builder.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require_relative "../patterns"
+require_relative "../publication"
+require_relative "../record_splitter"
+require_relative "../specialty_parser"
+require_relative "../shredout_parser"
+require_relative "../shredout_acronyms"
+require_relative "../title_degluer"
+require_relative "config"
+require_relative "section_slicer"
+require_relative "sdi_section_splitter"
+require_relative "sdi_card_parser"
+require_relative "ri_list_parser"
+
+module GovCodes
+  module Dafecd
+    module RiSdi
+      # Assembles the RI and SDI records of one publication into a single
+      # code-keyed index and surfaces the reconciliation data the CLI reports.
+      #
+      # Every section named by the Config is sliced out and parsed by the format
+      # it uses: SDI sections by the SdiSectionSplitter + SdiCardParser (plus the
+      # one embedded CEM ladder record, parsed by the AFSC SpecialtyParser and
+      # keyed by its CEM code), RI sections by the RiListParser.
+      #
+      # Verification gate: every emitted code must appear verbatim in the source,
+      # and every emitted acronym must appear as a "(ACRONYM)" parenthetical in
+      # its record's raw title (whitespace/dash/case tolerant). Titles are
+      # extracted verbatim (only sanitized) and then de-glued via human-verified
+      # overrides supplied by an injected TitleDegluer: each applied override is
+      # gated against the raw source title (spacing/case only), so drift lands in
+      # #unverified_titles and fails the build; a code with no override keeps its
+      # verbatim title and is reported in #codes_needing_deglue (not an error).
+      # #unverified? must be false before anything is written.
+      #
+      # Acronyms are captured from a title's trailing parenthetical, EXCEPT for
+      # the per-publication Config#acronym_exclusions (organization/sub-phrase
+      # abbreviations). Every candidate — shipped or excluded — is reported via
+      # #acronym_candidates for review.
+      class IndexBuilder
+        # A trailing parenthetical whose sole token is an uppercase abbreviation.
+        # Mirrors Dafecd::IndexBuilder::ACRONYM_PATTERN.
+        ACRONYM_PATTERN = /\(([A-Z][A-Z0-9]{1,7})\)\s*\z/
+
+        UNICODE_DASHES = Patterns::UNICODE_DASHES
+
+        attr_reader :index, :dropped_records, :collisions, :acronym_candidates,
+          :unverified_codes, :unverified_titles, :unverified_acronyms,
+          :codes_needing_deglue, :section_counts, :section_codes, :sequence_numbers
+
+        def initialize(full_text, config: Config.dafecd, degluer: TitleDegluer.empty)
+          @full_text = full_text
+          @config = config
+          @degluer = degluer
+        end
+
+        def build
+          @index = {}
+          @dropped_records = []
+          @collisions = []
+          @acronym_candidates = []
+          @unverified_codes = []
+          @unverified_titles = []
+          @unverified_acronyms = []
+          @codes_needing_deglue = []
+          @section_counts = Hash.new(0)
+          @section_codes = Hash.new { |h, k| h[k] = [] }
+          @sequence_numbers = Hash.new { |h, k| h[k] = [] }
+
+          SectionSlicer.new(@full_text, config: @config).sections.each do |section|
+            case section.kind
+            when :sdi then build_sdi(section)
+            when :ri then build_ri(section)
+            end
+          end
+
+          @index.each { |code, entry| apply_override(code, entry) }
+          @index.each { |code, entry| capture_acronym(code, entry) }
+          @index.each { |code, entry| verify(code, entry) }
+          @index
+        end
+
+        # True if any emitted value failed verification; the CLI must not write.
+        def unverified?
+          @unverified_codes.any? || @unverified_titles.any? || @unverified_acronyms.any?
+        end
+
+        # @return [Hash{Symbol=>Hash}] force => {present:, missing:, duplicates:}
+        def sequence_report
+          @sequence_numbers.transform_values do |numbers|
+            present = numbers.uniq.sort
+            duplicates = numbers.group_by(&:itself).select { |_, v| v.size > 1 }.keys.sort
+            missing = present.empty? ? [] : ((present.first..present.last).to_a - present)
+            {present: present, missing: missing, duplicates: duplicates}
+          end
+        end
+
+        # @return [Array<Hash>] {code:, raw_title:} in code order, for the
+        #   governed de-glue pass (titles are NOT cleaned here).
+        def title_inventory
+          @index.sort_by { |code, _| code.to_s }.map do |code, entry|
+            {code: code, name: entry[:name], raw_title: entry[:raw_title], glued: entry[:glued_title]}
+          end
+        end
+
+        private
+
+        # --- SDI sections --------------------------------------------------------
+
+        def build_sdi(section)
+          SdiSectionSplitter.new(section.text, config: @config).records.each do |record|
+            entries = SdiCardParser.new(record, config: @config).entries
+
+            if entries.empty?
+              if record.match?(@config.cem)
+                entries = [cem_entry(record)]
+              else
+                note_sdi_drop(record)
+                next
+              end
+            end
+
+            entries.each { |entry| add(section, entry) }
+          end
+        end
+
+        # The single embedded CEM ladder record (Premier Honor Guard, 8G000),
+        # parsed by the AFSC pipeline and keyed by its CEM code.
+        def cem_entry(record)
+          header = SpecialtyParser.new(record, publication: Publication.dafecd).parse
+          shredouts = ShredoutParser.new(record, publication: Publication.dafecd).parse
+          {
+            code: header[:cem_code],
+            name: header[:name],
+            raw_title: header[:raw_title],
+            changed_date: header[:changed_date],
+            glued_title: header[:glued_title],
+            shredouts: shredouts
+          }
+        end
+
+        def note_sdi_drop(record)
+          # Detect the anchor with the SAME structural regexes the splitter/parser
+          # use — a standalone "SDI <code>" line OR an inline "<code>,<title>" line
+          # (the latter now tolerates a missing "SDI " prefix), so a future dropped
+          # bare-format card is logged rather than silently skipped.
+          anchor = record.lines.find do |line|
+            line.match?(@config.sdi_anchor) || line.match?(@config.sdi_inline_anchor)
+          end
+          return unless anchor
+          @dropped_records << {
+            section: :sdi,
+            first_line: anchor.strip,
+            reason: "SDI anchor produced no valid entry"
+          }
+        end
+
+        # --- RI sections ---------------------------------------------------------
+
+        def build_ri(section)
+          RiListParser.new(section.text, config: @config).entries.each do |entry|
+            @sequence_numbers[section.force] << entry[:number]
+            add(section, entry)
+          end
+        end
+
+        # --- Index assembly ------------------------------------------------------
+
+        def add(section, entry)
+          code = entry[:code].to_sym
+          key = [section.force, section.kind]
+
+          if @index.key?(code)
+            @collisions << {code: code, section: key, kept: @index[code][:name], discarded: entry[:name]}
+            return
+          end
+
+          @index[code] = build_entry(section, entry)
+          @section_counts[key] += 1
+          @section_codes[key] << code
+        end
+
+        def build_entry(section, entry)
+          shredouts = entry[:shredouts] || {}
+          {
+            force: section.force,
+            kind: section.kind,
+            name: entry[:name],
+            raw_title: entry[:raw_title],
+            changed_date: entry[:changed_date],
+            glued_title: entry[:glued_title],
+            shredouts: shredouts,
+            shredout_acronyms: ShredoutAcronyms.from_table(shredouts)
+          }
+        end
+
+        # --- Title de-gluing (verified overrides) --------------------------------
+
+        # Apply a verified de-glued title override to +entry[:name]+, enforcing
+        # the same invariant as the AFSC pipeline: an override may differ from the
+        # raw source title only in spacing/case (TitleDegluer.matches_source?).
+        # Drift (a changed letter/digit/punctuation, or a stale rename) is
+        # recorded in #unverified_titles and never silently applied. A code with
+        # no override keeps its verbatim (glued) title and is merely reported in
+        # #codes_needing_deglue -- unlike the AFSC titles, full override coverage
+        # is not required for the build to pass.
+        def apply_override(code, entry)
+          override = @degluer.override_for(code)
+
+          if override.nil?
+            @codes_needing_deglue << code if entry[:name]
+            return
+          end
+
+          if TitleDegluer.matches_source?(override, entry[:raw_title])
+            entry[:name] = override
+          else
+            @unverified_titles << {
+              code: code,
+              applied: override,
+              raw_title: entry[:raw_title],
+              reason: "override differs from source title by more than spacing/case"
+            }
+          end
+        end
+
+        # --- Acronyms ------------------------------------------------------------
+
+        def capture_acronym(code, entry)
+          name = entry[:name]
+          return if name.nil?
+          match = name.match(ACRONYM_PATTERN)
+          return unless match
+
+          acronym = match[1]
+          shipped = !@config.acronym_exclusions.include?(code)
+          @acronym_candidates << {code: code, acronym: acronym, name: name, shipped: shipped}
+          entry[:acronym] = acronym if shipped
+        end
+
+        # --- Verification gate ---------------------------------------------------
+
+        def verify(code, entry)
+          @unverified_codes << code.to_s unless @full_text.include?(code.to_s)
+
+          raw = entry[:raw_title]
+          if raw && !norm(@full_text).include?(norm(raw))
+            @unverified_titles << {code: code, raw_title: raw}
+          end
+
+          acronym = entry[:acronym]
+          if acronym && !norm(raw).include?(norm("(#{acronym})"))
+            @unverified_acronyms << acronym
+          end
+        end
+
+        # Whitespace-, dash-, and case-insensitive normalization for the gate.
+        def norm(str)
+          str.to_s.gsub(UNICODE_DASHES, "-").gsub(/\s+/, "").downcase
+        end
+      end
+    end
+  end
+end

data/lib/gov_codes/dafecd/ri_sdi/ri_list_parser.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require_relative "../patterns"
+require_relative "../shredout_parser"
+require_relative "config"
+require_relative "change_date"
+require_relative "title"
+
+module GovCodes
+  module Dafecd
+    module RiSdi
+      # Parses a flat, numbered Reporting Identifiers (RI) list into per-code
+      # entries.
+      #
+      # Each top-level list item ("<n>. <code>, <title>. <date>") is one record,
+      # delimited by the next top-level number. The anchor tolerates the source's
+      # pdf-reader glue and decoration: an optional run of spaces (or none) after
+      # "<n>.", an optional leading decorative star, a comma OR a bare space
+      # before the title, and a list number glued to the code ("34.9T000").
+      #
+      # The title runs from the code to the FIRST of a change-date parenthetical
+      # or a sentence-ending period, and may wrap across physical lines. The
+      # officer directory omits the comma and always ends the title at the first
+      # period ("90G0 General Officer. Use this identifier ...").
+      #
+      # Rich records carry sub-paragraphs, sometimes including a
+      # "<n>.x Specialty Shredouts:" table, which is parsed by the reused
+      # ShredoutParser. Records are returned in document order; #entries also
+      # carries each item's :number so callers can run the 1..N completeness check.
+      class RiListParser
+        DECORATIVE = Patterns::DECORATIVE
+
+        # A sub-paragraph line ("24.1.", "56.2.1.") — where a record's body begins.
+        # Distinct from the anchor line, whose "<n>." is immediately followed by a
+        # code (letter in the second position), not another number-and-dot.
+        SUBPARAGRAPH = /\A\s*\d+\.\d/
+
+        # The title's trailing change-date parenthetical.
+        DATE_PAREN = /\((?:Changed|Established|Effective|Change)\b/
+
+        # A sentence-ending period: a "." followed by whitespace or end of string.
+        SENTENCE_PERIOD = /\.(?=\s|\z)/
+
+        def initialize(text, config: Config.dafecd)
+          @config = config
+          @lines = text.lines.reject { |line| line =~ config.header }
+        end
+
+        # @return [Array<Hash>] one entry per RI code, in document order
+        def entries
+          records.filter_map { |record| parse(record) }
+        end
+
+        private
+
+        # Split the section into records at each top-level anchor line.
+        def records
+          records = []
+          current = nil
+          @lines.each do |line|
+            if line.match?(@config.ri_anchor)
+              current = +""
+              records << current
+            end
+            current << line if current
+          end
+          records
+        end
+
+        def parse(record)
+          lines = record.lines
+          m = lines.first.match(@config.ri_anchor)
+          return nil unless m
+
+          number = m[1].to_i
+          code = m[2]
+          region = title_region(m[3], lines.drop(1))
+
+          raw_title = extract_title(region)
+          name = raw_title && Title.titleize(raw_title)
+
+          {
+            number: number,
+            code: code,
+            name: name,
+            raw_title: raw_title,
+            changed_date: ChangeDate.extract(region),
+            glued_title: Title.glued?(name),
+            shredouts: shredouts(record)
+          }
+        end
+
+        # The joined header region: the anchor line's title remainder plus every
+        # continuation line up to the first sub-paragraph, sanitized and with page
+        # numbers dropped. This holds the title and its trailing date.
+        def title_region(anchor_rest, rest_lines)
+          parts = [Title.sanitize(anchor_rest)]
+          rest_lines.each do |line|
+            break if line.match?(SUBPARAGRAPH)
+            stripped = Title.sanitize(line)
+            next if stripped.empty?
+            next if stripped.match?(/\A\d+\z/) # bare page number
+            parts << stripped
+          end
+          parts.reject(&:empty?).join(" ")
+        end
+
+        # The title: the region up to the first date parenthetical or sentence
+        # period, whichever comes first; the whole region when neither is present
+        # (e.g. "DAFWorld Class Athlete Program (WCAP)").
+        def extract_title(region)
+          cut = [region =~ DATE_PAREN, region =~ SENTENCE_PERIOD].compact.min
+          title = (cut ? region[0...cut] : region).sub(/[\s.,]+\z/, "").strip
+          title.empty? ? nil : title
+        end
+
+        def shredouts(record)
+          ShredoutParser.new(
+            record, publication: @config, pair: ShredoutParser::RI_SDI_PAIR
+          ).parse
+        end
+      end
+    end
+  end
+end

data/lib/gov_codes/dafecd/ri_sdi/sdi_card_parser.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require_relative "../patterns"
+require_relative "../shredout_parser"
+require_relative "config"
+require_relative "change_date"
+require_relative "title"
+
+module GovCodes
+  module Dafecd
+    module RiSdi
+      # Parses ONE "SDI card" record block into per-code entries.
+      #
+      # A card block is a run of one or more "SDI <code>" anchor lines followed by
+      # a shared body: a title (on its own line, or inline after the code), a
+      # change-date annotation, numbered sections, and (occasionally) a
+      # "Specialty Shredouts" table. Enlisted codes are 5 chars ("8A200"); officer
+      # codes are 4 ("80C0"); the shape is supplied by the injected Config.
+      #
+      # A multi-code block (e.g. the Air Advisor codes 8L100/8L200/8L300, each
+      # with its own inline title but one shared body) yields one entry per code.
+      #
+      # A wrapped-prose false positive — an "SDI <code>," anchor whose inline
+      # "title" is actually the lowercase continuation of a sentence, e.g.
+      # "SDI 8P000, completion of a current T5 Investigation ..." — is rejected:
+      # its title fails the real-title test, leaving the block with no valid
+      # anchor, so #entries returns [].
+      class SdiCardParser
+        DECORATIVE = Patterns::DECORATIVE
+
+        Anchor = Struct.new(:code, :inline_title, keyword_init: true)
+
+        # A bare (keyword-less) date parenthetical, e.g. "(31 Oct 24)" — the date
+        # annotation on the officer 88X0 card carries no Changed/Established word.
+        BARE_DATE = /\(\s*(\d{1,2}\s*[A-Za-z]{3,9}\s*\d{2,4})\s*\)/
+        BARE_DATE_LINE = /\A#{BARE_DATE}/
+
+        # A keyword date annotation ("(Changed ...)"), open paren optional.
+        KEYWORD_DATE_LINE = /\A\(?\s*(?:Changed|Established|Effective|Change)\b/
+
+        def initialize(record, config: Config.dafecd)
+          @config = config
+          @lines = record.lines.reject { |line| line =~ config.header }
+        end
+
+        # @return [Array<Hash>] one entry hash per code in the block
+        def entries
+          anchors = self.anchors
+          return [] if anchors.empty?
+
+          shared = shared_title
+          date = changed_date
+          shreds = shredouts
+
+          anchors.map do |anchor|
+            raw = anchor.inline_title || shared
+            name = raw && Title.titleize(raw)
+            {
+              code: anchor.code,
+              name: name,
+              raw_title: raw,
+              changed_date: date,
+              glued_title: Title.glued?(name),
+              shredouts: shreds
+            }
+          end
+        end
+
+        private
+
+        # The valid SDI anchors in document order. An inline-title anchor whose
+        # title is not a real title is treated as a non-anchor (the false-positive
+        # rejection); a standalone "SDI <code>" line is always a valid anchor.
+        def anchors
+          @lines.filter_map do |line|
+            if (m = line.match(@config.sdi_anchor))
+              Anchor.new(code: m[1], inline_title: nil)
+            elsif (m = line.match(@config.sdi_inline_anchor))
+              title = Title.sanitize(m[2])
+              Title.real?(title) ? Anchor.new(code: m[1], inline_title: title) : nil
+            end
+          end
+        end
+
+        # Index of the last VALID anchor line, or nil. A false-positive inline
+        # anchor in the body ("SDI 8P000, completion of ...") must not be treated
+        # as an anchor here, or the standalone card's title lookup would start
+        # after it and find nothing.
+        def last_anchor_index
+          @lines.each_index.reverse_each.find { |i| valid_anchor?(@lines[i]) }
+        end
+
+        def valid_anchor?(line)
+          return true if line.match?(@config.sdi_anchor)
+          m = line.match(@config.sdi_inline_anchor)
+          m && Title.real?(Title.sanitize(m[2]))
+        end
+
+        # The standalone title: title lines between the last anchor and the date /
+        # first numbered section, joined and sanitized. nil when the block carries
+        # only inline titles (or no title line).
+        def shared_title
+          last = last_anchor_index
+          return nil if last.nil?
+
+          collected = []
+          @lines[(last + 1)..].each do |line|
+            stripped = Title.sanitize(line)
+            next if stripped.empty?
+            next if stripped.match?(/\A\d+\z/)          # bare page number
+            break if date_line?(stripped)
+            break if stripped.match?(/\A\d+\./)         # numbered section
+            break unless Title.title_line?(stripped)
+            collected << stripped
+          end
+          collected.empty? ? nil : collected.join(" ")
+        end
+
+        # Everything up to the first numbered section, where the change date lives.
+        def head_region
+          cut = @lines.index { |line| line.match?(/\A\s*\d+\.\s/) }
+          (cut ? @lines[0...cut] : @lines).join
+        end
+
+        # The change date: a keyword annotation, or (officer 88X0) a keyword-less
+        # bare date parenthetical in the card's header region.
+        def changed_date
+          region = head_region
+          ChangeDate.extract(region) || bare_date(region)
+        end
+
+        def bare_date(region)
+          m = region.match(BARE_DATE)
+          m && ChangeDate.normalize(m[1])
+        end
+
+        def shredouts
+          ShredoutParser.new(
+            @lines.join, publication: @config, pair: ShredoutParser::RI_SDI_PAIR
+          ).parse
+        end
+
+        # A line that ends title collection: a keyword date annotation OR a bare
+        # date parenthetical ("(31 Oct 24)").
+        def date_line?(stripped)
+          stripped.match?(KEYWORD_DATE_LINE) || stripped.match?(BARE_DATE_LINE)
+        end
+      end
+    end
+  end
+end

data/lib/gov_codes/dafecd/ri_sdi/sdi_section_splitter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative "config"
+require_relative "title"
+
+module GovCodes
+  module Dafecd
+    module RiSdi
+      # Splits a Special Duty Identifiers (SDI) section into one string per record.
+      #
+      # A record begins at either:
+      #   * a run of "SDI <code>" card anchors (the first anchor of the run), or
+      #   * a "CEM Code <code>" line — the single ladder record (Premier Honor
+      #     Guard, 8G000) embedded in the enlisted AF SDI section. It is split out
+      #     as its own record so its body (notably its shredout table) can never
+      #     bleed into the preceding card; the AFSC pipeline parses it downstream.
+      #
+      # Running page headers are stripped first. Consecutive card anchors stay in
+      # one record (the multi-code Air Advisor blocks). A wrapped-prose false
+      # positive ("SDI 8P000, completion of a current T5 ...") does NOT start a
+      # record — its inline "title" fails the real-title test, so it is left as
+      # body of the surrounding record and later rejected by SdiCardParser.
+      class SdiSectionSplitter
+        def initialize(text, config: Config.dafecd)
+          @config = config
+          @lines = text.lines.reject { |line| line =~ config.header }
+        end
+
+        # @return [Array<String>] one string per record
+        def records
+          records = []
+          current = nil
+          prev_meaningful_was_anchor = false
+
+          @lines.each do |line|
+            is_card = card_anchor?(line)
+            is_cem = line.match?(@config.cem)
+            stripped = line.strip
+            neutral = stripped.empty? || stripped.match?(/\A\d+\z/)
+
+            starts_record = is_cem || (is_card && !prev_meaningful_was_anchor)
+
+            if starts_record
+              current = +""
+              records << current
+            end
+
+            current << line if current
+
+            unless neutral
+              # A CEM line's ladder ("AFSC 8G091 ...") keeps the run open so the
+              # card detector does not re-trigger inside the ladder record.
+              prev_meaningful_was_anchor = is_card || is_cem || ladder_line?(line)
+            end
+          end
+
+          records
+        end
+
+        private
+
+        # A card anchor: a standalone "SDI <code>" line, or an inline-title anchor
+        # whose title is a real title (rejecting the wrapped-prose false positive).
+        def card_anchor?(line)
+          return true if line.match?(@config.sdi_anchor)
+          m = line.match(@config.sdi_inline_anchor)
+          m && Title.real?(Title.sanitize(m[2]))
+        end
+
+        # A ladder line inside the embedded CEM record, e.g. "AFSC 8G091 ...".
+        def ladder_line?(line)
+          line.match?(/^\s*AFSC\s+\d[A-Z]\d\d\d/)
+        end
+      end
+    end
+  end
+end

data/lib/gov_codes/dafecd/ri_sdi/section_slicer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "config"
+
+module GovCodes
+  module Dafecd
+    module RiSdi
+      # Slices the full directory text into the RI/SDI sections named in the
+      # injected Config.
+      #
+      # Each section is located by its header line (e.g.
+      # "AIR FORCE REPORTING IDENTIFIERS (RI)"). The header regexes require the
+      # "(SDI)"/"(RI)"/"(SFSC)" parenthetical, which the table-of-contents entries
+      # lack, so the TOC is never mistaken for a section. A section runs from its
+      # header to the next located header. The out-of-scope Space Force Specialty
+      # Codes (SFSC) section is used only as a boundary (it terminates the AF RI
+      # section) and is dropped from the result.
+      class SectionSlicer
+        Section = Struct.new(:kind, :force, :text, keyword_init: true)
+
+        def initialize(text, config: Config.dafecd)
+          @text = text
+          @config = config
+        end
+
+        # @return [Array<Section>] located, in-scope sections in document order
+        def sections
+          located = @config.sections
+            .map { |spec| {spec: spec, at: (@text =~ spec[:header])} }
+            .reject { |m| m[:at].nil? }
+            .sort_by { |m| m[:at] }
+
+          located.each_with_index.map do |mark, i|
+            finish = (i + 1 < located.size) ? located[i + 1][:at] : @text.length
+            Section.new(
+              kind: mark[:spec][:kind],
+              force: mark[:spec][:force],
+              text: @text[mark[:at]...finish]
+            )
+          end.reject { |section| section.kind == :skip }
+        end
+      end
+    end
+  end
+end