ecoportal-api-graphql 1.3.9 → 1.3.11

data/lib/ecoportal/api/graphql/diff/id_resolver.rb

@@ -0,0 +1,64 @@
+module Ecoportal
+  module API
+    class GraphQL
+      module Diff
+        # Resolves the human keys a `VersionDiff` records (stage NAME, section HEADING) into the
+        # target-doc ids that move commands need (moveField -> section id, add/removeStageSection ->
+        # stage id). A structural diff only knows those human keys; this fills the gap WITHOUT
+        # guessing — it looks them up in a real target doc (the deploy destination), so
+        # `CommandSynthesizer` can emit faithful move commands.
+        #
+        # Answers `resolve(kind, key) => id | nil`. A nil (ambiguous / not found) keeps the move
+        # UNSUPPORTED rather than picking a wrong id.
+        #
+        #   resolver = IdResolver.from_doc(prod_template_doc)
+        #   CommandSynthesizer.new(changes, resolver: resolver)
+        #
+        # AMBIGUITY — if two stages share a name (or two sections a heading), the key is ambiguous
+        # and resolves to nil (the caller must disambiguate). Uniqueness is the safe default.
+        class IdResolver
+          # Build from an explicit map: { stage: { 'Report' => 'stg1' }, section: { 'Location' => 'sec1' } }
+          def initialize(index = {})
+            @index = index
+          end
+
+          # Build a resolver by indexing a page/template doc's stages (by name) and sections
+          # (by heading). Duplicate keys are dropped (resolve -> nil) so we never pick arbitrarily.
+          def self.from_doc(doc)
+            doc ||= {}
+            stages   = {}
+            sections = {}
+            Array(doc['stages']).each do |st|
+              mark(stages, st['name'], st['id'])
+              Array(st['sections']).each { |sec| mark(sections, sec['heading'], sec['id']) }
+            end
+            new(stage: strip_ambiguous(stages), section: strip_ambiguous(sections))
+          end
+
+          # @return [String,nil] the target id for (kind, key), or nil if unknown/ambiguous.
+          def resolve(kind, key)
+            @index.dig(kind.to_sym, key)
+          end
+
+          class << self
+            private
+
+            def mark(bucket, key, id)
+              return if key.nil? || id.nil?
+
+              (bucket[key] ||= []) << id
+            end
+
+            # Keep only keys that map to exactly one id; ambiguous keys are omitted (resolve -> nil).
+            def strip_ambiguous(bucket)
+              bucket.each_with_object({}) do |(key, ids), out|
+                uniq = ids.uniq
+                out[key] = uniq.first if uniq.size == 1
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/graphql/diff/pairing/candidate.rb

@@ -0,0 +1,32 @@
+module Ecoportal
+  module API
+    class GraphQL
+      module Diff
+        module Pairing
+          # A scored pairing proposal: source object (id-space A) <-> target object (id-space B),
+          # with the aggregate confidence and the per-signal breakdown that produced it.
+          #
+          # `source`/`target` are the raw field docs (Hashes) being paired. `score` is 0.0..1.0.
+          # `signals` maps signal-name => contribution (for transparency + the ledger + Product's
+          # Field-ID data). `matched_by` names the dominant signal (e.g. :genome, :label, :ledger).
+          Candidate = Struct.new(:source, :target, :score, :signals, :matched_by, keyword_init: true) do
+            def source_id
+              source && source['id']
+            end
+
+            def target_id
+              target && target['id']
+            end
+
+            def to_h
+              {
+                source_id: source_id, target_id: target_id, score: score,
+                matched_by: matched_by, signals: signals
+              }.compact
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/graphql/diff/pairing/engine.rb

@@ -0,0 +1,173 @@
+module Ecoportal
+  module API
+    class GraphQL
+      module Diff
+        module Pairing
+          # The equivalence-matching engine for CROSS-OBJECT pairing (UAT<->PROD, page<->template).
+          #
+          # Given two lists of field docs living in different id-spaces, it proposes pairings by:
+          #   1. consulting the LEDGER first — a previously-confirmed pair auto-resolves (method
+          #      :ledger, confidence 1.0), so pairing improves over time and only novelty is scored;
+          #   2. otherwise scoring every remaining source×target candidate with multi-signal
+          #      confidence (genome + type + label + options — see Signals), picking the best target
+          #      per source in a stable, greedy, one-to-one assignment (highest scores first).
+          #
+          # It then CLASSIFIES each best candidate:
+          #   - `accepted`  — score >= accept_threshold: high confidence, auto-paired;
+          #   - `ambiguous` — accept > score >= review_threshold, OR the top two candidates are within
+          #                   `tie_margin` (genuinely close): route to a HUMAN to adjudicate;
+          #   - `unmatched` — no target scored >= review_threshold: escalate as a novelty.
+          #
+          # NEVER guesses: only `accepted` pairs are safe to auto-apply / auto-record; ambiguous and
+          # unmatched are surfaced for human resolution. Confirmed decisions are written back to the
+          # ledger by the caller (or via #confirm!) so the next run needs no human on them.
+          class Engine
+            DEFAULTS = { accept_threshold: 0.85, review_threshold: 0.5, tie_margin: 0.1 }.freeze
+
+            Result = Struct.new(:accepted, :ambiguous, :unmatched, keyword_init: true) do
+              # source field docs with no acceptable target — the pure novelty set.
+              def resolved?
+                ambiguous.empty? && unmatched.empty?
+              end
+
+              def to_h
+                {
+                  accepted:  accepted.map(&:to_h),
+                  ambiguous: ambiguous.map(&:to_h),
+                  unmatched: unmatched.map { |f| { source_id: f['id'], label: f['label'] } }
+                }
+              end
+            end
+
+            # @param ledger [Ledger,nil] consulted first + written to on #confirm!.
+            # @param kind   [Symbol] entity kind recorded in the ledger (default :field).
+            def initialize(ledger: nil, kind: :field, **thresholds)
+              @ledger = ledger
+              @kind   = kind
+              @cfg    = DEFAULTS.merge(thresholds)
+            end
+
+            # Pair `sources` (id-space A) to `targets` (id-space B). Returns a Result.
+            def pair(sources, targets)
+              sources = Array(sources)
+              targets = Array(targets)
+
+              accepted, remaining_sources, remaining_targets = apply_ledger(sources, targets)
+              scored = score_all(remaining_sources, remaining_targets)
+              assign(scored, remaining_sources, accepted)
+            end
+
+            # Persist a confirmed pairing to the ledger (auto-accept or human decision). No-op
+            # without a ledger. `matched_by` overrides the candidate's dominant signal (e.g. :human
+            # when a person adjudicated). Returns the recorded Entry (or nil).
+            def confirm!(candidate, matched_by: nil)
+              return nil unless @ledger
+
+              @ledger.record(
+                kind: @kind, source_id: candidate.source_id, target_id: candidate.target_id,
+                matched_by: matched_by || candidate.matched_by, confidence: candidate.score,
+                signals: candidate.signals
+              )
+            end
+
+            private
+
+            # Pull out sources already confirmed in the ledger (auto-resolved), consuming the matched
+            # targets so they cannot be re-assigned. Returns [accepted, sources_left, targets_left].
+            def apply_ledger(sources, targets)
+              return [[], sources, targets] unless @ledger
+
+              by_id    = targets.each_with_object({}) { |t, h| h[t['id']] = t if t['id'] }
+              accepted = []
+              used     = []
+
+              left = sources.reject do |src|
+                tid = @ledger.target_for(@kind, src['id'])
+                tgt = tid && by_id[tid]
+                next false unless tgt
+
+                accepted << ledger_candidate(src, tgt)
+                used << tid
+                true
+              end
+
+              [accepted, left, targets.reject { |t| used.include?(t['id']) }]
+            end
+
+            def ledger_candidate(source, target)
+              Candidate.new(source: source, target: target, score: 1.0, matched_by: :ledger,
+                            signals: { ledger: 1.0 })
+            end
+
+            # Score every source×target pair; keep only those at/above the review threshold, best
+            # first, so the greedy assignment takes the strongest links first.
+            def score_all(sources, targets)
+              sources.flat_map do |src|
+                targets.map { |tgt| score_pair(src, tgt) }
+              end.select { |c| c.score >= @cfg[:review_threshold] }.sort_by { |c| -c.score }
+            end
+
+            def score_pair(source, target)
+              parts = {
+                genome:  Signals.genome(source, target),
+                type:    Signals.type(source, target),
+                label:   Signals.label(source, target),
+                options: Signals.options(source, target)
+              }
+              applied = parts.compact
+              score   = weighted_average(applied)
+              Candidate.new(source: source, target: target, score: score,
+                            signals: applied, matched_by: dominant(applied))
+            end
+
+            # Weighted mean over the signals that APPLIED (nil signals are excluded and their weight
+            # dropped), so a select field is not penalised for genome being absent, etc.
+            def weighted_average(applied)
+              return 0.0 if applied.empty?
+
+              num = applied.sum { |name, val| Signals::WEIGHTS.fetch(name, 0) * val }
+              den = applied.sum { |name, _| Signals::WEIGHTS.fetch(name, 0) }
+              den.zero? ? 0.0 : (num / den).round(4)
+            end
+
+            def dominant(applied)
+              return nil if applied.empty?
+
+              applied.max_by { |name, val| [val, Signals::WEIGHTS.fetch(name, 0)] }.first
+            end
+
+            # Greedy one-to-one assignment: walk best-scored candidates first, take a pair when both
+            # ends are still free. Then classify the taken pairs and collect unmatched sources.
+            def assign(scored, sources, accepted)
+              ambiguous   = []
+              taken_src   = accepted.map(&:source_id)
+              taken_tgt   = accepted.map(&:target_id)
+              best_by_src = scored.group_by(&:source_id)
+
+              scored.each do |cand|
+                next if taken_src.include?(cand.source_id) || taken_tgt.include?(cand.target_id)
+
+                bucket = accept?(cand, best_by_src[cand.source_id]) ? accepted : ambiguous
+                bucket << cand
+                taken_src << cand.source_id
+                taken_tgt << cand.target_id
+              end
+
+              unmatched = sources.reject { |s| taken_src.include?(s['id']) }
+              Result.new(accepted: accepted, ambiguous: ambiguous, unmatched: unmatched)
+            end
+
+            # Accept only when confidently above threshold AND not in a near-tie with the runner-up
+            # for the same source (a close second means the human should decide).
+            def accept?(cand, siblings)
+              return false if cand.score < @cfg[:accept_threshold]
+
+              runner_up = Array(siblings).reject { |c| c.target_id == cand.target_id }.map(&:score).max
+              runner_up.nil? || (cand.score - runner_up) >= @cfg[:tie_margin]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/graphql/diff/pairing/ledger.rb

@@ -0,0 +1,119 @@
+require 'json'
+require 'time'
+
+module Ecoportal
+  module API
+    class GraphQL
+      module Diff
+        module Pairing
+          # The LEARNING LEDGER — a first-class artifact that records CONFIRMED equivalences between
+          # objects that live in different id-spaces (UAT<->PROD, page<->template). Because MongoDB
+          # gives every object a distinct id and nothing is shared across counterparts, pairing is an
+          # equivalence / entity-resolution problem, not a lookup. Every decision the engine (or a
+          # human) makes is persisted here so pairing IMPROVES over time: on the next run the ledger
+          # is consulted FIRST and previously-resolved pairs auto-resolve, leaving only genuine
+          # novelty for the human to adjudicate.
+          #
+          # An entry is keyed by (kind, source_id) and records the paired target_id plus HOW it was
+          # resolved (the method, the confidence, the signals, a timestamp). This log of *how* is the
+          # bridge data Product's Field-ID / template-entity-id effort needs.
+          #
+          #   ledger = Ledger.load('pairings.json')      # or Ledger.new (in-memory)
+          #   ledger.record(kind: :field, source_id: 'a', target_id: 'b',
+          #                 method: :genome, confidence: 0.98, signals: {...})
+          #   ledger.lookup(:field, 'a')   # => Entry or nil
+          #   ledger.save                  # persist back to the same path
+          #
+          # SAFETY — the ledger stores only decisions that were CONFIRMED (auto-accepted at high
+          # confidence, or human-adjudicated). Ambiguous/low-confidence candidates are never written;
+          # they are escalated. The ledger is thus a growing store of ground truth, never guesses.
+          class Ledger
+            Entry = Struct.new(:kind, :source_id, :target_id, :matched_by, :confidence, :signals,
+                               :recorded_at, keyword_init: true) do
+              def to_h
+                {
+                  kind: kind, source_id: source_id, target_id: target_id, matched_by: matched_by,
+                  confidence: confidence, signals: signals, recorded_at: recorded_at
+                }.compact
+              end
+            end
+
+            # @param entries [Array<Entry,Hash>] existing entries (e.g. from a loaded file).
+            # @param path    [String,nil] where #save writes; defaults to the load path.
+            def initialize(entries: [], path: nil)
+              @path    = path
+              @entries = {}
+              Array(entries).each { |e| add_entry(coerce(e)) }
+            end
+
+            # Load a ledger from a JSON file. A missing file yields an empty (still writable) ledger.
+            def self.load(path)
+              data = File.exist?(path) ? JSON.parse(File.read(path)) : {}
+              new(entries: data['entries'] || [], path: path)
+            end
+
+            # Record a CONFIRMED equivalence. `matched_by` names how it was resolved (:genome,
+            # :label, :ledger, :human, ...). Later records for the same (kind, source_id) supersede
+            # earlier ones (a human can correct an auto-accept). Returns the stored Entry.
+            def record(kind:, source_id:, target_id:, matched_by:, confidence: nil, signals: nil)
+              entry = Entry.new(
+                kind: kind.to_sym, source_id: source_id, target_id: target_id,
+                matched_by: matched_by&.to_sym, confidence: confidence, signals: signals,
+                recorded_at: Time.now.utc.iso8601
+              )
+              add_entry(entry)
+              entry
+            end
+
+            # The confirmed Entry for (kind, source_id), or nil if never resolved.
+            def lookup(kind, source_id)
+              @entries[[kind.to_sym, source_id]]
+            end
+
+            # The confirmed target id for (kind, source_id), or nil.
+            def target_for(kind, source_id)
+              lookup(kind, source_id)&.target_id
+            end
+
+            def entries
+              @entries.values
+            end
+
+            def size
+              @entries.size
+            end
+
+            def to_h
+              { entries: entries.map(&:to_h) }
+            end
+
+            # Persist to `path` (or the load path). Returns the path written.
+            def save(path = @path)
+              raise ArgumentError, 'no path to save the ledger to' if path.nil?
+
+              File.write(path, JSON.pretty_generate(to_h))
+              path
+            end
+
+            private
+
+            def add_entry(entry)
+              @entries[[entry.kind, entry.source_id]] = entry
+            end
+
+            def coerce(entry)
+              return entry if entry.is_a?(Entry)
+
+              h = entry.transform_keys(&:to_sym)
+              Entry.new(
+                kind: h[:kind]&.to_sym, source_id: h[:source_id], target_id: h[:target_id],
+                matched_by: h[:matched_by]&.to_sym, confidence: h[:confidence], signals: h[:signals],
+                recorded_at: h[:recorded_at]
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/graphql/diff/pairing/signals.rb

@@ -0,0 +1,104 @@
+module Ecoportal
+  module API
+    class GraphQL
+      module Diff
+        module Pairing
+          # Weak pairing SIGNALS combined into a confidence score. Each signal scores a candidate
+          # (source-field-doc, target-field-doc) pair in 0.0..1.0. NONE is ground truth on its own —
+          # per the domain reference genomeSignature is strong-but-fallible (6 documented failure
+          # modes: manual/no-platform-support, new features never added, wrong for select options,
+          # wrongly syncs checklists, re-used/re-purposed fields, incomplete data-loss accounting).
+          # So genome is ONE signal among several, weighted highest but able to be outvoted /
+          # confirmed by type+label+options. The Engine aggregates these with `WEIGHTS`.
+          module Signals
+            module_function
+
+            # Relative weights (need not sum to 1; the Engine normalises by the weights that applied).
+            WEIGHTS = {
+              genome:  0.5,
+              type:    0.2,
+              label:   0.2,
+              options: 0.1
+            }.freeze
+
+            # genomeSignature match. Strong but fallible, so a MATCH scores high (not 1.0) and a
+            # MISMATCH scores 0 rather than vetoing (the field may have been re-purposed keeping its
+            # genome, or genome may be absent on newer field types). Returns nil when neither side
+            # carries a genome — the signal simply does not apply and is excluded from the average.
+            def genome(source, target)
+              a = source['genomeSignature']
+              b = target['genomeSignature']
+              return nil if blank?(a) && blank?(b)
+              return 0.0 if blank?(a) || blank?(b)
+
+              a == b ? 1.0 : 0.0
+            end
+
+            # Field type (__typename) equality. A type change is a strong DISQUALIFIER for a data
+            # pairing (you cannot faithfully migrate PlainText data into a Gauge), so a mismatch is 0.
+            def type(source, target)
+              ta = type_of(source)
+              tb = type_of(target)
+              return nil if ta.nil? && tb.nil?
+
+              ta == tb ? 1.0 : 0.0
+            end
+
+            # Label similarity — exact (case/space-insensitive) = 1.0, else a token-overlap
+            # (Jaccard) score, so `Date logged` vs `Date of sign-off` still gets partial credit
+            # (a re-purpose candidate the human should adjudicate rather than an auto-accept).
+            def label(source, target)
+              a = normalise(source['label'])
+              b = normalise(target['label'])
+              return nil if a.empty? && b.empty?
+              return 1.0 if a == b && !a.empty?
+
+              jaccard(a.split, b.split)
+            end
+
+            # Select-option overlap by VALUE (primary) / label (auxiliary) — the CORRECT option
+            # identity, never genome. Returns nil for non-select fields (no options either side).
+            def options(source, target)
+              a = option_keys(source)
+              b = option_keys(target)
+              return nil if a.empty? && b.empty?
+
+              jaccard(a, b)
+            end
+
+            def type_of(field)
+              field['__typename'] || field['type']
+            end
+
+            def normalise(value)
+              value.to_s.strip.downcase.gsub(/\s+/, ' ')
+            end
+
+            def option_keys(field)
+              Array(field['options']).filter_map do |o|
+                next unless o.is_a?(Hash)
+
+                normalise(o['value'] || o['label'] || o['name'])
+              end.reject(&:empty?)
+            end
+
+            # Jaccard similarity of two token collections (|∩| / |∪|). 0.0 when both empty.
+            def jaccard(list_a, list_b)
+              set_a = list_a.to_a.uniq
+              set_b = list_b.to_a.uniq
+              return 0.0 if set_a.empty? && set_b.empty?
+
+              inter = (set_a & set_b).size.to_f
+              union = (set_a | set_b).size
+              union.zero? ? 0.0 : inter / union
+            end
+
+            def blank?(value)
+              value.nil? || value.to_s.strip.empty?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/graphql/diff/strategy.rb

@@ -0,0 +1,113 @@
+module Ecoportal
+  module API
+    class GraphQL
+      module Diff
+        # Composable configuration for a diff. There is NO single diff — different processes need
+        # different modes (see `template_diff_pairing_domain.md` §8). A `Strategy` is the value object
+        # that names the axes, consumed by the diff front-end (`VersionDiff` / `CrossObjectDiff`) and
+        # by `Deploy`.
+        #
+        # Axes:
+        #
+        # * **pairing** — how counterpart objects are identified across the two snapshots:
+        #     :id         same object, retained Mongo ids (self-version) — NO equivalence needed;
+        #     :genome     pair by genomeSignature (cross-object; a strong-but-fallible signal);
+        #     :type_label pair by __typename + label (cross-object; the TypedFieldsPairing precedent);
+        #     :assisted   the full multi-signal `Pairing::Engine` (+ `Ledger`), human-in-the-loop
+        #                 for ambiguous/unmatched — the general cross-object case.
+        #   `:id` is the default and keeps the existing self-version behaviour untouched.
+        #
+        # * **scope** — which change kinds are emitted:
+        #     :structural    stages / sections / fields (add/remove/move/relabel/retype) + options + gauge;
+        #     :config_only   field configuration (byType), select options, gauge stops — NOT structure;
+        #     :data_migration field<->field data pairing only (structure-agnostic): pairings + type/label
+        #                     changes on paired fields, no add/remove of scaffolding.
+        #
+        # * **move_sensitive** — when false, section/stage MOVES are suppressed (a section-agnostic
+        #   diff that only cares about a field's data pairing, not where it now lives).
+        #
+        # * **intent** — the consuming process (documentation only; does not alter emission, but lets a
+        #   caller/`Deploy` branch on it):
+        #     :changelog     what changed, for a ticket / review checklist;
+        #     :deploy        UAT->PROD replay delta (drives placeholderId threading ON in Deploy);
+        #     :sync_readiness can this register/subset be synced to the active template.
+        #
+        # Cross-object pairings (:genome/:type_label/:assisted) require a pairing map to translate the
+        # two id-spaces before change emission — see `CrossObjectDiff`. `:id` needs no map.
+        class Strategy
+          PAIRINGS        = %i[id genome type_label assisted].freeze
+          SCOPES          = %i[structural config_only data_migration].freeze
+          INTENTS         = %i[changelog deploy sync_readiness].freeze
+
+          # Change KINDS considered structure vs configuration — used to filter by `scope`.
+          STRUCTURAL_KINDS = %i[stage section field].freeze
+          CONFIG_KINDS     = %i[field_config option gauge_stop].freeze
+
+          attr_reader :pairing, :scope, :intent
+
+          def initialize(pairing: :id, scope: :structural, move_sensitive: true, intent: :changelog)
+            @pairing        = validate!(:pairing, pairing, PAIRINGS)
+            @scope          = validate!(:scope, scope, SCOPES)
+            @move_sensitive = move_sensitive ? true : false
+            @intent         = validate!(:intent, intent, INTENTS)
+          end
+
+          # The default self-version strategy (id-paired, structural, move-aware, changelog). This is
+          # exactly the behaviour `VersionDiff` had before Phase 4 — used when no strategy is passed.
+          def self.default
+            new
+          end
+
+          def move_sensitive?
+            @move_sensitive
+          end
+
+          # True when pairing does NOT rely on retained ids — the two snapshots live in different
+          # id-spaces and need a pairing map (`CrossObjectDiff`) before change emission.
+          def cross_object?
+            @pairing != :id
+          end
+
+          # Should a change of this KIND be emitted under this scope?
+          def emit_kind?(kind)
+            case @scope
+            when :structural     then true
+            when :config_only    then CONFIG_KINDS.include?(kind)
+            when :data_migration then kind == :field # field-level data pairing only
+            end
+          end
+
+          # Should a :moved change be emitted? Suppressed when move-insensitive, or under a scope that
+          # ignores scaffolding placement (config_only / data_migration never care about moves).
+          def emit_move?
+            move_sensitive? && @scope == :structural
+          end
+
+          # Filter a raw change-set to what this strategy emits (scope + move-sensitivity). The diff
+          # front-ends compute the full change-set once and let the strategy decide what survives.
+          def filter(changes)
+            Array(changes).select do |change|
+              next false unless emit_kind?(change.kind)
+              next false if change.op == :moved && !emit_move?
+
+              true
+            end
+          end
+
+          def to_h
+            { pairing: @pairing, scope: @scope, move_sensitive: move_sensitive?, intent: @intent }
+          end
+
+          private
+
+          def validate!(axis, value, allowed)
+            sym = value&.to_sym
+            return sym if allowed.include?(sym)
+
+            raise ArgumentError, "invalid #{axis}: #{value.inspect} (expected one of #{allowed.inspect})"
+          end
+        end
+      end
+    end
+  end
+end