predictability-engine 0.6.6

data/lib/predictability_engine/config.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module PredictabilityEngine
+  # Configuration management for the predictability engine.
+  class Config
+    CONFIG_FILE = '.predictability_engine.yml'
+
+    # Single source of truth: config key → JIRA_ env var suffix.
+    JIRA_FIELDS = {
+      site: 'SITE', email: 'EMAIL', token: 'API_TOKEN', project: 'PROJECT',
+      context_path: 'CONTEXT_PATH', auth_mode: 'AUTH_MODE',
+      bearer_token: 'BEARER_TOKEN', auth_cookie: 'AUTH_COOKIE',
+      password: 'PASSWORD', totp_secret: 'TOTP_SECRET',
+      mfa_login_url: 'MFA_LOGIN_URL', mfa_token_field: 'MFA_TOKEN_FIELD',
+      idp_login_url: 'IDP_LOGIN_URL', idp_callback_port: 'IDP_CALLBACK_PORT'
+    }.freeze
+    private_constant :JIRA_FIELDS
+
+    def self.jira(profile_name = nil)
+      load_jira_config(profile_name)
+    end
+
+    def self.jira_client(profile = nil)
+      require 'jira-ruby'
+      instrument_jira_http! unless ::JIRA::HttpClient.ancestors.include?(JiraHttpLogger)
+      config = jira(profile)
+      validate_jira!(config)
+      origin, derived_path = split_jira_site(config[:site])
+      base = { site: origin, context_path: config[:context_path] || derived_path, default_headers: {} }
+      auth = JiraAuth.build(config)
+      options = auth.jira_options(base)
+      options[:http_debug] = true if jira_http_debug?
+      client = ::JIRA::Client.new(options)
+      auth.post_init(client)
+      client
+    end
+
+    # Splits a full Jira site URL (e.g. "https://host/jira") into the origin
+    # ("https://host") and context path ("/jira") that jira-ruby expects.
+    # This avoids 301/302 redirect errors when the server requires the path.
+    def self.split_jira_site(site_url)
+      uri = URI.parse(site_url.to_s.chomp('/'))
+      port_suffix = [80, 443].include?(uri.port) ? '' : ":#{uri.port}"
+      origin = "#{uri.scheme}://#{uri.host}#{port_suffix}"
+      [origin, uri.path]
+    end
+    private_class_method :split_jira_site
+
+    # Prepended to JIRA::HttpClient to trace every HTTP call at DEBUG level.
+    # SemanticLogger evaluates blocks only when debug is active → zero overhead at
+    # INFO or higher.
+    module JiraHttpLogger
+      def make_request(http_method, url, body = '', headers = {})
+        log = SemanticLogger['JIRA::HTTP']
+        log.debug { "→ #{http_method.upcase} #{url}" }
+        result = super
+        log.debug { "← #{result.code} #{result.message}" }
+        result
+      rescue StandardError => e
+        SemanticLogger['JIRA::HTTP'].debug { "✗ #{http_method.upcase} #{url} (#{e.class}: #{e.message})" }
+        raise
+      end
+    end
+    private_constant :JiraHttpLogger
+
+    def self.instrument_jira_http!
+      ::JIRA::HttpClient.prepend(JiraHttpLogger)
+    end
+    private_class_method :instrument_jira_http!
+
+    def self.jira_http_debug?
+      ENV.fetch('JIRA_HTTP_DEBUG', nil).to_s.match?(/\A(true|1|yes)\z/i)
+    end
+    private_class_method :jira_http_debug?
+
+    def self.validate_jira!(config)
+      raise_missing(config, :site)
+      case config[:auth_mode].to_s
+      when 'bearer'
+        raise_missing(config, :bearer_token)
+      when 'cookie'
+        raise_missing(config, :auth_cookie)
+      when 'mfa_api'
+        %i[email password totp_secret mfa_login_url].each { |k| raise_missing(config, k) }
+      when 'mfa_browser'
+        raise_missing(config, :idp_login_url)
+      else
+        raise_missing(config, :email)
+        raise_missing(config, :token)
+      end
+    end
+
+    def self.raise_missing(config, key)
+      return if config[key]
+
+      env_suffix = JIRA_FIELDS[key]
+      env_var = env_suffix ? "JIRA_#{env_suffix}" : "JIRA_#{key.to_s.upcase}"
+      raise Error, "Jira #{key} not configured (use #{env_var} env var or credentials file)"
+    end
+    private_class_method :raise_missing
+
+    def self.load_jira_config(profile_name = nil)
+      profile_name ||= default_profile_name
+      global = load_global_jira_config
+      local = load_local_jira_config
+
+      config = load_profile(profile_name, global, local) if profile_name
+      config || fallback_config(global, local)
+    end
+
+    def self.default_profile_name
+      ENV.fetch('JIRA_PROFILE', nil) || ENV.fetch('JIRA_PROJECT', nil)
+    end
+
+    def self.fallback_config(global, local)
+      JIRA_FIELDS.transform_values { |env_key| jira_val(env_key, global, local) }
+    end
+
+    def self.jira_val(name, global, local)
+      key = name.downcase.to_sym
+      ENV.fetch("JIRA_#{name}", nil) || local[key] || global[key]
+    end
+
+    def self.jira_credentials_file
+      File.expand_path('~/.config/jira/jira_credentials.yml')
+    end
+
+    def self.load_global_jira_config
+      load_jira_file(jira_credentials_file, global: true)
+    end
+
+    def self.load_local_jira_config
+      load_jira_file(CONFIG_FILE, global: false)
+    end
+
+    def self.load_yaml_file(path)
+      YAML.load_file(path)
+    rescue Psych::SyntaxError => e
+      raise Error, "Invalid YAML in #{path}: #{e.message}"
+    end
+
+    def self.load_jira_file(path, global: true)
+      raw = File.exist?(path) ? load_yaml_file(path) : {}
+      raw ||= {}
+      config = global ? (raw['jira'] || raw) : raw.fetch('jira', {})
+      extract_fields(config).merge(profiles: config.fetch('profiles', {}))
+    end
+
+    def self.load_profile(name, global, local)
+      profile = global[:profiles][name.to_s] || local[:profiles][name.to_s] || {}
+      return if profile.empty?
+
+      extract_fields(profile)
+    end
+
+    def self.extract_fields(hash)
+      JIRA_FIELDS.keys.to_h { |key| [key, hash[key.to_s]] }
+    end
+    private_class_method :extract_fields
+
+    private_class_method :load_jira_config, :load_global_jira_config,
+                         :load_local_jira_config, :load_profile, :default_profile_name,
+                         :fallback_config, :load_jira_file, :jira_val
+  end
+end

data/lib/predictability_engine/data_generator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module PredictabilityEngine
+  module DataGenerator
+    PRESETS = {
+      small: { completed: 10, wip: 4 },
+      medium: { completed: 40, wip: 10 },
+      large: { completed: 150, wip: 50 },
+      xl: { completed: 4000, wip: 400 }
+    }.freeze
+
+    def self.generate(output:, size: :medium, completed: nil, wip: nil)
+      File.binwrite(output, content(size: size, completed: completed, wip: wip))
+      output
+    end
+
+    def self.content(size: :medium, completed: nil, wip: nil)
+      preset = PRESETS.fetch(size.to_sym) do
+        raise ArgumentError, "Unknown size: #{size} (available: #{PRESETS.keys.join(', ')})"
+      end
+      build_csv(completed || preset[:completed], wip || preset[:wip])
+    end
+
+    def self.build_csv(completed_count, wip_count)
+      today = PredictabilityEngine.today
+      CSV.generate do |csv|
+        csv << %w[id title start_date end_date]
+        write_completed(csv, completed_count, today)
+        write_wip(csv, completed_count, wip_count, today)
+      end
+    end
+
+    def self.write_completed(csv, count, today)
+      (1..count).each do |i|
+        start_date = today - rand(200..400)
+        end_date   = start_date + rand(5..30)
+        csv << ["PROJ-#{i}", "Task #{i}",
+                PredictabilityEngine.format_date(start_date),
+                PredictabilityEngine.format_date(end_date)]
+      end
+    end
+
+    def self.write_wip(csv, offset, count, today)
+      ((offset + 1)..(offset + count)).each do |i|
+        start_date = today - rand(1..100)
+        csv << ["PROJ-#{i}", "In Progress Task #{i}",
+                PredictabilityEngine.format_date(start_date), nil]
+      end
+    end
+  end
+end

data/lib/predictability_engine/data_manager.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module PredictabilityEngine
+  class DataManager
+    attr_reader :work_items, :source
+
+    def initialize
+      @work_items = []
+      @source = nil
+    end
+
+    def load(spec, **)
+      @source = spec
+      @work_items = DataSources::Factory.for(spec, **).load(spec)
+    end
+
+    # Backward compatibility
+    alias load_csv load
+
+    def completed_items
+      @work_items.select(&:completed?)
+    end
+
+    def active_items
+      @work_items.reject(&:completed?)
+    end
+  end
+end

data/lib/predictability_engine/data_sources/base.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module PredictabilityEngine
+  module DataSources
+    class Base
+      def configure(opts)
+        @url_prefix = opts[:url_prefix]
+        self
+      end
+
+      def load(source_spec)
+        apply_project_config_defaults
+        perform_load(source_spec)
+      rescue StandardError => e
+        raise Error, "Failed to load from #{source_name}: #{e.message}"
+      end
+
+      protected
+
+      def source_name
+        self.class.name.split('::').last
+      end
+
+      def perform_load(_source_spec)
+        raise NotImplementedError, "#{self.class} must implement perform_load"
+      end
+
+      def build_work_items(data)
+        data.map do |row|
+          item_data = map_row(row)
+          item_id = item_data.delete(:id)
+          Models::WorkItem.new(item_id: item_id, **item_data)
+        end
+      end
+
+      def parse_date(val)
+        return val if val.is_a?(Date) || val.is_a?(Time)
+        return nil if val.nil? || val.to_s.strip.empty?
+
+        Date.parse(val.to_s)
+      rescue ArgumentError
+        nil
+      end
+
+      def load_data(iterator)
+        data = iterator.map do |row|
+          map_row(row)
+        end
+        build_work_items(data)
+      end
+
+      def map_row(row)
+        raw_id = row[:id] || row[:key] || row[:item_id]
+        {
+          id: raw_id,
+          title: row[:title] || row[:summary],
+          type: row[:type] || row[:issuetype],
+          priority: normalize_priority(row[:priority]),
+          start_date: parse_date(row[:start_date] || row[:created]),
+          end_date: parse_date(row[:end_date] || row[:resolutiondate] || row[:resolved]),
+          url: item_url(row[:url], raw_id)
+        }
+      end
+
+      def item_url(url, item_id)
+        url.presence || (@url_prefix && "#{@url_prefix}#{item_id}")
+      end
+
+      def normalize_priority(name)
+        return nil if name.nil?
+
+        (@priority_aliases || {})[name.to_s] || name
+      end
+
+      def mock_data(env_key)
+        json = ENV.fetch(env_key, '[]')
+        JSON.parse(json, symbolize_names: true)
+      end
+
+      private
+
+      def apply_project_config_defaults
+        return if @url_prefix
+        return unless File.exist?(Config::CONFIG_FILE)
+
+        config = YAML.load_file(Config::CONFIG_FILE) || {}
+        @url_prefix = config['url_prefix']
+      end
+    end
+  end
+end

data/lib/predictability_engine/data_sources/csv.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'yaml'
+
+module PredictabilityEngine
+  module DataSources
+    class Csv < Base
+      JIRA_HEADER_MAP = {
+        issue_key: :id,
+        issue_type: :type
+      }.freeze
+
+      def perform_load(path)
+        config = load_csv_config(path)
+        @url_prefix ||= config['url_prefix']
+        @done_statuses = load_done_statuses(config)
+        @source_url = "file://#{File.expand_path(path)}"
+        CSV.open(path, headers: true, header_converters: :symbol, encoding: 'bom|UTF-8', row_sep: :auto)
+           .then { |csv| load_data(csv.map { |row| apply_jira_header_map(row.to_h) }) }
+      end
+
+      private
+
+      def apply_jira_header_map(row_hash)
+        row_hash.transform_keys { |k| JIRA_HEADER_MAP.fetch(k, k) }
+      end
+
+      def map_row(row)
+        super.tap do |mapped|
+          mapped[:url] ||= @source_url unless @url_prefix
+          next if mapped[:end_date]
+          next unless @done_statuses.include?(row[:status].to_s.downcase)
+
+          mapped[:end_date] = parse_date(row[:updated])
+        end
+      end
+
+      def load_done_statuses(config)
+        if (wf_path = config['workflow_config_path'])
+          JiraWorkflow.load(File.expand_path(wf_path))&.departure_names.to_a.map(&:downcase)
+        elsif config.key?('statuses')
+          JiraWorkflow.new(statuses: Array(config['statuses'])).departure_names.map(&:downcase)
+        else
+          Array(config['done_statuses']).map(&:downcase)
+        end
+      end
+
+      def load_csv_config(csv_path)
+        sidecar = csv_path.sub(/\.csv$/i, '.yml')
+        return YAML.load_file(sidecar) || {} if File.exist?(sidecar)
+        return project_jira_csv_config if File.exist?(Config::CONFIG_FILE)
+
+        {}
+      end
+
+      def project_jira_csv_config
+        (YAML.load_file(Config::CONFIG_FILE) || {}).fetch('jira_csv', {})
+      end
+    end
+  end
+end

data/lib/predictability_engine/data_sources/excel.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'roo'
+
+module PredictabilityEngine
+  module DataSources
+    class Excel < Base
+      def perform_load(path)
+        return build_work_items(mock_data('MOCK_EXCEL_DATA')) if ENV['MOCK_EXCEL_DATA']
+
+        xlsx = Roo::Spreadsheet.open(path)
+        iterator = xlsx.sheet(0).each(id: 'id', start_date: 'start_date', end_date: 'end_date')
+                       .reject { |row| row[:id] == 'id' }
+        load_data(iterator)
+      end
+    end
+  end
+end

data/lib/predictability_engine/data_sources/factory.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module PredictabilityEngine
+  module DataSources
+    class Factory
+      def self.for(spec, **opts)
+        source = case spec
+                 when /^jira:/, /^jql:/, /\.yml$/, /\.yaml$/, 'jira', /^[A-Z][A-Z0-9]+$/
+                   Jira.new
+                 when /\.xlsx$/
+                   Excel.new
+                 else
+                   Csv.new
+                 end
+        source.configure(opts)
+        source
+      end
+    end
+  end
+end

data/lib/predictability_engine/data_sources/jira.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require 'jira-ruby'
+require_relative 'jira_yaml'
+
+module PredictabilityEngine
+  module DataSources
+    class Jira < Base
+      IN_PROGRESS_KEYWORDS = ['In Progress', 'Doing', 'Active', 'Development', 'Progress'].freeze
+      FIELDS = %w[summary issuetype created priority resolutiondate status].freeze
+
+      def perform_load(spec)
+        @priority_aliases = yaml_priority_aliases(spec)
+
+        if ENV['MOCK_JIRA'] == 'true'
+          data = mock_data('JIRA_MOCK_DATA')
+          # Even when mocked, we can validate the contract if requested
+          data.each { |row| validate_issue_contract!(row) } if ENV['JIRA_CONTRACT_CHECK'] == 'true'
+          return build_work_items(data)
+        end
+
+        profile, query, @workflow = resolve_source(spec)
+        @jira_site = Config.jira(profile)[:site]
+        client = build_client(profile)
+        issues = fetch_issues(client, query)
+
+        # Contract Validation
+        issues.each { |issue| validate_issue_contract!(issue) } if ENV['JIRA_CONTRACT_CHECK'] == 'true'
+
+        data = issues.map { |issue| map_issue(issue) }
+        build_work_items(data)
+      end
+
+      private
+
+      def validate_issue_contract!(issue)
+        # Handle both JIRA::Resource::Issue and Hash (for mocks)
+        is_hash = issue.is_a?(Hash)
+        key = get_field(issue, :key, is_hash)
+        validate_basic_fields(issue, key, is_hash)
+        validate_issuetype(issue, key, is_hash)
+        validate_created(issue, key, is_hash)
+        validate_changelog(issue, key, is_hash)
+      end
+
+      def validate_basic_fields(issue, key, is_hash)
+        summary = get_field(issue, :summary, is_hash)
+        raise Error, "Issue #{key} is missing 'key'" unless key
+        raise Error, "Issue #{key} is missing 'summary'" unless summary
+      end
+
+      def validate_issuetype(issue, key, is_hash)
+        issuetype = get_field(issue, :issuetype, is_hash)
+        raise Error, "Issue #{key} is missing 'issuetype'" unless issuetype
+
+        # Handle issuetype object or string
+        type_name = if issuetype.respond_to?(:name)
+                      issuetype.name
+                    elsif issuetype.is_a?(Hash)
+                      issuetype[:name] || issuetype['name']
+                    else
+                      issuetype
+                    end
+        raise Error, "Issue #{key} is missing 'issuetype.name'" unless type_name
+      end
+
+      def validate_created(issue, key, is_hash)
+        created = get_field(issue, :created, is_hash)
+        raise Error, "Issue #{key} is missing 'created'" unless created
+      end
+
+      def validate_changelog(issue, key, is_hash)
+        # Required for cycle time and aging
+        has_changelog = if is_hash
+                          issue[:changelog] || issue['changelog']
+                        else
+                          issue.respond_to?(:changelog) && issue.changelog
+                        end
+        return if has_changelog
+
+        PredictabilityEngine.logger.warn { "Issue #{key} is missing 'changelog' (expand=changelog failed?)" }
+      end
+
+      def get_field(issue, name, is_hash)
+        is_hash ? issue[name] || issue[name.to_s] : issue.send(name)
+      end
+
+      def yaml_priority_aliases(spec)
+        return {} unless spec.is_a?(String) && spec.match?(/\.ya?ml$/)
+
+        JiraYaml.new(spec).priority_aliases
+      end
+
+      def resolve_source(spec)
+        profile_name = ENV.fetch('JIRA_PROFILE', nil)
+        case spec
+        when /\.ya?ml$/
+          yaml = JiraYaml.new(spec)
+          [yaml.profile, yaml.query, load_workflow(yaml.workflow_config_path, yaml.profile)]
+        when 'jira'
+          [profile_name, query_from_config(profile_name), load_workflow(nil, profile_name)]
+        when /^[A-Z][A-Z0-9]+$/
+          [profile_name, "project = \"#{spec}\"", load_workflow(nil, profile_name)]
+        else
+          [nil, spec, nil]
+        end
+      end
+
+      def load_workflow(explicit_path, profile_name)
+        return JiraWorkflow.load(explicit_path) if explicit_path
+
+        profile_name && JiraWorkflow.load(JiraWorkflow.default_path(profile_name))
+      end
+
+      def query_from_config(profile_name)
+        config = Config.jira(profile_name)
+        project = config[:project]
+        query = ENV.fetch('JIRA_PROJECT_QUERY', nil) || (project ? "project = \"#{project}\"" : nil)
+        raise Error, 'No JIRA project specified (use JIRA_PROJECT env var or provide a query)' unless query
+
+        query
+      end
+
+      def build_client(profile = nil)
+        Config.jira_client(profile)
+      end
+
+      def fetch_issues(client, query)
+        jql = query.start_with?('jira:') ? "filter = #{query.sub('jira:', '')}" : query.sub('jql:', '')
+        logger.debug { "JIRA JQL: #{jql}" }
+        client.Issue.jql(jql, expand: 'changelog', fields: FIELDS)
+      rescue StandardError => e
+        logger.debug { "JIRA fetch failed: #{e.class} — #{e.message}" }
+        raise Error, "Failed to fetch issues from Jira: #{e.message}"
+      end
+
+      def map_issue(issue)
+        map_row({
+                  id: issue.key,
+                  url: "#{@jira_site.to_s.chomp('/')}/browse/#{issue.key}",
+                  summary: issue.summary,
+                  issuetype: issue.issuetype.name,
+                  priority: jira_priority_name(issue),
+                  created: issue.created,
+                  start_date: first_in_progress_date(issue),
+                  resolutiondate: last_departure_date(issue)
+                })
+      end
+
+      def jira_priority_name(issue)
+        return nil unless issue.respond_to?(:priority) && issue.priority
+
+        issue.priority.respond_to?(:name) ? issue.priority.name : issue.priority.to_s
+      end
+
+      def first_in_progress_date(issue)
+        names = @workflow&.arrival_names
+        return first_transition_matching(issue) { |t| names.any? { |n| t[:to].casecmp?(n) } } if names&.any?
+
+        warn_missing_workflow_once
+        first_transition_matching(issue) do |t|
+          IN_PROGRESS_KEYWORDS.any? { |kw| t[:to].downcase.include?(kw.downcase) }
+        end
+      end
+
+      def last_departure_date(issue)
+        names = @workflow&.departure_names
+        return issue.resolutiondate unless names&.any?
+
+        first_transition_matching(issue) { |t| names.any? { |n| t[:to].casecmp?(n) } } || issue.resolutiondate
+      end
+
+      def first_transition_matching(issue, &)
+        return nil unless issue.respond_to?(:changelog) && issue.changelog
+
+        transitions = status_transitions(issue.changelog.fetch('histories', [])).sort_by { |t| t[:date] }
+        match = transitions.find(&)
+        match ? match[:date] : nil
+      end
+
+      def warn_missing_workflow_once
+        return if @workflow_warning_emitted
+
+        @workflow_warning_emitted = true
+        PredictabilityEngine.logger.warn do
+          'No Jira workflow mapping found; falling back to in-progress keyword heuristic. ' \
+            'Run `./bin/predictability-engine jira_workflow <profile>` to generate an editable mapping.'
+        end
+      end
+
+      def status_transitions(histories)
+        histories.flat_map do |history|
+          created_at = history['created']
+          history['items'].select { |item| item['field'] == 'status' }.map do |item|
+            { date: created_at, to: item['toString'] }
+          end
+        end
+      end
+    end
+  end
+end