plan_driven 0.1.0 → 0.3.0

data/lib/plan_driven/renderer/markdown.rb

@@ -17,12 +17,15 @@ module PlanDriven
         "#{parts.join("\n\n")}\n"
       end
 
-      def report(plan)
+      # `charts` are the SVG files written next to the report, by name.
+      def report(plan, charts: [])
         parts = ["# #{plan.key}: #{plan.title} (delivery report)", meta(plan)]
-        parts << "## Summary\n\n#{summary(plan)}"
+        parts << "## Summary\n\n#{summary(plan)}#{chart(charts, "statistics-proof.svg", "Acceptance criteria proven")}"
         parts << "## Tickets and pull requests\n\n#{delivery_table(plan)}"
+        parts << "## Statistics\n\n#{statistics(plan, charts)}"
         parts << "## Acceptance criteria and proof\n\n#{Evidence.matrix_markdown(plan)}"
         parts << "## Checks run on each pull request\n\n#{guard_findings(plan)}"
+        parts << "## Tokens and cost\n\n#{usage_table(plan)}"
         parts << "## Approvals\n\n#{approval_history(plan)}"
         parts << "## Timeline\n\n#{timeline(plan)}"
         parts << "## The approved plan\n\nThe plan this delivery implements is in `plan.md`, revision #{plan.revision}."
@@ -88,6 +91,43 @@ module PlanDriven
         table(["#", "Ticket", "Status", "Pull request", "Merge commit", "Approved by"], rows)
       end
 
+      def statistics(plan, charts)
+        stats = Statistics.new(plan)
+        return "Statistics start when the tickets are approved." unless stats.started?
+
+        figures = [["statistics-burnup.svg", "Acceptance criteria merged and proven"],
+                   ["statistics-timeline.svg", "Where the time went, ticket by ticket"],
+                   ["statistics-time.svg", "Agents and people"]].map { |name, alt| chart(charts, name, alt) }.join
+        "#{table(%w[Measure Value], statistics_rows(stats))}#{figures}\n\n#{ticket_times(stats)}"
+      end
+
+      def statistics_rows(stats)
+        s = stats.summary
+        duration = ->(seconds) { Statistics.duration(seconds) }
+        rows = stats.phases.filter_map { |label, seconds| ["#{label} time", duration[seconds]] if seconds }
+        rows << ["Idea to delivery", duration[s[:lead_time]]]
+        rows << ["Pull requests approved the first time", "#{s[:first_time]} of #{s[:merged]}"]
+        rows << ["Review rounds (feedback sent to an agent)", s[:review_rounds].to_s]
+        rows << ["Agents' share of the time tickets were worked on", "#{s[:agent_share]}%"] if s[:agent_share]
+        rows << ["Estimated points", s[:points].to_s]
+        rows
+      end
+
+      def ticket_times(stats)
+        duration = ->(seconds) { seconds.positive? ? Statistics.duration(seconds) : "-" }
+        rows = stats.tickets.map do |row|
+          [row.ticket.key, row.ticket.estimate.to_s, duration[row.seconds("queued")], duration[row.seconds("agent")],
+           duration[row.seconds("review")], duration[row.seconds("fixes")], duration[row.seconds("merge")],
+           row.review_rounds.to_s, row.merged? ? Statistics.duration(row.cycle_time) : "open"]
+        end
+        table(["#", "Estimate", "Queued", "Agent coding", "Waiting for review", "Fixing feedback",
+               "Approved, not merged", "Review rounds", "Start to merge"], rows)
+      end
+
+      def chart(charts, name, alt)
+        charts.include?(name) ? "\n\n![#{alt}](#{name})" : ""
+      end
+
       def guard_findings(plan)
         lines = plan.tickets.map do |ticket|
           report = Guards::Report.from_h(ticket.guard_report)
@@ -116,8 +156,32 @@ module PlanDriven
         table(%w[When Subject Role Decision By Note], rows)
       end
 
+      def usage_table(plan)
+        rows = Usage.rows(plan)
+        return "No token usage recorded." if rows.empty?
+
+        "#{table(%w[Step Ticket Model Input Output Cache Time Cost], rows.map { |row| usage_row(row) })}\n\n" \
+          "#{usage_total(Usage.totals(rows))}"
+      end
+
+      def usage_row(row)
+        tokens = row.tokens.transform_values { |count| Usage.format_tokens(count) }
+        cache = Usage.format_tokens(row.tokens["cache_write_tokens"] + row.tokens["cache_read_tokens"])
+        time = row.duration_ms ? "#{(row.duration_ms / 60_000.0).round(1)} min" : "-"
+        [row.step.to_s, row.ticket || "-", row.model.to_s, tokens["input_tokens"], tokens["output_tokens"], cache, time,
+         Usage.format_cost(row.cost)]
+      end
+
+      def usage_total(totals)
+        tokens = Usage.format_tokens(totals[:total_tokens])
+        return "**Total: #{tokens} tokens, #{Usage.format_cost(totals[:cost])}**" if totals[:cost]
+
+        "**Total: #{tokens} tokens.** No price is set for #{totals[:unpriced].join(", ")}; " \
+          "add it to `config.token_prices` to see dollars."
+      end
+
       def timeline(plan)
-        plan.events.map do |event|
+        plan.events.where.not(name: %w[llm.usage agent.usage]).map do |event|
           target = event.ticket ? " #{event.ticket.key}" : ""
           "- #{event.created_at.strftime("%-d %b %Y %H:%M")} · #{event.name}#{target} · #{event.actor}"
         end.join("\n")

data/lib/plan_driven/renderer/style.css

@@ -20,3 +20,24 @@ th { background: #f6f8fa; font-weight: 600; }
 tr { break-inside: avoid; }
 a { color: #0969da; text-decoration: none; }
 h1, h2, h3 { break-after: avoid; }
+html { -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+h2 { color: #1f2330; border-left: 4px solid #cc342d; padding-left: 10px; }
+th { background: #f3f4f6; }
+.result { display: inline-block; padding: 1px 9px; border-radius: 10px; font-size: 8.5pt; font-weight: 600; white-space: nowrap; }
+.result::before { margin-right: 4px; }
+.result.passed, .result.merged { background: #dcfce7; color: #15803d; }
+.result.passed::before, .result.merged::before { content: "✓"; }
+.result.failed { background: #fee2e2; color: #b91c1c; }
+.result.failed::before { content: "✗"; }
+.result.not-run { background: #fef3c7; color: #b45309; }
+.result.not-run::before { content: "•"; }
+.result.no-scenario { background: #f3f4f6; color: #6b7280; }
+.result.no-scenario::before { content: "–"; }
+tr.passed td:first-child { border-left: 3px solid #16a34a; }
+tr.failed td { background: #fef2f2; }
+tr.failed td:first-child { border-left: 3px solid #dc2626; }
+tr.not-run td { background: #fffbeb; }
+tr.not-run td:first-child { border-left: 3px solid #f59e0b; }
+tr.no-scenario td:first-child { border-left: 3px solid #9ca3af; }
+figure.chart { margin: 12px 0 18px; border: 1px solid #e5e7eb; border-radius: 10px; overflow: hidden; break-inside: avoid; }
+figure.chart svg, figure.chart img { display: block; width: 100%; height: auto; }

data/lib/plan_driven/renderer.rb

@@ -17,23 +17,30 @@ module PlanDriven
       write(plan, "plan", Markdown.plan(plan, config: config), title: "#{plan.key} #{plan.title}", config: config)
     end
 
+    # The charts are SVG files beside the report, so GitHub shows them in the Markdown; the HTML
+    # and the PDF carry them inline.
     def write_report(plan, config: PlanDriven.configuration)
-      write(plan, "delivery-report", Markdown.report(plan),
-            title: "#{plan.key} delivery report", config: config)
+      charts = Charts.report(plan)
+      dir = directory(plan, config: config)
+      FileUtils.mkdir_p(dir)
+      Dir[dir.join("statistics-*.svg")].each { |file| File.delete(file) }
+      charts.each { |name, svg| File.write(dir.join(name), svg) }
+      write(plan, "delivery-report", Markdown.report(plan, charts: charts.keys),
+            title: "#{plan.key} delivery report", config: config, images: charts)
     end
 
     def directory(plan, config: PlanDriven.configuration)
       config.docs_root.join(plan.slug)
     end
 
-    def write(plan, name, markdown, title:, config:)
+    def write(plan, name, markdown, title:, config:, images: {})
       dir = directory(plan, config: config)
       FileUtils.mkdir_p(dir)
       md = dir.join("#{name}.md")
       html = dir.join("#{name}.html")
       pdf = dir.join("#{name}.pdf")
       File.write(md, markdown)
-      File.write(html, HTML.document(markdown, title: title))
+      File.write(html, HTML.document(markdown, title: title, images: images))
       written = PDF.render(html, pdf, config: config)
       { markdown: md, html: html, pdf: written ? pdf : nil }
     end

data/lib/plan_driven/statistics.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module PlanDriven
+  # Where a plan's time went, read from its audit trail: when each ticket was queued, when an
+  # agent was writing code, when it waited for a person, and when its acceptance criteria were
+  # merged and then proven. Arithmetic on recorded events, so it gives the same answer every time.
+  class Statistics
+    PHASES = {
+      "queued" => "Queued",
+      "agent" => "Agent coding",
+      "review" => "Waiting for review",
+      "fixes" => "Agent fixing feedback",
+      "merge" => "Approved, not merged"
+    }.freeze
+    WORK = %w[agent review fixes merge].freeze
+    AGENT = %w[agent fixes].freeze
+
+    # Each of these events starts the ticket's next phase; merging ends the last one.
+    NEXT_PHASE = {
+      "ticket.agent_started" => "agent", "ticket.pr_opened" => "review", "ticket.agent_failed" => "review",
+      "ticket.changes_requested" => "fixes", "ticket.pr_approved" => "merge", "ticket.merged" => nil
+    }.freeze
+
+    Segment = Struct.new(:phase, :from, :to) do
+      def seconds = to - from
+    end
+
+    TicketRow = Struct.new(:ticket, :segments, :review_rounds, :merged_at, keyword_init: true) do
+      def seconds(phase = nil)
+        segments.select { |segment| phase.nil? || segment.phase == phase }.sum(&:seconds)
+      end
+
+      def merged? = !merged_at.nil?
+      def first_time? = merged? && review_rounds.zero?
+      def started_at = segments.find { |segment| segment.phase != "queued" }&.from
+      def cycle_time = merged_at && started_at && (merged_at - started_at)
+    end
+
+    Point = Struct.new(:at, :value, :status)
+
+    attr_reader :plan, :now
+
+    def self.duration(seconds)
+      return "-" if seconds.nil?
+
+      seconds = seconds.round
+      return "#{seconds}s" if seconds < 60
+
+      minutes = seconds / 60
+      return "#{minutes} min" if minutes < 60
+
+      hours, minutes = minutes.divmod(60)
+      return "#{hours} h#{" #{minutes} min" if minutes.positive?}" if hours < 24
+
+      days, hours = hours.divmod(24)
+      "#{days} d#{" #{hours} h" if hours.positive?}"
+    end
+
+    def initialize(plan, now: Time.now)
+      @plan = plan
+      @now = now
+      @events = plan.events.includes(:ticket).to_a
+    end
+
+    def started? = !development_start.nil?
+
+    def development_start
+      @development_start ||= (first("tickets.approved") || first("ticket.agent_started"))&.created_at
+    end
+
+    def delivered_at
+      first("plan.delivered")&.created_at
+    end
+
+    # The span the charts draw: from approving the tickets to delivery, or to now while it runs.
+    def window
+      return unless started?
+
+      finish = [delivered_at || now, *plan.evidence_runs.map(&:created_at)].max
+      [development_start, [finish, development_start + 60].max]
+    end
+
+    def tickets
+      @tickets ||= plan.tickets.map { |ticket| ticket_row(ticket) }
+    end
+
+    def totals
+      PHASES.keys.to_h { |phase| [phase, tickets.sum { |row| row.seconds(phase) }] }
+    end
+
+    def work_seconds
+      totals.slice(*WORK).values.sum
+    end
+
+    # The agents' part of the time a ticket was being worked on, from 0 to 100.
+    def agent_share
+      work = work_seconds
+      return if work.zero?
+
+      (totals.slice(*AGENT).values.sum * 100.0 / work).round
+    end
+
+    # Planning phases, first to last, as [label, seconds].
+    def phases
+      drafted = first("plan.drafted")&.created_at
+      approved = last("plan.approved")&.created_at
+      proven = plan.evidence_runs.find(&:passed?)&.created_at
+      [["Planning", span(drafted, approved)], ["Tickets", span(approved, development_start)],
+       ["Development", span(development_start, delivered_at || (now if started?))],
+       ["Proof", span(delivered_at, proven)]]
+    end
+
+    # Criteria merged and criteria proven by a passing test, over time, against the plan's scope.
+    def burnup
+      return unless started?
+
+      { scope: plan.tickets.sum { |ticket| ticket.criteria.size }, from: window.first, to: window.last,
+        merged: merged_points, proven: proven_points }
+    end
+
+    def summary
+      rows = tickets
+      { lead_time: span(first("plan.drafted")&.created_at, delivered_at || now), development: phases[2].last,
+        merged: rows.count(&:merged?), tickets: rows.size, first_time: rows.count(&:first_time?),
+        review_rounds: rows.sum(&:review_rounds), agent_share: agent_share,
+        points: plan.tickets.sum { |ticket| ticket.estimate.to_i } }.merge(proof, usage)
+    end
+
+    private
+
+    def proof
+      matrix = Evidence.matrix(plan)
+      { proven: matrix.count { |row| row.status == "passed" }, criteria: matrix.size,
+        evidence: plan.evidence_runs.any? }
+    end
+
+    def usage
+      totals = Usage.totals(Usage.rows(plan))
+      { tokens: totals[:total_tokens], cost: totals[:cost] }
+    end
+
+    def ticket_row(ticket)
+      events = @events.select { |event| event.ticket_id == ticket.id && NEXT_PHASE.key?(event.name) }
+      TicketRow.new(ticket: ticket, segments: segments(events),
+                    review_rounds: events.count { |event| event.name == "ticket.changes_requested" },
+                    merged_at: events.reverse.find { |event| event.name == "ticket.merged" }&.created_at)
+    end
+
+    # Every ticket is queued from the moment the tickets are approved until its agent starts.
+    def segments(events)
+      return [] unless development_start
+
+      phase = "queued"
+      from = development_start
+      list = events.filter_map do |event|
+        segment = Segment.new(phase, from, event.created_at) if phase && event.created_at > from
+        phase = NEXT_PHASE[event.name]
+        from = [from, event.created_at].max
+        segment
+      end
+      finish = delivered_at || now
+      list << Segment.new(phase, from, finish) if phase && finish > from
+      list
+    end
+
+    def merged_points
+      seen = Set.new
+      count = 0
+      merges = @events.select { |event| event.name == "ticket.merged" && event.ticket && seen.add?(event.ticket_id) }
+      [Point.new(development_start, 0)] + merges.map do |event|
+        count += event.ticket.criteria.size
+        Point.new(event.created_at, count)
+      end
+    end
+
+    def proven_points
+      runs = plan.evidence_runs.to_a
+      [Point.new(development_start, 0)] + runs.map do |run|
+        Point.new(run.created_at, Evidence.matrix(plan, run).count { |row| row.status == "passed" }, run.status)
+      end
+    end
+
+    def first(name)
+      @events.find { |event| event.name == name }
+    end
+
+    def last(name)
+      @events.reverse.find { |event| event.name == name }
+    end
+
+    def span(from, to)
+      from && to && to >= from ? to - from : nil
+    end
+  end
+end

data/lib/plan_driven/template.rb

@@ -15,6 +15,11 @@ module PlanDriven
       def drafted?
         source == :draft
       end
+
+      # The question as the interview shows it, in the terminal and in the wizard.
+      def prompt
+        required ? question.to_s : "#{question} (optional)"
+      end
     end
 
     GROUPS = ["Overview", "Background", "Architectural changes", "Work overview", "Risks", "Testing"].freeze
@@ -68,7 +73,7 @@ module PlanDriven
       { key: "when", title: "When", group: "Overview", source: :ask, required: false, min_words: 0,
         question: "When is it needed? (date, milestone, or leave empty)", guidance: "Target date or milestone." },
       { key: "background", title: "Background", group: "Background", source: :ask, required: false, min_words: 0,
-        question: "Links, PRDs, existing tickets, earlier decisions (optional)",
+        question: "Links, PRDs, existing tickets, earlier decisions",
         guidance: "Links, product documents, related tickets and earlier decisions." },
       { key: "existing_data_structure", title: "Existing Data Structure", group: "Background", source: :draft,
         required: true, min_words: 30,
@@ -91,7 +96,7 @@ module PlanDriven
         source: :draft, required: true, min_words: 5,
         guidance: "Queues, external services, feature flags and rollout order. Say so when there are none." },
       { key: "out_of_scope", title: "Out of Scope", group: "Work overview", source: :ask, required: false,
-        min_words: 0, question: "What is explicitly out of scope? (optional)",
+        min_words: 0, question: "What is explicitly out of scope?",
         guidance: "What this plan deliberately doesn't do." },
       { key: "risks", title: "Risks", group: "Risks", source: :draft, required: true, min_words: 15,
         guidance: "The main risks as a list, and how each is mitigated." },

data/lib/plan_driven/usage.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module PlanDriven
+  # What a plan cost to deliver: tokens for every model call and agent run, recorded as events
+  # (`llm.usage`, `agent.usage`), and dollars when config.token_prices has a price for the model.
+  # Prices change and differ per account, so the gem ships none: put in what your provider charges.
+  module Usage
+    FIELDS = %w[input_tokens output_tokens cache_write_tokens cache_read_tokens].freeze
+
+    Row = Struct.new(:step, :ticket, :model, :calls, :tokens, :duration_ms, :cost, keyword_init: true) do
+      def total_tokens
+        tokens.values.sum
+      end
+    end
+
+    module_function
+
+    # Dollars for these tokens, or nil when the model has no price. Prices are per million tokens:
+    #   config.token_prices = { "your-model-id" => { input: 3.0, output: 15.0, cache_write: 3.75, cache_read: 0.3 } }
+    def cost(model, tokens, config: PlanDriven.configuration)
+      price = config.token_prices.to_h.transform_keys(&:to_s)[model.to_s] or return
+      price = price.transform_keys(&:to_s)
+      FIELDS.sum { |field| tokens[field].to_i * price.fetch(field.delete_suffix("_tokens"), 0).to_f } / 1_000_000.0
+    end
+
+    # What the metered model spent in one step, as an `llm.usage` event.
+    def record_llm(plan, llm, step, actor:)
+      spent = llm.take
+      return if spent["calls"].zero?
+
+      plan.log!("llm.usage", actor: actor, step: step, model: llm.model, **spent.symbolize_keys)
+    end
+
+    # One finished agent run. Usage is bookkeeping: a provider that can't report it mustn't stop
+    # the ticket moving.
+    def record_agent(ticket, run, agents, actor:, config: PlanDriven.configuration)
+      tokens = agents.respond_to?(:usage) ? agents.usage(ticket.agent_id, run.id) : {}
+      step = ticket.plan.events.exists?(name: "agent.usage", ticket: ticket) ? "agent follow-up" : "agent run"
+      ticket.plan.log!("agent.usage", actor: actor, ticket: ticket, step: step, run: run.id,
+                                      model: config.agent_model || "#{config.agent_provider} default",
+                                      calls: 1, duration_ms: run.duration_ms, **tokens.to_h.symbolize_keys)
+    rescue Error => e
+      ticket.plan.log!("agent.usage", actor: actor, ticket: ticket, run: run.id, error: e.message[0, 200])
+    end
+
+    def tokens_from(payload)
+      FIELDS.to_h { |field| [field, payload.to_h[field].to_i] }
+    end
+
+    def rows(plan, config: PlanDriven.configuration)
+      plan.events.where(name: %w[llm.usage agent.usage]).order(:created_at).map do |event|
+        payload = event.payload.to_h
+        tokens = tokens_from(payload)
+        Row.new(step: payload["step"], ticket: event.ticket&.key, model: payload["model"], calls: payload["calls"].to_i,
+                tokens: tokens, duration_ms: payload["duration_ms"],
+                cost: cost(payload["model"], tokens, config: config))
+      end
+    end
+
+    def totals(rows)
+      tokens = FIELDS.to_h { |field| [field, rows.sum { |row| row.tokens[field] }] }
+      costs = rows.map(&:cost)
+      { tokens: tokens, total_tokens: tokens.values.sum, cost: costs.any?(&:nil?) ? nil : costs.sum,
+        priced_cost: costs.compact.sum, unpriced: rows.select { |row| row.cost.nil? }.map(&:model).uniq }
+    end
+
+    def format_tokens(count)
+      count >= 1_000_000 ? format("%.2fM", count / 1_000_000.0) : count.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+    end
+
+    def format_cost(cost)
+      cost ? format("$%.2f", cost) : "-"
+    end
+  end
+
+  # Wraps any model client and counts what each step spends, so Delivery can record it.
+  class MeteredLLM
+    def initialize(llm)
+      @llm = llm
+      reset
+    end
+
+    def chat(**options)
+      reply = @llm.chat(**options)
+      @calls += 1
+      @input += reply.input_tokens.to_i
+      @output += reply.output_tokens.to_i
+      reply
+    end
+
+    def label
+      @llm.label
+    end
+
+    def model
+      label.split("/", 2).last
+    end
+
+    # What was spent since the last take, as an event payload.
+    def take
+      spent = { "calls" => @calls, "input_tokens" => @input, "output_tokens" => @output }
+      reset
+      spent
+    end
+
+    private
+
+    def reset
+      @calls = 0
+      @input = 0
+      @output = 0
+    end
+  end
+end

data/lib/plan_driven/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PlanDriven
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/plan_driven/wizard/engine.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module PlanDriven
+  module Wizard
+    # Mounted by the install generator, in development only:
+    #
+    #   mount PlanDriven::Wizard::Engine, at: "/plan_driven" if Rails.env.development?
+    class Engine < ::Rails::Engine
+      isolate_namespace PlanDriven::Wizard
+      engine_name "plan_driven_wizard"
+      config.root = File.expand_path("../../..", __dir__)
+
+      # Keys pasted on the Configuration page go to `plan-driven connect` on stdin, never to the log.
+      initializer "plan_driven_wizard.filter_parameters" do |app|
+        app.config.filter_parameters += [:api_key]
+      end
+    end
+  end
+end