hecks 3.5.0 → 3.6.0

data/lib/hecks/corpus.rb

@@ -1,4 +1,7 @@
 require_relative "fuzzing/target_capabilities"
+require_relative "corpus/routes"
+require_relative "corpus/rust"
+require_relative "corpus/sources"
 
 module Hecks
   # The corpus: every place in this repo that holds a real domain, named once.
@@ -40,36 +43,8 @@ module Hecks
 
     KINDS = (DIRECTORY_KINDS.keys + FILE_KINDS.keys).freeze
 
-    # A route sends what the sweep does not boot elsewhere; each entry's
-    # `check` (:named_in, :gitignored, :gap) is verified by
-    # spec/corpus_accounting_spec.rb, matched in order.
-    Route = Struct.new(:pattern, :check, :destination, :names, :why)
-
-    ROUTES = [
-      Route.new(%r{\Atmp/}, :gitignored, ".gitignore", "tmp/",
-                "scratch output — generated and mined candidate domains, fuzz failures — never committed"),
-      Route.new(%r{/\.aws-sam/}, :gitignored, ".gitignore", ".aws-sam/",
-                "SAM build output — a vendored copy of real sources, never committed"),
-      Route.new(%r{/data/eras/}, :gitignored, ".gitignore", "**/data/eras/",
-                "era snapshots a file adapter writes at runtime, never committed"),
-      Route.new(%r{\Arust/}, :named_in, "rust/parser/tests/gates.rs", :each_file,
-                "the Rust parser's own fixtures, each loaded by its gate tests"),
-      Route.new(%r{/translations/}, :named_in, "spec/translation/committed_edges_spec.rb", "bluebook/translations",
-                "translation edges, not chapters; each must load, chain era to era, and end at the storage " \
-                "shape its bluebook declares today"),
-      Route.new(%r{\Alib/hecks/forms/examples/}, :named_in, "spec/forms/app_spec.rb", :each_file,
-                "a Forms presentation config wearing the .bluebook extension"),
-      Route.new(%r{\Aspec/fixtures/eras/}, :named_in, "spec/runtime/storage_shape_spec.rb", "fixtures/eras",
-                "deliberately conflicting versions of one domain; each must classify as the verdict " \
-                "its filename declares (bump_* / same_*)"),
-      Route.new(%r{\Aspec/fixtures/model_check/}, :named_in, "spec/model_check_spec.rb", :each_file,
-                "domains broken on purpose; each must produce exactly the finding kinds it is built to trigger"),
-      # hecks fuzz sweeps it too, once Fuzzing::Replay coerces value-object args
-      # before recomputing givens — today it doesn't, so this given reads as
-      # wrongly admitted.
-      Route.new(%r{\Aspec/fixtures/rust_host/}, :named_in, "rust/host/src/web.rs", "checkout_fixture",
-                "the Rust host's checkout fixture, pinned by its web and /api tests")
-    ].freeze
+    extend Rust
+    extend Sources
 
     module_function
 
@@ -82,20 +57,32 @@ module Hecks
     # @raise [ArgumentError] if `kinds` names a kind not in `KINDS`
     def members(*kinds, root: ROOT)
       kinds = KINDS if kinds.empty?
-      kinds.flat_map do |kind|
-        if (glob = DIRECTORY_KINDS[kind])
-          Dir.glob(File.join(root, glob)).select { |path| File.directory?(path) }.sort
-             .map { |dir| Member.new(File.basename(dir), kind, dir) }
-        elsif (glob = FILE_KINDS[kind])
-          prefix = File.join(root, glob[/\A[^*]*/].chomp("/"))
-          Dir.glob(File.join(root, glob))
-             .map { |file| Member.new(file.delete_prefix("#{prefix}/").delete_suffix(".bluebook"), kind, file) }
-        else
-          raise ArgumentError, "unknown corpus kind #{kind.inspect} — known: #{KINDS.join(', ')}"
-        end
+      kinds.flat_map { |kind| kind_members(kind, root) }
+    end
+
+    # @return [Array<Member>] the members of one kind: a directory each, or a file each
+    # @raise [ArgumentError] if `kind` is not in `KINDS`
+    def kind_members(kind, root)
+      if (glob = DIRECTORY_KINDS[kind])
+        directory_members(kind, root, glob)
+      elsif (glob = FILE_KINDS[kind])
+        file_members(kind, root, glob)
+      else
+        raise ArgumentError, "unknown corpus kind #{kind.inspect} — known: #{KINDS.join(", ")}"
       end
     end
 
+    def directory_members(kind, root, glob)
+      Dir.glob(File.join(root, glob)).select { |path| File.directory?(path) }.sort
+         .map { |dir| Member.new(File.basename(dir), kind, dir) }
+    end
+
+    def file_members(kind, root, glob)
+      prefix = File.join(root, glob[/\A[^*]*/].chomp("/"))
+      Dir.glob(File.join(root, glob))
+         .map { |file| Member.new(file.delete_prefix("#{prefix}/").delete_suffix(".bluebook"), kind, file) }
+    end
+
     # What a boot loads for a member — a directory kind's bluebook
     # directory, or a file kind's own file.
     #
@@ -106,45 +93,6 @@ module Hecks
       DIRECTORY_KINDS.key?(member.kind) ? bluebook_dir(member.path) : member.path
     end
 
-    # How a domain's hecksagon names a chapter the gem carries and loads by name, as the QA
-    # ledger's does (`Hecks::Chapters.load!("QualityControl")`).
-    ATTACHED_CHAPTER = /Chapters\.load!\(\s*"([^"]+)"\s*\)/
-
-    # Where a domain path keeps its bluebooks — `<domain>/bluebook/*.bluebook`, or the directory
-    # itself when that holds none, or the chapter files its hecksagons load by name (the QA
-    # ledger, `qa/bluebook`, holds only wiring: its chapter ships in `lib/hecks/quality_control/`).
-    #
-    # @param domain_path [String] path to a domain directory
-    # @return [Array<String>, nil] `.bluebook` file paths found, or nil when none
-    def bluebook_files(domain_path)
-      [File.join(domain_path, "bluebook"), domain_path].each do |dir|
-        files = Dir[File.join(dir, "*.bluebook")]
-        return files unless files.empty?
-      end
-      attached_chapter_files(domain_path)
-    end
-
-    # The bluebook files of every chapter a domain's own hecksagons load by name.
-    #
-    # @param domain_path [String] path to a domain directory
-    # @return [Array<String>, nil] the chapters' `.bluebook` file paths, or nil when none is named
-    def attached_chapter_files(domain_path)
-      names = Dir[File.join(domain_path, "{bluebook/,}*.hecksagon")]
-              .flat_map { |path| File.read(path).scan(ATTACHED_CHAPTER).flatten }.uniq
-      files = names.flat_map { |name| Chapters.index.fetch(name, []) }
-      files.empty? ? nil : files
-    end
-
-    # The directory holding `domain_path`'s bluebooks.
-    #
-    # @param domain_path [String] path to a domain directory
-    # @return [String, nil] directory of the first `.bluebook` file found, or nil
-    #   when `domain_path` holds none
-    def bluebook_dir(domain_path)
-      files = bluebook_files(domain_path)
-      files && File.dirname(files.first)
-    end
-
     # The route a repo-relative path takes instead of the sweep — the
     # first one it matches — or `nil` when the sweep boots it.
     #
@@ -224,191 +172,6 @@ module Hecks
       File.basename(dir) == "bluebook" ? File.dirname(dir) : dir
     end
 
-    # Every Rust-facing list reads `rust/Cargo.toml`'s `[features]` as its
-    # one source, rather than each consumer typing it out by hand.
-    GENERATED_DIR = "rust/src/generated".freeze
-    RUST_DOMAIN_KINDS = %i[example stress fixture].freeze
-
-    RustDomain = Struct.new(:feature, :dir, :kind)
-
-    # `check: :named_in` — `destination` must name `names`.
-    Elsewhere = Struct.new(:check, :destination, :names, :why)
-
-    RUST_ELSEWHERE = {
-      "meta" => Elsewhere.new(:named_in, "lib/hecks/tools/regeneration_run.rb", "rust/src/generated",
-                              "the self-hosted grammar (lib/hecks/language), not a domain directory — every " \
-                              "hecks project_rust run rewrites it, so the drift check diffs it with the rest " \
-                              "of rust/src/generated, and there is no directory to fuzz")
-    }.freeze
-
-    # Every place a Rust-facing domain's own `attaches` attachment could be declared.
-    #
-    # @param root [String] repository root to search under
-    # @return [String] every reachable hecksagon file's own text, joined by newlines
-    def rust_attachment_hecksagon_text(root: ROOT)
-      rust_domains(root: root).flat_map { |domain| Dir.glob(File.join(domain.dir, "**", "*.hecksagon")) }
-                              .map { |path| File.read(path) }.join("\n")
-    end
-
-    # **Shrink-only**: a generated module `hecks rust_coverage` still flags.
-    # `hecks corpus_rust_coverage --rust-coverage` requires each entry to still fail, so a
-    # newly-passing module breaks the build until removed here.
-    RUST_COVERAGE_PENDING = {}.freeze
-
-    # The Cargo `[features]` table's raw text.
-    #
-    # @param root [String] repository root to search under
-    # @return [String] the table's text, or `""` when `rust/Cargo.toml` has none
-    def cargo_features_table(root: ROOT)
-      File.read(File.join(root, "rust/Cargo.toml"))[Fuzzing::TargetCapabilities::FEATURES_TABLE] || ""
-    end
-
-    # Every feature name Cargo declares with no dependency array of its own.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<String>] feature names, in file order
-    def cargo_features(root: ROOT)
-      cargo_features_table(root: root).scan(/^(\w+)\s*=\s*\[\]/).flatten
-    end
-
-    # The feature `hecks project_rust` last wrote as Cargo's `default`.
-    #
-    # @param root [String] repository root to search under
-    # @return [String, nil] the default feature's name, or nil when Cargo.toml
-    #   declares none
-    def cargo_default(root: ROOT)
-      cargo_features_table(root: root)[/^default\s*=\s*\["(\w+)"\]/, 1]
-    end
-
-    # Every in-repo domain directory whose name is a Cargo feature, sorted
-    # by path — disambiguated by the generated module's metadata.rs stamp when one exists.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<RustDomain>] each Rust-facing domain, sorted by directory path
-    def rust_domains(root: ROOT)
-      features = cargo_features(root: root)
-      members(*RUST_DOMAIN_KINDS, root: root)
-        .map { |member| [domain_dir_of(member), member.kind] }
-        .uniq(&:first)
-        .select { |dir, _| rust_domain_dir?(dir, features, root) }
-        .sort_by(&:first)
-        .map { |dir, kind| RustDomain.new(File.basename(dir).downcase, dir, kind) }
-    end
-
-    # Tells whether `dir` is the real source directory for its Cargo feature.
-    #
-    # @param dir [String] a candidate domain directory
-    # @param features [Array<String>] known Cargo feature names
-    # @param root [String] repository root `dir` is rooted under
-    # @return [Boolean]
-    def rust_domain_dir?(dir, features, root)
-      feature = File.basename(dir).downcase
-      source = generated_source(feature, root: root)
-      features.include?(feature) && (source.nil? || source == dir.delete_prefix("#{root}/"))
-    end
-
-    # The stamp hecks project_rust writes into metadata.rs, e.g.
-    # `examples/pizzas` or `the self-hosted language (...)`, with any
-    # ` (attaches "X")`/` (attaches "x", from: :vendor)` suffix stripped.
-    SOURCE_STAMP = /
-      GENERATED\ by\ hecks\ project_rust\ —\ (.+?)
-      (?:\ \(attaches\ "\w+"(?:,\ from:\ :vendor)?\))?
-      's\ own\ canonical\ IR,
-    /x
-
-    # Where a generated module came from, read off the stamp `hecks project_rust`
-    # writes into its metadata.rs.
-    #
-    # @param module_name [String] a generated module's name
-    # @param root [String] repository root to search under
-    # @return [String, nil] the source path the stamp names, or nil when the module
-    #   is not generated
-    def generated_source(module_name, root: ROOT)
-      metadata = File.join(root, GENERATED_DIR, module_name, "metadata.rs")
-      File.file?(metadata) ? File.read(metadata)[SOURCE_STAMP, 1] : nil
-    end
-
-    # Every module directory under `rust/src/generated`.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<String>] generated module names, sorted
-    def generated_modules(root: ROOT)
-      Dir.children(File.join(root, GENERATED_DIR)).select { |name| File.directory?(File.join(root, GENERATED_DIR, name)) }.sort
-    end
-
-    # Tells whether `feature` has already been generated (has a merged.rs).
-    #
-    # @param feature [String] a Cargo feature / domain name
-    # @param root [String] repository root to search under
-    # @return [Boolean]
-    def generated?(feature, root: ROOT)
-      File.file?(File.join(root, GENERATED_DIR, feature, "merged.rs"))
-    end
-
-    # Chapters attached via `kind` that get a generated module as a side
-    # effect of the attaching domain's regen, with no merged.rs of their own.
-    #
-    # @param kind [Symbol] :framework or :vendored
-    # @param root [String] repository root to search under
-    # @return [Array<String>] stems with a generated module but no merged.rs
-    #   of their own
-    def rust_side_chapters(kind, root: ROOT)
-      modules = generated_modules(root: root)
-      members(kind, root: root).map(&:stem)
-                               .select do |stem|
-                                 module_name = Naming.pascal(stem).downcase
-                                 modules.include?(module_name) && !generated?(module_name, root: root)
-                               end
-    end
-
-    # The generated module name a `:framework`/`:vendored` stem writes
-    # under — strips underscores, so it can differ from the raw stem.
-    #
-    # @param stem [String] a `:framework`/`:vendored` corpus member's stem
-    # @return [String] the generated module's own directory name
-    def rust_side_module_name(stem)
-      Naming.pascal(stem).downcase
-    end
-
-    # Names the framework chapters Rust carries as modules, beside its domains' own bluebooks.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<String>] generated module names for the `:framework` corpus members
-    def rust_framework_chapters(root: ROOT)
-      rust_side_chapters(:framework, root: root)
-    end
-
-    # Names the vendored chapters Rust carries as modules, beside its domains' own bluebooks.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<String>] generated module names for the `:vendored` corpus members
-    def rust_vendored_chapters(root: ROOT)
-      rust_side_chapters(:vendored, root: root)
-    end
-
-    # Chapters an in-repo Rust domain carries beside its own bluebook,
-    # written as a module by `hecks project_rust` with no merged.rs of its own.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<String>] generated module names with no merged.rs of their own
-    def rust_sibling_chapters(root: ROOT)
-      modules = generated_modules(root: root)
-      rust_domains(root: root)
-        .flat_map { |domain| Dir.glob(File.join(domain.dir, "bluebook", "*.bluebook")) }
-        .map { |path| Naming.pascal(File.basename(path, ".bluebook")).downcase }
-        .uniq
-        .select { |name| modules.include?(name) && !generated?(name, root: root) }
-    end
-
-    # The regeneration order the drift check runs, sorted by path — so which
-    # domain wins Cargo's `default` is derived, not hand-picked.
-    #
-    # @param root [String] repository root to search under
-    # @return [Array<RustDomain>] already-generated Rust domains, in regeneration order
-    def rust_regen_order(root: ROOT)
-      rust_domains(root: root).select { |domain| generated?(domain.feature, root: root) }
-    end
-
     # The chapter a bluebook declares — `Hecks.bluebook "<Name>"` — read off
     # the file rather than guessed from its name, since the two can differ.
     #

data/lib/hecks/doc/reference/coverage.rb

@@ -0,0 +1,70 @@
+module Hecks
+  module Doc
+    module Reference
+      # The two coverage gates over a reference's prose: every live word is written about, and
+      # every one has a runnable example. Extended onto `Reference`.
+      module Coverage
+        # Matches a runnable `ruby`/`ruby bluebook`/`ruby boot` fenced example;
+        # `ruby skip` and hidden boot blocks are setup, not an example, and don't count.
+        EXAMPLE_FENCE = /^```ruby(?: bluebook| boot)?[ \t]*$/
+
+        # Whether a word's prose carries a runnable example.
+        def exemplified?(prose) = prose.to_s.match?(EXAMPLE_FENCE)
+
+        # Every live word paired with its prose; shared by both coverage gates
+        # so their page-walks can't drift apart.
+        def live_words(directory)
+          rows = contexts.flat_map do |context|
+            prose = page_prose(directory, context)
+            keywords.select { |row| row[:context] == context && live?(row) }
+                    .map { |row| [row[:word], context, prose[row[:word]]] }
+          end
+          rows.uniq { |word, context, _| [word, context] }
+        end
+
+        # @return [Hash{String, Symbol => String}] the prose committed in a context's page, or
+        #   none when the page does not exist yet
+        def page_prose(directory, context)
+          path = File.join(directory, page_name(context))
+          File.exist?(path) ? harvest(File.read(path)) : {}
+        end
+
+        # Disambiguates a word by the context it is declared in.
+        def name_of(word, context) = "#{word} (#{context})"
+
+        # The coverage gate's question: every live word with no prose yet.
+        def undocumented(directory)
+          live_words(directory).reject { |_word, _context, prose| prose }
+                               .map { |word, context, _| name_of(word, context) }
+        end
+
+        # The second coverage gate: prose alone can go stale or describe
+        # unwired behavior; a runnable example is the only doc that can go red.
+        def unexemplified(directory)
+          live_words(directory).reject { |_word, _context, prose| exemplified?(prose) }
+                               .map { |word, context, _| name_of(word, context) }
+        end
+
+        # Both coverage gates, worded for a person: each gap section lists the words that owe it,
+        # and a word missing both kinds is listed twice, one repair each.
+        #
+        # @param directory [String] the reference pages
+        # @return [Array(Boolean, String)] whether every live word is covered, and what to print
+        def coverage_report(directory)
+          gaps = [[undocumented(directory), "no prose — write their sections"],
+                  [unexemplified(directory), "no running example — write one in each word's own section"]]
+          sections = gaps.reject { |words, _| words.empty? }.map { |words, owed| gap_section(words, owed) }
+          return [true, "every live word carries prose and a running example."] if sections.empty?
+
+          [false, sections.join]
+        end
+
+        # @return [String] the heading and list of the words that owe one kind of coverage
+        def gap_section(words, owed)
+          head = "#{words.size} live #{words.size == 1 ? "word carries" : "words carry"} #{owed}:\n"
+          "#{head}#{words.map { |word| "  #{word}\n" }.join}\n"
+        end
+      end
+    end
+  end
+end

data/lib/hecks/doc/reference/harvester.rb

@@ -0,0 +1,55 @@
+module Hecks
+  module Doc
+    module Reference
+      # Reads the hand-written prose back out of a committed reference page: everything between a
+      # section's generated region and the next `## ` heading, keyed by word.
+      #
+      # Starts collecting under `PREAMBLE` (not nil) so a page written before that region existed
+      # still parses unchanged. A `## ` line inside a fence is not a heading: it is a comment in
+      # the runnable example, not a section break.
+      class Harvester
+        def initialize
+          @prose = {}
+          @current = PREAMBLE
+          @collecting = false
+          @buffer = []
+          @in_fence = false
+        end
+
+        # @param text [String] a reference page
+        # @return [Hash{String, Symbol => String}] each word's prose (the preamble under
+        #   `PREAMBLE`), leaving out a word with none or only the TODO sentinel
+        def call(text)
+          text.each_line { |line| consume(line) }
+          flush
+          @prose.reject { |_word, body| body.empty? || body == TODO_SENTINEL }
+        end
+
+        private
+
+        def consume(line)
+          @in_fence = !@in_fence if line.start_with?("```")
+          heading = line.match(/\A## (\S+)\s*\z/) unless @in_fence
+          if heading
+            start_section(heading[1])
+          elsif line.include?(GENERATED_END)
+            @collecting = true
+          elsif @collecting
+            @buffer << line
+          end
+        end
+
+        def start_section(word)
+          flush
+          @current = word
+          @collecting = false
+          @buffer = []
+        end
+
+        def flush
+          @prose[@current] = @buffer.join.strip if @current && @collecting
+        end
+      end
+    end
+  end
+end

data/lib/hecks/doc/reference/readme.rb

@@ -0,0 +1,89 @@
+module Hecks
+  module Doc
+    module Reference
+      # README's own generated regions, keyed by region id instead of a word. Extended onto
+      # `Reference`.
+      module Readme
+        # README's own generated regions, keyed by region id instead of a word.
+        def readme_regions(root)
+          {
+            "guides"    => guide_index(root),
+            "reference" => reference_index(root),
+            "corpus"    => corpus_roster(root),
+            "diagrams"  => diagram_showcase(root)
+          }
+        end
+
+        # Lists every committed guide, linked and titled by its own heading.
+        def guide_index(root)
+          paths = Dir.glob(File.join(root, "docs/implemented/guides/*.md"))
+                     .reject { |p| %w[AUTHORING.md].include?(File.basename(p)) }
+          lines = paths.map do |path|
+            heading = File.foreach(path).find { |line| line.start_with?("# ") }
+            title = heading ? heading.sub(/\A#\s*/, "").strip : File.basename(path)
+            "- [#{title}](docs/implemented/guides/#{File.basename(path)})"
+          end
+          lines.join("\n")
+        end
+
+        # Links the reference index, with its context count.
+        def reference_index(_root)
+          count = contexts.size
+          "[The DSL reference](docs/implemented/reference/index.md) — #{count} contexts, generated from " \
+            "the aggregate-local tables under `lib/hecks/language/` and held to them by " \
+            "`spec/reference_golden_spec.rb`."
+        end
+
+        # Quotes the real, committed diagram file rather than re-deriving one,
+        # so this can't drift from `spec/diagrams_spec.rb`'s own check.
+        def diagram_showcase(root)
+          lifecycle = File.read(File.join(root, "docs/generated/diagrams/pizzas/Order_lifecycle.mmd")).strip
+          <<~MARKDOWN.strip
+            `hecks project_diagrams` reads a booted domain's own declaration and draws it as Mermaid — nine kinds so far: `<Name>_lifecycle.mmd`, `relationships.mmd`, `dispatch.mmd`, `roles.mmd`, `ports.mmd`, `read_models.mmd`, `<Name>_surface.mmd` (what a command does, and what it writes), `<Name>_saga.mmd`, and `frameworks.mmd`. Nothing hand-drawn — the same reason a domain is data at all. Order's own lifecycle, straight off the bluebook above:
+
+            ```mermaid
+            #{lifecycle}
+            ```
+
+            The full set for every domain in this checkout — `examples/pizzas`, `examples/banking` — lives in [`docs/generated/diagrams/`](docs/generated/diagrams/), held to the declaration by `spec/diagrams_spec.rb` the same drift-refusing way this page is held to its own source.
+          MARKDOWN
+        end
+
+        # Lists every example domain with a `.bluebook` file, with its own declared vision.
+        def corpus_roster(root)
+          Dir.glob(File.join(root, "examples/*/")).filter_map { |dir| roster_line(dir) }.join("\n")
+        end
+
+        # @return [String, nil] the example's name and vision as a list item; nil without a bluebook
+        def roster_line(dir)
+          bluebooks = example_bluebooks(dir)
+          return if bluebooks.empty?
+
+          vision = bluebooks.filter_map { |bluebook| File.read(bluebook)[/vision\s+"([^"]*)"/, 1] }.first
+          "- **#{File.basename(dir.chomp("/"))}** — #{vision}"
+        end
+
+        # @return [Array<String>] the example's bluebooks, under `bluebook/` or else beside it
+        def example_bluebooks(dir)
+          bluebooks = Dir.glob(File.join(dir, "bluebook/*.bluebook"))
+          bluebooks.empty? ? Dir.glob(File.join(dir, "*.bluebook")) : bluebooks
+        end
+
+        # Replaces every generated region inside `text` with its freshly rendered
+        # content, leaving the hand-written parts of README untouched.
+        def render_readme(root, text)
+          readme_regions(root).reduce(text) do |current, (id, content)|
+            pattern = /#{Regexp.escape(region_begin(id))}.*?#{Regexp.escape(GENERATED_END)}/m
+            current.sub(pattern) { "#{region_begin(id)}\n#{content}\n#{GENERATED_END}" }
+          end
+        end
+
+        # Regenerates README's generated regions in place.
+        def write_readme!(root)
+          path = File.join(root, "README.md")
+          File.write(path, render_readme(root, File.read(path)))
+        end
+      end
+    end
+  end
+end