jekyll-agent-audit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +17 -0
- data/README.md +180 -0
- data/Rakefile +38 -0
- data/doc/Jekyll/AgentAudit/BuildInventory.md +29 -0
- data/doc/Jekyll/AgentAudit/Configuration.md +42 -0
- data/doc/Jekyll/AgentAudit/ConfigurationError.md +6 -0
- data/doc/Jekyll/AgentAudit/Error.md +6 -0
- data/doc/Jekyll/AgentAudit/Extractor.md +20 -0
- data/doc/Jekyll/AgentAudit/Finding.md +30 -0
- data/doc/Jekyll/AgentAudit/InputError.md +6 -0
- data/doc/Jekyll/AgentAudit/LinkGraph.md +26 -0
- data/doc/Jekyll/AgentAudit/Registry.md +38 -0
- data/doc/Jekyll/AgentAudit/Report.md +27 -0
- data/doc/Jekyll/AgentAudit/ReportError.md +6 -0
- data/doc/Jekyll/AgentAudit/Reporters/Console.md +19 -0
- data/doc/Jekyll/AgentAudit/Reporters/JSON.md +9 -0
- data/doc/Jekyll/AgentAudit/Reporters.md +5 -0
- data/doc/Jekyll/AgentAudit/Rules/Provenance.md +127 -0
- data/doc/Jekyll/AgentAudit/Rules/Publication.md +73 -0
- data/doc/Jekyll/AgentAudit/Rules.md +5 -0
- data/doc/Jekyll/AgentAudit/Runner.md +13 -0
- data/doc/Jekyll/AgentAudit/UrlResolver.md +13 -0
- data/doc/Jekyll/AgentAudit.md +35 -0
- data/doc/Jekyll/Commands/AgentAudit/CommandParser.md +9 -0
- data/doc/Jekyll/Commands/AgentAudit/ParserErrors.md +9 -0
- data/doc/Jekyll/Commands/AgentAudit.md +13 -0
- data/doc/Jekyll/Commands.md +5 -0
- data/doc/Jekyll.md +5 -0
- data/doc/README.md +180 -0
- data/doc/index.csv +154 -0
- data/docs/example-report.json +281 -0
- data/docs/implementation-plan.md +26 -0
- data/docs/limitations.md +31 -0
- data/docs/verification.md +52 -0
- data/lib/jekyll/agent_audit/build_inventory.rb +196 -0
- data/lib/jekyll/agent_audit/command.rb +179 -0
- data/lib/jekyll/agent_audit/configuration.rb +196 -0
- data/lib/jekyll/agent_audit/errors.rb +10 -0
- data/lib/jekyll/agent_audit/extractor.rb +230 -0
- data/lib/jekyll/agent_audit/finding.rb +57 -0
- data/lib/jekyll/agent_audit/link_graph.rb +82 -0
- data/lib/jekyll/agent_audit/registry.rb +133 -0
- data/lib/jekyll/agent_audit/report.rb +79 -0
- data/lib/jekyll/agent_audit/reporters/console.rb +144 -0
- data/lib/jekyll/agent_audit/reporters/json.rb +16 -0
- data/lib/jekyll/agent_audit/rules/provenance.rb +412 -0
- data/lib/jekyll/agent_audit/rules/publication.rb +186 -0
- data/lib/jekyll/agent_audit/runner.rb +266 -0
- data/lib/jekyll/agent_audit/url_resolver.rb +98 -0
- data/lib/jekyll/agent_audit/version.rb +7 -0
- data/lib/jekyll-agent-audit.rb +24 -0
- data/llms.txt +35 -0
- data/schema/report-1.0.json +66 -0
- data/script/benchmark.rb +68 -0
- metadata +133 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "date"
|
|
6
|
+
|
|
7
|
+
module Jekyll
|
|
8
|
+
module AgentAudit
|
|
9
|
+
class BuildInventory
|
|
10
|
+
attr_reader :entries, :collisions, :diagnostics
|
|
11
|
+
|
|
12
|
+
def initialize(site, configuration)
|
|
13
|
+
@site, @configuration = site, configuration
|
|
14
|
+
@entries, @collisions, @diagnostics = [], [], []
|
|
15
|
+
@claims = Hash.new { |h, k| h[k] = [] }
|
|
16
|
+
@captured = @complete = false
|
|
17
|
+
@capture_failed = false
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def capture!
|
|
21
|
+
@captured = true
|
|
22
|
+
each_owner do |owner|
|
|
23
|
+
path = output_path(owner)
|
|
24
|
+
add_claim(owner, path) if path
|
|
25
|
+
end
|
|
26
|
+
self
|
|
27
|
+
rescue StandardError => e
|
|
28
|
+
@capture_failed = true
|
|
29
|
+
@diagnostics << {code: :inventory_capture_failed, message: e.message}
|
|
30
|
+
self
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def finalize!
|
|
34
|
+
capture! unless @captured
|
|
35
|
+
each_owner { |owner| add_claim(owner, output_path(owner)) if output_path(owner) }
|
|
36
|
+
destination = File.expand_path(@site.dest.to_s)
|
|
37
|
+
if Dir.exist?(destination)
|
|
38
|
+
Dir.glob(File.join(destination, "**", "*")).sort.each do |absolute|
|
|
39
|
+
next unless File.file?(absolute)
|
|
40
|
+
relative = "/#{Pathname.new(absolute).relative_path_from(Pathname.new(destination)).to_s.tr('\\', '/')}"
|
|
41
|
+
add_claim(nil, relative, final_artifact: true) unless @claims.key?(relative)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
@entries = @claims.values.flatten.uniq { |entry| [entry[:owner_token], entry[:output_path]] }
|
|
45
|
+
@collisions = @claims.map do |path, owners|
|
|
46
|
+
unique = owners.uniq { |entry| entry[:owner_token] }
|
|
47
|
+
{output_path: path, primary: unique.first, owners: unique} if unique.length > 1
|
|
48
|
+
end.compact
|
|
49
|
+
max = config_limit(:max_documents, 50_000)
|
|
50
|
+
html_count = @entries.count { |entry| entry[:output_path].end_with?(".html") || entry[:route].to_s.end_with?("/") }
|
|
51
|
+
if html_count > max
|
|
52
|
+
@diagnostics << {code: :resource_limit, limit: :max_documents, value: html_count}
|
|
53
|
+
@complete = false
|
|
54
|
+
else
|
|
55
|
+
@complete = !@capture_failed && @diagnostics.none? { |diagnostic| %i[ownership_outside_destination inventory_capture_failed].include?(diagnostic[:code]) }
|
|
56
|
+
end
|
|
57
|
+
self
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def complete?
|
|
61
|
+
@complete
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def each_owner
|
|
67
|
+
objects = %i[pages documents static_files].flat_map { |name| @site.respond_to?(name) ? Array(@site.public_send(name)) : [] }
|
|
68
|
+
seen = {}
|
|
69
|
+
objects.each do |owner|
|
|
70
|
+
next if seen[owner.object_id]
|
|
71
|
+
seen[owner.object_id] = true
|
|
72
|
+
yield owner
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def output_path(owner)
|
|
77
|
+
return owner[:output_path] if owner.is_a?(Hash) && owner[:output_path]
|
|
78
|
+
return owner.output_path.to_s if owner.respond_to?(:output_path) && owner.output_path
|
|
79
|
+
return owner.destination(@site.dest.to_s).to_s if owner.respond_to?(:destination)
|
|
80
|
+
nil
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def add_claim(owner, raw_path, final_artifact: false)
|
|
84
|
+
output = normalize_output(raw_path, final_artifact: final_artifact)
|
|
85
|
+
return if output.nil? || output.empty?
|
|
86
|
+
absolute = File.expand_path(output.delete_prefix("/"), @site.dest.to_s)
|
|
87
|
+
source = source_path(owner)
|
|
88
|
+
entry = {
|
|
89
|
+
identity: identity_for(owner, output, source), source_path: (source.to_s.empty? ? nil : source), output_path: output,
|
|
90
|
+
owner_token: owner ? owner.object_id : "artifact:#{output}",
|
|
91
|
+
absolute_path: absolute, url: public_url(route_for(owner, output)), route: route_for(owner, output),
|
|
92
|
+
destination_root: (File.realpath(@site.dest.to_s) rescue @site.dest.to_s),
|
|
93
|
+
data: data_for(owner), explicit_data: explicit_data_for(owner),
|
|
94
|
+
selected: selected?(source), final_artifact: final_artifact
|
|
95
|
+
}
|
|
96
|
+
@claims[output] << entry
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def normalize_output(path, final_artifact: false)
|
|
100
|
+
value = path.to_s.tr("\\", "/")
|
|
101
|
+
return nil if value.empty?
|
|
102
|
+
destination = @site.dest.to_s.tr('\\', '/').sub(%r{/\z}, '')
|
|
103
|
+
if value.start_with?("/")
|
|
104
|
+
if value == destination
|
|
105
|
+
value = ""
|
|
106
|
+
elsif value.start_with?("#{destination}/")
|
|
107
|
+
value = value.delete_prefix(destination)
|
|
108
|
+
elsif !destination.empty? && !final_artifact
|
|
109
|
+
@diagnostics << {code: :ownership_outside_destination, path: value}
|
|
110
|
+
return nil
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
value = "/#{value}" unless value.start_with?("/")
|
|
114
|
+
value = value.gsub(%r{/+}, "/")
|
|
115
|
+
return nil if Pathname.new(value).cleanpath.to_s.split("/").include?("..")
|
|
116
|
+
value
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def route_for(owner, output)
|
|
120
|
+
route = owner.is_a?(Hash) ? owner[:route] : (owner.respond_to?(:url) ? owner.url : nil)
|
|
121
|
+
route = output if route.to_s.empty?
|
|
122
|
+
route = "/#{route}" unless route.to_s.start_with?("/")
|
|
123
|
+
base = @site.respond_to?(:config) ? @site.config.fetch("baseurl", "").to_s.sub(%r{/\z}, "") : ""
|
|
124
|
+
route = "#{base}#{route}" if !base.empty? && route != base && !route.start_with?("#{base}/")
|
|
125
|
+
route.to_s
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def public_url(route)
|
|
129
|
+
config = @site.respond_to?(:config) ? @site.config : {}
|
|
130
|
+
host = config.fetch("url", "").to_s.sub(%r{\z/}, "")
|
|
131
|
+
"#{host}#{route}".sub(%r{(?<!:)//+}, "/")
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def source_path(owner)
|
|
135
|
+
return nil unless owner
|
|
136
|
+
return owner[:source_path] if owner.is_a?(Hash) && owner.key?(:source_path)
|
|
137
|
+
source_root = @site.respond_to?(:source) ? File.realpath(@site.source.to_s) : nil
|
|
138
|
+
candidate = if owner.respond_to?(:path) && owner.path && !owner.path.to_s.empty?
|
|
139
|
+
owner.path.to_s
|
|
140
|
+
elsif owner.respond_to?(:relative_path) && owner.relative_path && !owner.relative_path.to_s.empty? && source_root
|
|
141
|
+
File.join(source_root, owner.relative_path.to_s)
|
|
142
|
+
end
|
|
143
|
+
return owner.relative_path.to_s unless source_root && candidate && !owner.respond_to?(:path) && owner.relative_path && !owner.relative_path.to_s.empty?
|
|
144
|
+
return nil unless candidate && source_root && File.file?(candidate)
|
|
145
|
+
real = File.realpath(candidate)
|
|
146
|
+
return nil unless real == source_root || real.start_with?(source_root + File::SEPARATOR)
|
|
147
|
+
Pathname.new(real).relative_path_from(Pathname.new(source_root)).to_s
|
|
148
|
+
rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
|
|
149
|
+
nil
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def identity_for(owner, output, source)
|
|
153
|
+
return owner[:identity] if owner.is_a?(Hash) && owner[:identity]
|
|
154
|
+
source || "generated:#{owner.class}:#{output}"
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def data_for(owner)
|
|
158
|
+
return owner[:data] || {} if owner.is_a?(Hash)
|
|
159
|
+
owner.respond_to?(:data) ? owner.data.to_h : {}
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def explicit_data_for(owner)
|
|
163
|
+
return owner[:explicit_data] || {} if owner.is_a?(Hash)
|
|
164
|
+
path = source_path(owner)
|
|
165
|
+
source_root = @site.respond_to?(:source) ? @site.source.to_s : ""
|
|
166
|
+
raw_path = path && (File.file?(path) ? path : File.join(source_root, path))
|
|
167
|
+
return frontmatter_defaults_for(owner) unless raw_path && File.file?(raw_path)
|
|
168
|
+
raw = File.binread(raw_path)
|
|
169
|
+
return {} unless raw.start_with?("---")
|
|
170
|
+
front = raw.split(/^---\s*$\n?/, 3)[1]
|
|
171
|
+
YAML.safe_load(front.to_s, permitted_classes: [Date, Time], aliases: true) || {}
|
|
172
|
+
rescue Psych::Exception, Errno::ENOENT, Errno::EACCES
|
|
173
|
+
{}
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def frontmatter_defaults_for(owner)
|
|
177
|
+
return {} unless @site.respond_to?(:frontmatter_defaults) && @site.frontmatter_defaults.respond_to?(:all)
|
|
178
|
+
path = source_path(owner)
|
|
179
|
+
type = owner.respond_to?(:type) ? owner.type.to_s : "Page"
|
|
180
|
+
@site.frontmatter_defaults.all(path.to_s, type)
|
|
181
|
+
rescue StandardError
|
|
182
|
+
{}
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def config_limit(key, default)
|
|
186
|
+
config = @configuration.respond_to?(:to_h) ? @configuration.to_h : @configuration
|
|
187
|
+
limits = config[:limits] || config["limits"] || {}
|
|
188
|
+
limits[key] || limits[key.to_s] || default
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def selected?(source)
|
|
192
|
+
!@configuration.respond_to?(:selected?) || @configuration.selected?(source)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "tempfile"
|
|
6
|
+
|
|
7
|
+
module Jekyll
|
|
8
|
+
module Commands
|
|
9
|
+
class AgentAudit < Jekyll::Command
|
|
10
|
+
class << self
|
|
11
|
+
module ParserErrors
|
|
12
|
+
def parse!(*arguments)
|
|
13
|
+
super
|
|
14
|
+
rescue OptionParser::ParseError => error
|
|
15
|
+
config = instance_variable_get(:@agent_audit_parse_config) || {}
|
|
16
|
+
if config["format"] == "json"
|
|
17
|
+
report = Jekyll::AgentAudit::Report.new(
|
|
18
|
+
complete: false, operational_error: true,
|
|
19
|
+
diagnostics: [{"code" => "invalid_options", "message" => error.message}]
|
|
20
|
+
)
|
|
21
|
+
$stdout.puts Jekyll::AgentAudit::Reporters::JSON.render(report)
|
|
22
|
+
else
|
|
23
|
+
warn "jekyll-agent-audit: #{error.message}"
|
|
24
|
+
end
|
|
25
|
+
exit 2
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def init_with_program(prog)
|
|
30
|
+
prog.command(:"agent:audit") do |command|
|
|
31
|
+
command.singleton_class.prepend(CommandParser)
|
|
32
|
+
command.syntax "agent:audit [options]"
|
|
33
|
+
command.description "Audit rendered publication output"
|
|
34
|
+
add_build_options(command)
|
|
35
|
+
command.option "format", "--format FORMAT", "console or json"
|
|
36
|
+
command.option "output", "--output PATH", "Write the report atomically"
|
|
37
|
+
command.option "fail_on", "--fail-on LEVEL", "error, warning, or none"
|
|
38
|
+
command.option "only", "--only RULES", "Comma-separated rule IDs"
|
|
39
|
+
command.option "list_rules", "--list-rules", "List available rules"
|
|
40
|
+
command.option "explain", "--explain RULE", "Explain a rule"
|
|
41
|
+
command.option "verbose", "--verbose", "Include skipped checks"
|
|
42
|
+
command.action { |_, options| process(options) }
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def process(options)
|
|
47
|
+
options = options.transform_keys(&:to_s)
|
|
48
|
+
validate_only!(options)
|
|
49
|
+
if options["list_rules"]
|
|
50
|
+
write_output(render_rules(options), options["output"])
|
|
51
|
+
return 0
|
|
52
|
+
end
|
|
53
|
+
if options["explain"]
|
|
54
|
+
write_output(explain(options["explain"], options), options["output"])
|
|
55
|
+
return 0
|
|
56
|
+
end
|
|
57
|
+
report = run_audit(options)
|
|
58
|
+
format = options["format"] || report.to_h.dig("run", "effective_configuration", "format") || "console"
|
|
59
|
+
output = render_report(report, format)
|
|
60
|
+
write_output(output, options["output"])
|
|
61
|
+
exit(report_exit_code(report))
|
|
62
|
+
rescue Jekyll::AgentAudit::ConfigurationError, ArgumentError => e
|
|
63
|
+
if options && options["format"] == "json"
|
|
64
|
+
write_output(render_error(e), options["output"])
|
|
65
|
+
else
|
|
66
|
+
warn "jekyll-agent-audit: #{e.message}"
|
|
67
|
+
end
|
|
68
|
+
exit 2
|
|
69
|
+
rescue SystemExit
|
|
70
|
+
raise
|
|
71
|
+
rescue StandardError => e
|
|
72
|
+
if options && options["format"] == "json"
|
|
73
|
+
write_output(render_error(e), options["output"])
|
|
74
|
+
else
|
|
75
|
+
warn "jekyll-agent-audit: #{e.message}"
|
|
76
|
+
end
|
|
77
|
+
exit 2
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def validate_only!(options)
|
|
83
|
+
only = options["only"].to_s.split(",").map(&:strip).reject(&:empty?)
|
|
84
|
+
only.each do |id|
|
|
85
|
+
fail Jekyll::AgentAudit::ConfigurationError, "unknown or deferred rule: #{id}" unless Jekyll::AgentAudit::Registry.known?(id) && !Jekyll::AgentAudit::Registry.deferred?(id)
|
|
86
|
+
end
|
|
87
|
+
fail Jekyll::AgentAudit::ConfigurationError, "invalid fail_on" unless options["fail_on"].nil? || %w[error warning none].include?(options["fail_on"])
|
|
88
|
+
fail Jekyll::AgentAudit::ConfigurationError, "invalid format" unless options["format"].nil? || %w[console json].include?(options["format"])
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def render_rules(options)
|
|
92
|
+
if options["format"] == "json"
|
|
93
|
+
JSON.generate(Jekyll::AgentAudit::Registry.all.values)
|
|
94
|
+
else
|
|
95
|
+
Jekyll::AgentAudit::Registry.all.values.map { |rule| "#{rule[:id]} (#{rule[:release] == "M" ? "enabled" : "deferred"})" }.join("\n")
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def explain(id, options)
|
|
100
|
+
fail Jekyll::AgentAudit::ConfigurationError, "unknown or deferred rule: #{id}" unless Jekyll::AgentAudit::Registry.known?(id) && !Jekyll::AgentAudit::Registry.deferred?(id)
|
|
101
|
+
rule = Jekyll::AgentAudit::Registry[id]
|
|
102
|
+
if options["format"] == "json"
|
|
103
|
+
JSON.generate(rule)
|
|
104
|
+
else
|
|
105
|
+
[
|
|
106
|
+
"#{rule[:id]} — #{rule[:category]} (#{rule[:release] == "M" ? "MVP" : "deferred"})",
|
|
107
|
+
"Default severity: #{rule[:default_severity]}; evidence: #{rule[:evidence_level]}",
|
|
108
|
+
"Applicability: #{rule[:applicability]}",
|
|
109
|
+
"Required inputs: #{rule[:required_inputs].join(", ")}",
|
|
110
|
+
"Limitations: #{rule[:limitations].join("; ")}",
|
|
111
|
+
"Remediation: #{rule[:remediation]}",
|
|
112
|
+
"Evidence URLs: #{rule[:source_urls].join(", ")}"
|
|
113
|
+
].join("\n")
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def write_output(output, path)
|
|
118
|
+
if path
|
|
119
|
+
directory = File.dirname(File.expand_path(path))
|
|
120
|
+
FileUtils.mkdir_p(directory)
|
|
121
|
+
Tempfile.create(["agent-audit-", ".tmp"], directory) do |file|
|
|
122
|
+
file.write(output)
|
|
123
|
+
file.flush
|
|
124
|
+
file.fsync
|
|
125
|
+
File.rename(file.path, path)
|
|
126
|
+
end
|
|
127
|
+
else
|
|
128
|
+
$stdout.puts output
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def run_audit(options)
|
|
133
|
+
original_global_stdout = $stdout
|
|
134
|
+
original_stdout = STDOUT.dup
|
|
135
|
+
STDOUT.reopen(STDERR)
|
|
136
|
+
$stdout = STDOUT
|
|
137
|
+
report = Jekyll::AgentAudit::Runner.new(options).run
|
|
138
|
+
report
|
|
139
|
+
ensure
|
|
140
|
+
STDOUT.reopen(original_stdout) if original_stdout
|
|
141
|
+
original_stdout&.close
|
|
142
|
+
$stdout = original_global_stdout
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def render_report(report, format)
|
|
146
|
+
return Jekyll::AgentAudit::Reporters::Console.render(report) unless format == "json"
|
|
147
|
+
|
|
148
|
+
data = report.to_h
|
|
149
|
+
data["policy"]["exit_code"] = 130 if interrupted?(report)
|
|
150
|
+
JSON.generate(data)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def render_error(error)
|
|
154
|
+
report = Jekyll::AgentAudit::Report.new(
|
|
155
|
+
complete: false, operational_error: true,
|
|
156
|
+
diagnostics: [{"code" => "invalid_options", "message" => error.message}]
|
|
157
|
+
)
|
|
158
|
+
Jekyll::AgentAudit::Reporters::JSON.render(report)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def report_exit_code(report)
|
|
162
|
+
interrupted?(report) ? 130 : report.exit_code
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def interrupted?(report)
|
|
166
|
+
report.to_h.fetch("diagnostics", []).any? { |diagnostic| diagnostic["code"].to_s == "interrupted" }
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
module CommandParser
|
|
170
|
+
def go(arguments, parser, config)
|
|
171
|
+
parser.instance_variable_set(:@agent_audit_parse_config, config)
|
|
172
|
+
parser.singleton_class.prepend(ParserErrors) unless parser.singleton_class < ParserErrors
|
|
173
|
+
super
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Jekyll
|
|
8
|
+
module AgentAudit
|
|
9
|
+
class Configuration
|
|
10
|
+
DEFAULTS = {
|
|
11
|
+
"fail_on" => "error", "format" => "console", "include" => ["**/*"], "exclude" => [],
|
|
12
|
+
"content" => {"selector" => nil, "exclude_selectors" => %w[nav footer aside script style template]},
|
|
13
|
+
"metadata" => {"selectors" => {"author" => nil, "publisher" => nil, "published" => nil, "modified" => nil}},
|
|
14
|
+
"graph" => {"roots" => ["/"]}, "routes" => {"directory_index" => "index.html", "external_paths" => [], "aliases" => {}},
|
|
15
|
+
"rules" => {}, "suppressions" => [],
|
|
16
|
+
"limits" => {"max_html_bytes" => 5_242_880, "max_jsonld_bytes" => 1_048_576, "max_documents" => 50_000, "max_edges" => 1_000_000}
|
|
17
|
+
}.freeze
|
|
18
|
+
TOP_KEYS = DEFAULTS.keys.freeze
|
|
19
|
+
LIMITS = %w[max_html_bytes max_jsonld_bytes max_documents max_edges].freeze
|
|
20
|
+
|
|
21
|
+
attr_reader :raw
|
|
22
|
+
|
|
23
|
+
def initialize(raw_namespace = {}, overrides = {})
|
|
24
|
+
@raw = deep_stringify(raw_namespace || {})
|
|
25
|
+
@overrides = deep_stringify(overrides || {})
|
|
26
|
+
@effective = nil
|
|
27
|
+
validate!
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_h
|
|
31
|
+
@effective ||= deep_freeze(deep_merge(DEFAULTS, @raw).merge(cli_values))
|
|
32
|
+
@effective
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def [](key)
|
|
36
|
+
to_h[key.to_s]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def enabled_rules
|
|
40
|
+
registry = Registry.all
|
|
41
|
+
registry.each_with_object({}) do |(id, metadata), result|
|
|
42
|
+
next unless metadata[:release] == "M"
|
|
43
|
+
configured = self["rules"][id]
|
|
44
|
+
result[id] = metadata unless configured.is_a?(Hash) && configured["enabled"] == false
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def selected?(source_path)
|
|
49
|
+
path = source_path.to_s.sub(%r{\A/+}, "")
|
|
50
|
+
includes = Array(self["include"])
|
|
51
|
+
excludes = Array(self["exclude"])
|
|
52
|
+
includes.any? { |pattern| File.fnmatch?(pattern.to_s, path, File::FNM_PATHNAME | File::FNM_EXTGLOB) } &&
|
|
53
|
+
!excludes.any? { |pattern| File.fnmatch?(pattern.to_s, path, File::FNM_PATHNAME | File::FNM_EXTGLOB) }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def suppression_for(rule_id, source_path: nil, fingerprint: nil, now: Date.today)
|
|
57
|
+
Array(self["suppressions"]).filter_map do |suppression|
|
|
58
|
+
next unless suppression["rule"] == rule_id.to_s
|
|
59
|
+
if suppression["paths"]
|
|
60
|
+
next unless source_path
|
|
61
|
+
next unless Array(suppression["paths"]).any? { |p| File.fnmatch?(p, source_path.to_s, File::FNM_PATHNAME | File::FNM_EXTGLOB) }
|
|
62
|
+
end
|
|
63
|
+
next if suppression["fingerprint"] && suppression["fingerprint"] != fingerprint
|
|
64
|
+
next if suppression["until"] && Date.iso8601(suppression["until"].to_s) < now
|
|
65
|
+
suppression
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def expired_suppressions(now: Date.today)
|
|
70
|
+
Array(self["suppressions"]).select { |entry| entry["until"] && Date.iso8601(entry["until"].to_s) < now }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def cli_values
|
|
76
|
+
%w[fail_on format].each_with_object({}) { |key, values| values[key] = @overrides[key] if @overrides.key?(key) }
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def validate!
|
|
80
|
+
unknown = @raw.keys - TOP_KEYS
|
|
81
|
+
fail ConfigurationError, "unknown agent_audit key(s): #{unknown.join(', ')}" unless unknown.empty?
|
|
82
|
+
fail ConfigurationError, "fail_on must be error, warning, or none" unless %w[error warning none].include?(self["fail_on"])
|
|
83
|
+
fail ConfigurationError, "format must be console or json" unless %w[console json].include?(self["format"])
|
|
84
|
+
%w[include exclude].each { |key| fail ConfigurationError, "#{key} must be an array" unless @raw.fetch(key, DEFAULTS[key]).is_a?(Array) }
|
|
85
|
+
validate_hash_keys("content", %w[selector exclude_selectors])
|
|
86
|
+
content = @raw.fetch("content", {})
|
|
87
|
+
fail ConfigurationError, "content.exclude_selectors must be an array" unless content.fetch("exclude_selectors", DEFAULTS["content"]["exclude_selectors"]).is_a?(Array)
|
|
88
|
+
fail ConfigurationError, "content selectors must be strings or null" unless (content.fetch("selector", nil).nil? || content.fetch("selector", nil).is_a?(String)) && content.fetch("exclude_selectors", DEFAULTS["content"]["exclude_selectors"]).all? { |v| v.is_a?(String) }
|
|
89
|
+
validate_hash_keys("metadata", ["selectors"])
|
|
90
|
+
selectors = @raw.fetch("metadata", {}).fetch("selectors", {})
|
|
91
|
+
fail ConfigurationError, "metadata.selectors must be a mapping" unless selectors.is_a?(Hash)
|
|
92
|
+
fail ConfigurationError, "unknown metadata selector" unless (selectors.keys - %w[author publisher published modified]).empty?
|
|
93
|
+
fail ConfigurationError, "metadata selectors must be strings or null" unless selectors.values.all? { |value| value.nil? || value.is_a?(String) }
|
|
94
|
+
validate_hash_keys("routes", %w[directory_index external_paths aliases])
|
|
95
|
+
routes = @raw.fetch("routes", {})
|
|
96
|
+
fail ConfigurationError, "routes.external_paths must be an array" unless routes.fetch("external_paths", DEFAULTS["routes"]["external_paths"]).is_a?(Array)
|
|
97
|
+
fail ConfigurationError, "routes.directory_index must be a string" unless routes.fetch("directory_index", DEFAULTS["routes"]["directory_index"]).is_a?(String)
|
|
98
|
+
validate_hash_keys("limits", LIMITS)
|
|
99
|
+
validate_hash_keys("graph", ["roots"])
|
|
100
|
+
fail ConfigurationError, "graph.roots must be an array" unless @raw.fetch("graph", {}).fetch("roots", DEFAULTS["graph"]["roots"]).is_a?(Array)
|
|
101
|
+
validate_hash_keys("rules", Registry.all.keys)
|
|
102
|
+
validate_rules!
|
|
103
|
+
validate_suppressions!
|
|
104
|
+
validate_limits!
|
|
105
|
+
validate_aliases!
|
|
106
|
+
validate_selectors!
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def validate_hash_keys(section, allowed)
|
|
110
|
+
value = @raw.fetch(section, {})
|
|
111
|
+
fail ConfigurationError, "#{section} must be a mapping" unless value.is_a?(Hash)
|
|
112
|
+
unknown = value.keys - allowed
|
|
113
|
+
fail ConfigurationError, "unknown agent_audit.#{section} key(s): #{unknown.join(', ')}" unless unknown.empty?
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def validate_rules!
|
|
117
|
+
@raw.fetch("rules", {}).each do |id, value|
|
|
118
|
+
fail ConfigurationError, "rule #{id} must be a mapping" unless value.is_a?(Hash)
|
|
119
|
+
fail ConfigurationError, "unknown rule option for #{id}" unless (value.keys - ["enabled"]).empty?
|
|
120
|
+
fail ConfigurationError, "rule #{id} enabled must be boolean" unless [true, false].include?(value["enabled"])
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def validate_suppressions!
|
|
125
|
+
fail ConfigurationError, "suppressions must be an array" unless @raw.fetch("suppressions", []).is_a?(Array)
|
|
126
|
+
Array(@raw.fetch("suppressions", [])).each do |entry|
|
|
127
|
+
fail ConfigurationError, "suppression must be a mapping" unless entry.is_a?(Hash)
|
|
128
|
+
fail ConfigurationError, "unknown suppression key" unless (entry.keys - %w[rule paths fingerprint reason until]).empty?
|
|
129
|
+
fail ConfigurationError, "suppression requires known rule" unless Registry.known?(entry["rule"]) && !Registry.deferred?(entry["rule"])
|
|
130
|
+
fail ConfigurationError, "suppression requires a nonempty reason" if entry["reason"].to_s.strip.empty?
|
|
131
|
+
fail ConfigurationError, "suppression paths must be an array" if entry.key?("paths") && !entry["paths"].is_a?(Array)
|
|
132
|
+
if entry["until"]
|
|
133
|
+
Date.iso8601(entry["until"].to_s)
|
|
134
|
+
end
|
|
135
|
+
rescue Date::Error
|
|
136
|
+
fail ConfigurationError, "suppression until must be an ISO date"
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def validate_limits!
|
|
141
|
+
limits = deep_merge(DEFAULTS["limits"], @raw.fetch("limits", {}))
|
|
142
|
+
limits.each_value { |value| fail ConfigurationError, "limits must be positive integers" unless value.is_a?(Integer) && value.positive? }
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def validate_aliases!
|
|
146
|
+
aliases = self["routes"]["aliases"]
|
|
147
|
+
fail ConfigurationError, "routes.aliases must be a mapping" unless aliases.is_a?(Hash)
|
|
148
|
+
aliases.each_key do |path|
|
|
149
|
+
fail ConfigurationError, "route aliases must be site-relative" unless path.to_s.start_with?("/")
|
|
150
|
+
end
|
|
151
|
+
aliases.each_value do |target|
|
|
152
|
+
fail ConfigurationError, "route alias targets must be strings" unless target.is_a?(String) && target.start_with?("/")
|
|
153
|
+
end
|
|
154
|
+
aliases.keys.each do |key|
|
|
155
|
+
seen = {}
|
|
156
|
+
current = key
|
|
157
|
+
while aliases.key?(current)
|
|
158
|
+
fail ConfigurationError, "route aliases contain a cycle" if seen[current]
|
|
159
|
+
seen[current] = true
|
|
160
|
+
current = aliases[current]
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def validate_selectors!
|
|
166
|
+
selectors = self["metadata"]["selectors"].values.compact
|
|
167
|
+
selectors << self["content"]["selector"] if self["content"]["selector"]
|
|
168
|
+
return if selectors.empty?
|
|
169
|
+
require "nokogiri"
|
|
170
|
+
selectors.each { |selector| Nokogiri::HTML5.fragment("<x></x>").css(selector.to_s) }
|
|
171
|
+
rescue Nokogiri::CSS::SyntaxError, ArgumentError => e
|
|
172
|
+
fail ConfigurationError, "invalid CSS selector: #{e.message}"
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def deep_stringify(value)
|
|
176
|
+
case value
|
|
177
|
+
when Hash then value.to_h { |k, v| [k.to_s, deep_stringify(v)] }
|
|
178
|
+
when Array then value.map { |v| deep_stringify(v) }
|
|
179
|
+
else value
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def deep_merge(left, right)
|
|
184
|
+
left.merge(right) do |_key, old_value, new_value|
|
|
185
|
+
old_value.is_a?(Hash) && new_value.is_a?(Hash) ? deep_merge(old_value, new_value) : new_value
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def deep_freeze(value)
|
|
190
|
+
value.each { |v| deep_freeze(v) } if value.is_a?(Hash)
|
|
191
|
+
value.each { |v| deep_freeze(v) } if value.is_a?(Array)
|
|
192
|
+
value.freeze
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|