yuno-proofreader 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +13 -0
- data/LICENSE.txt +21 -0
- data/README.md +435 -0
- data/exe/yuno-proofreader +16 -0
- data/lib/yuno/proofreader/analysis/adjustments.rb +45 -0
- data/lib/yuno/proofreader/analysis/crawler.rb +95 -0
- data/lib/yuno/proofreader/analysis/entries.rb +73 -0
- data/lib/yuno/proofreader/analysis/extractor.rb +209 -0
- data/lib/yuno/proofreader/analysis/graph.rb +115 -0
- data/lib/yuno/proofreader/analysis/judge.rb +103 -0
- data/lib/yuno/proofreader/analysis/judge.yml +27 -0
- data/lib/yuno/proofreader/analysis/judge_prompt.rb +72 -0
- data/lib/yuno/proofreader/analysis/judgments.rb +70 -0
- data/lib/yuno/proofreader/analysis/links.rb +54 -0
- data/lib/yuno/proofreader/analysis/list_order.rb +33 -0
- data/lib/yuno/proofreader/analysis/reasons.rb +49 -0
- data/lib/yuno/proofreader/analysis/report.rb +110 -0
- data/lib/yuno/proofreader/analysis/runner.rb +171 -0
- data/lib/yuno/proofreader/analysis/scorer.rb +128 -0
- data/lib/yuno/proofreader/analysis/settings.rb +90 -0
- data/lib/yuno/proofreader/analysis/signals.rb +102 -0
- data/lib/yuno/proofreader/analysis/structured_data.rb +57 -0
- data/lib/yuno/proofreader/analysis/url_tree.rb +49 -0
- data/lib/yuno/proofreader/analysis/weights.rb +77 -0
- data/lib/yuno/proofreader/analysis/weights.yml +39 -0
- data/lib/yuno/proofreader/audit/agenda.rb +86 -0
- data/lib/yuno/proofreader/audit/agenda_report.rb +65 -0
- data/lib/yuno/proofreader/audit/answers.rb +74 -0
- data/lib/yuno/proofreader/audit/attempt.rb +69 -0
- data/lib/yuno/proofreader/audit/batch.rb +64 -0
- data/lib/yuno/proofreader/audit/batch_report.rb +87 -0
- data/lib/yuno/proofreader/audit/cadence.rb +67 -0
- data/lib/yuno/proofreader/audit/instructions.rb +30 -0
- data/lib/yuno/proofreader/audit/log_fields.rb +28 -0
- data/lib/yuno/proofreader/audit/prompt.rb +157 -0
- data/lib/yuno/proofreader/audit/prompts/instructions.yml +46 -0
- data/lib/yuno/proofreader/audit/prompts/steps.yml +67 -0
- data/lib/yuno/proofreader/audit/report.rb +153 -0
- data/lib/yuno/proofreader/audit/runner.rb +144 -0
- data/lib/yuno/proofreader/audit/steps.rb +56 -0
- data/lib/yuno/proofreader/cli/analysis_commands.rb +43 -0
- data/lib/yuno/proofreader/cli/audit_commands.rb +92 -0
- data/lib/yuno/proofreader/cli/command_log.rb +42 -0
- data/lib/yuno/proofreader/cli/flags.rb +48 -0
- data/lib/yuno/proofreader/cli/init_command.rb +116 -0
- data/lib/yuno/proofreader/cli/model_commands.rb +34 -0
- data/lib/yuno/proofreader/cli/queue_commands.rb +89 -0
- data/lib/yuno/proofreader/cli/sheet_commands.rb +57 -0
- data/lib/yuno/proofreader/cli.rb +218 -0
- data/lib/yuno/proofreader/config.rb +25 -0
- data/lib/yuno/proofreader/context.rb +55 -0
- data/lib/yuno/proofreader/env_file.rb +76 -0
- data/lib/yuno/proofreader/errors.rb +17 -0
- data/lib/yuno/proofreader/findings/finding.rb +95 -0
- data/lib/yuno/proofreader/findings/matching.rb +34 -0
- data/lib/yuno/proofreader/findings/merger.rb +43 -0
- data/lib/yuno/proofreader/findings/outline.rb +73 -0
- data/lib/yuno/proofreader/findings/plan.rb +24 -0
- data/lib/yuno/proofreader/findings/reconciler.rb +219 -0
- data/lib/yuno/proofreader/findings/source.rb +79 -0
- data/lib/yuno/proofreader/findings/text.rb +37 -0
- data/lib/yuno/proofreader/http/client.rb +131 -0
- data/lib/yuno/proofreader/http/net_http_transport.rb +55 -0
- data/lib/yuno/proofreader/http/robots.rb +72 -0
- data/lib/yuno/proofreader/init/ask.rb +67 -0
- data/lib/yuno/proofreader/init/detector.rb +100 -0
- data/lib/yuno/proofreader/init/interview.rb +210 -0
- data/lib/yuno/proofreader/init/parsing.rb +47 -0
- data/lib/yuno/proofreader/init/web.rb +33 -0
- data/lib/yuno/proofreader/llm/client.rb +140 -0
- data/lib/yuno/proofreader/llm/defaults.rb +22 -0
- data/lib/yuno/proofreader/llm/defaults.yml +6 -0
- data/lib/yuno/proofreader/llm/errors.rb +26 -0
- data/lib/yuno/proofreader/llm/factory.rb +45 -0
- data/lib/yuno/proofreader/llm/http_transport.rb +41 -0
- data/lib/yuno/proofreader/llm/json_text.rb +23 -0
- data/lib/yuno/proofreader/llm/model_commands.rb +34 -0
- data/lib/yuno/proofreader/llm/provider.rb +58 -0
- data/lib/yuno/proofreader/llm/providers/gemini.rb +146 -0
- data/lib/yuno/proofreader/llm/providers.rb +38 -0
- data/lib/yuno/proofreader/llm/types.rb +13 -0
- data/lib/yuno/proofreader/llm/usage.rb +17 -0
- data/lib/yuno/proofreader/log.rb +129 -0
- data/lib/yuno/proofreader/pages/fetcher.rb +75 -0
- data/lib/yuno/proofreader/pages/report.rb +45 -0
- data/lib/yuno/proofreader/pages/snapshot.rb +48 -0
- data/lib/yuno/proofreader/pages/trimmer.rb +36 -0
- data/lib/yuno/proofreader/settings_file.rb +51 -0
- data/lib/yuno/proofreader/site.rb +74 -0
- data/lib/yuno/proofreader/site_reader.rb +131 -0
- data/lib/yuno/proofreader/sitemap/aligner.rb +78 -0
- data/lib/yuno/proofreader/sitemap/importer.rb +31 -0
- data/lib/yuno/proofreader/sitemap/loader.rb +127 -0
- data/lib/yuno/proofreader/sitemap/mismatch.rb +76 -0
- data/lib/yuno/proofreader/sitemap/pairer.rb +223 -0
- data/lib/yuno/proofreader/sitemap/parser.rb +73 -0
- data/lib/yuno/proofreader/sitemap/report.rb +80 -0
- data/lib/yuno/proofreader/store/cells.rb +67 -0
- data/lib/yuno/proofreader/store/google_gateway.rb +198 -0
- data/lib/yuno/proofreader/store/memory.rb +90 -0
- data/lib/yuno/proofreader/store/records.rb +28 -0
- data/lib/yuno/proofreader/store/schema.rb +109 -0
- data/lib/yuno/proofreader/store/sheets.rb +183 -0
- data/lib/yuno/proofreader/store/tab_setup.rb +74 -0
- data/lib/yuno/proofreader/version.rb +7 -0
- data/lib/yuno/proofreader.rb +101 -0
- data/proofreader.example.yml +134 -0
- metadata +194 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# The command that starts the settings file of a site: init.
|
|
6
|
+
module InitCommand
|
|
7
|
+
EXAMPLE = File.expand_path("../../../../proofreader.example.yml", __dir__)
|
|
8
|
+
SWITCHES = %w[template yes offline].freeze
|
|
9
|
+
VALUES = %w[website business audience tone sitemap].freeze
|
|
10
|
+
LISTS = %w[language note skip-path rule ignore-term difference].freeze
|
|
11
|
+
HEADER = ["Settings of yuno-proofreader, written by `yuno-proofreader init`.",
|
|
12
|
+
"Every setting is explained in the template: run `yuno-proofreader init --template`.",
|
|
13
|
+
"`yuno-proofreader settings` shows what an audit still needs."].freeze
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def init(args)
|
|
18
|
+
flags = Flags.parse(args, switches: SWITCHES, values: VALUES, lists: LISTS)
|
|
19
|
+
problem = init_problem(flags)
|
|
20
|
+
return usage_error(problem) if problem
|
|
21
|
+
|
|
22
|
+
init_flow(flags)
|
|
23
|
+
0
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def init_problem(flags) = flags.problem || ("Too many arguments for init" if flags.rest.size > 1)
|
|
27
|
+
|
|
28
|
+
# Without --template the interview runs. With --template and no file name the template is only shown, and
|
|
29
|
+
# with a file name it is written there.
|
|
30
|
+
def init_flow(flags)
|
|
31
|
+
path = flags.rest.first
|
|
32
|
+
return init_interview(path || CLI::DEFAULT_SITE_FILE, flags) unless flags.switches.key?("template")
|
|
33
|
+
|
|
34
|
+
path ? init_template(path) : print_template
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# With no file name the template is only shown, so it can be read, or saved with a redirect.
|
|
38
|
+
def print_template = @out.write(example_text)
|
|
39
|
+
|
|
40
|
+
def init_template(path)
|
|
41
|
+
create_file(path, example_text)
|
|
42
|
+
@out.puts "Wrote #{path}. It is a template: every value that starts with [FILL IN] is for you to replace.",
|
|
43
|
+
"Then run: yuno-proofreader settings #{path}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def init_interview(path, flags)
|
|
47
|
+
ensure_free(path)
|
|
48
|
+
interview = Init::Interview.new(flags:, ask: Init::Ask.new(input: @input, out: @out),
|
|
49
|
+
web: Init::Web.new(http: http_client), out: @out,
|
|
50
|
+
offline: flags.switches.key?("offline"), yes: flags.switches.key?("yes"))
|
|
51
|
+
settings = interview.call
|
|
52
|
+
site = Site.from_hash(settings)
|
|
53
|
+
create_file(path, SettingsFile.render(settings, header: HEADER))
|
|
54
|
+
init_lines(path, site).each { |line| @out.puts line }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def ensure_free(path) = (raise ConfigError, taken_message(path) if File.exist?(path))
|
|
58
|
+
|
|
59
|
+
def taken_message(path)
|
|
60
|
+
"#{path} already exists, so it was left as it is. Delete it first, or choose another name: " \
|
|
61
|
+
"yuno-proofreader init NAME.yml"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Writes the file, and never over one that is already there.
|
|
65
|
+
def create_file(path, text)
|
|
66
|
+
File.open(path, File::WRONLY | File::CREAT | File::EXCL) { |file| file.write(text) }
|
|
67
|
+
rescue Errno::EEXIST
|
|
68
|
+
raise ConfigError, taken_message(path)
|
|
69
|
+
rescue SystemCallError => e
|
|
70
|
+
raise ConfigError, "Could not write #{path}: #{e.message}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def example_text
|
|
74
|
+
File.read(EXAMPLE)
|
|
75
|
+
rescue SystemCallError
|
|
76
|
+
raise Error, "The template settings file is missing from this installation (#{EXAMPLE})."
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# ---- what is printed afterwards ----
|
|
80
|
+
|
|
81
|
+
def init_lines(path, site)
|
|
82
|
+
problems = Array(site.audit_problems)
|
|
83
|
+
suffix = path == CLI::DEFAULT_SITE_FILE ? "" : " #{path}"
|
|
84
|
+
["", "Wrote #{path}.", *readiness(problems), "", "Outside this file you also need:", *environment_lines, "",
|
|
85
|
+
"Next:", *next_steps(suffix)]
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def readiness(problems)
|
|
89
|
+
return ["It is ready for an audit."] if problems.empty?
|
|
90
|
+
|
|
91
|
+
["Before an audit can run, this is still missing:", *problems.map { |problem| " - #{problem}" }]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def environment_lines
|
|
95
|
+
keys = Llm::Providers.ids.map { |id| [Llm::Providers.resolve(id).api_key_env, "the API key of #{id}"] }
|
|
96
|
+
lines = [["SPREADSHEET_ID", "the id of your Google Sheet"], *keys].map do |name, meaning|
|
|
97
|
+
environment_line(name, meaning)
|
|
98
|
+
end
|
|
99
|
+
[*lines, " Put them in a .env file in this folder, or export them.",
|
|
100
|
+
" Google credentials: see the README, section \"Connecting to Google\""]
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def environment_line(name, meaning)
|
|
104
|
+
state = @env[name].to_s.strip.empty? ? "not set" : "set"
|
|
105
|
+
" #{name.ljust(18)} #{state.ljust(8)} (#{meaning})"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def next_steps(suffix)
|
|
109
|
+
[" yuno-proofreader prepare#{suffix} creates the tabs in your Google Sheet",
|
|
110
|
+
" yuno-proofreader pages import#{suffix} adds your pages to the sheet",
|
|
111
|
+
" yuno-proofreader analyze#{suffix} ranks the pages by importance",
|
|
112
|
+
" yuno-proofreader audit --next#{suffix} audits the most important page that is due"]
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# The command that works with the model provider: models.
|
|
6
|
+
module ModelCommands
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def models(args)
|
|
10
|
+
file, problem = single_file(args, "models")
|
|
11
|
+
return usage_error(problem) if problem
|
|
12
|
+
|
|
13
|
+
site = Site.load(file)
|
|
14
|
+
provider = Llm.provider_for(site, env: @env, transport: @llm_transport)
|
|
15
|
+
selection = Llm.models_for(site, provider.class)
|
|
16
|
+
used = selection.names.map { |name| provider.normalize(name) }
|
|
17
|
+
print_models(provider, provider.models, used, selection.source)
|
|
18
|
+
0
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def print_models(provider, available, used, source)
|
|
22
|
+
@out.puts "Models your #{provider.class.id} key can use for text generation:"
|
|
23
|
+
available.each { |name| @out.puts " #{name}#{' (in use)' if used.include?(name)}" }
|
|
24
|
+
origin = source == :settings ? "from your settings" : "built-in default"
|
|
25
|
+
@out.puts "", "In use (#{origin}): #{used.join(', ')}"
|
|
26
|
+
missing = used - available
|
|
27
|
+
return if missing.empty?
|
|
28
|
+
|
|
29
|
+
@out.puts "Not available to this key: #{missing.join(', ')}. A model that fails is skipped, " \
|
|
30
|
+
"so set models in proofreader.yml to the ones you want."
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# The commands that decide which page to audit next: queue, and the batch form of audit (audit --next).
|
|
6
|
+
module QueueCommands
|
|
7
|
+
MAX_COUNT = 500
|
|
8
|
+
COUNT_FLAG = /\A--count(=|\z)/
|
|
9
|
+
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def queue(args)
|
|
13
|
+
count, remaining, problem = take_count(args)
|
|
14
|
+
return usage_error(problem) if problem
|
|
15
|
+
|
|
16
|
+
options, rest, unknown = split_args(remaining, %w[all])
|
|
17
|
+
problem = queue_args_problem(rest, unknown)
|
|
18
|
+
return usage_error(problem) if problem
|
|
19
|
+
|
|
20
|
+
site = Site.load(rest.first || CLI::DEFAULT_SITE_FILE)
|
|
21
|
+
picks = agenda_for(site, store_for(site))
|
|
22
|
+
report = Audit::AgendaReport.new(picks, shown: count || Audit::AgendaReport::SHOWN,
|
|
23
|
+
everything: options.key?("all"))
|
|
24
|
+
show_lines(report.lines)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def queue_args_problem(rest, unknown)
|
|
28
|
+
return "Unknown option: #{unknown.first}" unless unknown.empty?
|
|
29
|
+
|
|
30
|
+
"Too many arguments for queue" if rest.size > 1
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Takes --count=N or --count N out of the arguments. Returns the number, the other arguments and a problem.
|
|
34
|
+
def take_count(args)
|
|
35
|
+
remaining = args.dup
|
|
36
|
+
index = remaining.index { |arg| arg.match?(COUNT_FLAG) }
|
|
37
|
+
return [nil, remaining, nil] unless index
|
|
38
|
+
|
|
39
|
+
flag = remaining.delete_at(index)
|
|
40
|
+
number = Integer((flag[/\A--count=(.*)\z/, 1] || remaining.delete_at(index)).to_s, 10, exception: false)
|
|
41
|
+
return [number, remaining, nil] if number&.between?(1, MAX_COUNT)
|
|
42
|
+
|
|
43
|
+
[nil, remaining, "--count needs a whole number from 1 to #{MAX_COUNT}"]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def agenda_for(site, store)
|
|
47
|
+
Audit::Agenda.call(records: store.pages, cadence: site.cadence, now: Time.now.utc)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def show_lines(lines)
|
|
51
|
+
lines.each { |line| @out.puts line }
|
|
52
|
+
0
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The model clients are built first, so a missing API key stops the command before anything is read.
|
|
56
|
+
def audit_batch(site, options, count)
|
|
57
|
+
store = store_for(site)
|
|
58
|
+
runner = audit_runner(site, store)
|
|
59
|
+
picks = agenda_for(site, store)
|
|
60
|
+
pool = options.key?("include-not-due") ? picks : picks.select(&:due?)
|
|
61
|
+
chosen = pool.first(count || 1)
|
|
62
|
+
if chosen.empty?
|
|
63
|
+
log.info("nothing is due", pages: picks.size)
|
|
64
|
+
return show_lines(Audit::AgendaReport.new(picks).lines)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
run_batch(runner, store, chosen, picks, options)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def run_batch(runner, store, chosen, picks, options)
|
|
71
|
+
batch = Audit::Batch.new(runner:, store:, progress: ->(message) { @out.puts message }, log:)
|
|
72
|
+
dry_run = options.key?("dry-run")
|
|
73
|
+
outcome = batch.call(chosen, dry_run:, force: options.key?("force"))
|
|
74
|
+
still_due = [picks.count(&:due?) - outcome.entries.size, 0].max
|
|
75
|
+
log_batch(outcome, still_due:, dry_run:)
|
|
76
|
+
show_lines(Audit::BatchReport.new(outcome, dry_run:, still_due:, verbose: options.key?("verbose")).lines)
|
|
77
|
+
outcome.stopped ? 1 : 0
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def log_batch(outcome, still_due:, dry_run:)
|
|
81
|
+
statuses = outcome.entries.map(&:status).tally
|
|
82
|
+
calls = outcome.entries.filter_map(&:result).flat_map(&:calls)
|
|
83
|
+
log.info("batch finished", pages: outcome.entries.size, audited: statuses.fetch(:audited, 0),
|
|
84
|
+
skipped: statuses.fetch(:skipped, 0), failed: statuses.fetch(:failed, 0),
|
|
85
|
+
still_due:, dry_run:, stopped: outcome.stopped, **Llm::Usage.of(calls))
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# The commands that work with the Google Sheet: prepare and pages import.
|
|
6
|
+
module SheetCommands
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def prepare(args)
|
|
10
|
+
file, problem = single_file(args, "prepare")
|
|
11
|
+
return usage_error(problem) if problem
|
|
12
|
+
|
|
13
|
+
store_for(Site.load(file)).prepare.each { |line| @out.puts line }
|
|
14
|
+
0
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def pages(args)
|
|
18
|
+
action, *rest = args
|
|
19
|
+
return usage_error("Unknown pages command: #{action.inspect}. Try: pages import") unless action == "import"
|
|
20
|
+
|
|
21
|
+
file, problem = single_file(rest, "pages import")
|
|
22
|
+
return usage_error(problem) if problem
|
|
23
|
+
|
|
24
|
+
import_pages(Site.load(file))
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def import_pages(site)
|
|
28
|
+
loaded = Sitemap::Loader.call(site, http_client)
|
|
29
|
+
return refuse_import(site, loaded) unless loaded.pairing.resolved?
|
|
30
|
+
|
|
31
|
+
result = Sitemap::Importer.call(loaded.pairing, store_for(site), site)
|
|
32
|
+
print_import(result, loaded.warnings)
|
|
33
|
+
0
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def refuse_import(site, loaded)
|
|
37
|
+
Sitemap::Report.new(site, loaded).lines.each { |line| @out.puts line }
|
|
38
|
+
@out.puts "", "Nothing was written to the sheet."
|
|
39
|
+
1
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def print_import(result, warnings)
|
|
43
|
+
@out.puts "Added #{result.added} pages to the Pages tab (#{result.existing} were already there)."
|
|
44
|
+
if result.gone.positive?
|
|
45
|
+
@out.puts "#{result.gone} pages in the sheet are no longer in the sitemap and were left alone."
|
|
46
|
+
end
|
|
47
|
+
return if warnings.empty?
|
|
48
|
+
|
|
49
|
+
@out.puts "", "Warnings:", *warnings.map { |warning| " - #{warning}" }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def store_for(site)
|
|
53
|
+
@store || Store::Sheets.connect(Config.from_env(@env), site)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# Command-line entry point: parses arguments and returns an exit code.
|
|
6
|
+
class CLI
|
|
7
|
+
include SheetCommands
|
|
8
|
+
include ModelCommands
|
|
9
|
+
include AuditCommands
|
|
10
|
+
include AnalysisCommands
|
|
11
|
+
include InitCommand
|
|
12
|
+
prepend CommandLog
|
|
13
|
+
include QueueCommands
|
|
14
|
+
|
|
15
|
+
DEFAULT_SITE_FILE = "proofreader.yml"
|
|
16
|
+
STATE_LABELS = { set: "set", empty: "not set", placeholder: "still the [FILL IN] example text" }.freeze
|
|
17
|
+
COMMANDS = {
|
|
18
|
+
"help" => :help, "-h" => :help, "--help" => :help,
|
|
19
|
+
"version" => :version, "-v" => :version, "--version" => :version,
|
|
20
|
+
"doctor" => :doctor, "sitemap" => :sitemap, "fetch" => :fetch, "settings" => :settings,
|
|
21
|
+
"models" => :models, "prepare" => :prepare, "pages" => :pages, "prompt" => :prompt,
|
|
22
|
+
"analyze" => :analyze, "init" => :init, "queue" => :queue, "audit" => :audit
|
|
23
|
+
}.freeze
|
|
24
|
+
USAGE = <<~TEXT
|
|
25
|
+
Usage: yuno-proofreader <command>
|
|
26
|
+
|
|
27
|
+
Commands:
|
|
28
|
+
sitemap [FILE] [--all] Read the sitemap of the site described in FILE (default: proofreader.yml)
|
|
29
|
+
and show how its pages were paired by language
|
|
30
|
+
fetch PAGE [FILE] [--html=LANG]
|
|
31
|
+
Download all language versions of one page and show how the tool sees them.
|
|
32
|
+
PAGE is a name from `sitemap --all`, or any address of the page
|
|
33
|
+
prepare [FILE] Create the tabs the tool needs in your Google Sheet (safe to run again)
|
|
34
|
+
pages import [FILE] Add the pages of the sitemap to the Pages tab of the sheet
|
|
35
|
+
models [FILE] List the models your API key can use for text generation, and which are in use
|
|
36
|
+
init [FILE] [OPTIONS] Ask about your site and write a settings file (default proofreader.yml).
|
|
37
|
+
--template shows the full commented template, or with a FILE writes it there.
|
|
38
|
+
For scripts: --website, --language CODE[:PREFIX], --note CODE=TEXT, --business,
|
|
39
|
+
--audience, --tone, --sitemap, --skip-path, --rule, --ignore-term, --difference.
|
|
40
|
+
--yes accepts what was detected, --offline makes no request. Does not overwrite a file
|
|
41
|
+
audit PAGE [FILE] [--dry-run] [--force] [--verbose]
|
|
42
|
+
Audit one page: ask the model, compare the findings with the suggestions in the
|
|
43
|
+
sheet and update the sheet. --dry-run writes nothing (the model is still called
|
|
44
|
+
and costs the same), --force audits a page that has not changed, --verbose
|
|
45
|
+
lists everything that was decided
|
|
46
|
+
audit --next [FILE] [--count=N] [--include-not-due] [--dry-run] [--force] [--verbose]
|
|
47
|
+
Audit the pages that are due, the most important first (default 1 page)
|
|
48
|
+
queue [FILE] [--count=N] [--all]
|
|
49
|
+
Show which pages are due for an audit and why. Calls no model and fetches no page
|
|
50
|
+
analyze [FILE] [--dry-run] [--no-ai] [--reuse-ai]
|
|
51
|
+
Rank every page by importance and write it to the Pages tab. --dry-run writes
|
|
52
|
+
nothing, --no-ai skips the models, --reuse-ai takes their judgement from the
|
|
53
|
+
last run instead of asking again
|
|
54
|
+
settings [FILE] Show which settings that describe your site are filled in, and whether an
|
|
55
|
+
audit can start
|
|
56
|
+
doctor Check that the required environment variables are set
|
|
57
|
+
version Print the version
|
|
58
|
+
help Show this help
|
|
59
|
+
TEXT
|
|
60
|
+
|
|
61
|
+
def self.run(argv, **) = new(**).run(argv)
|
|
62
|
+
|
|
63
|
+
def initialize(out: $stdout, err: $stderr, env: ENV, client: nil, store: nil, llm_transport: nil,
|
|
64
|
+
input: $stdin)
|
|
65
|
+
@out = out
|
|
66
|
+
@err = err
|
|
67
|
+
@env = env
|
|
68
|
+
@http_client = client
|
|
69
|
+
@store = store
|
|
70
|
+
@llm_transport = llm_transport
|
|
71
|
+
@input = input
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Returns the exit code: 0 success, 1 runtime or configuration error, 2 usage error.
|
|
75
|
+
def run(argv)
|
|
76
|
+
command, *args = argv
|
|
77
|
+
return help if command.nil?
|
|
78
|
+
|
|
79
|
+
action = COMMANDS[command]
|
|
80
|
+
return usage_error("Unknown command: #{command}") unless action
|
|
81
|
+
|
|
82
|
+
send(action, args)
|
|
83
|
+
rescue Error => e
|
|
84
|
+
@err.puts "Error: #{e.message}"
|
|
85
|
+
1
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
def help(_args = nil)
|
|
91
|
+
@out.puts USAGE
|
|
92
|
+
0
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def version(_args = nil)
|
|
96
|
+
@out.puts "yuno-proofreader #{VERSION}"
|
|
97
|
+
0
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def doctor(_args = nil)
|
|
101
|
+
config = Config.from_env(@env)
|
|
102
|
+
@out.puts "Configuration looks complete."
|
|
103
|
+
@out.puts config.inspect
|
|
104
|
+
0
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def sitemap(args)
|
|
108
|
+
options, rest, unknown = split_args(args, %w[all])
|
|
109
|
+
return usage_error("Unknown option: #{unknown.first}") unless unknown.empty?
|
|
110
|
+
return usage_error("Too many arguments for sitemap") if rest.size > 1
|
|
111
|
+
|
|
112
|
+
site = Site.load(rest.first || DEFAULT_SITE_FILE)
|
|
113
|
+
result = Sitemap::Loader.call(site, http_client)
|
|
114
|
+
Sitemap::Report.new(site, result).lines(all: options.key?("all")).each { |line| @out.puts line }
|
|
115
|
+
result.pairing.resolved? ? 0 : 1
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def fetch(args)
|
|
119
|
+
options, rest, unknown = split_args(args, %w[html])
|
|
120
|
+
problem = fetch_args_problem(options, rest, unknown)
|
|
121
|
+
return usage_error(problem) if problem
|
|
122
|
+
|
|
123
|
+
site = Site.load(rest[1] || DEFAULT_SITE_FILE)
|
|
124
|
+
language = options["html"]
|
|
125
|
+
return usage_error("Unknown language for --html: #{language}") if language && !site.languages.include?(language)
|
|
126
|
+
|
|
127
|
+
show_page(site, rest.first, language)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def fetch_args_problem(options, rest, unknown)
|
|
131
|
+
return "Unknown option: #{unknown.first}" unless unknown.empty?
|
|
132
|
+
return "--html needs a language, for example --html=de" if options["html"] == true
|
|
133
|
+
return "fetch needs a page name or address" if rest.empty?
|
|
134
|
+
|
|
135
|
+
"Too many arguments for fetch" if rest.size > 2
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def settings(args)
|
|
139
|
+
file, problem = single_file(args, "settings")
|
|
140
|
+
return usage_error(problem) if problem
|
|
141
|
+
|
|
142
|
+
site = Site.load(file)
|
|
143
|
+
settings_lines(site, file).each { |line| @out.puts line }
|
|
144
|
+
site.audit_problems.empty? ? 0 : 1
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def settings_lines(site, file)
|
|
148
|
+
shown = site.prefixes.map { |code, prefix| prefix.empty? ? code : "#{code} (#{prefix})" }.join(", ")
|
|
149
|
+
problems = site.audit_problems
|
|
150
|
+
[
|
|
151
|
+
"Settings file: #{file}", "Website: #{site.website}", "Languages: #{shown}, default #{site.default_language}",
|
|
152
|
+
"", *site.states.map { |name, state| " #{name.ljust(26)}#{STATE_LABELS.fetch(state)}" }, "",
|
|
153
|
+
problems.empty? ? "Ready for an audit." : "Not ready for an audit (sitemap and fetch do not need this):",
|
|
154
|
+
*problems.map { |problem| " - #{problem}" }
|
|
155
|
+
]
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def show_page(site, query, language)
|
|
159
|
+
page = find_page(Sitemap::Loader.call(site, http_client).pairing, query)
|
|
160
|
+
fetched = Pages::Fetcher.new(site, http_client).fetch(page)
|
|
161
|
+
Pages::Report.new(fetched).lines.each { |line| @out.puts line }
|
|
162
|
+
print_html(fetched, language) if language
|
|
163
|
+
fetched.complete? ? 0 : 1
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def find_page(pairing, query)
|
|
167
|
+
unless pairing.resolved?
|
|
168
|
+
raise Error, "The pages could not be paired. Run the sitemap command and fix what it reports."
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
pairing.pages.find { |page| page.name == query || page.urls.value?(query) } ||
|
|
172
|
+
raise(Error, "No page named #{query.inspect}. List the names with: yuno-proofreader sitemap --all")
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def print_html(fetched, language)
|
|
176
|
+
snapshot = fetched.snapshot(language)
|
|
177
|
+
return @err.puts("No #{language} version to show.") unless snapshot&.ok?
|
|
178
|
+
|
|
179
|
+
@out.puts "", "--- trimmed HTML (#{language}) ---", snapshot.html
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Splits options (--name or --name=value) from positional arguments.
|
|
183
|
+
def split_args(args, known)
|
|
184
|
+
flags, positional = args.partition { |arg| arg.start_with?("-") }
|
|
185
|
+
options = {}
|
|
186
|
+
unknown = []
|
|
187
|
+
flags.each do |flag|
|
|
188
|
+
key, value = flag.sub(/\A-+/, "").split("=", 2)
|
|
189
|
+
if known.include?(key)
|
|
190
|
+
options[key] = value || true
|
|
191
|
+
else
|
|
192
|
+
unknown << flag
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
[options, positional, unknown]
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# One optional settings file and nothing else. Returns [path, nil], or [nil, what is wrong].
|
|
199
|
+
def single_file(args, command)
|
|
200
|
+
flags, rest = args.partition { |arg| arg.start_with?("-") }
|
|
201
|
+
return [nil, "Unknown option: #{flags.first}"] unless flags.empty?
|
|
202
|
+
return [nil, "Too many arguments for #{command}"] if rest.size > 1
|
|
203
|
+
|
|
204
|
+
[rest.first || DEFAULT_SITE_FILE, nil]
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def http_client
|
|
208
|
+
@http_client ||= Http::Client.new
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def usage_error(message)
|
|
212
|
+
@err.puts message
|
|
213
|
+
@err.puts USAGE
|
|
214
|
+
2
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# Settings that come from the environment: ids, never site content. The key of a model provider is not
|
|
6
|
+
# here: each provider names its own variable, and it is read when the provider is used. LOG_LEVEL is
|
|
7
|
+
# checked here too, so doctor can show it, and Log does the reading.
|
|
8
|
+
class Config < Data.define(:spreadsheet_id, :log_level)
|
|
9
|
+
REQUIRED = %w[SPREADSHEET_ID].freeze
|
|
10
|
+
LOG_LEVELS = %w[debug info warn error].freeze
|
|
11
|
+
|
|
12
|
+
def self.from_env(env = ENV)
|
|
13
|
+
missing = REQUIRED.select { |name| env[name].to_s.strip.empty? }
|
|
14
|
+
raise ConfigError, "Missing environment variables: #{missing.join(', ')}" unless missing.empty?
|
|
15
|
+
|
|
16
|
+
new(spreadsheet_id: env["SPREADSHEET_ID"].strip, log_level: Log.level_from(env["LOG_LEVEL"]))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def inspect
|
|
20
|
+
"#<#{self.class.name} spreadsheet_id=#{spreadsheet_id.inspect} log_level=#{log_level}>"
|
|
21
|
+
end
|
|
22
|
+
alias to_s inspect
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# What the model should know about the site, read from the settings file and sent with every page.
|
|
6
|
+
# Everything is optional, except that the audit needs a business description.
|
|
7
|
+
class Context < Data.define(:business, :audience, :tone, :rules, :ignore_terms, :intentional_differences)
|
|
8
|
+
TEXTS = %w[business audience tone].freeze
|
|
9
|
+
LISTS = %w[rules ignore_terms intentional_differences].freeze
|
|
10
|
+
KEYS = (TEXTS + LISTS).freeze
|
|
11
|
+
PLACEHOLDER = /\[fill in\]/i
|
|
12
|
+
MAX_TEXT = 500
|
|
13
|
+
MAX_ITEM = 200
|
|
14
|
+
MAX_ITEMS = 50
|
|
15
|
+
|
|
16
|
+
def self.from_hash(raw)
|
|
17
|
+
texts = TEXTS.to_h { |key| [key.to_sym, text(raw[key], key, MAX_TEXT)] }
|
|
18
|
+
lists = LISTS.to_h { |key| [key.to_sym, list(raw[key], key)] }
|
|
19
|
+
new(**texts, **lists)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.placeholder?(value) = value.to_s.match?(PLACEHOLDER)
|
|
23
|
+
|
|
24
|
+
# :set, :empty (nothing given) or :placeholder (the [FILL IN] example text is still there).
|
|
25
|
+
def self.state_of(value)
|
|
26
|
+
items = Array(value)
|
|
27
|
+
return :empty if items.all?(&:empty?)
|
|
28
|
+
|
|
29
|
+
items.any? { |item| placeholder?(item) } ? :placeholder : :set
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# One piece of text, trimmed. Nothing at all becomes an empty string.
|
|
33
|
+
def self.text(value, label, max)
|
|
34
|
+
raise ConfigError, "Site: #{label} must be plain text" if value.is_a?(Hash) || value.is_a?(Array)
|
|
35
|
+
|
|
36
|
+
cleaned = value.to_s.strip
|
|
37
|
+
raise ConfigError, "Site: #{label} is longer than #{max} characters" if cleaned.length > max
|
|
38
|
+
|
|
39
|
+
cleaned
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# A list of texts. A single text counts as a list of one. Blank and repeated entries are dropped.
|
|
43
|
+
def self.list(value, label)
|
|
44
|
+
items = Array(value).map { |item| text(item, "an entry of #{label}", MAX_ITEM) }.reject(&:empty?).uniq
|
|
45
|
+
raise ConfigError, "Site: #{label} has more than #{MAX_ITEMS} entries" if items.size > MAX_ITEMS
|
|
46
|
+
|
|
47
|
+
items
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def states
|
|
51
|
+
to_h.transform_keys(&:to_s).transform_values { |value| self.class.state_of(value) }
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Yuno
|
|
4
|
+
module Proofreader
|
|
5
|
+
# Reads the settings that live outside proofreader.yml (API keys, the id of the sheet, Google credentials) from
|
|
6
|
+
# a .env file, so they do not have to be typed into the shell. A variable that is already set wins over the file
|
|
7
|
+
module EnvFile
|
|
8
|
+
DEFAULT = ".env"
|
|
9
|
+
CHOICE = "PROOFREADER_ENV_FILE"
|
|
10
|
+
LINE = /\A(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\z/
|
|
11
|
+
DOUBLE = /\A"((?:[^"\\]|\\.)*)"\s*(?:#.*)?\z/
|
|
12
|
+
SINGLE = /\A'([^']*)'\s*(?:#.*)?\z/
|
|
13
|
+
ESCAPES = { "n" => "\n", "t" => "\t" }.freeze
|
|
14
|
+
|
|
15
|
+
# path: the file. names: the variables it set. skipped: the numbers of the lines that were not NAME=VALUE.
|
|
16
|
+
# loose: whether other users can read the file.
|
|
17
|
+
Loaded = Data.define(:path, :names, :skipped, :loose) do
|
|
18
|
+
def warnings = [skipped_warning, loose_warning].compact
|
|
19
|
+
|
|
20
|
+
def skipped_warning
|
|
21
|
+
return if skipped.empty?
|
|
22
|
+
|
|
23
|
+
"Ignored #{skipped.size} line(s) of #{path} that are not NAME=VALUE (line #{skipped.first(5).join(', ')})."
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def loose_warning = ("#{path} can be read by other users. Run: chmod 600 #{path}" if loose)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
module_function
|
|
30
|
+
|
|
31
|
+
# Loads the file into env without replacing what is already set. Returns what was loaded, or nil when there
|
|
32
|
+
# is no file to load. A file named in PROOFREADER_ENV_FILE that does not exist is an error.
|
|
33
|
+
def load(env = ENV, path: nil)
|
|
34
|
+
chosen = path || env[CHOICE].to_s.strip
|
|
35
|
+
file = chosen.empty? ? DEFAULT : chosen
|
|
36
|
+
unless File.file?(file)
|
|
37
|
+
raise ConfigError, "#{CHOICE} points to #{file}, which is not a file." unless chosen.empty?
|
|
38
|
+
|
|
39
|
+
return
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
pairs, skipped = parse(File.read(file))
|
|
43
|
+
names = pairs.keys.select { |name| env[name].to_s.empty? }
|
|
44
|
+
names.each { |name| env[name] = pairs[name] }
|
|
45
|
+
Loaded.new(path: file, names:, skipped:, loose: loose?(file))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# [the NAME=VALUE pairs, the numbers of the lines that were neither a pair, empty nor a comment]
|
|
49
|
+
def parse(text)
|
|
50
|
+
pairs = {}
|
|
51
|
+
skipped = []
|
|
52
|
+
text.each_line.with_index(1) do |raw, number|
|
|
53
|
+
line = raw.strip
|
|
54
|
+
next if line.empty? || line.start_with?("#")
|
|
55
|
+
|
|
56
|
+
match = LINE.match(line)
|
|
57
|
+
match ? pairs[match[1]] = value_of(match[2]) : skipped << number
|
|
58
|
+
end
|
|
59
|
+
[pairs, skipped]
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def value_of(raw)
|
|
63
|
+
text = raw.strip
|
|
64
|
+
if (match = DOUBLE.match(text)) then unescape(match[1])
|
|
65
|
+
elsif (match = SINGLE.match(text)) then match[1]
|
|
66
|
+
else text.sub(/\s+#.*\z/, "")
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def unescape(text) = text.gsub(/\\(.)/) { ESCAPES.fetch(Regexp.last_match(1), Regexp.last_match(1)) }
|
|
71
|
+
|
|
72
|
+
# Whether other users can read the file. Not checked where there are no unix permissions.
|
|
73
|
+
def loose?(file) = File::ALT_SEPARATOR.nil? && File.stat(file).mode.anybits?(0o077)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|