yuno-proofreader 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +13 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +435 -0
  5. data/exe/yuno-proofreader +16 -0
  6. data/lib/yuno/proofreader/analysis/adjustments.rb +45 -0
  7. data/lib/yuno/proofreader/analysis/crawler.rb +95 -0
  8. data/lib/yuno/proofreader/analysis/entries.rb +73 -0
  9. data/lib/yuno/proofreader/analysis/extractor.rb +209 -0
  10. data/lib/yuno/proofreader/analysis/graph.rb +115 -0
  11. data/lib/yuno/proofreader/analysis/judge.rb +103 -0
  12. data/lib/yuno/proofreader/analysis/judge.yml +27 -0
  13. data/lib/yuno/proofreader/analysis/judge_prompt.rb +72 -0
  14. data/lib/yuno/proofreader/analysis/judgments.rb +70 -0
  15. data/lib/yuno/proofreader/analysis/links.rb +54 -0
  16. data/lib/yuno/proofreader/analysis/list_order.rb +33 -0
  17. data/lib/yuno/proofreader/analysis/reasons.rb +49 -0
  18. data/lib/yuno/proofreader/analysis/report.rb +110 -0
  19. data/lib/yuno/proofreader/analysis/runner.rb +171 -0
  20. data/lib/yuno/proofreader/analysis/scorer.rb +128 -0
  21. data/lib/yuno/proofreader/analysis/settings.rb +90 -0
  22. data/lib/yuno/proofreader/analysis/signals.rb +102 -0
  23. data/lib/yuno/proofreader/analysis/structured_data.rb +57 -0
  24. data/lib/yuno/proofreader/analysis/url_tree.rb +49 -0
  25. data/lib/yuno/proofreader/analysis/weights.rb +77 -0
  26. data/lib/yuno/proofreader/analysis/weights.yml +39 -0
  27. data/lib/yuno/proofreader/audit/agenda.rb +86 -0
  28. data/lib/yuno/proofreader/audit/agenda_report.rb +65 -0
  29. data/lib/yuno/proofreader/audit/answers.rb +74 -0
  30. data/lib/yuno/proofreader/audit/attempt.rb +69 -0
  31. data/lib/yuno/proofreader/audit/batch.rb +64 -0
  32. data/lib/yuno/proofreader/audit/batch_report.rb +87 -0
  33. data/lib/yuno/proofreader/audit/cadence.rb +67 -0
  34. data/lib/yuno/proofreader/audit/instructions.rb +30 -0
  35. data/lib/yuno/proofreader/audit/log_fields.rb +28 -0
  36. data/lib/yuno/proofreader/audit/prompt.rb +157 -0
  37. data/lib/yuno/proofreader/audit/prompts/instructions.yml +46 -0
  38. data/lib/yuno/proofreader/audit/prompts/steps.yml +67 -0
  39. data/lib/yuno/proofreader/audit/report.rb +153 -0
  40. data/lib/yuno/proofreader/audit/runner.rb +144 -0
  41. data/lib/yuno/proofreader/audit/steps.rb +56 -0
  42. data/lib/yuno/proofreader/cli/analysis_commands.rb +43 -0
  43. data/lib/yuno/proofreader/cli/audit_commands.rb +92 -0
  44. data/lib/yuno/proofreader/cli/command_log.rb +42 -0
  45. data/lib/yuno/proofreader/cli/flags.rb +48 -0
  46. data/lib/yuno/proofreader/cli/init_command.rb +116 -0
  47. data/lib/yuno/proofreader/cli/model_commands.rb +34 -0
  48. data/lib/yuno/proofreader/cli/queue_commands.rb +89 -0
  49. data/lib/yuno/proofreader/cli/sheet_commands.rb +57 -0
  50. data/lib/yuno/proofreader/cli.rb +218 -0
  51. data/lib/yuno/proofreader/config.rb +25 -0
  52. data/lib/yuno/proofreader/context.rb +55 -0
  53. data/lib/yuno/proofreader/env_file.rb +76 -0
  54. data/lib/yuno/proofreader/errors.rb +17 -0
  55. data/lib/yuno/proofreader/findings/finding.rb +95 -0
  56. data/lib/yuno/proofreader/findings/matching.rb +34 -0
  57. data/lib/yuno/proofreader/findings/merger.rb +43 -0
  58. data/lib/yuno/proofreader/findings/outline.rb +73 -0
  59. data/lib/yuno/proofreader/findings/plan.rb +24 -0
  60. data/lib/yuno/proofreader/findings/reconciler.rb +219 -0
  61. data/lib/yuno/proofreader/findings/source.rb +79 -0
  62. data/lib/yuno/proofreader/findings/text.rb +37 -0
  63. data/lib/yuno/proofreader/http/client.rb +131 -0
  64. data/lib/yuno/proofreader/http/net_http_transport.rb +55 -0
  65. data/lib/yuno/proofreader/http/robots.rb +72 -0
  66. data/lib/yuno/proofreader/init/ask.rb +67 -0
  67. data/lib/yuno/proofreader/init/detector.rb +100 -0
  68. data/lib/yuno/proofreader/init/interview.rb +210 -0
  69. data/lib/yuno/proofreader/init/parsing.rb +47 -0
  70. data/lib/yuno/proofreader/init/web.rb +33 -0
  71. data/lib/yuno/proofreader/llm/client.rb +140 -0
  72. data/lib/yuno/proofreader/llm/defaults.rb +22 -0
  73. data/lib/yuno/proofreader/llm/defaults.yml +6 -0
  74. data/lib/yuno/proofreader/llm/errors.rb +26 -0
  75. data/lib/yuno/proofreader/llm/factory.rb +45 -0
  76. data/lib/yuno/proofreader/llm/http_transport.rb +41 -0
  77. data/lib/yuno/proofreader/llm/json_text.rb +23 -0
  78. data/lib/yuno/proofreader/llm/model_commands.rb +34 -0
  79. data/lib/yuno/proofreader/llm/provider.rb +58 -0
  80. data/lib/yuno/proofreader/llm/providers/gemini.rb +146 -0
  81. data/lib/yuno/proofreader/llm/providers.rb +38 -0
  82. data/lib/yuno/proofreader/llm/types.rb +13 -0
  83. data/lib/yuno/proofreader/llm/usage.rb +17 -0
  84. data/lib/yuno/proofreader/log.rb +129 -0
  85. data/lib/yuno/proofreader/pages/fetcher.rb +75 -0
  86. data/lib/yuno/proofreader/pages/report.rb +45 -0
  87. data/lib/yuno/proofreader/pages/snapshot.rb +48 -0
  88. data/lib/yuno/proofreader/pages/trimmer.rb +36 -0
  89. data/lib/yuno/proofreader/settings_file.rb +51 -0
  90. data/lib/yuno/proofreader/site.rb +74 -0
  91. data/lib/yuno/proofreader/site_reader.rb +131 -0
  92. data/lib/yuno/proofreader/sitemap/aligner.rb +78 -0
  93. data/lib/yuno/proofreader/sitemap/importer.rb +31 -0
  94. data/lib/yuno/proofreader/sitemap/loader.rb +127 -0
  95. data/lib/yuno/proofreader/sitemap/mismatch.rb +76 -0
  96. data/lib/yuno/proofreader/sitemap/pairer.rb +223 -0
  97. data/lib/yuno/proofreader/sitemap/parser.rb +73 -0
  98. data/lib/yuno/proofreader/sitemap/report.rb +80 -0
  99. data/lib/yuno/proofreader/store/cells.rb +67 -0
  100. data/lib/yuno/proofreader/store/google_gateway.rb +198 -0
  101. data/lib/yuno/proofreader/store/memory.rb +90 -0
  102. data/lib/yuno/proofreader/store/records.rb +28 -0
  103. data/lib/yuno/proofreader/store/schema.rb +109 -0
  104. data/lib/yuno/proofreader/store/sheets.rb +183 -0
  105. data/lib/yuno/proofreader/store/tab_setup.rb +74 -0
  106. data/lib/yuno/proofreader/version.rb +7 -0
  107. data/lib/yuno/proofreader.rb +101 -0
  108. data/proofreader.example.yml +134 -0
  109. metadata +194 -0
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # The command that starts the settings file of a site: init.
6
+ module InitCommand
7
+ EXAMPLE = File.expand_path("../../../../proofreader.example.yml", __dir__)
8
+ SWITCHES = %w[template yes offline].freeze
9
+ VALUES = %w[website business audience tone sitemap].freeze
10
+ LISTS = %w[language note skip-path rule ignore-term difference].freeze
11
+ HEADER = ["Settings of yuno-proofreader, written by `yuno-proofreader init`.",
12
+ "Every setting is explained in the template: run `yuno-proofreader init --template`.",
13
+ "`yuno-proofreader settings` shows what an audit still needs."].freeze
14
+
15
+ private
16
+
17
+ def init(args)
18
+ flags = Flags.parse(args, switches: SWITCHES, values: VALUES, lists: LISTS)
19
+ problem = init_problem(flags)
20
+ return usage_error(problem) if problem
21
+
22
+ init_flow(flags)
23
+ 0
24
+ end
25
+
26
+ def init_problem(flags) = flags.problem || ("Too many arguments for init" if flags.rest.size > 1)
27
+
28
+ # Without --template the interview runs. With --template and no file name the template is only shown, and
29
+ # with a file name it is written there.
30
+ def init_flow(flags)
31
+ path = flags.rest.first
32
+ return init_interview(path || CLI::DEFAULT_SITE_FILE, flags) unless flags.switches.key?("template")
33
+
34
+ path ? init_template(path) : print_template
35
+ end
36
+
37
+ # With no file name the template is only shown, so it can be read, or saved with a redirect.
38
+ def print_template = @out.write(example_text)
39
+
40
+ def init_template(path)
41
+ create_file(path, example_text)
42
+ @out.puts "Wrote #{path}. It is a template: every value that starts with [FILL IN] is for you to replace.",
43
+ "Then run: yuno-proofreader settings #{path}"
44
+ end
45
+
46
+ def init_interview(path, flags)
47
+ ensure_free(path)
48
+ interview = Init::Interview.new(flags:, ask: Init::Ask.new(input: @input, out: @out),
49
+ web: Init::Web.new(http: http_client), out: @out,
50
+ offline: flags.switches.key?("offline"), yes: flags.switches.key?("yes"))
51
+ settings = interview.call
52
+ site = Site.from_hash(settings)
53
+ create_file(path, SettingsFile.render(settings, header: HEADER))
54
+ init_lines(path, site).each { |line| @out.puts line }
55
+ end
56
+
57
+ def ensure_free(path) = (raise ConfigError, taken_message(path) if File.exist?(path))
58
+
59
+ def taken_message(path)
60
+ "#{path} already exists, so it was left as it is. Delete it first, or choose another name: " \
61
+ "yuno-proofreader init NAME.yml"
62
+ end
63
+
64
+ # Writes the file, and never over one that is already there.
65
+ def create_file(path, text)
66
+ File.open(path, File::WRONLY | File::CREAT | File::EXCL) { |file| file.write(text) }
67
+ rescue Errno::EEXIST
68
+ raise ConfigError, taken_message(path)
69
+ rescue SystemCallError => e
70
+ raise ConfigError, "Could not write #{path}: #{e.message}"
71
+ end
72
+
73
+ def example_text
74
+ File.read(EXAMPLE)
75
+ rescue SystemCallError
76
+ raise Error, "The template settings file is missing from this installation (#{EXAMPLE})."
77
+ end
78
+
79
+ # ---- what is printed afterwards ----
80
+
81
+ def init_lines(path, site)
82
+ problems = Array(site.audit_problems)
83
+ suffix = path == CLI::DEFAULT_SITE_FILE ? "" : " #{path}"
84
+ ["", "Wrote #{path}.", *readiness(problems), "", "Outside this file you also need:", *environment_lines, "",
85
+ "Next:", *next_steps(suffix)]
86
+ end
87
+
88
+ def readiness(problems)
89
+ return ["It is ready for an audit."] if problems.empty?
90
+
91
+ ["Before an audit can run, this is still missing:", *problems.map { |problem| " - #{problem}" }]
92
+ end
93
+
94
+ def environment_lines
95
+ keys = Llm::Providers.ids.map { |id| [Llm::Providers.resolve(id).api_key_env, "the API key of #{id}"] }
96
+ lines = [["SPREADSHEET_ID", "the id of your Google Sheet"], *keys].map do |name, meaning|
97
+ environment_line(name, meaning)
98
+ end
99
+ [*lines, " Put them in a .env file in this folder, or export them.",
100
+ " Google credentials: see the README, section \"Connecting to Google\""]
101
+ end
102
+
103
+ def environment_line(name, meaning)
104
+ state = @env[name].to_s.strip.empty? ? "not set" : "set"
105
+ " #{name.ljust(18)} #{state.ljust(8)} (#{meaning})"
106
+ end
107
+
108
+ def next_steps(suffix)
109
+ [" yuno-proofreader prepare#{suffix} creates the tabs in your Google Sheet",
110
+ " yuno-proofreader pages import#{suffix} adds your pages to the sheet",
111
+ " yuno-proofreader analyze#{suffix} ranks the pages by importance",
112
+ " yuno-proofreader audit --next#{suffix} audits the most important page that is due"]
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # The command that works with the model provider: models.
6
+ module ModelCommands
7
+ private
8
+
9
+ def models(args)
10
+ file, problem = single_file(args, "models")
11
+ return usage_error(problem) if problem
12
+
13
+ site = Site.load(file)
14
+ provider = Llm.provider_for(site, env: @env, transport: @llm_transport)
15
+ selection = Llm.models_for(site, provider.class)
16
+ used = selection.names.map { |name| provider.normalize(name) }
17
+ print_models(provider, provider.models, used, selection.source)
18
+ 0
19
+ end
20
+
21
+ def print_models(provider, available, used, source)
22
+ @out.puts "Models your #{provider.class.id} key can use for text generation:"
23
+ available.each { |name| @out.puts " #{name}#{' (in use)' if used.include?(name)}" }
24
+ origin = source == :settings ? "from your settings" : "built-in default"
25
+ @out.puts "", "In use (#{origin}): #{used.join(', ')}"
26
+ missing = used - available
27
+ return if missing.empty?
28
+
29
+ @out.puts "Not available to this key: #{missing.join(', ')}. A model that fails is skipped, " \
30
+ "so set models in proofreader.yml to the ones you want."
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # The commands that decide which page to audit next: queue, and the batch form of audit (audit --next).
6
+ module QueueCommands
7
+ MAX_COUNT = 500
8
+ COUNT_FLAG = /\A--count(=|\z)/
9
+
10
+ private
11
+
12
+ def queue(args)
13
+ count, remaining, problem = take_count(args)
14
+ return usage_error(problem) if problem
15
+
16
+ options, rest, unknown = split_args(remaining, %w[all])
17
+ problem = queue_args_problem(rest, unknown)
18
+ return usage_error(problem) if problem
19
+
20
+ site = Site.load(rest.first || CLI::DEFAULT_SITE_FILE)
21
+ picks = agenda_for(site, store_for(site))
22
+ report = Audit::AgendaReport.new(picks, shown: count || Audit::AgendaReport::SHOWN,
23
+ everything: options.key?("all"))
24
+ show_lines(report.lines)
25
+ end
26
+
27
+ def queue_args_problem(rest, unknown)
28
+ return "Unknown option: #{unknown.first}" unless unknown.empty?
29
+
30
+ "Too many arguments for queue" if rest.size > 1
31
+ end
32
+
33
+ # Takes --count=N or --count N out of the arguments. Returns the number, the other arguments and a problem.
34
+ def take_count(args)
35
+ remaining = args.dup
36
+ index = remaining.index { |arg| arg.match?(COUNT_FLAG) }
37
+ return [nil, remaining, nil] unless index
38
+
39
+ flag = remaining.delete_at(index)
40
+ number = Integer((flag[/\A--count=(.*)\z/, 1] || remaining.delete_at(index)).to_s, 10, exception: false)
41
+ return [number, remaining, nil] if number&.between?(1, MAX_COUNT)
42
+
43
+ [nil, remaining, "--count needs a whole number from 1 to #{MAX_COUNT}"]
44
+ end
45
+
46
+ def agenda_for(site, store)
47
+ Audit::Agenda.call(records: store.pages, cadence: site.cadence, now: Time.now.utc)
48
+ end
49
+
50
+ def show_lines(lines)
51
+ lines.each { |line| @out.puts line }
52
+ 0
53
+ end
54
+
55
+ # The model clients are built first, so a missing API key stops the command before anything is read.
56
+ def audit_batch(site, options, count)
57
+ store = store_for(site)
58
+ runner = audit_runner(site, store)
59
+ picks = agenda_for(site, store)
60
+ pool = options.key?("include-not-due") ? picks : picks.select(&:due?)
61
+ chosen = pool.first(count || 1)
62
+ if chosen.empty?
63
+ log.info("nothing is due", pages: picks.size)
64
+ return show_lines(Audit::AgendaReport.new(picks).lines)
65
+ end
66
+
67
+ run_batch(runner, store, chosen, picks, options)
68
+ end
69
+
70
+ def run_batch(runner, store, chosen, picks, options)
71
+ batch = Audit::Batch.new(runner:, store:, progress: ->(message) { @out.puts message }, log:)
72
+ dry_run = options.key?("dry-run")
73
+ outcome = batch.call(chosen, dry_run:, force: options.key?("force"))
74
+ still_due = [picks.count(&:due?) - outcome.entries.size, 0].max
75
+ log_batch(outcome, still_due:, dry_run:)
76
+ show_lines(Audit::BatchReport.new(outcome, dry_run:, still_due:, verbose: options.key?("verbose")).lines)
77
+ outcome.stopped ? 1 : 0
78
+ end
79
+
80
+ def log_batch(outcome, still_due:, dry_run:)
81
+ statuses = outcome.entries.map(&:status).tally
82
+ calls = outcome.entries.filter_map(&:result).flat_map(&:calls)
83
+ log.info("batch finished", pages: outcome.entries.size, audited: statuses.fetch(:audited, 0),
84
+ skipped: statuses.fetch(:skipped, 0), failed: statuses.fetch(:failed, 0),
85
+ still_due:, dry_run:, stopped: outcome.stopped, **Llm::Usage.of(calls))
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # The commands that work with the Google Sheet: prepare and pages import.
6
+ module SheetCommands
7
+ private
8
+
9
+ def prepare(args)
10
+ file, problem = single_file(args, "prepare")
11
+ return usage_error(problem) if problem
12
+
13
+ store_for(Site.load(file)).prepare.each { |line| @out.puts line }
14
+ 0
15
+ end
16
+
17
+ def pages(args)
18
+ action, *rest = args
19
+ return usage_error("Unknown pages command: #{action.inspect}. Try: pages import") unless action == "import"
20
+
21
+ file, problem = single_file(rest, "pages import")
22
+ return usage_error(problem) if problem
23
+
24
+ import_pages(Site.load(file))
25
+ end
26
+
27
+ def import_pages(site)
28
+ loaded = Sitemap::Loader.call(site, http_client)
29
+ return refuse_import(site, loaded) unless loaded.pairing.resolved?
30
+
31
+ result = Sitemap::Importer.call(loaded.pairing, store_for(site), site)
32
+ print_import(result, loaded.warnings)
33
+ 0
34
+ end
35
+
36
+ def refuse_import(site, loaded)
37
+ Sitemap::Report.new(site, loaded).lines.each { |line| @out.puts line }
38
+ @out.puts "", "Nothing was written to the sheet."
39
+ 1
40
+ end
41
+
42
+ def print_import(result, warnings)
43
+ @out.puts "Added #{result.added} pages to the Pages tab (#{result.existing} were already there)."
44
+ if result.gone.positive?
45
+ @out.puts "#{result.gone} pages in the sheet are no longer in the sitemap and were left alone."
46
+ end
47
+ return if warnings.empty?
48
+
49
+ @out.puts "", "Warnings:", *warnings.map { |warning| " - #{warning}" }
50
+ end
51
+
52
+ def store_for(site)
53
+ @store || Store::Sheets.connect(Config.from_env(@env), site)
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # Command-line entry point: parses arguments and returns an exit code.
6
+ class CLI
7
+ include SheetCommands
8
+ include ModelCommands
9
+ include AuditCommands
10
+ include AnalysisCommands
11
+ include InitCommand
12
+ prepend CommandLog
13
+ include QueueCommands
14
+
15
+ DEFAULT_SITE_FILE = "proofreader.yml"
16
+ STATE_LABELS = { set: "set", empty: "not set", placeholder: "still the [FILL IN] example text" }.freeze
17
+ COMMANDS = {
18
+ "help" => :help, "-h" => :help, "--help" => :help,
19
+ "version" => :version, "-v" => :version, "--version" => :version,
20
+ "doctor" => :doctor, "sitemap" => :sitemap, "fetch" => :fetch, "settings" => :settings,
21
+ "models" => :models, "prepare" => :prepare, "pages" => :pages, "prompt" => :prompt,
22
+ "analyze" => :analyze, "init" => :init, "queue" => :queue, "audit" => :audit
23
+ }.freeze
24
+ USAGE = <<~TEXT
25
+ Usage: yuno-proofreader <command>
26
+
27
+ Commands:
28
+ sitemap [FILE] [--all] Read the sitemap of the site described in FILE (default: proofreader.yml)
29
+ and show how its pages were paired by language
30
+ fetch PAGE [FILE] [--html=LANG]
31
+ Download all language versions of one page and show how the tool sees them.
32
+ PAGE is a name from `sitemap --all`, or any address of the page
33
+ prepare [FILE] Create the tabs the tool needs in your Google Sheet (safe to run again)
34
+ pages import [FILE] Add the pages of the sitemap to the Pages tab of the sheet
35
+ models [FILE] List the models your API key can use for text generation, and which are in use
36
+ init [FILE] [OPTIONS] Ask about your site and write a settings file (default proofreader.yml).
37
+ --template shows the full commented template, or with a FILE writes it there.
38
+ For scripts: --website, --language CODE[:PREFIX], --note CODE=TEXT, --business,
39
+ --audience, --tone, --sitemap, --skip-path, --rule, --ignore-term, --difference.
40
+ --yes accepts what was detected, --offline makes no request. Does not overwrite a file
41
+ audit PAGE [FILE] [--dry-run] [--force] [--verbose]
42
+ Audit one page: ask the model, compare the findings with the suggestions in the
43
+ sheet and update the sheet. --dry-run writes nothing (the model is still called
44
+ and costs the same), --force audits a page that has not changed, --verbose
45
+ lists everything that was decided
46
+ audit --next [FILE] [--count=N] [--include-not-due] [--dry-run] [--force] [--verbose]
47
+ Audit the pages that are due, the most important first (default 1 page)
48
+ queue [FILE] [--count=N] [--all]
49
+ Show which pages are due for an audit and why. Calls no model and fetches no page
50
+ analyze [FILE] [--dry-run] [--no-ai] [--reuse-ai]
51
+ Rank every page by importance and write it to the Pages tab. --dry-run writes
52
+ nothing, --no-ai skips the models, --reuse-ai takes their judgement from the
53
+ last run instead of asking again
54
+ settings [FILE] Show which settings that describe your site are filled in, and whether an
55
+ audit can start
56
+ doctor Check that the required environment variables are set
57
+ version Print the version
58
+ help Show this help
59
+ TEXT
60
+
61
+ def self.run(argv, **) = new(**).run(argv)
62
+
63
+ def initialize(out: $stdout, err: $stderr, env: ENV, client: nil, store: nil, llm_transport: nil,
64
+ input: $stdin)
65
+ @out = out
66
+ @err = err
67
+ @env = env
68
+ @http_client = client
69
+ @store = store
70
+ @llm_transport = llm_transport
71
+ @input = input
72
+ end
73
+
74
+ # Returns the exit code: 0 success, 1 runtime or configuration error, 2 usage error.
75
+ def run(argv)
76
+ command, *args = argv
77
+ return help if command.nil?
78
+
79
+ action = COMMANDS[command]
80
+ return usage_error("Unknown command: #{command}") unless action
81
+
82
+ send(action, args)
83
+ rescue Error => e
84
+ @err.puts "Error: #{e.message}"
85
+ 1
86
+ end
87
+
88
+ private
89
+
90
+ def help(_args = nil)
91
+ @out.puts USAGE
92
+ 0
93
+ end
94
+
95
+ def version(_args = nil)
96
+ @out.puts "yuno-proofreader #{VERSION}"
97
+ 0
98
+ end
99
+
100
+ def doctor(_args = nil)
101
+ config = Config.from_env(@env)
102
+ @out.puts "Configuration looks complete."
103
+ @out.puts config.inspect
104
+ 0
105
+ end
106
+
107
+ def sitemap(args)
108
+ options, rest, unknown = split_args(args, %w[all])
109
+ return usage_error("Unknown option: #{unknown.first}") unless unknown.empty?
110
+ return usage_error("Too many arguments for sitemap") if rest.size > 1
111
+
112
+ site = Site.load(rest.first || DEFAULT_SITE_FILE)
113
+ result = Sitemap::Loader.call(site, http_client)
114
+ Sitemap::Report.new(site, result).lines(all: options.key?("all")).each { |line| @out.puts line }
115
+ result.pairing.resolved? ? 0 : 1
116
+ end
117
+
118
+ def fetch(args)
119
+ options, rest, unknown = split_args(args, %w[html])
120
+ problem = fetch_args_problem(options, rest, unknown)
121
+ return usage_error(problem) if problem
122
+
123
+ site = Site.load(rest[1] || DEFAULT_SITE_FILE)
124
+ language = options["html"]
125
+ return usage_error("Unknown language for --html: #{language}") if language && !site.languages.include?(language)
126
+
127
+ show_page(site, rest.first, language)
128
+ end
129
+
130
+ def fetch_args_problem(options, rest, unknown)
131
+ return "Unknown option: #{unknown.first}" unless unknown.empty?
132
+ return "--html needs a language, for example --html=de" if options["html"] == true
133
+ return "fetch needs a page name or address" if rest.empty?
134
+
135
+ "Too many arguments for fetch" if rest.size > 2
136
+ end
137
+
138
+ def settings(args)
139
+ file, problem = single_file(args, "settings")
140
+ return usage_error(problem) if problem
141
+
142
+ site = Site.load(file)
143
+ settings_lines(site, file).each { |line| @out.puts line }
144
+ site.audit_problems.empty? ? 0 : 1
145
+ end
146
+
147
+ def settings_lines(site, file)
148
+ shown = site.prefixes.map { |code, prefix| prefix.empty? ? code : "#{code} (#{prefix})" }.join(", ")
149
+ problems = site.audit_problems
150
+ [
151
+ "Settings file: #{file}", "Website: #{site.website}", "Languages: #{shown}, default #{site.default_language}",
152
+ "", *site.states.map { |name, state| " #{name.ljust(26)}#{STATE_LABELS.fetch(state)}" }, "",
153
+ problems.empty? ? "Ready for an audit." : "Not ready for an audit (sitemap and fetch do not need this):",
154
+ *problems.map { |problem| " - #{problem}" }
155
+ ]
156
+ end
157
+
158
+ def show_page(site, query, language)
159
+ page = find_page(Sitemap::Loader.call(site, http_client).pairing, query)
160
+ fetched = Pages::Fetcher.new(site, http_client).fetch(page)
161
+ Pages::Report.new(fetched).lines.each { |line| @out.puts line }
162
+ print_html(fetched, language) if language
163
+ fetched.complete? ? 0 : 1
164
+ end
165
+
166
+ def find_page(pairing, query)
167
+ unless pairing.resolved?
168
+ raise Error, "The pages could not be paired. Run the sitemap command and fix what it reports."
169
+ end
170
+
171
+ pairing.pages.find { |page| page.name == query || page.urls.value?(query) } ||
172
+ raise(Error, "No page named #{query.inspect}. List the names with: yuno-proofreader sitemap --all")
173
+ end
174
+
175
+ def print_html(fetched, language)
176
+ snapshot = fetched.snapshot(language)
177
+ return @err.puts("No #{language} version to show.") unless snapshot&.ok?
178
+
179
+ @out.puts "", "--- trimmed HTML (#{language}) ---", snapshot.html
180
+ end
181
+
182
+ # Splits options (--name or --name=value) from positional arguments.
183
+ def split_args(args, known)
184
+ flags, positional = args.partition { |arg| arg.start_with?("-") }
185
+ options = {}
186
+ unknown = []
187
+ flags.each do |flag|
188
+ key, value = flag.sub(/\A-+/, "").split("=", 2)
189
+ if known.include?(key)
190
+ options[key] = value || true
191
+ else
192
+ unknown << flag
193
+ end
194
+ end
195
+ [options, positional, unknown]
196
+ end
197
+
198
+ # One optional settings file and nothing else. Returns [path, nil], or [nil, what is wrong].
199
+ def single_file(args, command)
200
+ flags, rest = args.partition { |arg| arg.start_with?("-") }
201
+ return [nil, "Unknown option: #{flags.first}"] unless flags.empty?
202
+ return [nil, "Too many arguments for #{command}"] if rest.size > 1
203
+
204
+ [rest.first || DEFAULT_SITE_FILE, nil]
205
+ end
206
+
207
+ def http_client
208
+ @http_client ||= Http::Client.new
209
+ end
210
+
211
+ def usage_error(message)
212
+ @err.puts message
213
+ @err.puts USAGE
214
+ 2
215
+ end
216
+ end
217
+ end
218
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # Settings that come from the environment: ids, never site content. The key of a model provider is not
6
+ # here: each provider names its own variable, and it is read when the provider is used. LOG_LEVEL is
7
+ # checked here too, so doctor can show it, and Log does the reading.
8
+ class Config < Data.define(:spreadsheet_id, :log_level)
9
+ REQUIRED = %w[SPREADSHEET_ID].freeze
10
+ LOG_LEVELS = %w[debug info warn error].freeze
11
+
12
+ def self.from_env(env = ENV)
13
+ missing = REQUIRED.select { |name| env[name].to_s.strip.empty? }
14
+ raise ConfigError, "Missing environment variables: #{missing.join(', ')}" unless missing.empty?
15
+
16
+ new(spreadsheet_id: env["SPREADSHEET_ID"].strip, log_level: Log.level_from(env["LOG_LEVEL"]))
17
+ end
18
+
19
+ def inspect
20
+ "#<#{self.class.name} spreadsheet_id=#{spreadsheet_id.inspect} log_level=#{log_level}>"
21
+ end
22
+ alias to_s inspect
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # What the model should know about the site, read from the settings file and sent with every page.
6
+ # Everything is optional, except that the audit needs a business description.
7
+ class Context < Data.define(:business, :audience, :tone, :rules, :ignore_terms, :intentional_differences)
8
+ TEXTS = %w[business audience tone].freeze
9
+ LISTS = %w[rules ignore_terms intentional_differences].freeze
10
+ KEYS = (TEXTS + LISTS).freeze
11
+ PLACEHOLDER = /\[fill in\]/i
12
+ MAX_TEXT = 500
13
+ MAX_ITEM = 200
14
+ MAX_ITEMS = 50
15
+
16
+ def self.from_hash(raw)
17
+ texts = TEXTS.to_h { |key| [key.to_sym, text(raw[key], key, MAX_TEXT)] }
18
+ lists = LISTS.to_h { |key| [key.to_sym, list(raw[key], key)] }
19
+ new(**texts, **lists)
20
+ end
21
+
22
+ def self.placeholder?(value) = value.to_s.match?(PLACEHOLDER)
23
+
24
+ # :set, :empty (nothing given) or :placeholder (the [FILL IN] example text is still there).
25
+ def self.state_of(value)
26
+ items = Array(value)
27
+ return :empty if items.all?(&:empty?)
28
+
29
+ items.any? { |item| placeholder?(item) } ? :placeholder : :set
30
+ end
31
+
32
+ # One piece of text, trimmed. Nothing at all becomes an empty string.
33
+ def self.text(value, label, max)
34
+ raise ConfigError, "Site: #{label} must be plain text" if value.is_a?(Hash) || value.is_a?(Array)
35
+
36
+ cleaned = value.to_s.strip
37
+ raise ConfigError, "Site: #{label} is longer than #{max} characters" if cleaned.length > max
38
+
39
+ cleaned
40
+ end
41
+
42
+ # A list of texts. A single text counts as a list of one. Blank and repeated entries are dropped.
43
+ def self.list(value, label)
44
+ items = Array(value).map { |item| text(item, "an entry of #{label}", MAX_ITEM) }.reject(&:empty?).uniq
45
+ raise ConfigError, "Site: #{label} has more than #{MAX_ITEMS} entries" if items.size > MAX_ITEMS
46
+
47
+ items
48
+ end
49
+
50
+ def states
51
+ to_h.transform_keys(&:to_s).transform_values { |value| self.class.state_of(value) }
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Yuno
4
+ module Proofreader
5
+ # Reads the settings that live outside proofreader.yml (API keys, the id of the sheet, Google credentials) from
6
+ # a .env file, so they do not have to be typed into the shell. A variable that is already set wins over the file
7
+ module EnvFile
8
+ DEFAULT = ".env"
9
+ CHOICE = "PROOFREADER_ENV_FILE"
10
+ LINE = /\A(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\z/
11
+ DOUBLE = /\A"((?:[^"\\]|\\.)*)"\s*(?:#.*)?\z/
12
+ SINGLE = /\A'([^']*)'\s*(?:#.*)?\z/
13
+ ESCAPES = { "n" => "\n", "t" => "\t" }.freeze
14
+
15
+ # path: the file. names: the variables it set. skipped: the numbers of the lines that were not NAME=VALUE.
16
+ # loose: whether other users can read the file.
17
+ Loaded = Data.define(:path, :names, :skipped, :loose) do
18
+ def warnings = [skipped_warning, loose_warning].compact
19
+
20
+ def skipped_warning
21
+ return if skipped.empty?
22
+
23
+ "Ignored #{skipped.size} line(s) of #{path} that are not NAME=VALUE (line #{skipped.first(5).join(', ')})."
24
+ end
25
+
26
+ def loose_warning = ("#{path} can be read by other users. Run: chmod 600 #{path}" if loose)
27
+ end
28
+
29
+ module_function
30
+
31
+ # Loads the file into env without replacing what is already set. Returns what was loaded, or nil when there
32
+ # is no file to load. A file named in PROOFREADER_ENV_FILE that does not exist is an error.
33
+ def load(env = ENV, path: nil)
34
+ chosen = path || env[CHOICE].to_s.strip
35
+ file = chosen.empty? ? DEFAULT : chosen
36
+ unless File.file?(file)
37
+ raise ConfigError, "#{CHOICE} points to #{file}, which is not a file." unless chosen.empty?
38
+
39
+ return
40
+ end
41
+
42
+ pairs, skipped = parse(File.read(file))
43
+ names = pairs.keys.select { |name| env[name].to_s.empty? }
44
+ names.each { |name| env[name] = pairs[name] }
45
+ Loaded.new(path: file, names:, skipped:, loose: loose?(file))
46
+ end
47
+
48
+ # [the NAME=VALUE pairs, the numbers of the lines that were neither a pair, empty nor a comment]
49
+ def parse(text)
50
+ pairs = {}
51
+ skipped = []
52
+ text.each_line.with_index(1) do |raw, number|
53
+ line = raw.strip
54
+ next if line.empty? || line.start_with?("#")
55
+
56
+ match = LINE.match(line)
57
+ match ? pairs[match[1]] = value_of(match[2]) : skipped << number
58
+ end
59
+ [pairs, skipped]
60
+ end
61
+
62
+ def value_of(raw)
63
+ text = raw.strip
64
+ if (match = DOUBLE.match(text)) then unescape(match[1])
65
+ elsif (match = SINGLE.match(text)) then match[1]
66
+ else text.sub(/\s+#.*\z/, "")
67
+ end
68
+ end
69
+
70
+ def unescape(text) = text.gsub(/\\(.)/) { ESCAPES.fetch(Regexp.last_match(1), Regexp.last_match(1)) }
71
+
72
+ # Whether other users can read the file. Not checked where there are no unix permissions.
73
+ def loose?(file) = File::ALT_SEPARATOR.nil? && File.stat(file).mode.anybits?(0o077)
74
+ end
75
+ end
76
+ end