run_ical 0.5.0 → 0.7.0

data/run_ical.gemspec

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "run_ical/version"
 
@@ -9,30 +10,34 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Sean Cashin"]
   spec.email         = ["scashin133@gmail.com"]
 
-  spec.summary       = %q{Generate an ical file for a race}
-  spec.description   = %q{Uses Hal Higdon's plans for creating a calendar}
+  spec.summary       = "Generate an ical file for a race"
+  spec.description   = "Uses Hal Higdon's plans for creating a calendar"
   spec.homepage      = "https://github.com/scashin133/run_ical"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  # The data loaders use YAML.safe_load_file, which arrived in Psych 3.3 / Ruby 3.0.
+  spec.required_ruby_version = ">= 3.1"
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "yaml"
+  spec.add_dependency "activesupport", ">= 5.0.0"
   spec.add_dependency "base64"
   spec.add_dependency "bigdecimal"
+  spec.add_dependency "icalendar", ">= 2.0.0"
   spec.add_dependency "logger"
   spec.add_dependency "mutex_m"
   spec.add_dependency "ostruct"
-  spec.add_dependency "icalendar", ">= 2.0.0"
-  spec.add_dependency "activesupport", ">= 5.0.0"
+  spec.add_dependency "yaml"
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "irb"
+  spec.add_development_dependency "nokogiri" # rake data:refresh
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rdoc"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 1.0"
 end

data/tasks/higdon_programs.yml

@@ -0,0 +1,211 @@
+# Source manifest for `rake data:refresh`.
+#
+# One entry per training program, in the order they should be presented to users:
+# by distance ascending, then by difficulty. `rake data:refresh` emits POSITION
+# from this order, so reordering here reorders RunIcal::Data.programs.
+#
+# `class` is the public API name passed to RunIcal::Generator.generate(program:)
+# -- never rename an existing one. DESCRIPTION is derived as "<distance> - <level>".
+# `weeks` is asserted against the scraped table so a silent site change fails loudly.
+---
+- class: Novice5K
+  file: novice_5_k
+  distance: "5-K"
+  level: "Novice"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/5k-training/novice-5k/
+- class: Intermediate5K
+  file: intermediate_5_k
+  distance: "5-K"
+  level: "Intermediate"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/5k-training/intermediate-5k/
+- class: Advanced5K
+  file: advanced_5_k
+  distance: "5-K"
+  level: "Advanced"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/5k-training/advanced-5k/
+- class: Walkers5K
+  file: walkers_5_k
+  distance: "5-K"
+  level: "Walkers"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/5k-training/walkers-5k/
+- class: Novice8K
+  file: novice_8_k
+  distance: "8-K"
+  level: "Novice"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/8k-training/novice-8k/
+- class: Intermediate8K
+  file: intermediate_8_k
+  distance: "8-K"
+  level: "Intermediate"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/8k-training/intermediate-8k/
+- class: Advanced8K
+  file: advanced_8_k
+  distance: "8-K"
+  level: "Advanced"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/8k-training/advanced-8k/
+- class: Novice10K
+  file: novice_10_k
+  distance: "10-K"
+  level: "Novice"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/10k-training/novice-10k/
+- class: Intermediate10K
+  file: intermediate_10_k
+  distance: "10-K"
+  level: "Intermediate"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/10k-training/intermediate-10k/
+- class: Advanced10K
+  file: advanced_10_k
+  distance: "10-K"
+  level: "Advanced"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/10k-training/advanced-10k/
+- class: Walking10K
+  file: walking_10_k
+  distance: "10-K"
+  level: "Walkers"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/10k-training/walkers-10k/
+- class: Novice15K
+  file: novice_15_k
+  distance: "15-K"
+  level: "Novice"
+  weeks: 10
+  url: https://www.halhigdon.com/training-programs/15k-10-mile-training/novice-15k-10-mile/
+- class: Intermediate15K
+  file: intermediate_15_k
+  distance: "15-K"
+  level: "Intermediate"
+  weeks: 10
+  url: https://www.halhigdon.com/training-programs/15k-10-mile-training/intermediate-15k-10-mile/
+- class: Advanced15K
+  file: advanced_15_k
+  distance: "15-K"
+  level: "Advanced"
+  weeks: 10
+  url: https://www.halhigdon.com/training-programs/15k-10-mile-training/advanced-15k-10-mile/
+- class: HalfMarathonNovice1
+  file: half_marathon_novice1
+  distance: "Half Marathon"
+  level: "Novice 1"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/novice-1-half-marathon/
+- class: HalfMarathonNovice2
+  file: half_marathon_novice2
+  distance: "Half Marathon"
+  level: "Novice 2"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/novice-2-half-marathon/
+- class: HalfMarathonIntermediate1
+  file: half_marathon_intermediate1
+  distance: "Half Marathon"
+  level: "Intermediate 1"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/intermediate-1-half-marathon/
+- class: HalfMarathonIntermediate2
+  file: half_marathon_intermediate2
+  distance: "Half Marathon"
+  level: "Intermediate 2"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/intermediate-2-half-marathon/
+- class: HalfMarathonAdvanced
+  file: half_marathon_advanced
+  distance: "Half Marathon"
+  level: "Advanced"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/advanced-half-marathon/
+- class: HalfMarathonHM3
+  file: half_marathon_hm3
+  distance: "Half Marathon"
+  level: "HM3"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/half-marathon-3/
+- class: HalfMarathonWalkers
+  file: half_marathon_walkers
+  distance: "Half Marathon"
+  level: "Walkers"
+  weeks: 12
+  url: https://www.halhigdon.com/training-programs/half-marathon-training/walkers-half-marathon/
+- class: MarathonNovice1
+  file: marathon_novice1
+  distance: "Marathon"
+  level: "Novice 1"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/novice-1-marathon/
+- class: MarathonNovice2
+  file: marathon_novice2
+  distance: "Marathon"
+  level: "Novice 2"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/novice-2-marathon/
+- class: MarathonIntermediate1
+  file: marathon_intermediate1
+  distance: "Marathon"
+  level: "Intermediate 1"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/intermediate-1-marathon/
+- class: MarathonIntermediate2
+  file: marathon_intermediate2
+  distance: "Marathon"
+  level: "Intermediate 2"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/intermediate-2-marathon/
+- class: MarathonAdvanced1
+  file: marathon_advanced1
+  distance: "Marathon"
+  level: "Advanced 1"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/advanced-1-marathon/
+- class: MarathonAdvanced2
+  file: marathon_advanced2
+  distance: "Marathon"
+  level: "Advanced 2"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/advanced-2-marathon/
+- class: MarathonNoviceSupreme
+  file: marathon_novice_supreme
+  distance: "Marathon"
+  level: "Novice Supreme"
+  weeks: 30
+  url: https://www.halhigdon.com/training-programs/marathon-training/novice-supreme/
+- class: MarathonPersonalBest
+  file: marathon_personal_best
+  distance: "Marathon"
+  level: "Personal Best"
+  weeks: 30
+  url: https://www.halhigdon.com/training-programs/marathon-training/personal-best/
+- class: MarathonSenior
+  file: marathon_senior
+  distance: "Marathon"
+  level: "Senior"
+  weeks: 8
+  url: https://www.halhigdon.com/training-programs/marathon-training/senior/
+- class: Marathon3
+  file: marathon_3
+  distance: "Marathon"
+  level: "Marathon 3"
+  weeks: 24
+  url: https://www.halhigdon.com/training-programs/marathon-training/marathon-3/
+- class: MarathonDopeyChallenge
+  file: marathon_dopey_challenge
+  distance: "Marathon"
+  level: "Dopey Challenge"
+  weeks: 18
+  url: https://www.halhigdon.com/training-programs/marathon-training/dopey-challenge/
+# Boston is run on Patriots Day (a Monday), so its final table row holds a single
+# Monday cell instead of a full week. The scraper truncates to 12 weeks + race day.
+- class: MarathonBostonBound
+  file: marathon_boston_bound
+  distance: "Marathon"
+  level: "Boston Bound"
+  weeks: 13
+  partial_final_week: true
+  url: https://www.halhigdon.com/training-programs/marathon-training/boston-bound/

data/tasks/refresh_data.rake

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "yaml"
+
+# Re-scrapes the training plans in lib/run_ical/data from halhigdon.com.
+#
+#   rake data:refresh            # every program in tasks/higdon_programs.yml
+#   rake data:refresh[Novice5K]  # just one
+#
+# Every page renders the schedule as a static table with a `Week | Mon..Sun` header
+# row, in miles first and kilometres second. We take the miles table, flatten it
+# row-major into one activity per day, and write a YAML sidecar plus a loader stub.
+module HigdonScraper
+  DATA_DIR = File.expand_path("../lib/run_ical/data", __dir__)
+  MANIFEST = File.expand_path("higdon_programs.yml", __dir__)
+  DAYS_PER_WEEK = 7
+
+  # The final cell of a plan is the race itself. Higdon isn't consistent about it:
+  # most read "<distance> Race", but Novice 10K ends on "10K Run", Walking 10K on
+  # "10K Walk", and the marathons on a bare "Marathon" / "Boston Marathon".
+  RACE = /\A(\d+K (Race|Run|Walk)|(Boston |Half )?Marathon)\z/i
+
+  class ScrapeError < StandardError; end
+
+  module_function
+
+  # Every program in the manifest, or the single one named. Position is taken from
+  # the full manifest so refreshing one program still writes the right ordering.
+  def programs(only = nil)
+    all = YAML.safe_load_file(MANIFEST)
+    all.each_with_index { |program, index| program["position"] = index }
+    return all unless only
+
+    selected = all.select { |program| program["class"] == only }
+    raise ScrapeError, "unknown program #{only}" if selected.empty?
+
+    selected
+  end
+
+  def scrape(program)
+    rows = schedule_rows(Nokogiri::HTML(fetch(program["url"])), program)
+    unless rows.size == program["weeks"]
+      raise ScrapeError, "#{program['class']}: expected #{program['weeks']} weeks, got #{rows.size}"
+    end
+
+    activities = rows.flat_map { |row| week_activities(row) }
+    activities = trim_partial_final_week(activities) if program["partial_final_week"]
+    validate!(program, activities)
+    activities
+  end
+
+  def fetch(url)
+    uri = URI(url)
+    response = Net::HTTP.get_response(uri, "User-Agent" => "run_ical data refresh")
+    raise ScrapeError, "#{url}: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+    response.body
+  end
+
+  def schedule_rows(doc, program)
+    table = schedule_table(doc)
+    raise ScrapeError, "#{program['class']}: no schedule table found" unless table
+
+    table.css("tbody tr").map { |tr| tr.css("td").map { |td| cell_text(td) } }.reject(&:empty?)
+  end
+
+  # The first table whose header starts "Week | Mon" -- the walkers half marathon page
+  # leads with an unrelated pace legend, and the kilometre table always follows the
+  # miles one, so first-match gives us miles.
+  def schedule_table(doc)
+    doc.css("table").find do |candidate|
+      headers = candidate.css("th").map { |th| cell_text(th) }
+      headers[0] == "Week" && headers[1] == "Mon"
+    end
+  end
+
+  # Cells are taken by position, not by header: a few pages ship an empty <th> where
+  # "Wed" belongs. The 8K pages add a ninth "Miles" total column, and on the novice
+  # page the race sits in it rather than under Sunday -- pull it back onto race day.
+  def week_activities(row)
+    week = row[1, DAYS_PER_WEEK]
+    total = row[DAYS_PER_WEEK + 1]
+    week[-1] = total if total&.match?(RACE)
+    week
+  end
+
+  # Boston's last row is a lone Monday race day padded with six empty cells.
+  def trim_partial_final_week(activities)
+    race = activities.reverse.find { |activity| !activity.empty? }
+    activities.take(activities.size - DAYS_PER_WEEK) << race
+  end
+
+  def validate!(program, activities)
+    blank = activities.index(&:empty?)
+    raise ScrapeError, "#{program['class']}: empty activity at index #{blank}" if blank
+
+    expected = expected_size(program)
+    unless activities.size == expected
+      raise ScrapeError, "#{program['class']}: expected #{expected} days, got #{activities.size}"
+    end
+    return if activities.last.match?(RACE)
+
+    raise ScrapeError, "#{program['class']}: plan ends on #{activities.last.inspect}, not a race"
+  end
+
+  def expected_size(program)
+    weeks = program["weeks"]
+    return weeks * DAYS_PER_WEEK unless program["partial_final_week"]
+
+    ((weeks - 1) * DAYS_PER_WEEK) + 1
+  end
+
+  # Higdon's markup is littered with non-breaking spaces and hard line breaks.
+  def cell_text(node)
+    node.text.gsub(/[[:space:]]+/, " ").strip
+  end
+
+  def write(program, activities)
+    write_yaml(program, activities)
+    write_loader(program, program["position"])
+  end
+
+  # Ruby's emitter quotes numeric-looking strings, which keeps the marathon plans'
+  # bare mileage cells ("6", "20") loading back as Strings.
+  def write_yaml(program, activities)
+    body = activities.map { |activity| { "activity" => activity } }
+    File.write(File.join(DATA_DIR, "#{program['file']}.yaml"), body.to_yaml)
+  end
+
+  def write_loader(program, position)
+    File.write(File.join(DATA_DIR, "#{program['file']}.rb"), loader_source(program, position))
+  end
+
+  # POSITION is the program's index in the manifest, which is ordered by distance
+  # then difficulty -- see RunIcal::Data.programs.
+  def loader_source(program, position)
+    <<~RUBY
+      module RunIcal
+        module Data
+          class #{program['class']}
+            DISTANCE = #{program['distance'].inspect}
+            LEVEL = #{program['level'].inspect}
+            POSITION = #{position}
+            DESCRIPTION = "\#{DISTANCE} - \#{LEVEL}"
+            DATA = YAML.safe_load_file(File.join(__dir__, '#{program['file']}.yaml'), symbolize_names: true)
+          end
+        end
+      end
+    RUBY
+  end
+end
+
+namespace :data do
+  desc "Re-scrape training plan data from halhigdon.com (optionally one program)"
+  task :refresh, [:program] do |_t, args|
+    require "nokogiri"
+
+    HigdonScraper.programs(args[:program]).each do |program|
+      activities = HigdonScraper.scrape(program)
+      HigdonScraper.write(program, activities)
+      puts format("%<name>-26s %<days>3d days  %<race>s",
+                  name: program["class"], days: activities.size, race: activities.last)
+    end
+  end
+end