recipe_scrapers 0.1.0

data/lib/recipe_scrapers/parsers/vocabulary.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module RecipeScrapers
+  module Parsers
+    # The words {Ingredients} reads: units, pinch words, size adjectives, qualifiers and number words.
+    # The gem bundles one vocabulary per language, plus a common one with the metric units.
+    #
+    # @api private
+    class Vocabulary
+      WORD_LISTS = %i[units pinches sizes qualifiers].freeze
+      BUNDLED = File.expand_path("vocabulary/*.yml", __dir__)
+      NEVER = "(?!)"
+      private_constant :WORD_LISTS, :BUNDLED, :NEVER
+
+      # @return [Array<String>] units, where a space matches any run of spaces or none
+      attr_reader :units
+
+      # @return [Array<String>] words that are a unit and an amount of one, such as "pinch"
+      attr_reader :pinches
+
+      # @return [Array<String>] adjectives that can come before a unit, such as "large"
+      attr_reader :sizes
+
+      # @return [Array<String>] words before an amount that are not part of it, such as "about"
+      attr_reader :qualifiers
+
+      # @return [Hash{String => Numeric}] amounts written as words, such as "two" => 2
+      attr_reader :number_words
+
+      class << self
+        # Adds a vocabulary to the catalog, next to the bundled ones.
+        #
+        # @param name [Symbol]
+        # @param languages [Array<String>] the language codes it is used for, such as %w[pl]
+        # @param words [Hash] the word lists, as {#initialize} takes them
+        # @return [Hash] the catalog entry
+        def define(name, languages: [], **words)
+          catalog[name] = entry(languages, words)
+        end
+
+        # @param language [String, nil] a language tag, such as "en-US"
+        # @return [Array<Symbol>] the common vocabulary plus those of the language, or every vocabulary
+        #   when no file declares the language
+        #
+        # @api private
+        def names_for(language)
+          code = language.to_s.downcase[/\A[a-z]+/]
+          spoken = catalog.filter_map { |name, entry| name if entry[:languages].include?(code) }
+          spoken.empty? ? catalog.keys.sort : [:common, *spoken.sort]
+        end
+
+        # @param language [String, nil]
+        # @return [Vocabulary] the vocabularies of the language combined
+        def for(language)
+          combine(names_for(language))
+        end
+
+        # @param names [Array<Symbol>]
+        # @return [Vocabulary] the named vocabularies added together
+        #
+        # @api private
+        def combine(names)
+          names.map { |name| catalog.fetch(name)[:vocabulary] }.reduce(:+)
+        end
+
+        # @return [Hash{Symbol => Hash}] every vocabulary by name, the bundled ones loaded on first use
+        #
+        # @api private
+        def catalog
+          @catalog ||= load_bundled
+        end
+
+        # @param words [Array<String>]
+        # @return [String] a regexp alternation of the words, longest first, that never matches when empty
+        #
+        # @api private
+        def alternation(words)
+          return NEVER if words.empty?
+
+          words.sort_by { |word| -word.length }.map { |word| Regexp.escape(word).gsub("\\ ", "\\s*") }.join("|")
+        end
+
+        private
+
+        def load_bundled
+          Dir[BUNDLED].to_h do |path|
+            words = YAML.safe_load_file(path).transform_keys(&:to_sym)
+            [File.basename(path, ".yml").to_sym, entry(words.delete(:languages), words)]
+          end
+        end
+
+        def entry(languages, words)
+          { languages: Array(languages), vocabulary: new(**words) }
+        end
+      end
+
+      # @param units [Array<String>]
+      # @param pinches [Array<String>]
+      # @param sizes [Array<String>]
+      # @param qualifiers [Array<String>]
+      # @param number_words [Hash{String => Numeric}]
+      def initialize(units: [], pinches: [], sizes: [], qualifiers: [], number_words: {})
+        @units = units.uniq.freeze
+        @pinches = pinches.uniq.freeze
+        @sizes = sizes.uniq.freeze
+        @qualifiers = qualifiers.uniq.freeze
+        @number_words = number_words.freeze
+      end
+
+      # @param other [Vocabulary]
+      # @return [Vocabulary] both word lists without repeats. The other one wins for a number word in both
+      def +(other)
+        lists = WORD_LISTS.to_h { |list| [list, public_send(list) + other.public_send(list)] }
+        Vocabulary.new(**lists, number_words: number_words.merge(other.number_words))
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/parsers/yields.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module Parsers
+    module Yields
+      SERVING_WORDS = /serv|portion|порц|person|people/i
+      NUMBER = /(\d+)/
+
+      class << self
+        def parse(value)
+          return nil if value.nil?
+
+          text = Text.normalize(value.is_a?(Array) ? value.first : value)
+          return nil if text.nil?
+
+          count = text[NUMBER, 1]
+          return nil if count.nil?
+
+          "#{count} #{noun_for(text, count)}"
+        end
+
+        private
+
+        def noun_for(text, count)
+          text.match?(SERVING_WORDS) || text == count ? "servings" : "items"
+        end
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/registry.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module RecipeScrapers
+  # Maps hosts to the sites the gem supports. A host maps to a {Declaration}, or to a {Scraper}
+  # subclass for a site that needs code. A leading "www." is ignored.
+  module Registry
+    class << self
+      # @return [Hash{String => Declaration, Class}] every registered host and what reads it
+      #
+      # @api private
+      def entries
+        @entries ||= {}
+      end
+
+      # Registers a site by declaration.
+      #
+      # @param host [String] the host of the site
+      # @param also [Array<String>] more hosts read the same way, such as a regional domain
+      # @yield the declaration body, evaluated on the {Declaration}
+      # @return [Declaration]
+      def register(host, also: [], &)
+        declaration = Declaration.build(host, &)
+        ([host] + Array(also)).each { |name| entries[normalize(name)] = declaration }
+        declaration
+      end
+
+      # Registers a {Scraper} subclass under the host it declares with {Scraper.host}.
+      #
+      # @param klass [Class<Scraper>]
+      # @return [Class<Scraper>] the class it was given
+      def register_class(klass)
+        entries[normalize(klass.declared_host)] = klass
+        klass
+      end
+
+      # Finds what reads a URL.
+      #
+      # @param url [String]
+      # @return [Declaration, Class<Scraper>, nil] nil when the host is not registered or the URL does not parse
+      def for(url)
+        entries[normalize(URI.parse(url).host.to_s)]
+      rescue URI::Error
+        nil
+      end
+
+      # @return [Array<String>] every supported host
+      def hosts
+        entries.keys
+      end
+
+      # Forgets every registered site.
+      #
+      # @return [void]
+      #
+      # @api private
+      def clear!
+        @entries = {}
+      end
+
+      # A copy of the entries, to put back with {restore}. Specs use it to register a site for one example.
+      #
+      # @return [Hash{String => Declaration, Class}]
+      #
+      # @api private
+      def snapshot
+        entries.dup
+      end
+
+      # Puts back entries a {snapshot} took.
+      #
+      # @param saved [Hash{String => Declaration, Class}]
+      # @return [void]
+      #
+      # @api private
+      def restore(saved)
+        @entries = saved
+      end
+
+      private
+
+      def normalize(host)
+        host.to_s.downcase.delete_prefix("www.")
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/scraper/parsed_fields.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  class Scraper
+    # The fields that split text into value objects, with the parsers {Configuration#parsers} holds.
+    module ParsedFields
+      # @param scraper [Class<Scraper>] gets {ClassMethods}
+      # @return [void]
+      #
+      # @api private
+      def self.included(scraper)
+        scraper.extend(ClassMethods)
+      end
+
+      # The class methods a scraper class gets from {ParsedFields}.
+      module ClassMethods
+        # Sets the parsers of a field for every page of this scraper class. It wins over the site
+        # declaration and the configuration.
+        #
+        # @param field [Symbol] :ingredients or :nutrients
+        # @param parsers [Array<#call>] tried in order, the first result that is not nil wins
+        # @raise [ArgumentError] for a field that takes no parser
+        def parser(field, *parsers)
+          own_parsers[Parsers::Chain.field!(field)] = parsers
+        end
+
+        # @return [Hash{Symbol => Array<#call>}] the parsers this class sets with {#parser}
+        #
+        # @api private
+        def own_parsers
+          @own_parsers ||= {}
+        end
+      end
+
+      # (see Models::Recipe#parsed_ingredients)
+      def parsed_ingredients
+        spoken = language
+        ingredients&.map { |line| parse(:ingredients, line, spoken) }
+      end
+
+      # (see Models::Recipe#parsed_nutrients)
+      def parsed_nutrients
+        spoken = language
+        parsed = nutrients.to_h.filter_map do |name, text|
+          nutrient = parse(:nutrients, text, spoken)
+          nutrient && (nutrient.name ? nutrient : nutrient.with(name: name))
+        end
+        parsed.empty? ? nil : parsed
+      end
+
+      private
+
+      def with_parsed_ingredients(groups)
+        spoken = language
+        groups.map do |group|
+          group.with(parsed_ingredients: group.ingredients.map { |line| parse(:ingredients, line, spoken) })
+        end
+      end
+
+      def parse(field, text, spoken)
+        Parsers::Chain.run(field, parsers_for(field), text, language: spoken)
+      end
+
+      def parsers_for(field)
+        self.class.own_parsers[field] || @declaration&.parsers_for(field) || RecipeScrapers.config.parsers.fetch(field)
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/scraper.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "uri"
+
+module RecipeScrapers
+  # Reads the recipe fields of one page. {RecipeScrapers.parse} builds one and returns its
+  # {#to_recipe}, so an application gets a {Models::Recipe} and never the page itself. A field the
+  # page does not publish is nil.
+  #
+  # Every field reads the site declaration first, then the schema.org recipe, then OpenGraph where
+  # that applies. A site that needs code subclasses Scraper, names its host with {.host} and
+  # registers with {Registry.register_class}.
+  #
+  # @example A site with its own schema.org reader
+  #   class Chefkoch < RecipeScrapers::Scraper
+  #     class SchemaReader < RecipeScrapers::Sources::SchemaOrg
+  #       def ingredients = super&.map(&:strip)
+  #     end
+  #
+  #     host "chefkoch.de"
+  #     schema_reader SchemaReader
+  #   end
+  #   RecipeScrapers::Registry.register_class(Chefkoch)
+  class Scraper
+    include ParsedFields
+
+    # Every field a scraper answers, in the order {#to_h} returns them.
+    CONTRACT = (Models::Recipe.members - %i[url]).freeze
+
+    class << self
+      # Names the host a subclass reads, or returns it.
+      #
+      # @param value [String, nil]
+      # @return [String, nil]
+      def host(value = nil)
+        @declared_host = value if value
+        @declared_host
+      end
+
+      alias declared_host host
+
+      # Sets the class that reads the schema.org recipe for a subclass, or returns it.
+      #
+      # @param value [Class<Sources::SchemaOrg>, nil]
+      # @return [Class<Sources::SchemaOrg>] {Sources::SchemaOrg} unless a subclass sets its own
+      def schema_reader(value = nil)
+        @schema_reader = value if value
+        @schema_reader || Sources::SchemaOrg
+      end
+    end
+
+    # @return [String] the address the page was read from
+    attr_reader :url
+
+    # @return [Nokogiri::HTML5::Document] the parsed page, for a subclass that reads it directly
+    attr_reader :document
+
+    # @param html [String] the page source
+    # @param url [String] the address of the page
+    # @param declaration [Declaration, nil] the declaration of the site
+    def initialize(html, url:, declaration: nil)
+      @url = url
+      @declaration = declaration
+      @document = Nokogiri::HTML5(Text.recode(html, declaration&.encoding_name))
+      @schema = self.class.schema_reader.new(Sources::JsonLd.new(@document), Sources::Microdata.new(@document))
+      @open_graph = Sources::OpenGraph.new(@document)
+      @declared = Sources::Declared.new(@document, declaration)
+    end
+
+    # (see Models::Recipe#host)
+    def host
+      URI.parse(url).host.to_s.delete_prefix("www.")
+    end
+
+    # (see Models::Recipe#canonical_url)
+    def canonical_url
+      link = document.at_css('link[rel="canonical"][href]')
+      return url unless link
+
+      URI.join(url, link["href"]).to_s
+    rescue URI::Error
+      url
+    end
+
+    # (see Models::Recipe#title)
+    def title = @declared.text(:title) || @schema.title || @open_graph.title
+    # (see Models::Recipe#author)
+    def author = @declared.text(:author) || @schema.author
+    # (see Models::Recipe#site_name)
+    def site_name = @declared.text(:site_name) || @schema.site_name || @open_graph.site_name
+    # (see Models::Recipe#language)
+    def language = @declared.text(:language) || @schema.language || document_language
+    # (see Models::Recipe#description)
+    def description = @declared.text(:description) || @schema.description || @open_graph.description
+    # (see Models::Recipe#category)
+    def category = @declared.text(:category) || @schema.category
+    # (see Models::Recipe#cuisine)
+    def cuisine = @declared.text(:cuisine) || @schema.cuisine
+    # (see Models::Recipe#cooking_method)
+    def cooking_method = @declared.text(:cooking_method) || @schema.cooking_method
+    # (see Models::Recipe#keywords)
+    def keywords = @declared.rows(:keywords) || @schema.keywords
+    # (see Models::Recipe#equipment)
+    def equipment = @declared.rows(:equipment)
+    # (see Models::Recipe#nutrients)
+    def nutrients = @schema.nutrients
+    # (see Models::Recipe#dietary_restrictions)
+    def dietary_restrictions = @schema.dietary_restrictions
+    # (see Models::Recipe#ratings)
+    def ratings = @schema.ratings
+    # (see Models::Recipe#ratings_count)
+    def ratings_count = @schema.ratings_count
+
+    # (see Models::Recipe#yields)
+    def yields = Parsers::Yields.parse(@declared.text(:yields)) || @schema.yields
+    # (see Models::Recipe#total_time)
+    def total_time = Parsers::Durations.minutes(@declared.text(:total_time)) || @schema.total_time
+    # (see Models::Recipe#cook_time)
+    def cook_time = Parsers::Durations.minutes(@declared.text(:cook_time)) || @schema.cook_time
+    # (see Models::Recipe#prep_time)
+    def prep_time = Parsers::Durations.minutes(@declared.text(:prep_time)) || @schema.prep_time
+
+    # (see Models::Recipe#ingredients)
+    def ingredients
+      lines = ingredient_sections.flat_map(&:last)
+      lines.empty? ? nil : lines
+    end
+
+    # (see Models::Recipe#instructions_list)
+    def instructions_list = @declared.rows(:instructions) || @schema.instructions_list
+    # (see Models::Recipe#instructions)
+    def instructions = instructions_list&.join("\n")
+
+    # (see Models::Recipe#ingredient_groups)
+    def ingredient_groups
+      lines = ingredients
+      return nil if lines.nil?
+
+      marked = marked_groups(lines)
+      groups = marked.any?(&:purpose) ? marked : Sources::IngredientGroups.from_sections(ingredient_sections)
+      with_parsed_ingredients(groups)
+    end
+
+    # (see Models::Recipe#image)
+    def image
+      address = @declared.text(:image) || @schema.image || @open_graph.image
+      address && URI.join(url, address).to_s
+    rescue URI::Error
+      address
+    end
+
+    # (see Models::Recipe#links)
+    def links
+      document.css("a[href]").map { |anchor| anchor["href"] }
+    end
+
+    # @return [Hash{Symbol => Object}] every field of {CONTRACT} with its value
+    #
+    # @api private
+    def to_h
+      CONTRACT.to_h { |field| [field, public_send(field)] }
+    end
+
+    # Reads every field once and returns them without the page.
+    #
+    # @return [Models::Recipe]
+    #
+    # @api private
+    def to_recipe
+      Models::Recipe.new(url: url, **to_h)
+    end
+
+    # @return [Boolean] whether the page has a title and ingredients
+    #
+    # @api private
+    def recipe?
+      !title.nil? && !ingredients.nil?
+    end
+
+    private
+
+    def ingredient_sections
+      @ingredient_sections ||=
+        Sources::IngredientGroups.sections(@declared.rows(:ingredients) || @schema.ingredients || [])
+    end
+
+    def marked_groups(lines)
+      rule = @declared.rule(:ingredient_groups)
+      return Sources::IngredientGroups.detect(document: document, ingredients: lines) if rule.nil?
+
+      Sources::IngredientGroups.call(document: document, ingredients: lines, heading: rule[:heading], item: rule[:item])
+    end
+
+    def document_language
+      Text.normalize(document.at_css("html")&.[]("lang"))
+    end
+  end
+end

data/lib/recipe_scrapers/site_path.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module SitePath
+    class << self
+      def for(host)
+        "#{directory(host)}/#{slug(host)}"
+      end
+
+      private
+
+      def directory(host)
+        host.split(".").last
+      end
+
+      def slug(host)
+        host.split(".").first.tr("-", "_")
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sites/au/bestrecipes.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "bestrecipes.com.au"

data/lib/recipe_scrapers/sites/be/15gram.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "15gram.be"

data/lib/recipe_scrapers/sites/br/panelinha.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "panelinha.com.br"

data/lib/recipe_scrapers/sites/br/receitasnestle.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "receitasnestle.com.br"

data/lib/recipe_scrapers/sites/br/receiteria.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "receiteria.com.br"

data/lib/recipe_scrapers/sites/br/tudogostoso.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "tudogostoso.com.br"

data/lib/recipe_scrapers/sites/ch/bettybossi.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "bettybossi.ch"

data/lib/recipe_scrapers/sites/ch/migusto.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "migusto.migros.ch"

data/lib/recipe_scrapers/sites/ch/swissmilk.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "swissmilk.ch"

data/lib/recipe_scrapers/sites/co/tasty.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "tasty.co"

data/lib/recipe_scrapers/sites/com/365daysofbakingandmore.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "365daysofbakingandmore.com"

data/lib/recipe_scrapers/sites/com/40aprons.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "40aprons.com"

data/lib/recipe_scrapers/sites/com/750g.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "750g.com"

data/lib/recipe_scrapers/sites/com/abeautifulmess.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "abeautifulmess.com"

data/lib/recipe_scrapers/sites/com/abuelascounter.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "abuelascounter.com"

data/lib/recipe_scrapers/sites/com/acozykitchen.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "acozykitchen.com"

data/lib/recipe_scrapers/sites/com/addapinch.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "addapinch.com"

data/lib/recipe_scrapers/sites/com/adozensundays.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "adozensundays.com"

data/lib/recipe_scrapers/sites/com/aflavorjournal.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "aflavorjournal.com"

data/lib/recipe_scrapers/sites/com/afullliving.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "afullliving.com"

data/lib/recipe_scrapers/sites/com/ahealthysliceoflife.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "ahealthysliceoflife.com" do
+  ingredients rows: ".tasty-recipes-ingredients-body p"
+end

data/lib/recipe_scrapers/sites/com/alexandracooks.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "alexandracooks.com"

data/lib/recipe_scrapers/sites/com/allnutritious.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "allnutritious.com"

data/lib/recipe_scrapers/sites/com/allthehealthythings.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "allthehealthythings.com"

data/lib/recipe_scrapers/sites/com/altonbrown.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "altonbrown.com"

data/lib/recipe_scrapers/sites/com/amazingribs.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "amazingribs.com"

data/lib/recipe_scrapers/sites/com/amberskitchencooks.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "amberskitchencooks.com"

data/lib/recipe_scrapers/sites/com/andy_cooks.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "andy-cooks.com"