recipe_scrapers 0.1.0

data/lib/recipe_scrapers/sites/recipes/howtocook.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "howtocook.recipes"

data/lib/recipe_scrapers/sites/ru/eda.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "eda.ru"

data/lib/recipe_scrapers/sites/ru/edimdoma.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "edimdoma.ru"

data/lib/recipe_scrapers/sites/ru/gastronom.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "gastronom.ru"

data/lib/recipe_scrapers/sites/ru/iamcook.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "iamcook.ru" do
+  ingredients rows: "div.ingredients div.ilist > div > p"
+  instructions rows: "div.instructions > p"
+end

data/lib/recipe_scrapers/sites/ru/povar.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "povar.ru"

data/lib/recipe_scrapers/sites/ru/russianfood.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "russianfood.com" do
+  encoding "windows-1251"
+  title "h1"
+  ingredients rows: "tr.ingr_tr_0, tr.ingr_tr_1"
+  instructions rows: "div.step_n"
+end

data/lib/recipe_scrapers/sites/se/alltommat.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "alltommat.se", also: %w[alltommat.expressen.se]

data/lib/recipe_scrapers/sites/se/ica.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "ica.se"

data/lib/recipe_scrapers/sites/se/kiddokitchen.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "kiddokitchen.se"

data/lib/recipe_scrapers/sites/se/koket.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "koket.se"

data/lib/recipe_scrapers/sites/se/recept.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "recept.se"

data/lib/recipe_scrapers/sites/sk/receptyprevas.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+RecipeScrapers.register "receptyprevas.sk"

data/lib/recipe_scrapers/sources/declared.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module Sources
+    class Declared
+      def initialize(document, declaration)
+        @document = document
+        @declaration = declaration
+      end
+
+      def rule(field)
+        @declaration&.rule(field)
+      end
+
+      def text(field)
+        selector = rule(field)&.fetch(:selector, nil)
+        return nil if selector.nil?
+
+        Text.normalize(Text.content(@document.at_css(selector)))
+      end
+
+      def rows(field)
+        selector = rule(field)&.fetch(:rows, nil)
+        return nil if selector.nil?
+
+        lines = @document.css(selector).filter_map { |node| row_text(node) }
+        lines.empty? ? nil : lines
+      end
+
+      private
+
+      def row_text(node)
+        parts = node.children.filter_map { |child| Text.normalize(Text.content(child)) }
+        parts.empty? ? nil : parts.join(" ")
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/ingredient_groups.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module Sources
+    module IngredientGroups
+      LAYOUTS = [
+        { heading: ".wprm-recipe-ingredient-group-name",
+          item: ".wprm-recipe-ingredient" },
+        { heading: ".tasty-recipes-ingredients p strong, .tasty-recipes-ingredients h4",
+          item: ".tasty-recipes-ingredients li" },
+        { heading: ".mv-create-ingredients h4, .mv-create-ingredients h3:not(.mv-create-ingredients-title)",
+          item: ".mv-create-ingredients li" }
+      ].freeze
+      SEPARATOR = /\A[^\p{L}\p{N}]*\z/
+      HEADING = /\A(?<purpose>\D+?)\s*:\z/
+
+      class << self
+        def sections(lines)
+          found = lines.grep_v(SEPARATOR).each_with_object([]) do |line, sections|
+            purpose = line[HEADING, :purpose]
+            next sections << [purpose, []] if purpose
+
+            sections << [nil, []] if sections.empty?
+            sections.last.last << line
+          end
+          found.reject { |_, section_lines| section_lines.empty? }
+        end
+
+        def from_sections(sections)
+          sections.map { |purpose, lines| Models::IngredientGroup.new(purpose: purpose, ingredients: lines) }
+        end
+
+        def detect(document:, ingredients:)
+          named = LAYOUTS.lazy.
+                  map { |layout| call(document: document, ingredients: ingredients, **layout) }.
+                  find { |groups| groups.any?(&:purpose) }
+
+          named || [ungrouped(ingredients)]
+        end
+
+        def call(document:, ingredients:, heading:, item:)
+          headings = document.css(heading)
+          return [ungrouped(ingredients)] if headings.empty?
+          return [ungrouped(ingredients)] unless document.css(item).size == ingredients.size
+
+          build(document.css("#{heading}, #{item}"), headings.to_a, ingredients)
+        end
+
+        private
+
+        def ungrouped(ingredients)
+          Models::IngredientGroup.new(purpose: nil, ingredients: ingredients)
+        end
+
+        def build(nodes, headings, ingredients)
+          remaining = ingredients.dup
+          sections = []
+
+          nodes.each { |node| absorb(sections, node, headings, remaining) }
+
+          sections.reject { |_, lines| lines.empty? }.
+            map { |purpose, lines| Models::IngredientGroup.new(purpose: purpose, ingredients: lines) }
+        end
+
+        def absorb(sections, node, headings, remaining)
+          return sections << [Text.normalize(node.text), []] if headings.include?(node)
+
+          sections << [nil, []] if sections.empty?
+          sections.last.last << (remaining.shift || Text.normalize(node.text))
+        end
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/json_ld.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RecipeScrapers
+  module Sources
+    class JsonLd
+      SELECTOR = 'script[type="application/ld+json"]'
+      ESCAPED_CONTROL = { "\b" => "\\b", "\f" => "\\f", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t" }.freeze
+
+      def initialize(document)
+        @nodes = flatten(document.css(SELECTOR).flat_map { |tag| parse(tag.text) })
+        @by_id = @nodes.select { |node| node["@id"] }.to_h { |node| [node["@id"], node] }
+      end
+
+      def recipe
+        @recipe ||= @nodes.find { |node| type?(node, "Recipe") } || main_entity_recipe
+      end
+
+      def person(id)
+        entity(id, "Person")
+      end
+
+      def rating(id)
+        entity(id, "AggregateRating")
+      end
+
+      def website_name
+        site = @nodes.find { |node| type?(node, "WebSite") }
+        site && Text.normalize(site["name"])
+      end
+
+      private
+
+      def entity(id, schema_type)
+        node = @by_id[id]
+        node if node && type?(node, schema_type)
+      end
+
+      def main_entity_recipe
+        page = @nodes.find { |node| type?(node, "WebPage") && node["mainEntity"].is_a?(Hash) }
+        return nil unless page
+
+        candidate = page["mainEntity"]
+        candidate if type?(candidate, "Recipe")
+      end
+
+      def parse(text)
+        JSON.parse(text)
+      rescue JSON::ParserError
+        parse_leniently(text)
+      end
+
+      def parse_leniently(text)
+        JSON.parse(escape_control_characters(text))
+      rescue JSON::ParserError
+        []
+      end
+
+      def escape_control_characters(text)
+        inside = false
+        escaped = false
+        text.each_char.map do |character|
+          next character.tap { escaped = false } if escaped
+          next ESCAPED_CONTROL.fetch(character, " ") if inside && character.ord < 0x20
+
+          escaped = inside && character == "\\"
+          inside = !inside if character == '"'
+          character
+        end.join
+      end
+
+      def flatten(value)
+        case value
+        when Array then value.flat_map { |item| flatten(item) }
+        when Hash then [value] + flatten(graph_nodes(value)) + flatten(value["mainEntity"])
+        else []
+        end
+      end
+
+      def graph_nodes(value)
+        graph = value["@graph"]
+        graph.is_a?(Hash) ? graph.values : graph
+      end
+
+      def type?(node, schema_type)
+        return false unless node.is_a?(Hash)
+
+        Array(node["@type"]).any? { |name| name.to_s.casecmp?(schema_type) }
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/microdata.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module RecipeScrapers
+  module Sources
+    class Microdata
+      VALUE_ATTRIBUTES = {
+        "meta" => "content",
+        "audio" => "src", "embed" => "src", "iframe" => "src", "img" => "src",
+        "source" => "src", "track" => "src", "video" => "src",
+        "a" => "href", "area" => "href", "link" => "href",
+        "object" => "data",
+        "data" => "value", "meter" => "value",
+        "time" => "datetime"
+      }.freeze
+      INLINE = %w[
+        a abbr b bdi bdo big cite code data del dfn em font i img ins kbd label link mark meta
+        q s samp small span strong sub sup svg time tt u var wbr
+      ].freeze
+
+      def initialize(document)
+        @nodes = roots(document).flat_map { |element| flatten(item_for(element)) }
+        @by_id = @nodes.select { |node| node["@id"] }.to_h { |node| [node["@id"], node] }
+      end
+
+      def recipe = @recipe ||= node_of_type("Recipe")
+      def person(id) = entity(id, "Person")
+      def rating(id) = entity(id, "AggregateRating")
+
+      def website_name
+        site = node_of_type("WebSite")
+        site && Text.normalize(site["name"])
+      end
+
+      private
+
+      def roots(document)
+        document.css("[itemscope]").reject { |element| element.ancestors.any? { |parent| parent.key?("itemscope") } }
+      end
+
+      def flatten(node)
+        [node] + node.values.flatten.grep(Hash).flat_map { |value| flatten(value) }
+      end
+
+      def node_of_type(schema_type)
+        @nodes.find { |node| node["@type"].to_s.casecmp?(schema_type) }
+      end
+
+      def entity(id, schema_type)
+        node = @by_id[id]
+        node if node && node["@type"].to_s.casecmp?(schema_type)
+      end
+
+      def item_for(element)
+        node = {}
+        type = type_name(element["itemtype"])
+        node["@type"] = type if type
+        node["@id"] = element["itemid"] if element["itemid"]
+        properties_of(element).each { |name, value| assign(node, name, value) }
+        node
+      end
+
+      def type_name(itemtype)
+        name = itemtype.to_s.split(/\s+/).first.to_s.split("/").last.to_s
+        name.empty? ? nil : name
+      end
+
+      def properties_of(element)
+        element.element_children.flat_map { |child| properties_in(child) }
+      end
+
+      def properties_in(element)
+        names = element["itemprop"].to_s.split(/\s+/)
+        return names.map { |name| [name, item_for(element)] } if element.key?("itemscope")
+
+        names.map { |name| [name, value_of(element)] } + properties_of(element)
+      end
+
+      def assign(node, name, value)
+        return node[name] = value unless node.key?(name)
+
+        existing = node[name]
+        node[name] = existing.is_a?(Array) ? existing + [value] : [existing, value]
+      end
+
+      def value_of(element)
+        attribute = VALUE_ATTRIBUTES[element.name]
+        return element[attribute] if attribute && element[attribute]
+        return element["content"] if element["content"]
+
+        lines_of(element)
+      end
+
+      def lines_of(element)
+        text_of(element).split("\n").map(&:strip).reject(&:empty?).join("\n")
+      end
+
+      def text_of(element)
+        element.children.filter_map { |child| text_from(child) }.join
+      end
+
+      def text_from(child)
+        return nil if Text::TEXTLESS.include?(child.name)
+        return child.text if child.text? || child.cdata?
+        return nil unless child.element?
+        return text_of(child) if INLINE.include?(child.name)
+
+        "\n#{text_of(child)}\n"
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/open_graph.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module Sources
+    class OpenGraph
+      def initialize(document)
+        @document = document
+      end
+
+      def title
+        property("og:title") || Text.normalize(@document.at_css("title")&.text)
+      end
+
+      def image = property("og:image")
+      def description = property("og:description")
+      def site_name = property("og:site_name")
+
+      private
+
+      def property(name)
+        tag = @document.at_css(%(meta[property="#{name}"][content]))
+        tag && Text.normalize(tag["content"])
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/schema_org/ingredient_list.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module Sources
+    class SchemaOrg
+      module IngredientList
+        PROPERTY_VALUE = "PropertyValue"
+
+        class << self
+          def parse(value)
+            lines = Array(value).flatten(1).filter_map { |item| Text.normalize(line_for(item)) }
+            lines.empty? ? nil : lines
+          end
+
+          private
+
+          def line_for(item)
+            return item unless item.is_a?(Hash)
+            return item.to_s unless property_value?(item)
+
+            measured(item).join(" ")
+          end
+
+          def property_value?(item)
+            Array(item["@type"]).any? { |name| name.to_s.casecmp?(PROPERTY_VALUE) }
+          end
+
+          def measured(item)
+            [item["value"], item["unitText"] || item["unitCode"], item["name"]].
+              filter_map { |part| part.to_s unless part.to_s.empty? }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/schema_org/nutrition_facts.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  module Sources
+    class SchemaOrg
+      module NutritionFacts
+        class << self
+          def parse(nutrition)
+            return nil if nutrition.nil?
+
+            values = nutrition.filter_map do |name, value|
+              text = Text.normalize(value)
+              [name, text] if text && !name.empty? && !name.start_with?("@")
+            end
+            values.empty? ? nil : values.to_h
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recipe_scrapers/sources/schema_org.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module RecipeScrapers
+  # Read what a page says. {SchemaOrg} is the one to subclass for a site.
+  module Sources
+    # Reads the fields of the schema.org Recipe a page publishes, from JSON-LD or from microdata.
+    #
+    # Every method answers the field of the same name on {Scraper}. A site that structures its markup
+    # in its own way subclasses this class and overrides a method, then names the subclass with
+    # {Scraper.schema_reader}. The steps of recipeInstructions are read by private methods that can be
+    # overridden one at a time: steps_in, section_steps, step_steps and step_heading.
+    class SchemaOrg
+      FACET = /\A[\w-]+: /
+      private_constant :FACET
+      # @param readers [Array<Sources::JsonLd, Sources::Microdata>] the first one that finds a Recipe wins
+      def initialize(*readers)
+        @source = readers.find(&:recipe) || readers.first
+        @data = @source.recipe || {}
+      end
+
+      # (see Models::Recipe#title)
+      def title = Text.normalize(@data["name"])
+      # (see Models::Recipe#description)
+      def description = Text.normalize(@data["description"])
+      # (see Models::Recipe#language)
+      def language = Text.normalize(@data["inLanguage"])
+      # (see Models::Recipe#site_name)
+      def site_name = @source.website_name
+      # (see Models::Recipe#category)
+      def category = first_text(@data["recipeCategory"])
+      # (see Models::Recipe#cuisine)
+      def cuisine = first_text(@data["recipeCuisine"])
+      # (see Models::Recipe#cooking_method)
+      def cooking_method = first_text(@data["cookingMethod"])
+      # (see Models::Recipe#yields)
+      def yields = Parsers::Yields.parse(@data["recipeYield"])
+      # (see Models::Recipe#total_time)
+      def total_time = Parsers::Durations.minutes(@data["totalTime"])
+      # (see Models::Recipe#cook_time)
+      def cook_time = Parsers::Durations.minutes(@data["cookTime"])
+      # (see Models::Recipe#prep_time)
+      def prep_time = Parsers::Durations.minutes(@data["prepTime"])
+
+      # (see Models::Recipe#author)
+      def author
+        names = [@data["author"]].flatten.filter_map { |entry| author_name(dereference(entry, :person)) }
+        names.empty? ? nil : names.join(", ")
+      end
+
+      # (see Models::Recipe#ingredients)
+      def ingredients = IngredientList.parse(@data["recipeIngredient"] || @data["ingredients"])
+
+      # (see Models::Recipe#instructions_list)
+      def instructions_list
+        steps = steps_in(@data["recipeInstructions"])
+        steps.empty? ? nil : steps
+      end
+
+      # (see Models::Recipe#instructions)
+      def instructions = instructions_list&.join("\n")
+      # (see Models::Recipe#image)
+      def image = Text.normalize(url_in(@data["image"]))
+
+      # (see Models::Recipe#keywords)
+      def keywords
+        words = [@data["keywords"]].flatten.flat_map { |entry| entry.to_s.split(",") }
+        cleaned = words.filter_map { |word| Text.normalize(word) }.grep_v(FACET)
+        cleaned.empty? ? nil : cleaned
+      end
+
+      # (see Models::Recipe#nutrients)
+      def nutrients = NutritionFacts.parse(first_hash(@data["nutrition"]))
+
+      # (see Models::Recipe#dietary_restrictions)
+      def dietary_restrictions
+        diets = Array(@data["suitableForDiet"]).filter_map do |diet|
+          Text.normalize(diet.is_a?(Hash) ? diet["name"] : diet.to_s.split("/").last)
+        end
+        diets.empty? ? nil : diets
+      end
+
+      # (see Models::Recipe#ratings)
+      def ratings = Parsers::Ratings.average(rating_node)
+      # (see Models::Recipe#ratings_count)
+      def ratings_count = Parsers::Ratings.count(rating_node)
+
+      private
+
+      def rating_node
+        @rating_node ||= first_hash(dereference(first_hash(@data["aggregateRating"]), :rating))
+      end
+
+      def author_name(value) = Text.normalize(value.is_a?(Hash) ? value["name"] : value)
+
+      def first_hash(value) = [value].flatten.grep(Hash).first
+
+      def dereference(value, kind)
+        return value unless value.is_a?(Hash) && value.keys == ["@id"]
+
+        @source.public_send(kind, value["@id"])
+      end
+
+      def first_text(value) = Text.normalize(value.is_a?(Array) ? value.first : value)
+
+      def url_in(value)
+        case value
+        when Array then url_in(value.first)
+        when Hash then url_in(value["url"])
+        else value
+        end
+      end
+
+      def steps_in(value)
+        case value
+        when String then value.split("\n").filter_map { |line| Text.normalize(line) }
+        when Array then value.flat_map { |item| listed_step(item) }
+        when Hash then steps_in_hash(value)
+        else []
+        end
+      end
+
+      def listed_step(value) = value.is_a?(String) ? [Text.normalize(value)].compact : steps_in(value)
+
+      def steps_in_hash(value)
+        types = Array(value["@type"]).map { |name| name.to_s.downcase }
+        return section_steps(value) if types.include?("howtosection")
+        return step_steps(value) if types.include?("howtostep")
+
+        untyped_steps(value)
+      end
+
+      def section_steps(value)
+        heading = Text.normalize(value["name"] || value["Name"])
+        [heading].compact + steps_in(value["itemListElement"])
+      end
+
+      def step_steps(value)
+        nested = value["itemListElement"]
+        return [step_heading(value)].compact + steps_in(nested) if nested.is_a?(Array)
+
+        text = nested.is_a?(Hash) ? nested["text"] : value["text"]
+        [step_heading(value), Text.normalize(text)].compact
+      end
+
+      def step_heading(value)
+        name = value["name"]
+        return nil unless name.is_a?(String)
+        return nil if value["text"].to_s.start_with?(name.sub(/\.+\z/, ""))
+
+        Text.normalize(name)
+      end
+
+      def untyped_steps(value)
+        return steps_in(value["itemListElement"]) if value["itemListElement"]
+
+        [Text.normalize(value["text"] || value["name"])].compact
+      end
+    end
+  end
+end