wovnrb 1.0.13 → 1.1.0

data/lib/wovnrb/html_replacers/unified_values/text_replacer.rb

@@ -0,0 +1,35 @@
+module Wovnrb
+  module UnifiedValues
+    class TextReplacer < ReplacerBase
+      def initialize(store, text_index)
+        super(store)
+        @text_index = text_index
+      end
+
+      def replace(dom, lang)
+        translated_nodes_with_targets = NodeSwappingTargetsCreator.new(TextScraper.new(@ignored_class_set).run(dom)).run!
+        text_index_with_targets = DstSwappingTargetsCreator.new(@text_index).run!
+
+        translated_nodes_with_targets.each do |translated_nodes_with_target|
+          dst_swapping_targets =
+              text_index_with_targets[translated_nodes_with_target[:dst]]
+                  .try(:fetch, lang.lang_code, nil)
+                  .try(:first)
+                  .try(:fetch, 'swapping_targets', nil)
+          next unless dst_swapping_targets
+
+          translated_nodes_with_target[:swapping_targets].each_with_index do |node_swapping_target, index|
+            # NOTE: current logic to swap back search text node and find wovn-src base on the text node.
+            #       if `translated_text` is empry string, translated html don't have text node. it means that widget can't find wovn-src and swap back
+            #       so we use `\u200b`(ZERO WIDTH SPACE) instead of empty string
+            translated_text = dst_swapping_targets[index].blank? ? "\u200b" : dst_swapping_targets[index]
+            original_text = node_swapping_target.content
+
+            node_swapping_target.content = translated_text
+            add_comment_node(node_swapping_target, original_text)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wovnrb/html_replacers/unified_values/text_scraper.rb

@@ -0,0 +1,152 @@
+module Wovnrb
+  module UnifiedValues
+    class TextScraper
+      def initialize(ignored_class_set)
+        @ignored_class_set = ignored_class_set
+        @values = []
+        @values_stack = nil
+      end
+
+      def run(dom)
+        refresh_all!
+        @values_stack = ValuesStack.new(dom.path, 1)
+        scrape(dom)
+      end
+
+      def create_text_value(src, node_stack)
+        { dst: ValueAgent.normalize_text(src.gsub(/ /i, ' ')), nodes: node_stack }
+      end
+
+      private
+
+      def scrape(dom)
+        type = stop_recursion_type(dom)
+        if type
+          case type
+          when 'ignore_element', 'skip_element'
+            next_stack = @values_stack.build_next_stack
+            build_src
+            @values_stack = next_stack
+          when 'text_element'
+            @values_stack.add_text_element(dom, dom.content)
+          when 'empty_element'
+            @values_stack.add(dom, empty_tag(dom))
+          when 'comment_element'
+            # do nothing
+          else
+            raise 'Unsupported type'
+          end
+        elsif inline_element?(dom)
+          @values_stack.add(dom, start_tag(dom))
+          scrape_children_of(dom)
+          @values_stack.add(dom, end_tag(dom)) unless @values_stack.blank?
+        elsif block_element?(dom)
+          next_stack = @values_stack.build_next_stack
+          build_src
+          @values_stack = ValuesStack.new(dom.path, 1)
+          scrape_children_of(dom)
+          build_src
+          @values_stack = next_stack
+        else
+          next_stack = @values_stack.build_next_stack
+          build_src
+          @values_stack = ValuesStack.new(dom.path, 1)
+          scrape_children_of(dom)
+          build_src
+          @values_stack = next_stack
+        end
+
+        @values
+      end
+
+      def scrape_children_of(dom)
+        dom.children.each { |c| scrape(c) } unless wovn_ignore_element?(dom)
+      end
+
+      def build_src
+        return nil if @values_stack.blank?
+
+        src = @values_stack.src
+        @values << create_text_value(src, @values_stack.node_stack) if src.present? && !tag_only?(src)
+        @values_stack = nil
+      end
+
+      def tag_only?(src)
+        Nokogiri::HTML5.fragment(src).text.blank?
+      end
+
+      def start_tag(dom)
+        if wovn_ignore_element?(dom)
+          "<#{dom.name} wovn-ignore>"
+        else
+          "<#{dom.name}>"
+        end
+      end
+
+      def end_tag(dom)
+        "</#{dom.name}>"
+      end
+
+      def empty_tag(dom)
+        start_tag(dom)
+      end
+
+      def block_element?(element)
+        ElementCategory::BLOCK_ELEMENTS.include?(element.name)
+      end
+
+      def inline_element?(element)
+        ElementCategory::INLINE_ELEMENTS.include?(element.name)
+      end
+
+      def empty_element?(element)
+        ElementCategory::EMPTY_ELEMENTS.include?(element.name)
+      end
+
+      def ignore_element?(element)
+        ElementCategory::IGNORE_ELEMENTS.include?(element.name)
+      end
+
+      def skip_element?(element)
+        ElementCategory::SKIP_ELEMENTS.include?(element.name)
+      end
+
+      def comment_element?(element)
+        element.comment?
+      end
+
+      def text_element?(element)
+        element.text?
+      end
+
+      def wovn_ignore_element?(element)
+        return false unless element
+
+        return true if element.attribute('wovn-ignore')
+
+        class_attribute = element.attribute('class')
+        return false unless class_attribute
+        class_attribute.value.split.any? { |c| @ignored_class_set.include?(c) }
+      end
+
+      def stop_recursion_type(element)
+        return 'ignore_element'  if ignore_element?(element)
+        return 'skip_element'    if skip_element?(element)
+        return 'text_element'    if text_element?(element)
+        return 'empty_element'   if empty_element?(element)
+        return 'comment_element' if comment_element?(element)
+
+        nil
+      end
+
+      def refresh_all!
+        @values_stack = nil
+        refresh_values!
+      end
+
+      def refresh_values!
+        @values = []
+      end
+    end
+  end
+end

data/lib/wovnrb/html_replacers/unified_values/values_stack.rb

@@ -0,0 +1,65 @@
+require 'wovnrb/services/value_agent'
+
+module Wovnrb
+  module UnifiedValues
+    class ValuesStack
+      attr_reader :node_stack
+
+      # @param head_path [String]
+      # @param index [Number]
+      #
+      # Be careful xpath's index starts with 1
+      def initialize(head_path, index)
+        @head_path = head_path
+        @index = index
+        @node_stack = []
+        @src_stack = []
+        @src_without_tag_stack = []
+      end
+
+      # @param node [Nokogiri::XML::Element]
+      # @param src [String]
+      def add(node, src)
+        @node_stack << node if node.name != 'text' || node.content.present?
+        @src_stack << Wovnrb::ValueAgent.normalize_text(src)
+      end
+
+      # @param node [Nokogiri::XML::Element]
+      # @param dom_content [String]
+      def add_text_element(node, dom_content)
+        add(node, CGI.escapeHTML(dom_content))
+        @src_without_tag_stack << Wovnrb::ValueAgent.normalize_text(dom_content)
+      end
+
+      # @return [Bool]
+      def blank?
+        @src_stack.blank?
+      end
+
+      # @return [String]
+      def path
+        return @head_path if @head_path.end_with?('title')
+
+        # Ends with "text()" because some type checking takes path as a normal text when the path ends with "text()"
+        p = "#{@head_path}/text()"
+
+        @index == 1 ? p : "#{p}[#{@index}]"
+      end
+
+      # @return [String]
+      def src
+        @src_stack.inject(:+)
+      end
+
+      # @return [String]
+      def src_without_tag
+        @src_without_tag_stack.inject(:+)
+      end
+
+      # @return [ValuesStack]
+      def build_next_stack
+        ValuesStack.new(@head_path, @index + 1)
+      end
+    end
+  end
+end

data/lib/wovnrb/lang.rb

@@ -204,6 +204,7 @@ module Wovnrb
 
     def replace_dom_values(dom, values, store, url, headers)
       text_index = values['text_vals'] || {}
+      html_text_index = values['html_text_vals'] || {}
       src_index = values['img_vals'] || {}
       img_src_prefix = values['img_src_prefix'] || ''
       host_aliases = values['host_aliases'] || []
@@ -215,7 +216,11 @@ module Wovnrb
         replacers << LinkReplacer.new(store, pattern, headers)
       end
 
-      replacers << TextReplacer.new(store, text_index)
+      unless html_text_index.empty?
+        replacers << UnifiedValues::TextReplacer.new(store, html_text_index)
+      else
+        replacers << TextReplacer.new(store, text_index)
+      end
       replacers << MetaReplacer.new(store, text_index, pattern, headers)
       replacers << InputReplacer.new(store, text_index)
       replacers << ImageReplacer.new(store, url, text_index, src_index, img_src_prefix, host_aliases)

data/lib/wovnrb/services/value_agent.rb

@@ -0,0 +1,9 @@
+module Wovnrb
+  class ValueAgent
+    def self.normalize_text(src)
+      src.gsub(/[\ufffd]/, "\b")
+          .gsub(/[\n \t\u0020\u0009\u000C\u200B\u000D\u000A]+/, ' ')
+          .gsub(/^[\s\u00A0\uFEFF\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]+|[\s\u00A0\uFEFF\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]+$/, '')
+    end
+  end
+end

data/lib/wovnrb/store.rb

@@ -6,6 +6,7 @@ require 'wovnrb/services/wovn_logger'
 require 'wovnrb/services/glob'
 require 'wovnrb/settings'
 require 'active_support'
+require 'active_support/core_ext'
 
 module Wovnrb
   class Store
@@ -64,43 +65,35 @@ module Wovnrb
       errors = [];
       #if valid_token?(!settings.has_key?('project_token') || settings['project_token'].length < 5 || settings['project_token'].length > 6
       if !valid_token?(settings['project_token'])
-        valid = false
         errors.push("Project token #{settings['project_token']} is not valid.")
       end
       if settings.has_key?('ignore_paths') && !settings['ignore_paths'].kind_of?(Array)
-        valid = false
         errors.push("Ignore Paths #{settings['ignore_paths']} should be Array.")
       end
       if !settings.has_key?('url_pattern') || settings['url_pattern'].length == 0
-        valid = false
         errors.push("Url pattern #{settings['url_pattern']} is not valid.")
       end
       if !settings.has_key?('query') || !settings['query'].kind_of?(Array)
-        valid = false
         errors.push("query config #{settings['query']} is not valid.")
       end
       if !settings.has_key?('ignore_class') || !settings['ignore_class'].kind_of?(Array)
-        valid = false
         errors.push("ignore_class config #{settings['ignore_class']} should be Array.")
       end
       if !settings.has_key?('api_url') || settings['api_url'].length == 0
-        valid = false
         errors.push("API URL is not configured.")
       end
       if !settings.has_key?('default_lang') || settings['default_lang'].nil?
-        valid = false
         errors.push("Default lang #{settings['default_lang']} is not valid.")
       end
       if !settings.has_key?('supported_langs') || !settings['supported_langs'].kind_of?(Array) || settings['supported_langs'].size < 1
-        valid = false
         errors.push("Supported langs configuration is not valid.")
       end
       if !settings.has_key?('custom_lang_aliases') || !settings['custom_lang_aliases'].kind_of?(Hash)
-        valid = false
         errors.push("Custom lang aliases is not valid.")
       end
       # log errors
       if errors.length > 0
+        valid = false
         errors.each do |e|
           WovnLogger.instance.error(e)
         end

data/lib/wovnrb/version.rb

@@ -1,3 +1,3 @@
 module Wovnrb
-  VERSION = "1.0.13"
+  VERSION = "1.1.0"
 end

data/test/fixtures/unified_values/site_html/simple_actual.html

@@ -0,0 +1,96 @@
+<html>
+<script>console.log('hello')</script>
+<title>SIMPLE TITLE</title>
+<meta name="description" content="meta description">
+<body>
+<div>
+  <form role="search">
+    <div class="search-control">
+      <input type="search" id="site-search" name="q"
+             placeholder="Search the site..."
+             aria-label="Search through site content">
+      <button>Search</button>
+    </div>
+  </form>
+
+  <form role="search" method="get" class="search-form wb-flex" action="https://www.wovn.io/">
+    <label>
+      <span>Search for:</span>
+      <input type="search" placeholder="Search …" value="" name="s" title="Search for:">
+    </label>
+  </form>
+</div>
+<div>
+  <span>
+    <p>Well</p>
+  </span>
+</div>
+<a class="home">
+    <span><h1>scrap</h1>only
+    <img src="/assets/hedgehog.jpg">
+    this too</span>
+</a><br>
+<span href="/" class="home">
+    <a><img src="/assets/hedgehog.jpg"></a>
+    <a>"Storm hunters" it is to see.</a>
+    <a>BLABLA5</a>
+  </span>
+<div>
+  <form class="ftr-locator-form">
+    <input name="hdr-ftr-locator-term" placeholder="Street, City, State or ZIP" type="text">
+    <input type="submit">
+
+  </form>
+</div>
+<h1>
+  <div style="opacity: 1;">Welcome to my Website!</div>
+</h1>
+<div>
+  <div class="content-asset"><div class="heading1">LOST AF?</div>
+    <div class="error-info">
+      <p>WE'LL TAKE THE BLAME.</p>
+      <p>LET US MAKE IT UP TO YOU WITH SOME LINKS.</p>
+    </div>
+
+    <div class="shopping-link">
+      <nav class="gender-links">
+        <ul>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/mens/" class="nav-link">SHOP MENS</a></li>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/womens/" class="nav-link">SHOP WOMENS</a></li>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/kids/boys/" class="nav-link">shop boys</a></li>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/kids/girls/" class="nav-link">shop girls</a></li>
+        </ul>
+      </nav>
+    </div></div>
+  <div class="error-custom-message noshow">"<div class="heading1">LOST AF?</div>
+    <div class="error-info">
+      <p>WE'LL TAKE THE BLAME.</p>
+      <p>LET US MAKE IT UP TO YOU WITH SOME LINKS.</p>
+    </div>
+
+    <div class="shopping-link">
+      <nav class="gender-links">
+        <ul>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/mens/" class="nav-link">SHOP MENS</a></li>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/womens/" class="nav-link">SHOP WOMENS</a></li>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/kids/boys/" class="nav-link">shop boys</a></li>
+          <li class="link-text"><a href="https://www.abercrombie.co.jp/en_JP/kids/girls/" class="nav-link">shop girls</a></li>
+        </ul>
+      </nav>
+    </div></div>"
+</div>
+
+<p>This is a paragraph! Here's how you make a link: <a href="https://neocities.org">Neocities</a>.</p>
+
+<p>Here's how you can make <strong>bold</strong> and <em>italic</em> text.</p>
+<p>Here's how you can add an image:</p>
+<p>Here's how to make a list:</p>
+<ul>
+  <li href="/">First thing</li>
+  <li>Second thing</li>
+  <li>Third thing</li>
+</ul>
+
+<p>To learn more HTML/CSS, check out these <a href="https://neocities.org/tutorials">tutorials</a>!</p>
+</body>
+</html>

data/test/fixtures/unified_values/site_html/simple_expected.json

@@ -0,0 +1,251 @@
+[
+  {
+    "xpath": "/html/head/title",
+    "srcs": [
+      "SIMPLE TITLE"
+    ]
+  },
+  {
+    "xpath": "/html/body/div/form/div/text()",
+    "srcs": [
+      "<input>",
+      "<button>",
+      "Search",
+      "</button>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div/form[2]/text()",
+    "srcs": [
+      "<label>",
+      "<span>",
+      "Search for:",
+      "</span>",
+      "<input>",
+      "</label>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[2]/span/p/text()",
+    "srcs": [
+      "Well"
+    ]
+  },
+  {
+    "xpath": "/html/body/a/span/h1/text()",
+    "srcs": [
+      "scrap"
+    ]
+  },
+  {
+    "xpath": "/html/body/text()[4]",
+    "srcs": [
+      "only"
+    ]
+  },
+  {
+    "xpath": "/html/body/text()[5]",
+    "srcs": [
+      "this too",
+      "</span>",
+      "</a>",
+      "<br>",
+      "<span>",
+      "<a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/text()[6]",
+    "srcs": [
+      "<a>",
+      "&quot;Storm hunters&quot; it is to see.",
+      "</a>",
+      "<a>",
+      "BLABLA5",
+      "</a>",
+      "</span>"
+    ]
+  },
+  {
+    "xpath": "/html/body/h1/div/text()",
+    "srcs": [
+      "Welcome to my Website!"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div/text()",
+    "srcs": [
+      "LOST AF?"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div[2]/p/text()",
+    "srcs": [
+      "WE&#39;LL TAKE THE BLAME."
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div[2]/p[2]/text()",
+    "srcs": [
+      "LET US MAKE IT UP TO YOU WITH SOME LINKS."
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div[3]/nav/ul/li/text()",
+    "srcs": [
+      "<a>",
+      "SHOP MENS",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div[3]/nav/ul/li[2]/text()",
+    "srcs": [
+      "<a>",
+      "SHOP WOMENS",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div[3]/nav/ul/li[3]/text()",
+    "srcs": [
+      "<a>",
+      "shop boys",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div/div[3]/nav/ul/li[4]/text()",
+    "srcs": [
+      "<a>",
+      "shop girls",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/text()",
+    "srcs": [
+      "&quot;"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div/text()",
+    "srcs": [
+      "LOST AF?"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div[2]/p/text()",
+    "srcs": [
+      "WE&#39;LL TAKE THE BLAME."
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div[2]/p[2]/text()",
+    "srcs": [
+      "LET US MAKE IT UP TO YOU WITH SOME LINKS."
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div[3]/nav/ul/li/text()",
+    "srcs": [
+      "<a>",
+      "SHOP MENS",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div[3]/nav/ul/li[2]/text()",
+    "srcs": [
+      "<a>",
+      "SHOP WOMENS",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div[3]/nav/ul/li[3]/text()",
+    "srcs": [
+      "<a>",
+      "shop boys",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/div[2]/div[3]/nav/ul/li[4]/text()",
+    "srcs": [
+      "<a>",
+      "shop girls",
+      "</a>"
+    ]
+  },
+  {
+    "xpath": "/html/body/div[4]/text()[3]",
+    "srcs": [
+      "&quot;"
+    ]
+  },
+  {
+    "xpath": "/html/body/p/text()",
+    "srcs": [
+      "This is a paragraph! Here&#39;s how you make a link:",
+      "<a>",
+      "Neocities",
+      "</a>",
+      "."
+    ]
+  },
+  {
+    "xpath": "/html/body/p[2]/text()",
+    "srcs": [
+      "Here&#39;s how you can make",
+      "<strong>",
+      "bold",
+      "</strong>",
+      "and",
+      "<em>",
+      "italic",
+      "</em>",
+      "text."
+    ]
+  },
+  {
+    "xpath": "/html/body/p[3]/text()",
+    "srcs": [
+      "Here&#39;s how you can add an image:"
+    ]
+  },
+  {
+    "xpath": "/html/body/p[4]/text()",
+    "srcs": [
+      "Here&#39;s how to make a list:"
+    ]
+  },
+  {
+    "xpath": "/html/body/ul/li/text()",
+    "srcs": [
+      "First thing"
+    ]
+  },
+  {
+    "xpath": "/html/body/ul/li[2]/text()",
+    "srcs": [
+      "Second thing"
+    ]
+  },
+  {
+    "xpath": "/html/body/ul/li[3]/text()",
+    "srcs": [
+      "Third thing"
+    ]
+  },
+  {
+    "xpath": "/html/body/p[5]/text()",
+    "srcs": [
+      "To learn more HTML/CSS, check out these",
+      "<a>",
+      "tutorials",
+      "</a>",
+      "!"
+    ]
+  }
+]