parsergem 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb26a9c2d1591965320607c103b70fcfc231fcbb67532224c0affea0cba59443
-  data.tar.gz: 644bffc80b1f88dec4f3523eac8bd96213b6f6dd80090b8693965347b2970849
+  metadata.gz: 7f4b5dcf97aec9f152486fa05c4264780bb9f4b2b9a890c4dd1ac66458cba31a
+  data.tar.gz: c073c2e151d5521f1870b8f94daa4068ceb618d991fb8ce5503d8c3f936b026a
 SHA512:
-  metadata.gz: 82e0b832ff9df26b5ba0fd65894a6eccf1d0605d176f23c5643883680d4231d740db4fca5cdf0680ffe0d78860cdf69533fdc08d05d59b44e5e9de42102af87a
-  data.tar.gz: 9d91c3fb09f112327897c7ef38217edef2f5a1e20158454113d07125ff383ef12896a97586dd442d3d6ae0ac14f200a6083c9507170702e66fabb35aa2e537dd
+  metadata.gz: 225ddfb601c1102ab8af2f9acef55007e6a93df5aa59e31cee827398f692ea0e9010f229363bdbe978f9117160c96ce0b57800a23c13b04955e3010a61dab717
+  data.tar.gz: 4dbc5f4fd706778bd438d099165b06922c6d19dfad50a23a1412eac152b6abb0f91bb00973dbd0b42012a3bfef7dde62db452292d2c93720c22a0538d1b1264d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parsergem (0.1.2)
+    parsergem (0.1.3)
       activesupport (>= 5.0.0)
       aws-sdk-translate
       nokogiri

data/README.md

@@ -42,7 +42,7 @@ To create language versions (You must fill in all parameters): <br />
 `--aws_public_key` - The public key of your AWS TRANSLATE API <br />
 `--aws_secret_key` - The secret key of your AWS TRANSLATE API <br />
 
-After the generator has finished its work, you will most likely need to modify the header and footer parshals to fix the links.
+After the generator has finished its work, you will most likely need to modify the header and footer parshals to fix the links. And in current version some media files includes in css may be broken.
 
 ## Contributing
 
@@ -92,7 +92,7 @@ Rails 7.0.4.3
         `--aws_public_key` - Публичный ключ вашего AWS TRANSLATE API <br />
         `--aws_secret_key` - Секретный ключ вашего AWS TRANSLATE API <br />
 
-После того как генератор закончит свою работу, вам, скорее всего, придётся доработать паршалы хедеров и футеров для исправления ссылок. 
+После того как генератор закончит свою работу, вам, скорее всего, придётся доработать паршалы хедеров и футеров для исправления ссылок. Так же в текущей версии некоторые ссылки на подключение фотографий в css файлах могут не работать.
 
 ## Содействие
 

data/lib/generators/parser_gem/test_generator.rb

@@ -17,12 +17,21 @@ module ParserGem
     class_option :aws_region, type: :string, default: nil # Set aws region for translate API
     class_option :aws_public_key, type: :string, default: nil # Set aws api public key
     class_option :aws_secret_key, type: :string, default: nil # Set aws api secret key
+    class_option :to_partials_blocks_ids, type: :array, default: [] # Set blocks ids which must be partial
+    class_option :media_to_ignore, type: :array, default: [] # Set media filenames which must be ignored
+    class_option :media_to_ignore_for_langs, type: :array, default: [] # Set media filenames which must be ignored in lang versions
+    class_option :create_meta_og, type: :boolean, default: false # Create meta og: tags from standart meta tags
+    class_option :inline_styles_to_files, type: :boolean, default: false
+    class_option :inline_scripts_to_files, type: :boolean, default: false
+    class_option :meta_tags_names_to_ignore, type: :array, default: []
 
     def clone # Main method 
     sitemap = Nokogiri::XML(URI.open("http://#{options[:target_url]}/sitemap.xml")) # Load sitemap.xml
     controller_name = options[:target_url].split('/').last.delete('.').delete('-') # Set controller name from clear domain name
-    actions = set_actions(sitemap) # Set actions for controller
+    actions = set_actions(sitemap)[0..1] # Set actions for controller
     
+    puts options[:to_partials_blocks_ids].to_s
+
     actions.each do |action| # Create routes
       if action[:rails_route] == '/' 
         route "root \"#{controller_name}##{action[:action_name]}\""
@@ -51,12 +60,21 @@ module ParserGem
       client = Aws::Translate::Client.new( region: options[:aws_region],
                                            credentials: credentials)
     end
+
+    inline_styles = []
+    inline_scripts = []
             
     actions.each do |action| 
       file_path = File.join('app/views', controller_name, "#{action[:action_name]}.html.erb") # Set view
       File.open(file_path, 'wb') do |file|
         page = Nokogiri::HTML(URI.open("#{action[:native_url]}")) # Parse page from url in sitemap
         
+        page.css('meta').each do |meta|
+          if options[:meta_tags_names_to_ignore].include?(meta['name'].to_s)
+            meta.replace('')
+          end
+        end
+
         if action[:lang]
           page.css('title').each do |title|
             if title.inner_html.length > 1
@@ -75,12 +93,12 @@ module ParserGem
               media['alt'] = translate(media['alt'], action[:lang], client)
             end
           end
+        end
 
-          page.traverse do |node|
-            if node.text? && !node.parent.name.in?(%w[script style])
-              if node.text.length > 1
-                node.content = translate(node.text, action[:lang], client)
-              end
+        if options[:media_to_ignore_for_langs] && action[:lang]
+          page.css('img, video').each do |media|
+            if options[:media_to_ignore_for_langs].include?(File.basename(media['src'])) 
+              media.replace('')
             end
           end
         end
@@ -97,6 +115,12 @@ module ParserGem
           file.puts meta
         end
 
+        if options[:create_meta_og]
+          file.puts "<meta property=\"og:titile\" content=\"#{head.css('title').first.inner_html}\" >"
+          file.puts "<meta property=\"og:description\" content=\"#{head.css('meta[name=\"description\"]').first['content']}\" >"
+          file.puts "<meta property=\"og:image\" content=\" \" >"
+        end
+
         page.css('link[rel="stylesheet"]').each do |link| # Repair css including for Rails
           css_file_path = download_css(path_to_download(link['href']), controller_name)
           file.puts "<%= stylesheet_link_tag '#{css_file_path}' %>"
@@ -109,26 +133,94 @@ module ParserGem
           end
         end
 
-        page.css('style').each do |style_node| # Move all style tags to head
-          file.puts style_node
+        if options[:inline_styles_to_files]
+          page.css('style').each do |style_node| 
+            node_hash = { filename: "inline_style_#{inline_styles.size + 1}.css", 
+                          content: style_node.content }
+            
+            if inline_styles.size > 0
+              existing_hash = inline_styles.find { |elem| elem[:content] == node_hash[:content] }
+              if existing_hash
+                file.puts "<%= stylesheet_link_tag '#{existing_hash[:filename]}' %>"
+              else
+                inline_styles << node_hash  
+                file.puts "<%= stylesheet_link_tag '#{node_hash[:filename]}' %>"      
+              end
+            else
+              inline_styles << node_hash 
+              file.puts "<%= stylesheet_link_tag '#{node_hash[:filename]}' %>"
+            end
+          end
+        else
+          page.css('style').each do |style_node| # Move all style tags to head
+            file.puts style_node
+          end
         end
 
-        page.css('script').each do |script_node| # Move all script tags to head
-          file.puts script_node
+        if options[:inline_scripts_to_files]
+          page.css('script').each do |script_node|
+            if script_node.content && script_node.content.length > 1 
+              node_hash = { filename: "inline_script_#{inline_scripts.size + 1}.js", 
+                            content: script_node.content }
+            
+              if inline_scripts.size > 0
+                existing_hash = inline_scripts.find { |elem| elem[:content] == node_hash[:content] }
+                if existing_hash
+                  file.puts "<%= javascript_tag '#{existing_hash[:filename]}' %>"
+                else
+                  inline_scripts << node_hash  
+                  file.puts "<%= javascript_tag '#{node_hash[:filename]}' %>"
+                end
+              else
+                inline_scripts << node_hash 
+                file.puts "<%= javascript_tag '#{node_hash[:filename]}' %>"
+              end
+            end
+          end
+        else
+          page.css('script').each do |script_node| # Move all script tags to head
+            file.puts script_node
+          end
         end
 
         file.puts "<% end %>"
 
-        if !File.exist?("app/views/layouts/_#{action[:lang]}header.html.erb") || !File.exist?("app/views/layouts/_#{action[:lang]}footer.html.erb") # Create footer and header partials
+        if !File.exist?("app/views/layouts/_#{action[:lang]}header.html.erb") # Create footer and header partials
           if options[:header_class_name] # Find and save header to partial
             File.open(File.join('app/views/layouts/', "_#{action[:lang]}header.html.erb"), 'w') do |file| 
-              file.puts body.css("div[id=\"#{options[:header_class_name]}\"]").to_html
+              if action[:lang]
+                translated_block = body.css("div[id=\"#{options[:header_class_name]}\"]").first
+                translated_block.traverse do |node| 
+                  if node.text? && !node.parent.name.in?(%w[script style])
+                    if node.text.length > 1
+                      node.content = translate(node.text, action[:lang], client)
+                    end
+                  end
+                end
+                file.puts translated_block.to_html
+              else
+                file.puts body.css("div[id=\"#{options[:header_class_name]}\"]").first.to_html
+              end
             end
           end
+        end
 
+        if !File.exist?("app/views/layouts/_#{action[:lang]}footer.html.erb")
           if options[:footer_class_name] # Find and save footer to partial
             File.open(File.join('app/views/layouts/', "_#{action[:lang]}footer.html.erb"), 'w') do |file| 
-              file.puts body.css("div[id=\"#{options[:footer_class_name]}\"]").to_html
+              if action[:lang]
+                translated_block = body.css("div[id=\"#{options[:footer_class_name]}\"]").first
+                translated_block.traverse do |node| 
+                  if node.text? && !node.parent.name.in?(%w[script style])
+                    if node.text.length > 1
+                      node.content = translate(node.text, action[:lang], client)
+                    end
+                  end
+                end
+                file.puts translated_block.to_html
+              else
+                file.puts body.css("div[id=\"#{options[:footer_class_name]}\"]").first.to_html
+              end
             end
           end
         end
@@ -158,16 +250,63 @@ module ParserGem
           end
         end
 
+        options[:to_partials_blocks_ids].each do |block_id|
+          body.css("div[id=\"#{block_id}\"]").each do |block|
+            if !File.exist?("app/views/layouts/_#{action[:lang]}#{block_id}.html.erb")
+              File.open(File.join('app/views/layouts/', "_#{action[:lang]}#{block_id}.html.erb"), 'w') do |file| 
+              if action[:lang]
+                translated_block = body.css("div[id=\"#{block_id}\"]").first
+                translated_block.traverse do |node| 
+                  if node.text? && !node.parent.name.in?(%w[script style])
+                    if node.text.length > 1
+                      node.content = translate(node.text, action[:lang], client)
+                    end
+                  end
+                end
+                file.puts translated_block.to_html
+              else
+                file.puts body.css("div[id=\"#{block_id}\"]").first.to_html
+              end
+              end
+            end
+            block.replace("<%= render \"layouts/#{action[:lang]}#{block_id}\" %>")
+          end
+        end
+
         if action[:lang]
           body.css('a').each do |link| # Repair links for lang versions
             if link['href']
-              if !link['href'].include?('http') 
-                if link['href'][0] == '/'
+              if !link['href'].include?('http') && !link['href'].include?('mailto:') && !link['href'].include?('tel:')
+                if link['href'][0] == '/' && ( !link['href'].include?('.jpg') || !link['href'].include?('.jpeg') || !link['href'].include?('.png') || !link['href'].include?('.webp'))
                   link['href'] = "/#{action[:lang]}#{link['href']}"
                 else
                   link['href'] = "/#{action[:lang]}/#{link['href']}"
                 end
               end
+              if link['title']
+                link['title'] = translate(link['title'], action[:lang], client)
+              end
+            end
+          end
+        end
+
+        body.css('a').each do |link|
+          if link['href']
+            if link['href'][0] != '/'
+              link['href'] = "/#{link['href']}"
+            end
+            link['href'] = link['href'].gsub('/..', '')
+          end
+        end
+
+        if action[:lang]
+          body.traverse do |node|
+            if node.text? && !node.parent.name.in?(%w[script style])
+              if node.text.length > 1
+                if !node.content.include?('<%= render')
+                  node.content = translate(node.text, action[:lang], client)
+                end
+              end
             end
           end
         end
@@ -177,7 +316,27 @@ module ParserGem
       repair_css(file_path, controller_name) # Repair links in inline css
       progress.increment
     end
-    
+
+    if options[:inline_styles_to_files]
+      inline_styles.each do |style|
+        File.open(File.join('app/assets/stylesheets', style[:filename]), 'wb') do |file|
+          file.puts style[:content]
+        end
+        repair_css("app/assets/stylesheets/#{style[:filename]}", controller_name)
+      end
+    end
+
+    if options[:inline_scripts_to_files]
+      inline_scripts.each do |script|
+        File.open(File.join('app/javascript/', script[:filename]), 'wb') do |file|
+          file.puts script[:content]
+        end
+        File.open('app/javascript/application.js', 'a') do |file|
+          file.puts("import \"#{script[:filename]}\"") # add downloaded js file to js including file
+        end
+      end
+    end
+
     # editing app/views/layouts/application.html.erb template
     File.open("app/views/layouts/application.html.erb", 'w') do |file|
       new_layout = "<!DOCTYPE html><html><head><%= yield :head %></head><body><%= yield %></body></html>"
@@ -190,10 +349,10 @@ module ParserGem
     private
 
     def translate(text_to_translate, target_lang, client)
-        client.translate_text({ text: "#{text_to_translate}", # required
-        source_language_code: "#{options[:target_site_language]}", # required
-        target_language_code: "#{target_lang}", # required
-        settings: { formality: "FORMAL" } }).translated_text
+      client.translate_text({ text: "#{text_to_translate}", # required
+      source_language_code: "#{options[:target_site_language]}", # required
+      target_language_code: "#{target_lang}", # required
+      settings: { formality: "FORMAL" } }).translated_text
     end
 
     def download_css(css_url, controller_name) # Method for save css table to file

data/lib/parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Parser
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parsergem
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Дмитрий Герасименко
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-12 00:00:00.000000000 Z
+date: 2023-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor