parsergem 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +13 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +70 -0
- data/LICENSE.txt +21 -0
- data/README.md +105 -0
- data/Rakefile +7 -0
- data/lib/generators/parser_gem/test_generator.rb +340 -0
- data/lib/parser/version.rb +5 -0
- data/lib/parser.rb +7 -0
- data/sig/parser.rbs +4 -0
- metadata +141 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 631bcfdba7aac41738b184b905b8b1b9ee47bc907666a297afd8f8a0d89d854f
|
4
|
+
data.tar.gz: bd3bfac34984ea9cd940fd874379cd60ecf4cf7cd0edf3811e824640a580a3f4
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1159dc126a68bf509c1ffc9b0ca74e37c3ae6c5a639cfd065ff1e80d73ecf99d80b796582998213584562bb1f82cbbc06b3677882c57166a15172e3780173aa4
|
7
|
+
data.tar.gz: aa186c04bd4ecbe2628544fb8ee625324304cf418ca2de153d13969baebaf641ea00db09ea5a6a59f71b5ed7b1396b4e2b09f41204943084a0a309c9dcbe0642
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
parsergem (0.1.1)
|
5
|
+
activesupport (>= 5.0.0)
|
6
|
+
aws-sdk-translate
|
7
|
+
nokogiri
|
8
|
+
rack (>= 1.4.0)
|
9
|
+
ruby-progressbar
|
10
|
+
thor
|
11
|
+
|
12
|
+
GEM
|
13
|
+
remote: https://rubygems.org/
|
14
|
+
specs:
|
15
|
+
activesupport (7.0.4.3)
|
16
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
17
|
+
i18n (>= 1.6, < 2)
|
18
|
+
minitest (>= 5.1)
|
19
|
+
tzinfo (~> 2.0)
|
20
|
+
aws-eventstream (1.2.0)
|
21
|
+
aws-partitions (1.763.0)
|
22
|
+
aws-sdk-core (3.172.0)
|
23
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
24
|
+
aws-partitions (~> 1, >= 1.651.0)
|
25
|
+
aws-sigv4 (~> 1.5)
|
26
|
+
jmespath (~> 1, >= 1.6.1)
|
27
|
+
aws-sdk-translate (1.50.0)
|
28
|
+
aws-sdk-core (~> 3, >= 3.165.0)
|
29
|
+
aws-sigv4 (~> 1.1)
|
30
|
+
aws-sigv4 (1.5.2)
|
31
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
32
|
+
concurrent-ruby (1.2.2)
|
33
|
+
diff-lcs (1.5.0)
|
34
|
+
i18n (1.13.0)
|
35
|
+
concurrent-ruby (~> 1.0)
|
36
|
+
jmespath (1.6.2)
|
37
|
+
minitest (5.18.0)
|
38
|
+
nokogiri (1.14.3-x86_64-linux)
|
39
|
+
racc (~> 1.4)
|
40
|
+
racc (1.6.2)
|
41
|
+
rack (3.0.7)
|
42
|
+
rake (13.0.6)
|
43
|
+
rspec (3.12.0)
|
44
|
+
rspec-core (~> 3.12.0)
|
45
|
+
rspec-expectations (~> 3.12.0)
|
46
|
+
rspec-mocks (~> 3.12.0)
|
47
|
+
rspec-core (3.12.1)
|
48
|
+
rspec-support (~> 3.12.0)
|
49
|
+
rspec-expectations (3.12.2)
|
50
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
51
|
+
rspec-support (~> 3.12.0)
|
52
|
+
rspec-mocks (3.12.5)
|
53
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
54
|
+
rspec-support (~> 3.12.0)
|
55
|
+
rspec-support (3.12.0)
|
56
|
+
ruby-progressbar (1.13.0)
|
57
|
+
thor (1.2.1)
|
58
|
+
tzinfo (2.0.6)
|
59
|
+
concurrent-ruby (~> 1.0)
|
60
|
+
|
61
|
+
PLATFORMS
|
62
|
+
x86_64-linux
|
63
|
+
|
64
|
+
DEPENDENCIES
|
65
|
+
parsergem!
|
66
|
+
rake (~> 13.0)
|
67
|
+
rspec (~> 3.0)
|
68
|
+
|
69
|
+
BUNDLED WITH
|
70
|
+
2.4.9
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2023 d1mentor
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
# English
|
2
|
+
|
3
|
+
# Parsergemv 0.1
|
4
|
+
|
5
|
+
The gem is intended for copying and automatically integrating the front-end of the target site into Rails. It is also possible to automatically translate site pages, and create a multilingual site structure in Rails.
|
6
|
+
|
7
|
+
Runtime versions tested:
|
8
|
+
Ruby 3.0.0
|
9
|
+
Rails 7.0.4.3
|
10
|
+
|
11
|
+
## Installation
|
12
|
+
|
13
|
+
Install the gem and add it to the Gemfile:
|
14
|
+
|
15
|
+
$ bundle add parsergem
|
16
|
+
|
17
|
+
If you are not using bundler, you can install the gem like this:
|
18
|
+
|
19
|
+
$ gem install parsergem
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
It is important that the target site meets certain requirements:
|
24
|
+
1) The presence of a sitemap, at "target.com/sitemap.xml"
|
25
|
+
2) If you need to make language versions, the target site must be monolingual
|
26
|
+
|
27
|
+
At this stage, the gem provides one generator to perform the declared functions.
|
28
|
+
After installing the gem, run the generator:
|
29
|
+
|
30
|
+
$ rails g parser_gem:test --target-url "target.com"
|
31
|
+
|
32
|
+
The generator takes the following parameters: <br />
|
33
|
+
For normal cloning: <br />
|
34
|
+
`--target_url` - Domain name of the target site <br />
|
35
|
+
To template the header and footer of the site: <br />
|
36
|
+
`--header_class_name` - ID of the block in which the header is located <br />
|
37
|
+
`--footer_class_name` - ID of the block containing the footer <br />
|
38
|
+
To create language versions (You must fill in all parameters): <br />
|
39
|
+
`--target_site_language` - Current site language <br />
|
40
|
+
`--languages` - "en es de" format string, with a list of language codes to which the original site should be translated <br />
|
41
|
+
`--aws_region` - AWS Region <br />
|
42
|
+
`--aws_public_key` - The public key of your AWS TRANSLATE API <br />
|
43
|
+
`--aws_secret_key` - The secret key of your AWS TRANSLATE API <br />
|
44
|
+
|
45
|
+
After the generator has finished its work, you will most likely need to modify the header and footer parshals to fix the links.
|
46
|
+
|
47
|
+
## Contributing
|
48
|
+
|
49
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/d1mentor/parser.
|
50
|
+
|
51
|
+
# Russian
|
52
|
+
|
53
|
+
# Parsergem v 0.1
|
54
|
+
|
55
|
+
Гем предназначен для копирования и автоматической интеграции в Rails фронтенда целевого сайта. Так-же есть возможность автоматического перевода страниц сайта, и создания в Rails структуры мультиязычного сайта.
|
56
|
+
|
57
|
+
Проверенные версии среды выполнения:
|
58
|
+
Ruby 3.0.0
|
59
|
+
Rails 7.0.4.3
|
60
|
+
|
61
|
+
## Установка
|
62
|
+
|
63
|
+
Установить гем и добавить его в Gemfile:
|
64
|
+
|
65
|
+
$ bundle add parsergem
|
66
|
+
|
67
|
+
Если вы не используете bundler, установить гем можно так:
|
68
|
+
|
69
|
+
$ gem install parsergem
|
70
|
+
|
71
|
+
## Использование
|
72
|
+
|
73
|
+
Важно, что-бы целевой сайт соответствовал некоторым требованиям:
|
74
|
+
1) Наличие карты сайта, по адресу "target.com/sitemap.xml"
|
75
|
+
2) Если необходимо сделать языковые версии, целевой сайт должен быть моноязычным
|
76
|
+
|
77
|
+
На данном этапе гем предоставляет один генератор, для выполнения заявленных функций.
|
78
|
+
После установки гема, запустите генератор:
|
79
|
+
|
80
|
+
$ rails g parser_gem:test --target-url "target.com"
|
81
|
+
|
82
|
+
Генератор принимает следующие параметры: <br />
|
83
|
+
Для обычного клонирования: <br />
|
84
|
+
`--target_url` - Доменное имя целевого сайта <br />
|
85
|
+
Для шаблонизации хедера и футера сайта: <br />
|
86
|
+
`--header_class_name` - ID блока в котором находится хедер <br />
|
87
|
+
`--footer_class_name` - ID блока в котором находится футер <br />
|
88
|
+
Для создания языковых версий(Необходимо заполнить все параметры): <br />
|
89
|
+
`--target_site_language` - Текущий язык сайта <br />
|
90
|
+
`--languages` - Строка формата "en es de", с перечнем кодов языков, на которые стоит перевести оригинальный сайт <br />
|
91
|
+
`--aws_region` - Регион AWS <br />
|
92
|
+
`--aws_public_key` - Публичный ключ вашего AWS TRANSLATE API <br />
|
93
|
+
`--aws_secret_key` - Секретный ключ вашего AWS TRANSLATE API <br />
|
94
|
+
|
95
|
+
После того как генератор закончит свою работу, вам, скорее всего, придётся доработать паршалы хедеров и футеров для исправления ссылок.
|
96
|
+
|
97
|
+
## Содействие
|
98
|
+
|
99
|
+
Сообщения об ошибках и запросы на доработку гема приветствуются на GitHub по адресу https://github.com/d1mentor/parser
|
100
|
+
|
101
|
+
## License
|
102
|
+
|
103
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
104
|
+
|
105
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,340 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'openssl'
|
4
|
+
require 'ruby-progressbar'
|
5
|
+
require 'aws-sdk-translate'
|
6
|
+
OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE # For dodge SSL error
|
7
|
+
|
8
|
+
module ParserGem
|
9
|
+
class TestGenerator < Rails::Generators::Base
|
10
|
+
desc 'Clone site frontend from url'
|
11
|
+
class_option :target_url, type: :string, default: nil # Target site url, site must have sitemap.xml
|
12
|
+
class_option :target_site_language, type: :string, default: nil # Set target site language
|
13
|
+
class_option :languages, type: :string, default: nil # Additional languages
|
14
|
+
class_option :google_api_key, type: :string, default: nil # User google API key for translate pages
|
15
|
+
class_option :header_class_name, type: :string, default: nil # Header block id, for create partial
|
16
|
+
class_option :footer_class_name, type: :string, default: nil # Footer block id, for create partial
|
17
|
+
class_option :aws_region, type: :string, default: nil # Set aws region for translate API
|
18
|
+
class_option :aws_public_key, type: :string, default: nil # Set aws api public key
|
19
|
+
class_option :aws_secret_key, type: :string, default: nil # Set aws api secret key
|
20
|
+
|
21
|
+
def clone # Main method
|
22
|
+
sitemap = Nokogiri::XML(URI.open("http://#{options[:target_url]}/sitemap.xml")) # Load sitemap.xml
|
23
|
+
controller_name = options[:target_url].split('/').last.delete('.').delete('-') # Set controller name from clear domain name
|
24
|
+
actions = set_actions(sitemap) # Set actions for controller
|
25
|
+
|
26
|
+
actions.each do |action| # Create routes
|
27
|
+
if action[:rails_route] == '/'
|
28
|
+
route "root \"#{controller_name}##{action[:action_name]}\""
|
29
|
+
else
|
30
|
+
route "get \'#{action[:rails_route]}\', to: \"#{controller_name}##{action[:action_name]}\""
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
actions_list = ""
|
35
|
+
actions.each do |action| # Create actions list for rails generator
|
36
|
+
actions_list += " " + action[:action_name]
|
37
|
+
end
|
38
|
+
|
39
|
+
generate "controller", "#{controller_name}#{actions_list} --skip-routes" # Create controller with action list
|
40
|
+
|
41
|
+
puts "Start cloning #{options[:target_url]}"
|
42
|
+
|
43
|
+
progress = ProgressBar.create(:format => "%a %b\u{15E7}%i %p%% %t", # Setup progressbar
|
44
|
+
:progress_mark => ' ',
|
45
|
+
:remainder_mark => "\u{FF65}",
|
46
|
+
:starting_at => 0,
|
47
|
+
:total => actions.size)
|
48
|
+
|
49
|
+
credentials = Aws::Credentials.new( options[:aws_public_key], options[:aws_secret_key])
|
50
|
+
client = Aws::Translate::Client.new( region: options[:aws_region],
|
51
|
+
credentials: credentials)
|
52
|
+
|
53
|
+
actions.each do |action|
|
54
|
+
file_path = File.join('app/views', controller_name, "#{action[:action_name]}.html.erb") # Set view
|
55
|
+
File.open(file_path, 'wb') do |file|
|
56
|
+
page = Nokogiri::HTML(URI.open("#{action[:native_url]}")) # Parse page from url in sitemap
|
57
|
+
|
58
|
+
if action[:lang]
|
59
|
+
page.css('title').each do |title|
|
60
|
+
if title.inner_html.length > 1
|
61
|
+
title.inner_html = translate(title.inner_html, action[:lang], client)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
page.css('meta').each do |meta|
|
66
|
+
if meta['content'].length > 1
|
67
|
+
meta['content'] = translate(meta['content'], action[:lang], client)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
page.css('img, video').each do |media|
|
72
|
+
if media['alt'].length > 1
|
73
|
+
media['alt'] = translate(media['alt'], action[:lang], client)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
page.traverse do |node|
|
78
|
+
if node.text? && !node.parent.name.in?(%w[script style])
|
79
|
+
if node.text.length > 1
|
80
|
+
node.content = translate(node.text, action[:lang], client)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
head = page.at_css('head')
|
87
|
+
body = page.at_css('body')
|
88
|
+
|
89
|
+
file.puts "<% content_for :head do %>" # Saving uniq page meta and title
|
90
|
+
head.css('title').each do |title|
|
91
|
+
file.puts title
|
92
|
+
end
|
93
|
+
|
94
|
+
head.css('meta').each do |meta|
|
95
|
+
file.puts meta
|
96
|
+
end
|
97
|
+
|
98
|
+
page.css('link[rel="stylesheet"]').each do |link| # Repair css including for Rails
|
99
|
+
css_file_path = download_css(path_to_download(link['href']), controller_name)
|
100
|
+
file.puts "<%= stylesheet_link_tag '#{css_file_path}' %>"
|
101
|
+
end
|
102
|
+
|
103
|
+
page.css('script[type="text/javascript"]').each do |link| # Repair js including for Rails
|
104
|
+
if link['src'] != nil
|
105
|
+
js_file_name = download_js(path_to_download(link['src']), controller_name)
|
106
|
+
file.puts "<%= javascript_tag '#{js_file_name}' %>"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
page.css('style').each do |style_node| # Move all style tags to head
|
111
|
+
file.puts style_node
|
112
|
+
end
|
113
|
+
|
114
|
+
page.css('script').each do |script_node| # Move all script tags to head
|
115
|
+
file.puts script_node
|
116
|
+
end
|
117
|
+
|
118
|
+
file.puts "<% end %>"
|
119
|
+
|
120
|
+
if !File.exist?("app/views/layouts/_#{action[:lang]}header.html.erb") || !File.exist?("app/views/layouts/_#{action[:lang]}footer.html.erb") # Create footer and header partials
|
121
|
+
if options[:header_class_name] # Find and save header to partial
|
122
|
+
File.open(File.join('app/views/layouts/', "_#{action[:lang]}header.html.erb"), 'w') do |file|
|
123
|
+
file.puts body.css("div[id=\"#{options[:header_class_name]}\"]").to_html
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
if options[:footer_class_name] # Find and save footer to partial
|
128
|
+
File.open(File.join('app/views/layouts/', "_#{action[:lang]}footer.html.erb"), 'w') do |file|
|
129
|
+
file.puts body.css("div[id=\"#{options[:footer_class_name]}\"]").to_html
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
body.css('img, video').each do |media| # Save media files from page, and repair src
|
135
|
+
media_file_path = download_media(path_to_download(media['src']), controller_name)
|
136
|
+
media['src'] = media_file_path
|
137
|
+
end
|
138
|
+
|
139
|
+
body.css('script, style').each do |trash| # Clear already moved to header styles and scripts
|
140
|
+
trash.replace('')
|
141
|
+
end
|
142
|
+
|
143
|
+
body.css("div[id=\"#{options[:header_class_name]}\"]").each do |header| # Call partial instead of old header block
|
144
|
+
header.replace("<%= render \"layouts/#{action[:lang]}header\" %>")
|
145
|
+
end
|
146
|
+
|
147
|
+
body.css("div[id=\"#{options[:footer_class_name]}\"]").each do |footer| # Call partial instead of old footer block
|
148
|
+
footer.replace("<%= render \"layouts/#{action[:lang]}footer\" %>")
|
149
|
+
end
|
150
|
+
|
151
|
+
body.css('a').each do |link| # Self absolute links to media repair
|
152
|
+
if link['href']
|
153
|
+
if link['href'].include?("http://#{options[:target_url]}") && (link['href'].include?('.png') || link['href'].include?('.jpg') || link['href'].include?('.jpeg') || link['href'].include?('.webp'))
|
154
|
+
link['href'] = download_media(link['href'], controller_name)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
if action[:lang]
|
160
|
+
body.css('a').each do |link| # Repair links for lang versions
|
161
|
+
if link['href']
|
162
|
+
if !link['href'].include?('http')
|
163
|
+
if link['href'][0] == '/'
|
164
|
+
link['href'] = "/#{action[:lang]}#{link['href']}"
|
165
|
+
else
|
166
|
+
link['href'] = "/#{action[:lang]}/#{link['href']}"
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
file.puts body.to_s.gsub(/(<%|%>)/) {|x| x=='<%' ? '<%' : '%>'} # Put edited body to view file, and repair ERB tags
|
174
|
+
end
|
175
|
+
repair_css(file_path, controller_name) # Repair links in inline css
|
176
|
+
progress.increment
|
177
|
+
end
|
178
|
+
|
179
|
+
# editing app/views/layouts/application.html.erb template
|
180
|
+
File.open("app/views/layouts/application.html.erb", 'w') do |file|
|
181
|
+
new_layout = "<!DOCTYPE html><html><head><%= yield :head %></head><body><%= yield %></body></html>"
|
182
|
+
file.write(new_layout.to_s.gsub(/(<%|%>)/) {|x| x=='<%' ? '<%' : '%>'})
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
# СОЗДАНИЕ ЯЗЫКОВЫХ ВЕРСИЙ
|
187
|
+
|
188
|
+
private
|
189
|
+
|
190
|
+
def translate(text_to_translate, target_lang, client)
|
191
|
+
client.translate_text({ text: "#{text_to_translate}", # required
|
192
|
+
source_language_code: "#{options[:target_site_language]}", # required
|
193
|
+
target_language_code: "#{target_lang}", # required
|
194
|
+
settings: { formality: "FORMAL" } }).translated_text
|
195
|
+
end
|
196
|
+
|
197
|
+
def download_css(css_url, controller_name) # Method for save css table to file
|
198
|
+
if !File.exist?("app/assets/stylesheets/#{File.basename(css_url)}") # Check, for dont download one file many times
|
199
|
+
begin # need catch errors, sometimes links can be broken
|
200
|
+
css_file_path = File.join('app/assets/stylesheets', File.basename(css_url))
|
201
|
+
File.open(css_file_path, 'wb') do |file|
|
202
|
+
file.write(URI.open(css_url).read) # Save css in file
|
203
|
+
end
|
204
|
+
rescue
|
205
|
+
else
|
206
|
+
repair_css(css_file_path, controller_name) # Repair links in css
|
207
|
+
"#{File.basename(css_url)}"
|
208
|
+
end
|
209
|
+
else
|
210
|
+
if File.zero?("app/assets/stylesheets/#{File.basename(css_url)}")
|
211
|
+
''
|
212
|
+
else
|
213
|
+
"#{File.basename(css_url)}" # If file already exist, just return his name
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
def download_js(js_url, controller_name) # This method working like download_css
|
219
|
+
if !File.exist?("app/javascript/#{File.basename(js_url)}") # Check, for dont download one file many times
|
220
|
+
begin # need catch errors, sometimes links can be broken
|
221
|
+
js_file_path = File.join('app/javascript', File.basename(js_url))
|
222
|
+
File.open(js_file_path, 'wb') do |file|
|
223
|
+
file.write(URI.open(js_url).read)
|
224
|
+
end
|
225
|
+
File.open('app/javascript/application.js', 'a') do |file|
|
226
|
+
file.puts("import \"#{File.basename(js_url)}\"") # add downloaded js file to js including file
|
227
|
+
end
|
228
|
+
rescue
|
229
|
+
else
|
230
|
+
"#{File.basename(js_url)}"
|
231
|
+
end
|
232
|
+
else
|
233
|
+
"#{File.basename(js_url)}"
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
def download_media(media_url, controller_name) # This method work like download_css and download_js
|
238
|
+
if !File.exist?("public/#{File.basename(media_url)}") # Check, for dont download one file many times
|
239
|
+
begin # need catch errors, sometimes links can be broken
|
240
|
+
media_file_path = File.join('public', File.basename(media_url))
|
241
|
+
File.open(media_file_path, 'wb') do |file|
|
242
|
+
file.write(URI.open(media_url).read)
|
243
|
+
end
|
244
|
+
rescue
|
245
|
+
else
|
246
|
+
"/#{File.basename(media_url)}"
|
247
|
+
end
|
248
|
+
else
|
249
|
+
"/#{File.basename(media_url)}"
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
def repair_css(css_path, controller_name) # Method for repair links in css
|
254
|
+
css_content = File.read(css_path)
|
255
|
+
css_content.gsub!(/url\((.*?)\)/i) do |match|
|
256
|
+
url = $1.gsub(/['"]/, '') # Remove quotes from url
|
257
|
+
if url.start_with?('http') # Check if url is a relative path
|
258
|
+
match # Return original link, if its third party resource
|
259
|
+
else
|
260
|
+
new_url = download_media(path_to_download(url), controller_name) #download media included in css
|
261
|
+
"url(#{new_url})"
|
262
|
+
end
|
263
|
+
end
|
264
|
+
File.write(css_path, css_content) # Rewrite file
|
265
|
+
end
|
266
|
+
|
267
|
+
def path_to_download(path) # Repair links for download files
|
268
|
+
begin
|
269
|
+
result = ""
|
270
|
+
if path.include?("http://") || path.include?("https://") || path.include?(".com")
|
271
|
+
return path
|
272
|
+
else
|
273
|
+
if path[0] == '/'
|
274
|
+
result = "http://#{options['target_url']}#{path}"
|
275
|
+
else
|
276
|
+
result = "http://#{options['target_url']}/#{path}"
|
277
|
+
end
|
278
|
+
end
|
279
|
+
rescue
|
280
|
+
else
|
281
|
+
normalize_link(result.gsub(/\?.*/, ''))
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
def normalize_link(link) # Repair links for download files
|
286
|
+
link_parts = link.split("/")
|
287
|
+
result_parts = []
|
288
|
+
|
289
|
+
link_parts.each do |part|
|
290
|
+
if part != ".."
|
291
|
+
result_parts << part
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
normalized_link = result_parts.join("/")
|
296
|
+
return normalized_link
|
297
|
+
end
|
298
|
+
|
299
|
+
|
300
|
+
def set_actions(sitemap) # Set actions from sitemap
|
301
|
+
actions = []
|
302
|
+
urls = []
|
303
|
+
languages = options[:languages].split(' ') if options[:languages]
|
304
|
+
sitemap.xpath('//xmlns:url/xmlns:loc').each do |url|
|
305
|
+
urls << url.text
|
306
|
+
end
|
307
|
+
|
308
|
+
urls.each do |url|
|
309
|
+
element = { rails_route: "#{url.gsub('http://', '').gsub('https://', '').gsub("#{options[:target_url]}", '').delete('.')}",
|
310
|
+
action_name: "#{url.gsub('http://', '').gsub('https://', '').gsub("#{options[:target_url]}", '').delete('.').delete('/').gsub('-', '_')}",
|
311
|
+
native_url: "#{url}",
|
312
|
+
lang: nil }
|
313
|
+
|
314
|
+
if element[:rails_route] == '/'
|
315
|
+
element[:action_name] = "index"
|
316
|
+
end
|
317
|
+
|
318
|
+
actions << element
|
319
|
+
|
320
|
+
if options[:languages]
|
321
|
+
languages.each do |lng_ver|
|
322
|
+
element = { rails_route: "#{lng_ver}#{url.gsub('http://', '').gsub('https://', '').gsub("#{options[:target_url]}", '').delete('.')}",
|
323
|
+
action_name: "#{lng_ver}_#{url.gsub('http://', '').gsub('https://', '').gsub("#{options[:target_url]}", '').delete('.').delete('/').gsub('-', '_')}",
|
324
|
+
native_url: "#{url}",
|
325
|
+
lang: lng_ver }
|
326
|
+
|
327
|
+
if element[:rails_route] == "#{lng_ver}/"
|
328
|
+
element[:action_name] = "#{lng_ver}_index"
|
329
|
+
end
|
330
|
+
|
331
|
+
actions << element
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
actions
|
336
|
+
end
|
337
|
+
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
data/lib/parser.rb
ADDED
data/sig/parser.rbs
ADDED
metadata
ADDED
@@ -0,0 +1,141 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: parsergem
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Дмитрий Герасименко
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-05-12 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: nokogiri
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: activesupport
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 5.0.0
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 5.0.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rack
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.4.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.4.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: ruby-progressbar
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: aws-sdk-translate
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: Ruby gem for clone site.
|
98
|
+
email:
|
99
|
+
- gerasimenkot92@gmail.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- ".rspec"
|
105
|
+
- ".rubocop.yml"
|
106
|
+
- CHANGELOG.md
|
107
|
+
- Gemfile
|
108
|
+
- Gemfile.lock
|
109
|
+
- LICENSE.txt
|
110
|
+
- README.md
|
111
|
+
- Rakefile
|
112
|
+
- lib/generators/parser_gem/test_generator.rb
|
113
|
+
- lib/parser.rb
|
114
|
+
- lib/parser/version.rb
|
115
|
+
- sig/parser.rbs
|
116
|
+
homepage: https://github.com/d1mentor/parser
|
117
|
+
licenses:
|
118
|
+
- MIT
|
119
|
+
metadata:
|
120
|
+
homepage_uri: https://github.com/d1mentor/parser
|
121
|
+
source_code_uri: https://github.com/d1mentor/parser
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: 2.6.0
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubygems_version: 3.2.3
|
138
|
+
signing_key:
|
139
|
+
specification_version: 4
|
140
|
+
summary: Ruby gem for clone site.
|
141
|
+
test_files: []
|