gscomp 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +24 -0
- data/Gemfile +6 -0
- data/README.md +4 -0
- data/Rakefile +1 -0
- data/bin/gscomp +8 -0
- data/gscomp.gemspec +24 -0
- data/lib/gscomp/version.rb +3 -0
- data/lib/gscomp.rb +305 -0
- metadata +54 -0
data/.gitignore
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
*.gem
|
2
|
+
<<<<<<< HEAD
|
3
|
+
.bundle
|
4
|
+
Gemfile.lock
|
5
|
+
pkg/*
|
6
|
+
=======
|
7
|
+
*.rbc
|
8
|
+
.bundle
|
9
|
+
.config
|
10
|
+
coverage
|
11
|
+
InstalledFiles
|
12
|
+
lib/bundler/man
|
13
|
+
pkg
|
14
|
+
rdoc
|
15
|
+
spec/reports
|
16
|
+
test/tmp
|
17
|
+
test/version_tmp
|
18
|
+
tmp
|
19
|
+
|
20
|
+
# YARD artifacts
|
21
|
+
.yardoc
|
22
|
+
_yardoc
|
23
|
+
doc/
|
24
|
+
>>>>>>> 9632f3f284daadba0f9055f01f538b1f2b5e6dd3
|
data/Gemfile
ADDED
data/README.md
ADDED
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/gscomp
ADDED
data/gscomp.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "gscomp/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "gscomp"
|
7
|
+
s.version = Gscomp::VERSION
|
8
|
+
s.authors = ["Global System"]
|
9
|
+
s.email = ["gs@gmail.com"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{GS computation}
|
12
|
+
s.description = %q{GS computation}
|
13
|
+
|
14
|
+
s.rubyforge_project = "gscomp"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
# specify any dependencies here; for example:
|
22
|
+
# s.add_development_dependency "rspec"
|
23
|
+
# s.add_runtime_dependency "rest-client"
|
24
|
+
end
|
data/lib/gscomp.rb
ADDED
@@ -0,0 +1,305 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "gscomp/version"
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'open-uri'
|
7
|
+
require 'net/https'
|
8
|
+
require 'net/http'
|
9
|
+
require 'json'
|
10
|
+
require 'gtk2'
|
11
|
+
|
12
|
+
|
13
|
+
module Gscomp
|
14
|
+
class Taras
|
15
|
+
private
|
16
|
+
def initialize
|
17
|
+
# Разрешаем отображение окна со всеми дочерними элементами
|
18
|
+
@status = Gtk::StatusIcon.new
|
19
|
+
@status.stock = Gtk::Stock::EXECUTE
|
20
|
+
@status.tooltip = "Обработчик данных GS"
|
21
|
+
|
22
|
+
# Связывание сигналов иконки с обработчиками
|
23
|
+
@status.signal_connect('activate') { on_activate }
|
24
|
+
@status.signal_connect('popup-menu') {|statusicon, button, time| on_right_click statusicon, button, time }
|
25
|
+
|
26
|
+
# Создание меню
|
27
|
+
@menu = Gtk::Menu.new
|
28
|
+
@menu.append(preferences = Gtk::ImageMenuItem.new(Gtk::Stock::PREFERENCES))
|
29
|
+
@menu.append(about = Gtk::ImageMenuItem.new(Gtk::Stock::ABOUT))
|
30
|
+
@menu.append(Gtk::SeparatorMenuItem.new)
|
31
|
+
@menu.append(quit = Gtk::ImageMenuItem.new(Gtk::Stock::QUIT))
|
32
|
+
|
33
|
+
# Связывание обработчиков с пунктами меню
|
34
|
+
preferences.signal_connect('activate') { on_click_preferences }
|
35
|
+
about.signal_connect('activate') { on_click_about }
|
36
|
+
quit.signal_connect('activate') { Gtk.main_quit }
|
37
|
+
|
38
|
+
# @window = Gtk::Window.new
|
39
|
+
# @window.border_width = 10
|
40
|
+
# @window.signal_connect('delete_event') { @window.hide_all; true }
|
41
|
+
|
42
|
+
@processed = 0
|
43
|
+
@power = 3
|
44
|
+
end
|
45
|
+
protected
|
46
|
+
def on_right_click(statusicon, button, time)
|
47
|
+
@menu.popup(nil, nil, button, time) {|menu, x, y, push_in| @status.position_menu(@menu)}
|
48
|
+
@menu.show_all
|
49
|
+
@menu.reposition
|
50
|
+
end
|
51
|
+
|
52
|
+
def on_click_preferences
|
53
|
+
pref = Gtk::Dialog.new(
|
54
|
+
"Параметры",
|
55
|
+
nil,
|
56
|
+
Gtk::Dialog::MODAL,
|
57
|
+
[ Gtk::Stock::APPLY, Gtk::Dialog::RESPONSE_APPLY ],
|
58
|
+
[ Gtk::Stock::CLOSE, Gtk::Dialog::RESPONSE_CLOSE ]
|
59
|
+
)
|
60
|
+
pref.default_response = Gtk::Dialog::RESPONSE_APPLY
|
61
|
+
|
62
|
+
label = Gtk::Label.new ("Нагрузка:")
|
63
|
+
hscale = Gtk::HScale.new(10, 100, 10)
|
64
|
+
hscale.width_request = 350
|
65
|
+
hscale.value = @power * 10
|
66
|
+
table = Gtk::Table.new(1, 2)
|
67
|
+
table.attach_defaults(label, 0, 1, 0, 1)
|
68
|
+
table.attach_defaults(hscale, 1, 2, 0, 1)
|
69
|
+
table.row_spacings = 5
|
70
|
+
table.column_spacings = 5
|
71
|
+
table.border_width = 10
|
72
|
+
pref.vbox.add table
|
73
|
+
pref.show_all
|
74
|
+
|
75
|
+
exit = true
|
76
|
+
while exit do
|
77
|
+
pref.run do |response|
|
78
|
+
case response
|
79
|
+
when Gtk::Dialog::RESPONSE_APPLY
|
80
|
+
@power = hscale.value / 10
|
81
|
+
when Gtk::Dialog::RESPONSE_CLOSE
|
82
|
+
exit = false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
pref.destroy
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
def on_click_about
|
91
|
+
about = Gtk::AboutDialog.new
|
92
|
+
about.name = "GSComp"
|
93
|
+
about.version = "0.0.1"
|
94
|
+
about.copyright = "(c) Global System, 2012"
|
95
|
+
about.comments = "Программа является частью распределенной сети обработки данных"
|
96
|
+
about.signal_connect('response') { about.destroy }
|
97
|
+
about.show_all
|
98
|
+
end
|
99
|
+
|
100
|
+
def on_activate
|
101
|
+
# @window.show_all
|
102
|
+
end
|
103
|
+
|
104
|
+
# Инициализация
|
105
|
+
def get_client_id
|
106
|
+
# Регистрация на сервере
|
107
|
+
uri = URI.parse('http://localhost:3000/idfactory')
|
108
|
+
uri.user = 'tester'
|
109
|
+
uri.password = 'qwzxas'
|
110
|
+
begin
|
111
|
+
Net::HTTP.get(uri)
|
112
|
+
rescue
|
113
|
+
nil
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
# Запрос задания
|
118
|
+
def get_job (client_id)
|
119
|
+
uri = URI.parse('http://localhost:3000/products')
|
120
|
+
uri.user = 'tester'
|
121
|
+
uri.password = 'qwzxas'
|
122
|
+
uri.query = URI.encode_www_form({:id => client_id})
|
123
|
+
begin
|
124
|
+
JSON::parse(Net::HTTP.get(uri))
|
125
|
+
rescue
|
126
|
+
nil
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# Выбрать название товара
|
131
|
+
def get_product_name(node = nil, recurse_level = 0)
|
132
|
+
# Выйти если рекурсия достигла предела
|
133
|
+
return if (recurse_level == 0)
|
134
|
+
|
135
|
+
n = node
|
136
|
+
images = nil
|
137
|
+
recurse_level.times do
|
138
|
+
break unless (images = n.css("a img")).nil?
|
139
|
+
n = n.parent
|
140
|
+
end
|
141
|
+
|
142
|
+
# images = node.css("a img")
|
143
|
+
result = []
|
144
|
+
node.css("a").each do |x|
|
145
|
+
weight = 0
|
146
|
+
# Признак: цифры в ссылке (обычно это id товара)
|
147
|
+
weight += 1 if (not x["href"].nil?) and x["href"].partition(/\d{0,}/)[1].length > 0
|
148
|
+
# Признак: цифры, -, () в названии товара
|
149
|
+
weight += 10 if x.content.partition(/[\-\.\(\)\d+\"]/u)[1].length > 0
|
150
|
+
# Признак: ссылка в картинке совпадает с ссылкой в названии товара
|
151
|
+
weight += 100 if (not images.nil?) and (images.map { |i| i.parent["href"] == x["href"] }).find_index(true)
|
152
|
+
# Если сработал хотя-бы один признак записать результат
|
153
|
+
result << {:content => x.content, :weight => weight, :href => x["href"]} if weight >= 10
|
154
|
+
end
|
155
|
+
|
156
|
+
|
157
|
+
max = {:weight => 0}
|
158
|
+
result.each do |i|
|
159
|
+
max = i if i[:weight] > max[:weight]
|
160
|
+
end
|
161
|
+
|
162
|
+
# puts result.to_s
|
163
|
+
return max if max[:weight] > 10
|
164
|
+
|
165
|
+
# Выйти если признаки не обнаружены
|
166
|
+
# return result if (not result[:content].nil?) and result[:content].length > 0
|
167
|
+
|
168
|
+
# Если названий нет, то переместиться выше
|
169
|
+
get_product_name node.parent, recurse_level - 1
|
170
|
+
end
|
171
|
+
|
172
|
+
# Выбрать цену товара
|
173
|
+
def get_product_price(node = nil, recurse_level = 0, result = {:price => "", :currency => ""})
|
174
|
+
# Выйти если рекурсия достигла предела
|
175
|
+
return result if (recurse_level == 0)
|
176
|
+
|
177
|
+
# Выбрать суммы
|
178
|
+
node.parent.children.each do |n|
|
179
|
+
result[:price] = n.content.partition(/(?i)\d+.*\.{0,}\,{0,}\d{2}+/)[1].gsub(/\s/,'') if result[:price].length == 0
|
180
|
+
result[:currency] = n.content.partition(/(?i)\b\D{3}\b/)[1] if result[:currency].length == 0
|
181
|
+
|
182
|
+
exit =
|
183
|
+
(not result[:price].length == 0) and
|
184
|
+
(not result[:currency].length == 0)
|
185
|
+
return result if exit
|
186
|
+
end
|
187
|
+
# Если сумм нет, то переместиться на уровень выше
|
188
|
+
get_product_price node.parent, recurse_level - 1, result
|
189
|
+
end
|
190
|
+
|
191
|
+
# Загузить содежимое станицы по URL
|
192
|
+
def html_get(uri)
|
193
|
+
page = nil
|
194
|
+
begin
|
195
|
+
Timeout::timeout(80) do
|
196
|
+
open(URI(uri), {:redirect => true, :read_timeout => 60}) do |cite|
|
197
|
+
# Получть заголовок
|
198
|
+
cite.meta
|
199
|
+
break unless cite.content_type["text/html"]
|
200
|
+
page = cite.read
|
201
|
+
end
|
202
|
+
# @debug_log.debug "Страница #{uri} загружена успешно..."
|
203
|
+
end
|
204
|
+
rescue Exception => err
|
205
|
+
# error = "file: #{__FILE__} line: #{__LINE__} comment: load html page '#{uri.to_s}'; #{err}"
|
206
|
+
# @log.error error
|
207
|
+
return nil
|
208
|
+
end
|
209
|
+
return page
|
210
|
+
end
|
211
|
+
|
212
|
+
|
213
|
+
# Обработка задания
|
214
|
+
def process_job(job)
|
215
|
+
records = job['records']
|
216
|
+
|
217
|
+
prices = []
|
218
|
+
records.each do |r|
|
219
|
+
# @debug_log.debug "Обработка страницы: #{url.to_s}"
|
220
|
+
|
221
|
+
html_data = html_get(r['link'])
|
222
|
+
# puts r['link']
|
223
|
+
# Парсинг страницы
|
224
|
+
data = html_data.to_s.force_encoding('UTF-8').encode("utf-16be", :undef => :replace, :invalid => :replace, :replace => "?").encode("utf-8", :undef => :replace, :invalid => :replace, :replace => "?")
|
225
|
+
html = Nokogiri::HTML(data, nil, 'UTF-8')
|
226
|
+
html.search('script').remove
|
227
|
+
|
228
|
+
# Выбрать все ноды содержимое которых = грн.
|
229
|
+
nodes = html.css("[text()*='грн.']")
|
230
|
+
|
231
|
+
# Попытаться попдобрать цены и название товаров
|
232
|
+
nodes.each do |node|
|
233
|
+
name = get_product_name(node, 7)
|
234
|
+
price = get_product_price(node, 2)
|
235
|
+
|
236
|
+
# next if name.length == 0 or price.length == 0
|
237
|
+
|
238
|
+
prices << {
|
239
|
+
:name => name.nil? ? '' : name,
|
240
|
+
:price => price.nil? ? '' : price[:price],
|
241
|
+
:currency => price.nil? ? '' : price[:currency],
|
242
|
+
:page_id => r['page_id'].to_s,
|
243
|
+
:link => r['link']
|
244
|
+
}
|
245
|
+
end
|
246
|
+
@processed += 1
|
247
|
+
end
|
248
|
+
return prices
|
249
|
+
end
|
250
|
+
|
251
|
+
# Отправка результатов
|
252
|
+
def send_job(result)
|
253
|
+
uri = URI.parse('http://localhost:3000/products')
|
254
|
+
uri.user = 'tester'
|
255
|
+
uri.password = 'qwzxas'
|
256
|
+
return Net::HTTP.post_form(uri, {:result => result.to_json})
|
257
|
+
end
|
258
|
+
public
|
259
|
+
def process_packet
|
260
|
+
sleep 1 while not (client_id = get_client_id())
|
261
|
+
#puts client_id
|
262
|
+
|
263
|
+
attempt = 10
|
264
|
+
sleep 1 while not (job = get_job(client_id) and attempt -= 1)
|
265
|
+
false unless job
|
266
|
+
# puts job
|
267
|
+
|
268
|
+
records = process_job(job)
|
269
|
+
# puts records
|
270
|
+
|
271
|
+
result = {
|
272
|
+
:client_id => client_id,
|
273
|
+
:records_count => records.length,
|
274
|
+
:records => records
|
275
|
+
}
|
276
|
+
|
277
|
+
send_result = send_job(result)
|
278
|
+
case send_result
|
279
|
+
when Net::HTTPSuccess, Net::HTTPRedirection
|
280
|
+
p 'Good'
|
281
|
+
else
|
282
|
+
# p result.error
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
def main
|
287
|
+
uu = Thread.new do
|
288
|
+
# Запускаем цикл обработки событий
|
289
|
+
loop do
|
290
|
+
process_packet()
|
291
|
+
sleep @power
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
GLib::Timeout.add(1000) { on_timer }
|
296
|
+
|
297
|
+
Gtk.main
|
298
|
+
end
|
299
|
+
|
300
|
+
def on_timer
|
301
|
+
@status.tooltip = "Global System\nОбработано: #{@processed.to_s} единц"
|
302
|
+
true
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
metadata
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gscomp
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Global System
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-06-16 00:00:00.000000000Z
|
13
|
+
dependencies: []
|
14
|
+
description: GS computation
|
15
|
+
email:
|
16
|
+
- gs@gmail.com
|
17
|
+
executables:
|
18
|
+
- gscomp
|
19
|
+
extensions: []
|
20
|
+
extra_rdoc_files: []
|
21
|
+
files:
|
22
|
+
- .gitignore
|
23
|
+
- Gemfile
|
24
|
+
- README.md
|
25
|
+
- Rakefile
|
26
|
+
- bin/gscomp
|
27
|
+
- gscomp.gemspec
|
28
|
+
- lib/gscomp.rb
|
29
|
+
- lib/gscomp/version.rb
|
30
|
+
homepage: ''
|
31
|
+
licenses: []
|
32
|
+
post_install_message:
|
33
|
+
rdoc_options: []
|
34
|
+
require_paths:
|
35
|
+
- lib
|
36
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
37
|
+
none: false
|
38
|
+
requirements:
|
39
|
+
- - ! '>='
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
43
|
+
none: false
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
requirements: []
|
49
|
+
rubyforge_project: gscomp
|
50
|
+
rubygems_version: 1.8.17
|
51
|
+
signing_key:
|
52
|
+
specification_version: 3
|
53
|
+
summary: GS computation
|
54
|
+
test_files: []
|