gscomp 0.0.1

data/.gitignore

@@ -0,0 +1,24 @@
+*.gem
+<<<<<<< HEAD
+.bundle
+Gemfile.lock
+pkg/*
+=======
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+>>>>>>> 9632f3f284daadba0f9055f01f538b1f2b5e6dd3

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in gscomp.gemspec
+gemspec
+
+gem nokogiri

data/README.md

@@ -0,0 +1,4 @@
+gscomp
+======
+
+gscomp

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/gscomp

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+system("gem install gscomp")
+
+require 'gscomp'
+
+tt = Gscomp::Taras.new()
+tt.main()

data/gscomp.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "gscomp/version"
+
+Gem::Specification.new do |s|
+  s.name        = "gscomp"
+  s.version     = Gscomp::VERSION
+  s.authors     = ["Global System"]
+  s.email       = ["gs@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{GS computation}
+  s.description = %q{GS computation}
+
+  s.rubyforge_project = "gscomp"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/gscomp/version.rb

@@ -0,0 +1,3 @@
+module Gscomp
+  VERSION = "0.0.1"
+end

data/lib/gscomp.rb

@@ -0,0 +1,305 @@
+# encoding: utf-8
+require "gscomp/version"
+
+require 'rubygems'
+require 'nokogiri'
+require 'open-uri'
+require 'net/https'
+require 'net/http'
+require 'json'
+require 'gtk2'
+
+
+module Gscomp
+  class Taras
+    private 
+      def initialize
+        # Разрешаем отображение окна со всеми дочерними элементами
+        @status = Gtk::StatusIcon.new
+        @status.stock = Gtk::Stock::EXECUTE
+        @status.tooltip = "Обработчик данных GS"
+
+        # Связывание сигналов иконки с обработчиками
+        @status.signal_connect('activate') { on_activate }
+        @status.signal_connect('popup-menu') {|statusicon, button, time| on_right_click statusicon, button, time }
+        
+        # Создание меню
+        @menu = Gtk::Menu.new
+        @menu.append(preferences = Gtk::ImageMenuItem.new(Gtk::Stock::PREFERENCES))
+        @menu.append(about = Gtk::ImageMenuItem.new(Gtk::Stock::ABOUT))
+        @menu.append(Gtk::SeparatorMenuItem.new)
+        @menu.append(quit = Gtk::ImageMenuItem.new(Gtk::Stock::QUIT))
+        
+        # Связывание обработчиков с пунктами меню
+        preferences.signal_connect('activate') { on_click_preferences }
+        about.signal_connect('activate') { on_click_about }
+        quit.signal_connect('activate') { Gtk.main_quit }
+
+        # @window = Gtk::Window.new
+        # @window.border_width = 10
+        # @window.signal_connect('delete_event') { @window.hide_all; true } 
+
+        @processed = 0 
+        @power = 3
+      end
+    protected
+      def on_right_click(statusicon, button, time)
+        @menu.popup(nil, nil, button, time) {|menu, x, y, push_in| @status.position_menu(@menu)}
+        @menu.show_all
+        @menu.reposition
+      end
+      
+      def on_click_preferences
+        pref = Gtk::Dialog.new(
+          "Параметры",
+          nil,
+          Gtk::Dialog::MODAL,
+          [ Gtk::Stock::APPLY, Gtk::Dialog::RESPONSE_APPLY ],
+          [ Gtk::Stock::CLOSE, Gtk::Dialog::RESPONSE_CLOSE ]
+        )
+        pref.default_response = Gtk::Dialog::RESPONSE_APPLY
+
+        label = Gtk::Label.new ("Нагрузка:")
+        hscale = Gtk::HScale.new(10, 100, 10)
+        hscale.width_request = 350
+        hscale.value = @power * 10        
+        table = Gtk::Table.new(1, 2)
+        table.attach_defaults(label, 0, 1, 0, 1)
+        table.attach_defaults(hscale, 1, 2, 0, 1)
+        table.row_spacings = 5
+        table.column_spacings = 5
+        table.border_width = 10
+        pref.vbox.add table
+        pref.show_all
+
+        exit = true
+        while exit do 
+          pref.run do |response|
+            case response
+              when Gtk::Dialog::RESPONSE_APPLY
+                @power = hscale.value / 10
+              when Gtk::Dialog::RESPONSE_CLOSE
+                exit = false
+            end
+          end
+        end
+        pref.destroy
+
+      end
+
+      def on_click_about
+        about = Gtk::AboutDialog.new
+        about.name = "GSComp"
+        about.version = "0.0.1"
+        about.copyright = "(c) Global System, 2012"
+        about.comments = "Программа является частью распределенной сети обработки данных"
+        about.signal_connect('response') { about.destroy }
+        about.show_all
+      end
+      
+      def on_activate
+        # @window.show_all
+      end
+
+      # Инициализация
+      def get_client_id 
+        # Регистрация на сервере
+        uri = URI.parse('http://localhost:3000/idfactory')
+        uri.user = 'tester'
+        uri.password = 'qwzxas'
+        begin
+          Net::HTTP.get(uri)
+        rescue
+          nil 
+        end 
+      end
+
+      # Запрос задания
+      def get_job (client_id)
+        uri = URI.parse('http://localhost:3000/products')
+        uri.user = 'tester'
+        uri.password = 'qwzxas'
+        uri.query = URI.encode_www_form({:id => client_id})
+        begin     
+          JSON::parse(Net::HTTP.get(uri))
+        rescue
+          nil
+        end
+      end
+     
+      # Выбрать название товара
+      def get_product_name(node = nil, recurse_level = 0)
+        # Выйти если рекурсия достигла предела
+        return if (recurse_level == 0)
+
+        n = node
+        images = nil
+        recurse_level.times do
+          break unless (images = n.css("a img")).nil?    
+          n = n.parent
+        end
+
+        # images = node.css("a img")
+        result = []
+        node.css("a").each do |x|
+          weight = 0
+          # Признак: цифры в ссылке (обычно это id товара)
+          weight += 1 if (not x["href"].nil?) and x["href"].partition(/\d{0,}/)[1].length > 0
+          # Признак: цифры, -, () в названии товара
+          weight += 10 if x.content.partition(/[\-\.\(\)\d+\"]/u)[1].length > 0
+          # Признак:  ссылка в картинке совпадает с ссылкой в названии товара
+          weight += 100 if (not images.nil?) and (images.map { |i| i.parent["href"] == x["href"] }).find_index(true)
+          # Если сработал хотя-бы один признак записать результат
+          result << {:content => x.content, :weight => weight, :href => x["href"]} if weight >= 10
+        end
+
+
+        max = {:weight => 0}
+        result.each do |i|
+          max = i if i[:weight] > max[:weight]
+        end
+
+        # puts result.to_s
+        return max if max[:weight] > 10
+
+        # Выйти если признаки не обнаружены
+        # return result if (not result[:content].nil?) and result[:content].length > 0
+
+        # Если названий нет, то переместиться выше
+        get_product_name node.parent, recurse_level - 1
+      end
+
+      # Выбрать цену товара
+      def get_product_price(node = nil, recurse_level = 0, result = {:price => "", :currency => ""})
+        # Выйти если рекурсия достигла предела
+        return result if (recurse_level == 0)
+
+        # Выбрать суммы
+        node.parent.children.each do |n|
+          result[:price] = n.content.partition(/(?i)\d+.*\.{0,}\,{0,}\d{2}+/)[1].gsub(/\s/,'') if result[:price].length == 0
+          result[:currency] = n.content.partition(/(?i)\b\D{3}\b/)[1] if result[:currency].length == 0
+          
+          exit = 
+            (not result[:price].length == 0) and 
+            (not result[:currency].length == 0)
+          return result if exit
+        end
+        # Если сумм нет, то переместиться на уровень выше
+        get_product_price node.parent, recurse_level - 1, result
+      end
+
+      # Загузить содежимое станицы по URL
+      def html_get(uri)
+        page = nil
+        begin
+          Timeout::timeout(80) do
+            open(URI(uri), {:redirect => true, :read_timeout => 60}) do |cite|
+              # Получть заголовок
+              cite.meta
+              break unless cite.content_type["text/html"]
+              page = cite.read
+            end
+            # @debug_log.debug "Страница #{uri} загружена успешно..."
+          end
+        rescue Exception => err
+          # error = "file: #{__FILE__} line: #{__LINE__} comment: load html page '#{uri.to_s}'; #{err}"
+          # @log.error error
+          return nil
+        end
+        return page
+      end
+
+        
+      # Обработка задания
+      def process_job(job)
+        records = job['records']
+
+        prices = []
+        records.each do |r|
+          #  @debug_log.debug "Обработка страницы: #{url.to_s}"
+    
+          html_data = html_get(r['link'])
+          # puts r['link']
+          # Парсинг страницы
+          data = html_data.to_s.force_encoding('UTF-8').encode("utf-16be", :undef => :replace, :invalid => :replace, :replace => "?").encode("utf-8", :undef => :replace, :invalid => :replace, :replace => "?")
+          html = Nokogiri::HTML(data, nil, 'UTF-8')
+          html.search('script').remove
+
+          # Выбрать все ноды содержимое которых = грн.
+          nodes = html.css("[text()*='грн.']")
+
+          # Попытаться попдобрать цены и название товаров
+          nodes.each do |node|
+            name = get_product_name(node, 7)
+            price = get_product_price(node, 2)
+
+            # next if name.length == 0 or price.length == 0
+
+            prices << {
+              :name => name.nil? ? '' : name, 
+              :price => price.nil? ? '' : price[:price],
+              :currency => price.nil? ? '' : price[:currency],
+              :page_id => r['page_id'].to_s,
+              :link => r['link']
+            }
+          end
+          @processed += 1
+        end
+        return prices
+      end
+
+      # Отправка результатов
+      def send_job(result)
+        uri = URI.parse('http://localhost:3000/products')
+        uri.user = 'tester'
+        uri.password = 'qwzxas'
+        return Net::HTTP.post_form(uri, {:result => result.to_json})
+      end
+    public
+      def process_packet
+        sleep 1 while not (client_id = get_client_id())
+        #puts client_id
+
+        attempt = 10
+        sleep 1 while not (job = get_job(client_id) and attempt -= 1)
+        false unless job
+        # puts job
+
+        records = process_job(job)
+        # puts records
+
+        result = {
+           :client_id => client_id,
+           :records_count => records.length,
+           :records => records
+        }
+
+        send_result = send_job(result) 
+        case send_result
+          when Net::HTTPSuccess, Net::HTTPRedirection
+            p 'Good'
+          else
+            # p result.error
+        end
+      end    
+
+      def main
+        uu = Thread.new do
+          # Запускаем цикл обработки событий
+          loop do
+            process_packet()
+            sleep @power
+          end
+        end
+
+        GLib::Timeout.add(1000) { on_timer }
+
+        Gtk.main
+      end
+
+      def on_timer
+        @status.tooltip = "Global System\nОбработано: #{@processed.to_s} единц"
+        true
+      end
+  end
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: gscomp
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Global System
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-16 00:00:00.000000000Z
+dependencies: []
+description: GS computation
+email:
+- gs@gmail.com
+executables:
+- gscomp
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/gscomp
+- gscomp.gemspec
+- lib/gscomp.rb
+- lib/gscomp/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: gscomp
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: GS computation
+test_files: []