sentra 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5aae7672bc44a10bda509b23c28cfa305ba59f9
-  data.tar.gz: 7a14358e04931d5823e4ff24a227b404e0f01024
+  metadata.gz: 53eebda466208eb48275bd61128748e7ca7f2ff6
+  data.tar.gz: 5ad2fe6740404c2667add696685efcb306d7a2e4
 SHA512:
-  metadata.gz: 5ad86147736ec4b9b897f1550c15534908e12204d35f5c0b986b66b1ba9dd91cbd7ab6b40f9484daa9f0a714fd3a949f2336975a8d4661a151fa148d582636ec
-  data.tar.gz: 3a49675508ee76ad2de61fa111577a1219e3741d7b56638917034d436f915ca2d22702f62d6546a2660afafb27dcebda61c64dad8fc3e0f4ad03ab3651a53209
+  metadata.gz: a98e98d92cbc68ed9ccb509e7be8d81023bbbedb89f34269def7457f25255c49a9f4a034ae20a2cfbe0e8031013988244d000f66ba864f1232ef7c7e4c26ff02
+  data.tar.gz: 275514cc4bb7a06c80fdffa984ff5783d0c8093063f10d06c9d4e676353a340c7cb57908ff586f72f002b193fbf7f1aa3bca5e9336daaae877a22cf86c63bcad

data/bin/config.ru

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+
 require "sentra"
 
 run Sentra::SentraService

data/bin/config.yml

@@ -0,0 +1 @@
+parallelism: 4

data/lib/sentra/opr.rb

@@ -34,6 +34,22 @@ module SemanticService
 
 end
 
+module Storm
+  include_package "crriskdata.opr.nlp.storm"
+  import "crriskdata.opr.nlp.storm"
+end
+
+module StormDRPC
+  include_package "crriskdata.opr.nlp.storm.drpc"
+  import "crriskdata.opr.nlp.storm.drpc"
+end
+
+module StormDrpcTopology
+  include_package "crriskdata.opr.nlp.storm.rabbitmq"
+  import "crriskdata.opr.nlp.storm.rabbitmq"
+end
+
+
 module ArticleTools
   class ArticleSerializer
     def initialize
@@ -54,4 +70,36 @@ module ArticleTools
       @simpleFormat.parse(input)
     end
   end
+end
+
+def prepare_article content, url, title, domain='GENERAL'
+  tweet = OprData::Article.new
+
+  calendar = Util::Calendar.getInstance();
+  tweet.setUrl(Net::URL.new(url));
+  tweet.setDate(calendar.getTime)
+  tweet.setTitle(title)
+  tweet.setChannel(domain)
+  tweet.setContent(content)
+  tweet
+end
+
+
+def process_result statistics
+  response = {}
+  # puts 'here statistics'
+  # p statistics
+  if statistics[3].getValue.to_f > 0
+    response['negative'] = statistics[0].getValue.to_f / statistics[3].getValue.to_f
+    response['positive'] = statistics[1].getValue.to_f / statistics[3].getValue.to_f
+    response['neutral'] = statistics[2].getValue.to_f / statistics[3].getValue.to_f
+
+  else
+    response['negative'] = 0
+    response['positive'] = 0
+    response['neutral'] = 1
+
+  end
+  response['details'] = {}
+  response
 end

data/lib/sentra/version.rb

@@ -1,3 +1,3 @@
 module Sentra
-  VERSION = "0.2.2"
+  VERSION = "0.2.4"
 end

data/lib/sentra.rb

@@ -7,6 +7,7 @@ require 'sentra/opr'
 require 'sentra/rest_client'
 require "sentra/sents"
 require 'securerandom'
+require 'yaml'
 
 
 module Sentra
@@ -15,8 +16,37 @@ module Sentra
   puts "Hello, gem "
   class SentraService < Sinatra::Base
     configure do
+
+      config_path = Dir.pwd  + "/config.yml"
+      p config_path
+      if File.exists? config_path
+        topolog_settings = YAML.load_file config_path
+        puts 'Config file exists and read'
+      else
+        topolog_settings = {}
+        topolog_settings['parallelism'] = 4
+        puts 'Config file does not exist parallelism set to 3'
+      end
+      p topolog_settings
+
+      s = java.util.HashSet.new
+      s.add(StormDrpcTopology::STAGES::NORMALIZE_STAGE.name());
+      s.add(StormDrpcTopology::STAGES::SENTENCES_STAGE.name());
+      s.add(StormDrpcTopology::STAGES::POSTAG_STAGE.name());
+      s.add(StormDrpcTopology::STAGES::ENTITIES_STAGE.name());
+      s.add(StormDrpcTopology::STAGES::SENTRA_STAGE.name());
+      s.add(StormDrpcTopology::STAGES::TWITTER_STAGE.name());
+
+
+      spout = StormDRPC::ArticlesMapSpout
+      bolt = StormDRPC::ArticlesMapBolt
+      cluster = StormDrpcTopology::ServiceSemanticToplogy.createLocalCluster(StormDRPC::ArticlesMapSpout.new,
+                                                                             StormDRPC::ArticlesMapBolt.new, s, topolog_settings['parallelism'].to_i)
       set :server, :trinidad
       set :bind, '0.0.0.0'
+      set :spout, spout
+      set :bolt, bolt
+      set :cluster, cluster
     end
 
 
@@ -30,39 +60,8 @@ module Sentra
     end
 
 
-    post '/sentra/analyzer/atomic' do
-      query = request.body.read.to_s
-      begin
-        query = JSON.parse(query)
-      rescue Exception => e
-        puts e.message
-        query = {}
-        query['content'] = nil
-      end
-      if params['key'] and params['key'] =="QD3F7Yr2u098RfaB" and query['content']
-        processing_client = Sentra::ProcessingClient.new
-        processing_client.prepare_article query['content'],
-                                          "http://" +SecureRandom.urlsafe_base64 + ".com/" + SecureRandom.urlsafe_base64,
-                                          "Test1"
-        processing_client.send
-        response = processing_client.receive
-        begin
-          processing_client.process_result(response).to_json
-        rescue Exception => e
-          puts e.message
-          response = {'error' => e.message}
-          response.to_json
-        end
-
-      elsif params['key'] and params['key'] =="QD3F7Yr2u098RfaB" and not query['content']
-        return "There is no text to process"
-      else
-        return "Not authorized"
-      end
-    end
-
-
     post '/sentra/analyzer/batch' do
+
       query = request.body.read.to_s
       begin
         query = JSON.parse(query)
@@ -70,44 +69,26 @@ module Sentra
         puts 'Error with JSON'
         query = nil
       end
-      if params['key'] and params['key'] =="QD3F7Yr2u098RfaB" and query
-        p query
-        responses = []
-        clients = []
-        query.each_with_index do |tweet, index|
-          begin
-            processing_client = Sentra::ProcessingClient.new
-
-            processing_client.prepare_article tweet['content'],
-                                              "http://" + SecureRandom.urlsafe_base64 + ".com/" + SecureRandom.urlsafe_base64,
-                                              "Test" + index.to_s
-            processing_client.send
-            clients.push processing_client
-
-          rescue
-            puts 'Errors with getting result from socket or queue'
-          end
-        end
-        clients.each do |client|
-          begin
-
-            response = client.receive
-            p response
-            responses.push(client.process_result(response))
-          rescue
-          end
-        end
-
-        begin
-          responses.to_json
-        rescue Exception => e
-          puts 'Errors with converting result to json'
-        end
-      elsif params['key'] and params['key'] =="QD3F7Yr2u098RfaB" and not query and not query['content']
-        return "There is no text to process"
-      else
-        return "Not authorized"
+
+      list = java.util.ArrayList.new
+      query.each_with_index do |tweet, index|
+        list.add(tweet['content'])
       end
+
+      responses = []
+
+      key = SecureRandom.urlsafe_base64
+      settings.spout.putArticle(key, list);
+      article_cl = settings.bolt.getArticle(key)
+
+      # puts "next"
+      # p article_cl
+      article_cl.each { |a|
+        facts = a.getMetadata('twitter').getFacts()
+        responses.push(process_result(facts))
+      }
+
+      responses.to_json
     end
 
 

data/tools/multithread_batch.rb

@@ -0,0 +1,60 @@
+require 'tools_helper'
+
+class CheckBatch
+  def initialize id, host='localhost:4567'
+    @id=id
+    mycsv = CSV_MODULE::MyCSV.new('auxillary/testdata.csv')
+    @queries = mycsv.get_queries
+    @iterations = 100
+    @url = "http://#{host}/sentra/analyzer/batch"
+  end
+
+  def call
+    puts 'STARTED THREAD NUMBER: ' + @id.to_s
+
+    r = Random.new
+
+    batch_query = []
+
+    @iterations.times do |time|
+      q = {}
+      q['content'] = @queries[r.rand(0..490)]['content']
+      batch_query.push q
+    end
+
+
+    response = RestClient2.post(@url, batch_query.to_json)
+
+    json = JSON.parse(response.body)
+    p json
+
+  end
+end
+
+
+
+num_threads = 1
+host = '10.1.2.230:4567'
+
+tasks = []
+executor = ThreadPoolExecutor.new(num_threads, # core_pool_treads
+                                  num_threads + 10, # max_pool_threads
+                                  60, # keep_alive_time
+                                  TimeUnit::SECONDS,
+                                  LinkedBlockingQueue.new)
+
+
+num_threads.times do |i|
+  crawler = CheckBatch.new(i) do |selff|
+  end
+
+  task = FutureTask.new(crawler)
+  executor.execute(task)
+  tasks << task
+end
+
+tasks.each do |t|
+  t.get
+end
+
+executor.shutdown()

data/tools/multithread_check.rb

@@ -35,7 +35,7 @@ class CheckLoad
       q['content'] = @queries[r.rand(0..490)]['content']
       puts "QUERY TO SEND: '" + q['content'] + "'"
 
-      url = "http://localhost:4567/sentra/analyzer/atomic?key=QD3F7Yr2u098RfaB"
+      url = "http://10.1.2.230:4567/sentra/analyzer/atomic?key=QD3F7Yr2u098RfaB"
       response = RestClient2.post(url, q.to_json)
 
       json = JSON.parse(response.body)
@@ -46,8 +46,8 @@ class CheckLoad
 end
 
 
-executor = ThreadPoolExecutor.new(100, # core_pool_treads
-                                  110, # max_pool_threads
+executor = ThreadPoolExecutor.new(20, # core_pool_treads
+                                  30, # max_pool_threads
                                   60, # keep_alive_time
                                   TimeUnit::SECONDS,
                                   LinkedBlockingQueue.new)

data/tools/singlethread_check.rb

@@ -9,7 +9,7 @@ q = {}
 q['content'] = 'Hans Morris makes excellent point about P2P risks should be.'
 puts "QUERY TO SEND: '" +  q['content'] + "'"
 
-url = "http://localhost:4567/sentra/analyzer/atomic?key=QD3F7Yr2u098RfaB"
+url = "http://10.1.2.230:4567/sentra/analyzer/atomic?key=QD3F7Yr2u098RfaB"
 response = RestClient2.post(url, q.to_json)
 
 json = JSON.parse(response.body)

data/tools/tools_helper.rb

@@ -0,0 +1,8 @@
+java_import 'java.util.concurrent.Callable'
+java_import 'java.util.concurrent.FutureTask'
+java_import 'java.util.concurrent.LinkedBlockingQueue'
+java_import 'java.util.concurrent.ThreadPoolExecutor'
+java_import 'java.util.concurrent.TimeUnit'
+
+require 'sentra/rest_client'
+require "sentra/sents"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sentra
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - kshakirov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2015-05-07 00:00:00.000000000 Z
+date: 2015-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -138,6 +138,7 @@ files:
 - README_INSTALL
 - Rakefile
 - bin/config.ru
+- bin/config.yml
 - bin/console
 - bin/setup
 - lib/sentra.rb
@@ -150,8 +151,10 @@ files:
 - tools/auxillary/tweets.csv
 - tools/calculate_sents_diff.rb
 - tools/compare_sentiments.rb
+- tools/multithread_batch.rb
 - tools/multithread_check.rb
 - tools/singlethread_check.rb
+- tools/tools_helper.rb
 homepage: https://rubygems.org
 licenses:
 - MIT