norikra 0.0.21-java → 0.0.22-java

data/lib/norikra/engine.rb

@@ -44,7 +44,7 @@ module Norikra
     def statistics
       s = @statistics
       {
-        started: s[:started].httpdate,
+        started: s[:started].rfc2822,
         uptime: self.uptime,
         memory: self.memory_statistics,
         input_events: s[:events][:input],

data/lib/norikra/output_pool.rb

@@ -30,6 +30,14 @@ module Norikra
       end
     end
 
+    # returns [[time, event], ...], but not remove from pool
+    def fetch(query_name)
+      events = @mutex.synchronize do
+        @pool.fetch(query_name, [])
+      end
+      events.reduce(&:+) || []
+    end
+
     # returns [[time(int from epoch), event], ...], event: hash
     def pop(query_name)
       events = @mutex.synchronize do

data/lib/norikra/query.rb

@@ -264,6 +264,7 @@ module Norikra
       traverse_fields(rewriter, recaller, statement_model)
     end
 
+    ### Targets and fields re-writing supports (*) nodes
     # model.methods.select{|m| m.to_s.start_with?('get')}
     # :getContextName,
     # :getCreateContext,
@@ -278,7 +279,7 @@ module Norikra
     # :getForClause,
     # (*) :getFromClause,
     # (*) :getGroupByClause,
-    # :getHavingClause,
+    # (*) :getHavingClause,
     # :getInsertInto,
     # :getMatchRecognizeClause,
     # :getOnExpr,
@@ -293,8 +294,6 @@ module Norikra
 
     def self.traverse_fields(rewriter, recaller, statement_model)
       #NOTICE: SQLStream is not supported yet.
-      #TODO: other clauses with fields, especially: Having, For
-
       dig = lambda {|node|
         rewriter.call(node)
 
@@ -368,6 +367,10 @@ module Norikra
         end
       end
 
+      if statement_model.getHavingClause
+        dig.call(statement_model.getHavingClause)
+      end
+
       statement_model
     end
   end

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "0.0.21"
+  VERSION = "0.0.22"
 end

data/lib/norikra/webui/handler.rb

@@ -5,11 +5,13 @@ require 'norikra/logger'
 require 'norikra/webui'
 
 require 'sinatra/base'
+require 'sinatra/json'
+require 'erubis'
 
 class Norikra::WebUI::Handler < Sinatra::Base
   set :public_folder, File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'public'))
   set :views, File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'views'))
-  # set :erb, :escape_html => true # hmmm, this doesn't works...
+  set :erb, :escape_html => true
 
   enable :sessions
 
@@ -111,4 +113,53 @@ class Norikra::WebUI::Handler < Sinatra::Base
     end
   end
 
+  get '/json/query/:name' do
+    query_name = params[:name]
+    logging(:show, :json_query, [query_name]) do
+      query = engine.queries.select{|q| q.name == query_name}.first
+      if query
+        content = {
+          name: query.name,
+          group: query.group || "(default)",
+          targets: query.targets,
+          expression: query.expression,
+          events: engine.output_pool.fetch(query.name).size
+        }
+        json content
+      else
+        halt 404
+      end
+    end
+  end
+
+  get '/json/events_sample/:query_name' do
+    query_name = params[:query_name]
+    logging(:show, :json_query, [query_name]) do
+      query = engine.queries.select{|q| q.name == query_name}.first
+      if query
+        events = engine.output_pool.fetch(query.name).last(5)
+        json events
+      else
+        halt 404
+      end
+    end
+  end
+
+  get '/json/target/:name' do
+    target_name = params[:name]
+    logging(:show, :json_target, [target_name]) do
+      target = engine.targets.select{|t| t.name == target_name}.first
+      if target
+        content = {
+          name: target.name,
+          auto_field: target.auto_field,
+          fields: engine.typedef_manager.field_list(target.name)
+        }
+        json content
+      else
+        halt 404
+      end
+    end
+  end
+
 end

data/norikra.gemspec

@@ -21,10 +21,12 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "mizuno", "~> 0.6"
   spec.add_runtime_dependency "rack"
-  spec.add_runtime_dependency "sinatra"
-  spec.add_runtime_dependency "msgpack-rpc-over-http-jruby", ">= 0.0.5"
   spec.add_runtime_dependency "thor"
+  spec.add_runtime_dependency "msgpack-rpc-over-http-jruby", ">= 0.0.5"
   spec.add_runtime_dependency "norikra-client-jruby", ">= 0.0.7"
+  spec.add_runtime_dependency "sinatra"
+  spec.add_runtime_dependency "sinatra-contrib"
+  spec.add_runtime_dependency "erubis"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/public/js/norikra.webui.js

@@ -0,0 +1,74 @@
+$(function(){
+
+  $('.show-query-expression').each(function(i,e){
+    $(this).bind('click', function(e){
+      var button = $(this);
+      if (! button.data('loaded')) {
+        $.get(button.data('load'), function(data){
+          button.attr('data-loaded', 'true');
+          button.popover({
+            placement: 'left',
+            html: true,
+            title: 'name:' + data.name + ', group:' + data.group,
+            content: '<pre class="query-expression" style="border: none; width: 100%;">' + data.expression + '</pre>'
+          }).popover('toggle');
+          $('pre.query-expression').closest('div').css('padding', '0');
+        });
+        e.preventDefault();
+      }
+    });
+  });
+
+  $('.show-query-events-sample').each(function(i,e){
+    $(this).bind('click', function(e){
+      var button = $(this);
+      if (! button.data('loaded')) {
+        $.get(button.data('load'), function(data){
+          var events_texts = data.map(function(d){ return JSON.stringify(d); }).join("\n");
+          button.attr('data-loaded', 'true');
+          button.popover({
+            placement: 'left',
+            html: true,
+            title: 'Events sample',
+            content: '<pre class="query-events-sample" style="border: none; width: 100%;">' + events_texts + '</pre>'
+          }).popover('toggle');
+          $('pre.query-events-sample').closest('div').css('padding', '0');
+        });
+        e.preventDefault();
+      }
+    });
+  });
+
+  $('.show-target-fields').each(function(i,e){
+    $(this).bind('click', function(e){
+      var button = $(this);
+      if (! button.data('loaded')) {
+        $.get(button.data('load'), function(data){
+          var field_rows_html = data.fields.sort(function(a,b){
+            if (a.name > b.name) return 1;
+            if (a.name < b.name) return -1;
+            return 0;
+          }).map(function(t){
+            return '<tr style="border:none;"><td>' + t.name
+              + '</td><td>' + t.type + '</td><td>' + (t.optional ? '(optional)' : '') + '</td></tr>';
+          }).join('');
+          var table_html = '<table class="target-fields" style="border: 0; width: 100%;">'
+                + '<tr><th>field</th><th>type</th><th></th></tr>'
+                + field_rows_html
+                + '</table>';
+          button.attr('data-loaded', 'true');
+          button.popover({
+            placement: 'top',
+            html: true,
+            content: table_html
+          }).popover('toggle');
+          $('table.target-fields').closest('div').css('padding', '0');
+        });
+        e.preventDefault();
+      }
+    });
+  });
+
+  $('#query_add_editor_toggle').click(function(e){ $('#query_add_editor').toggle(); });
+});
+

data/spec/query_spec.rb

@@ -358,17 +358,26 @@ describe Norikra::Query do
           mapping = {'StreamA' => 'S1', 'StreamB' => 'S2'}
           expect(Norikra::Query.rewrite_query(model, mapping).toEPL).to eql(x10)
 
-          e11 = 'select campaign.id as campaign_id, member.region as region, member.lang as lang, count(*) as click, count(distinct member.id) as uu from applog_linereward.win:time_batch(1 minutes) where type = "click" group by campaign.id, member.region, member.lang'
-          x11 = 'select campaign$id as campaign_id, member$region as region, member$lang as lang, count(*) as click, count(distinct member$id) as uu from A1.win:time_batch(1 minutes) where type = "click" group by campaign$id, member$region, member$lang'
+          # GROUP BY clause
+          e11 = 'select applog.campaign.id as campaign_id, member.region as region, member.lang as lang, count(*) as click, count(distinct member.id) as uu from applog.win:time_batch(1 minutes) where type = "click" group by applog.campaign.id, member.region, member.lang'
+          x11 = 'select A1.campaign$id as campaign_id, member$region as region, member$lang as lang, count(*) as click, count(distinct member$id) as uu from A1.win:time_batch(1 minutes) where type = "click" group by A1.campaign$id, member$region, member$lang'
           model = administrator.compileEPL(e11)
-          mapping = {'applog_linereward' => 'A1'}
+          mapping = {'applog' => 'A1'}
           expect(Norikra::Query.rewrite_query(model, mapping).toEPL).to eql(x11)
 
-          e11 = 'select campaign.id, member.region as region, member.lang as lang, count(*) as click, count(distinct member.id) as uu from applog_linereward.win:time_batch(1 minutes) where type = "click" group by campaign.id, member.region, member.lang order by campaign.id'
-          x11 = 'select campaign$id, member$region as region, member$lang as lang, count(*) as click, count(distinct member$id) as uu from A1.win:time_batch(1 minutes) where type = "click" group by campaign$id, member$region, member$lang order by campaign$id'
-          model = administrator.compileEPL(e11)
-          mapping = {'applog_linereward' => 'A1'}
-          expect(Norikra::Query.rewrite_query(model, mapping).toEPL).to eql(x11)
+          # ORDER BY clause
+          e12 = 'select campaign.id, member.region as region, member.lang as lang, count(*) as click, count(distinct member.id) as uu from applog.win:time_batch(1 minutes) where type = "click" group by campaign.id, member.region, member.lang order by campaign.id'
+          x12 = 'select campaign$id, member$region as region, member$lang as lang, count(*) as click, count(distinct member$id) as uu from A1.win:time_batch(1 minutes) where type = "click" group by campaign$id, member$region, member$lang order by campaign$id'
+          model = administrator.compileEPL(e12)
+          mapping = {'applog' => 'A1'}
+          expect(Norikra::Query.rewrite_query(model, mapping).toEPL).to eql(x12)
+
+          # HAVING clause
+          e13 = 'select path, max(response.duration) from logs.win:time_batch(10 seconds) where path.startsWith("/api/") group by path having max(response.duration) >= 100'
+          x13 = 'select path, max(response$duration) from L111.win:time_batch(10 seconds) where path.startsWith("/api/") group by path having max(response$duration) >= 100'
+          model = administrator.compileEPL(e13)
+          mapping = {'logs' => 'L111'}
+          expect(Norikra::Query.rewrite_query(model, mapping).toEPL).to eql(x13)
         end
       end
     end

data/views/base.erb

@@ -52,7 +52,7 @@
     </div>
 
     <div class="container theme-showcase">
-      <%= yield %>
+      <%== yield %>
     </div> <!-- /container -->
 
     <!-- Bootstrap core JavaScript
@@ -61,12 +61,7 @@
     <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
     <script src="//code.jquery.com/jquery.js"></script>
     <script src="/js/bootstrap.min.js"></script>
-    <script>
-      $(function(){
-        $('.show-query-expression').each(function(i){ console.log(this); $(this).popover() });
-        $('.show-target-fields').each(function(i){ console.log(this); $(this).popover() });
-      })
-    </script>
+    <script src="/js/norikra.webui.js"></script>
   </body>
 </html>
 

data/views/index.erb

@@ -1,5 +1,6 @@
 <div class="page-header">
   <h1 id="summary">Runtime Summary</h1>
+  at: <%= Time.now.rfc2822 %>
 </div>
 
 <% [["HEAP", :heap],["NON-HEAP", :nonheap]].each do |name,sym| %>
@@ -64,20 +65,27 @@
 
 <% if queries.size > 0 %>
 <table class="table">
-  <tr><th>Group</th><th>Query name</th><th>Targets</th><th>Query</th><th>Events</th><th></th></tr>
+  <tr><th>Group</th><th>Query name</th><th>Targets</th><th>Query</th><th style="text-align:right;">Events</th><th></th><th></th></tr>
   <% queries.each do |query| %>
   <tr>
     <td><%= query.group || "(default)" %></td>
     <td><%= query.name %></td>
     <td><%= query.targets.join(", ") %></td>
     <td>
-      <button class="btn btn-default btn-xs show-query-expression"
-        data-toggle="popover" data-placement="left" data-html="true"
-        data-content="query:<pre><%= escape_html query.expression %></pre>">show query</button>
+      <button class="btn btn-default btn-xs show-query-expression" data-load="/json/query/<%= query.name %>">show query</button>
     </td>
-    <td><%= pool[query.name] %></td>
+    <td style="text-align: right;"><%= pool[query.name] %></td>
     <td>
-      <a class="btn btn-danger btn-xs" data-toggle="modal" href="#removeQuery<%= query.name %>"><span class="glyphicon glyphicon-trash"></span></a>
+      <% if pool[query.name].to_i > 0 %>
+      <button class="btn btn-info btn-xs show-query-events-sample" data-load="/json/events_sample/<%= query.name %>">
+        <span class="glyphicon glyphicon-search"></span>
+      </button>
+      <% end %>
+    </td>
+    <td>
+      <a class="btn btn-danger btn-xs" data-toggle="modal" href="#removeQuery<%= query.name %>">
+        <span class="glyphicon glyphicon-trash"></span>
+      </a>
       <div class="modal fade"
         id="removeQuery<%= query.name %>"
         tabindex="-1" role="dialog" aria-labelledby="removeQueryLabel<%= query.name %>" aria-hidden="true">
@@ -109,10 +117,9 @@
 <p>No queries found.</p>
 <% end %>
 
-<h3 id="query_add">Add Query</h3>
-<!-- TODO: toggle button this "Add Query", and opened with input_data[:query_add] -->
-<div class="well">
-  <% input_data ||= {}; input_data[:query_add] ||= {} %>
+<h3 id="query_add">Add Query <button class="btn btn-warning btn-xs" id="query_add_editor_toggle">editor</button></h3>
+<% input_data ||= {}; input_data[:query_add] ||= {} %>
+<div id="query_add_editor" class="well" style="display: <%= input_data[:query_add].empty? ? "none" : "block" %>;">
   <% if input_data[:query_add][:error] %>
     <div class="alert alert-danger"><%= input_data[:query_add][:error] %></div>
   <% end %>
@@ -151,15 +158,12 @@
     <td><%= target[:auto_field] %></td>
     <td>
       <% if target[:fields].size > 0 %>
-        <% field_html = target[:fields].map{|t| "<tr><td>#{t[:name]}</td><td>#{t[:type]}</td><td>#{t[:optional] ? '(optional)' : ''}</td></tr>"}.join %>
-        <button class="btn btn-default btn-xs show-target-fields"
-          data-toggle="popover" data-placement="top" data-html="true"
-          data-content="<table><tr><th>name</th><th>type</th><th></th></tr><%= field_html %></table>">show fields</button>
+        <button class="btn btn-default btn-xs show-target-fields" data-load="/json/target/<%= target[:name] %>">show fields</button>
       <% else %>
         (lazy target)
       <% end %>
     </td>
-    <td>TODO: reserve fields, remove targets</td>
+    <td style="font-size: small; color: #c0c0c0;">TODO: change auto_fields, reserve fields, remove targets</td>
   </tr>
   <% end %>
 </table>

metadata

@@ -2,14 +2,14 @@
 name: norikra
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.0.21
+  version: 0.0.22
 platform: java
 authors:
 - TAGOMORI Satoshi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-10-25 00:00:00.000000000 Z
+date: 2013-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mizuno
@@ -44,7 +44,7 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: sinatra
+  name: thor
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -76,7 +76,23 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: norikra-client-jruby
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: sinatra
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -92,18 +108,34 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: norikra-client-jruby
+  name: sinatra-contrib
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.7
+        version: '0'
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.7
+        version: '0'
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: erubis
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
     none: false
   prerelease: false
   type: :runtime
@@ -313,6 +345,7 @@ files:
 - public/fonts/glyphicons-halflings-regular.woff
 - public/js/bootstrap.js
 - public/js/bootstrap.min.js
+- public/js/norikra.webui.js
 - script/spec_server_pry
 - spec/field_spec.rb
 - spec/fieldset_spec.rb