tupelo 0.19 → 0.20

data/example/riemann/producer.rb

@@ -1,9 +1,8 @@
 class Tupelo::Client
-  # Generate some events.
-  def run_producer i
-    event = {
+  def base_event
+    @base_event ||= {
       host:         `hostname`.chomp,
-      service:      "service #{i}",
+      service:      "process #$$",
       state:        "",
       time:         0,
       description:  "",
@@ -12,21 +11,43 @@ class Tupelo::Client
       ttl:          0,
       custom:       nil
     }.freeze
+  end
+
+  def write_event event
+    if event[:ttl] == 0.0
+      pulse event # no need to bother with expiration
+    else
+      write event
+    end
 
-    e_ok = event.merge(
+    log.info "created event #{event}"
+  end
+
+  # Generate some events.
+  def run_producer i
+    e_ok = base_event.merge(
+      service:  "service #{i}",
       state:    "ok",
       time:     Time.now.to_f,
       ttl:      0.2
     )
+    write_event e_ok
 
-    if e_ok[:ttl] == 0.0
-      pulse e_ok # no need to bother with expiration
-    else
-      write e_ok
+    30.times do |ei|
+      e_cpu = base_event.merge(
+        service:  "service #{i}",
+        state:    ei==10 ? "critical" : "ok",
+        time:     Time.now.to_f,
+        ttl:      0.2,
+        tags:     ["cpu", "cumulative"],
+        metric:   Process.times.utime + Process.times.stime
+      )
+      write_event e_cpu
+      sleep 0.2
     end
-    
-    log "created event #{e_ok}"
-    
-    sleep 0.5 # Let it expire
+
+    sleep 0.5
+      # Make sure whole set of processes stay alive long enough to see
+      # expiration happen.
   end
 end

data/example/riemann/v1/expirer.rb

@@ -0,0 +1,28 @@
+class Tupelo::Client
+  # Expire old events. Uses a scheduler (which is a thread plus an rbtree)
+  # to keep track of the expiration times and events.
+  def run_expirer_v1
+    scheduler = make_scheduler
+    
+    read subspace("event") do |event|
+      next if event["state"] == "expired"
+
+      event_exp = event["time"] + event["ttl"]
+      scheduler.at event_exp do
+        transaction do
+          take_nowait event or break
+            # Be cautious, in case of other expirer. If you can rule out
+            # this possibility, then `take event` is fine.
+          pulse event.merge("state" => "expired")
+            # Not sure if this is riemann semantics. Using #pulse rather
+            # than #write means that the expired event exists in the
+            # space only while the transaction is executing, but that
+            # momentary existence is enough to trigger any client that
+            # is waiting on a template that matches the event. Use the
+            # --debug-expiration switch to see this happening (and use
+            # -v to make log messages verbose, showing timestamps).
+        end
+      end
+    end
+  end
+end

data/example/riemann/{riemann-v1.rb → v1/riemann.rb}

@@ -4,17 +4,15 @@
 # for searching.
 
 require 'tupelo/app'
-require_relative 'event-subspace'
-require_relative 'producer'
-require_relative 'expirer-v1'
+require_relative '../event-subspace'
+require_relative '../producer'
+require_relative 'expirer'
 
 N_PRODUCERS = 3
 N_CONSUMERS = 2
 
 Tupelo.application do
-
   local do
-    use_subspaces!
     define_event_subspace
   end
 
@@ -45,11 +43,10 @@ Tupelo.application do
 
   if argv.include?("--debug-expiration")
     # expired event debugger
+    require_relative '../expiration-dbg'
     child subscribe: "event", passive: true do
       log.progname = "expiration debugger"
-      read Tupelo::Client::EXPIRED_EVENT do |event|
-        log event
-      end
+      run_expiration_debugger
     end
   end
 

data/example/riemann/v2/expirer.rb

@@ -0,0 +1,31 @@
+require_relative 'ordered-event-store'
+
+class Tupelo::Client
+  # Expire old events.
+  #
+  # A little more complex than v1, but more efficient.
+  #
+  # This version uses the rbtree to manage the space itself (in the worker
+  # thread), instead of using an rbtree to manage the scheduler (in the client
+  # thread). It also uses a custom template class to perform range-based queries
+  # of the rbtree key, which is not explicitly stored in the tuples. The rbree
+  # key is, however, the sum of two values (time + ttl) in the tuples, so the
+  # template is still selecting tuples based on their contents.
+  #
+  def run_expirer_v2
+    loop do
+      event = read subspace("event") # event with lowest time+ttl
+      dt_next_expiration = event["time"] + event["ttl"] - Time.now.to_f
+      begin
+        transaction timeout: dt_next_expiration do
+          read OrderedEventTemplate.before(event)
+        end
+      rescue TimeoutError
+        transaction do
+          take_nowait event or break # see note in v1/expirer.rb
+          pulse event.merge("state" => "expired")
+        end
+      end
+    end
+  end
+end

data/example/riemann/v2/hash-store.rb

@@ -0,0 +1,33 @@
+require 'tupelo/archiver/tuplespace'
+
+# Store for any kind of tuple, and faster for lookups of literal tuples,
+# rather than matching with templates or other queries.
+# See also example/multitier/kvspace.rb.
+class HashStore < Tupelo::Archiver::Tuplespace
+  def initialize zero_tolerance: 1000
+    super
+  end
+
+  def find_match_for template, distinct_from: []
+    case template
+    when Array, Hash # just a tuple
+      super
+
+    else
+      # We added this case to Archiver::Tuplespace just so the read(..)
+      # will work correctly on tuples that are already in the space
+      # when this process starts up. After that point, incoming tuples
+      # are matched directly against the CRITICAL_EVENT template without
+      # searching the space.
+
+      # We're not going to use Client#take in this client, so there's no need
+      # to handle the distinct_from keyword argument.
+      raise "internal error" unless distinct_from.empty?
+
+      find do |tuple|
+        template === tuple
+      end
+    end
+  end
+end
+

data/example/riemann/v2/http-mode.rb

@@ -0,0 +1,53 @@
+# Run a web client -- depends only on http, not tupelo.
+def start_web_client
+  fork do
+    require 'http'
+
+    url = 'http://localhost:4567'
+
+    print "trying server at #{url}"
+    begin
+      print "."
+      HTTP.get url
+    rescue Errno::ECONNREFUSED
+      sleep 0.2
+      retry
+    end
+
+    puts
+    puts HTTP.get "#{url}/read"
+    HTTP.get "#{url}/exit"
+  end
+end
+
+# Run a little API server using +client+ to access tupelo.
+def run_web_server(client)
+  require 'sinatra/base'
+  require 'json'
+
+  Class.new(Sinatra::Base).class_eval do
+    get '/read' do
+      host = params["host"] # nil is ok -- matches all hosts
+      resp = client.read(
+        host:         host,
+        service:      nil,
+        state:        nil,
+        time:         nil,
+        description:  nil,
+        tags:         nil,
+        metric:       nil,
+        ttl:          nil,
+        custom:       nil
+      )
+      resp.to_json + "\n"
+    end
+    ## need way to query by existence of tag
+
+    get '/exit' do
+      Thread.new {sleep 1; exit}
+      "bye\n"
+    end
+
+    run!
+  end
+end

data/example/riemann/v2/ordered-event-store.rb

@@ -0,0 +1,128 @@
+require 'rbtree'
+
+# Hard-coded to work with tuples belonging to the "event" subspace defined in
+# event-subspace.rb and with the OrderedEventStore data structure defined below.
+class OrderedEventTemplate
+  attr_reader :expiration
+
+  def self.before event
+    new(event: event)
+  end
+
+  def initialize event: event
+    time = event["time"] || event[:time]
+    ttl = event["ttl"] || event[:ttl]
+       # We check symbol keys in case of manually specified event, rather than
+       # tuples that passed thru tupelo.
+    @expiration = time + ttl
+  end
+
+  def === other
+    begin
+      other["time"] + other["ttl"] < @expiration
+    rescue
+      false
+    end
+    # Should check that subspace("event") === other, but in this example
+    # we don't need to, we just make sure it's got time and ttl keys.
+    # Anyway, the #=== method is not used, since in this case
+    # we just lookup using RBTree#first. See the special case for
+    # OrderedEventTemplate in OrderedEventStore#find_match_for.
+    # This implementation is just for completeness.
+  end
+end
+
+# A tuple store (in-memory) that is optimized for events and for searching them
+# in expiration order. This is very much a special case and not reusable for
+# other spaces/stores without modification.
+#
+# This store also manages meta tuples, which it keeps in an array,
+# just like the default Tuplespace class does. Actually, any tuple for which
+# `tuple["time"] + tuple["ttl"]` raises an exception will go in the metas,
+# but in this example the process only subscribes to events and metas.
+#
+class OrderedEventStore
+  include Enumerable
+
+  attr_reader :tree, :metas
+
+  def initialize
+    clear
+  end
+
+  def clear
+    @tree = RBTree.new{|t,k| t[k] = []}
+    @metas = []
+      # We are automatically subscribed to tupelo metadata (subspace defs), so
+      # we need to keep them somewhere.
+  end
+
+  def each
+    tree.each do |k, events|
+      events.each do |event|
+        yield event
+      end
+    end
+    metas.each do |tuple|
+      yield tuple
+    end
+  end
+
+  def insert tuple
+    k = tuple["time"] + tuple["ttl"]
+  rescue
+    metas << tuple
+  else
+    tree[k] << tuple
+  end
+
+  def delete_once tuple
+    k = tuple["time"] + tuple["ttl"]
+  rescue
+    if i=metas.index(tuple)
+      metas.delete_at i
+    end
+  else
+    if tree.key?(k) and tree[k].include? tuple
+      tree[k].delete tuple
+      tree.delete k if tree[k].empty?
+      true
+    else
+      false
+    end
+  end
+
+  def transaction inserts: [], deletes: [], tick: nil
+    deletes.each do |tuple|
+      delete_once tuple or raise "bug"
+    end
+
+    inserts.each do |tuple|
+      insert tuple.freeze ## should be deep_freeze
+    end
+  end
+
+  def find_distinct_matches_for templates
+    templates.inject([]) do |tuples, template|
+      tuples << find_match_for(template, distinct_from: tuples)
+    end
+  end
+
+  def find_match_for template, distinct_from: []
+    case template
+    when OrderedEventTemplate
+      k, firsts = tree.first
+      k && k < template.expiration &&
+        firsts.find {|tuple| distinct_from.all? {|t| !t.equal? tuple}}
+      # The `find` clause isn't really needed, since OrderedEventTemplate is
+      # only used for reads, not takes, and anyway we never take multiple
+      # tuples in a transaction on the event space so the array would be empty.
+      # But let's be correct... Note the use of #equal?.
+    else
+      # Fall back to linear search, same as default tuplestore.
+      find do |tuple|
+        template === tuple and not distinct_from.any? {|t| t.equal? tuple}
+      end
+    end
+  end
+end

data/example/riemann/{riemann-v2.rb → v2/riemann.rb}

@@ -7,29 +7,47 @@
 #
 #   * the expiration manager need to sort by expiration time
 #
-#   * the critical event alerter doesn't need to sort at all.
+#   * the critical event alerter doesn't need to sort or search at all,
+#     it just needs efficient insert and delete. 
+#
+# Run with --http to expose a web API and run a test web client.
+#
+# You will need to `gem install rbtree sqlite3 sequel`.
+# For the --http option, you'll also need to `gem install http json sinatra`.
 
-abort "work in progress"
+USE_HTTP = ARGV.delete("--http")
+
+if USE_HTTP
+  require_relative 'http-mode'
+  start_web_client
+  at_exit {Process.waitall}
+end
 
 require 'tupelo/app'
-require_relative 'event-subspace'
-require_relative 'producer'
-require_relative 'expirer-v2'
+require_relative '../event-subspace'
+require_relative '../producer'
+require_relative 'expirer'
 
 N_PRODUCERS = 3
 N_CONSUMERS = 2
 
 Tupelo.application do
-
   local do
-    use_subspaces!
     define_event_subspace
   end
 
+  if USE_HTTP
+    # Web API using sinata to access the index of events.
+    child subscribe: "event" do |client|
+      log.progname = "web server"
+      run_web_server(client)
+    end
+  end
+
   N_PRODUCERS.times do |i|
     child subscribe: [] do # N.b., no subscriptions
       log.progname = "producer #{i}"
-      run_producer i ### V2: manual tagging
+      run_producer i
     end
   end
 
@@ -38,13 +56,14 @@ Tupelo.application do
     child subscribe: "event", passive: true do ### tuplespace: sqlite
       log.progname = "consumer #{i}"
       read subspace("event") do |event|
-        log event ### need filtering, actions, etc.
+        log.info event ### need filtering, actions, etc.
       end
     end
   end
   
   # critical event alerter
-  child subscribe: "event", passive: true do ### tuplespace: bag?
+  require_relative 'hash-store'
+  child tuplespace: HashStore, subscribe: "event", passive: true do
     log.progname = "alerter"
     read Tupelo::Client::CRITICAL_EVENT do |event|
       log.error event
@@ -53,20 +72,16 @@ Tupelo.application do
 
   if argv.include?("--debug-expiration")
     # expired event debugger
+    require_relative '../expiration-dbg'
     child subscribe: "event", passive: true do
       log.progname = "expiration debugger"
-      read Tupelo::Client::EXPIRED_EVENT do |event|
-        log event
-      end
+      run_expiration_debugger
     end
   end
 
   # expirer: stores current events and looks for events that can be expired.
-  child subscribe: "event", passive: true do
+  child tuplespace: OrderedEventStore, subscribe: "event", passive: true do
     log.progname = "expirer"
     run_expirer_v2
-    ### use rbtree
   end
-  
-  ### Add sinatra app.
 end

data/example/sqlite/poi-store.rb

@@ -0,0 +1,160 @@
+require 'sequel'
+
+# Hard-coded to work with tuples belonging to the "poi" subspace 
+# and with the PoiStore table defined below.
+class PoiTemplate
+  attr_reader :lat, :lng, :poispace
+
+  # lat and lng can be intervals or single values or nil to match any value
+  def initialize lat: nil, lng: nil, poispace: nil
+    @lat = lat
+    @lng = lng
+    @poispace = poispace
+  end
+
+  # we only need to define this method if we plan to wait for poi tuples
+  # locally using this template, i.e. read(template) or take(template)
+  def === tuple
+    @poispace === tuple and
+    !@lat || @lat === tuple["lat"] and
+    !@lng || @lng === tuple["lng"]
+  end
+
+  def find_in table, distinct_from: []
+    where_terms = {}
+    where_terms[:lat] = @lat if @lat
+    where_terms[:lng] = @lng if @lng
+
+    matches = table.
+      select(:lat, :lng, :desc).
+      where(where_terms).
+      limit(distinct_from.size + 1).all
+
+    distinct_from.each do |tuple|
+      if i=matches.index(tuple)
+        matches.delete_at i
+      end
+    end
+
+    matches.first
+  end
+end
+
+# A tuple store that is optimized for POI data. These tuples are stored in an
+# in-memory sqlite database table. Tuples that do not fit this pattern (such
+# as metatuples) are stored in an array, as in the default Tuplespace class.
+class PoiStore
+  include Enumerable
+
+  attr_reader :table, :metas, :poispace
+
+  def self.define_poispace client
+    client.define_subspace("poi",
+      lat:  Numeric,
+      lng:  Numeric,
+      desc: String
+    )
+    client.subspace("poi") ## is this awkward?
+  end
+
+  # poispace should be client.subspace("poi"), but really we only need
+  # poispace.pot
+  def initialize poispace
+    @poispace = poispace
+    clear
+  end
+  
+  def clear
+    @db = Sequel.sqlite
+    @db.create_table "poi" do
+      primary_key   :id # id is not significant to our app
+      float         :lat, null: false
+      float         :lng, null: false
+      text          :desc
+
+      ## spatial_index [:lat, :lng] # by default sqlite doesn't support this
+      index         :lat
+      index         :lng
+    end
+
+    @table = @db[:poi]
+    @metas = []
+  end
+
+  def each
+    table.each do |row|
+      yield "lat" => row[:lat], "lng" => row[:lng], "desc" => row[:desc]
+    end
+    metas.each do |tuple|
+      yield tuple
+    end
+  end
+
+  def insert tuple
+    case tuple
+    when poispace
+      table << tuple
+    else
+      metas << tuple
+    end
+  end
+
+  def delete_once tuple
+    case tuple
+    when poispace
+      id = table.select(:id).where(
+        lat:  tuple["lat"],
+        lng:  tuple["lng"],
+        desc: tuple["desc"]
+      ).limit(1)
+      count = table.where(id: id).delete
+
+      if count == 0
+        false
+      elsif count == 1
+        true
+      else
+        raise "internal error: primary key, id, was not unique"
+      end
+
+    else
+      if i=metas.index(tuple)
+        metas.delete_at i
+      end
+    end
+  end
+
+  def transaction inserts: [], deletes: [], tick: nil
+    deletes.each do |tuple|
+      delete_once tuple or raise "bug"
+    end
+
+    inserts.each do |tuple|
+      insert tuple.freeze ## should be deep_freeze
+    end
+  end
+
+  def find_distinct_matches_for templates
+    templates.inject([]) do |tuples, template|
+      tuples << find_match_for(template, distinct_from: tuples)
+    end
+  end
+
+  def find_match_for template, distinct_from: []
+    case template
+    when PoiTemplate
+      template.find_in table, distinct_from: distinct_from
+    else
+      ## if template can match subspace("poi")
+      # Fall back to linear search, same as default tuplestore.
+      find do |tuple|
+        template === tuple and not distinct_from.any? {|t| t.equal? tuple}
+      end
+      ## else
+      ##   metas.find do |tuple|
+      ##     template === tuple and not distinct_from.any? {|t| t.equal? tuple}
+      ##   end
+      ## end
+    end
+  end
+end

data/example/sqlite/poi-v2.rb

@@ -0,0 +1,58 @@
+# POI -- Points Of Interest
+#
+# This example creates a sqlite db in memory with a table of locations and
+# descriptions of points of interest, and attaches the db to a subspace of the
+# tuplespace. The process which manages that subspace can now do two things:
+#
+# 1. accept inserts (via write)
+#
+# 2. custom queries, accessed by write to a different subspace
+#
+# You can have redundant instances of this, and that will distribute load
+# in #2 above.
+#
+# See example/subspaces/addr-book.rb for more comments on the command/response
+# subspace pattern.
+#
+# gem install sequel sqlite3
+
+require 'tupelo/app'
+require_relative 'poi-store'
+
+Tupelo.application do
+  local do
+    POISPACE = PoiStore.define_poispace(self)
+    define_subspace("cmd", {id: nil, cmd: String, arg: nil})
+    define_subspace("rsp", {id: nil, result: nil})
+  end
+
+  child tuplespace: [PoiStore, POISPACE],
+        subscribe: ["poi", "cmd"], passive: true do
+    log.progname = "poi-store #{client_id}"
+
+    # handle custom queries here, using poi template
+    loop do
+      req = take subspace("cmd")
+      case req["cmd"]
+      when "find box"
+        arg = req["arg"] ## validate this
+        lat = arg["lat"]; lng = arg["lng"]
+        template = PoiTemplate.new(poispace: subspace("poi"),
+          lat: lat[0]..lat[1], lng: lng[0]..lng[1])
+        write id: req["id"], result: read_all(template)
+      end
+    end
+  end
+  
+  child subscribe: "rsp" do
+    write lat: 12, lng: 34, desc: "foo"
+    write lat: 56, lng: 78, desc: "bar"
+    write lat: 12, lng: 34, desc: "foo" # dup is ok
+    write lat: 13, lng: 35, desc: "baz"
+    
+    req_id = [client_id, 1]
+    write id: req_id, cmd: "find box", arg: {lat: [10, 14], lng: [30, 40]}
+    rsp = take id: req_id, result: nil
+    log rsp["result"]
+  end
+end