tsuga 0.0.1

data/lib/tsuga/model/point.rb

@@ -0,0 +1,206 @@
+require 'tsuga'
+
+module Tsuga::Model
+
+  # Represents a position in the 0..1 x 0..1 square, modeling points on the
+  # Earth as represented by their longitude/latitude coordinates.
+  # 
+  # Concretions have the following accessors:
+  # - :geohash (64-bit integer)
+  # 
+  # - :lat (float, -90..90)
+  # - :lng (float, -180..180)
+  # 
+  module PointTrait
+
+
+
+    def distance_to(other)
+      Math.sqrt((self.lat - other.lat) ** 2 + (self.lng - other.lng) ** 2)
+    end
+
+
+    def =~(other)
+      self.geohash == other.geohash
+    end
+
+
+    def &(other)
+      distance_to(other)
+    end
+
+
+    def geohash=(value)
+      super(value)
+      _updating_coords { _set_latlng_from_geohash }
+      geohash
+    end
+
+
+    def lat=(value)
+      _validate_lat(value) if value
+      super(value)
+      _updating_coords { _set_geohash_from_latlng }
+      lat
+    end
+
+
+    def lng=(value)
+      _validate_lng(value) if value
+      super(value)
+      _updating_coords { _set_geohash_from_latlng }
+      lng
+    end
+
+
+    def inspect
+      "<%s lat:%s lng:%s geohash:%s>" % [
+        (self.class.name || 'Point').gsub(/.*::/, ''),
+        lat ? ("%.3f" % lat) : 'nil',
+        lng ? ("%.3f" % lng) : 'nil',
+        geohash ? geohash : 'nil'
+      ]
+    end
+
+    def prefix(depth)
+      geohash[0...depth]
+    end
+
+    private
+
+    # only the outmost call yields.
+    # prevents infinite loops of latlng <-> geohash updates
+    def _updating_coords
+      return if @_updating
+      @_updating = true
+      yield
+      @_updating = false
+    end
+
+
+    def _validate_lat(_lat)
+      raise ArgumentError, 'bad lat' unless ( -90.0 ...  90.0).include?(_lat)
+    end
+
+    def _validate_lng(_lng)
+      raise ArgumentError, 'bad lng' unless (-180.0 ... 180.0).include?(_lng)
+    end
+
+
+    def _validate_geohash(value)
+      raise ArgumentError, 'bad geohash' unless /^[0-3]{32}$/ =~ value
+    end
+
+
+    def _geohash_to_int(value)
+      value.to_i(4)
+    end
+
+    def _int_to_geohash(value)
+      value.to_s(4).rjust(32,'0')
+    end
+
+    # Convert the geohash into lat/lng
+    def _set_latlng_from_geohash
+      geohash = self.geohash
+      if geohash.nil?
+        self.lat = self.lng = nil
+        return
+      end
+      _validate_geohash(geohash)
+
+      geohash_i = _geohash_to_int(geohash)
+      lat,lng = _deinterleave_bits(geohash_i)
+      lat = lat * 180.0 / (1<<32) -  90.0
+      lng = lng * 360.0 / (1<<32) - 180.0
+      self.lat = lat
+      self.lng = lng
+      return
+    end
+
+
+    def _set_geohash_from_latlng
+      lat = self.lat
+      lng = self.lng
+      if lat.nil? || lng.nil?
+        self.geohash = nil
+        return
+      end
+      _validate_lat(lat)
+      _validate_lng(lng)
+      normalized_lat = ((lat +  90.0) * (1<<32) / 180.0).to_i
+      normalized_lng = ((lng + 180.0) * (1<<32) / 360.0).to_i
+
+      geohash_i = _interleave_bits(normalized_lat, normalized_lng)
+      self.geohash = _int_to_geohash(geohash_i)
+      return
+    end
+
+
+    def _interleave_bits(a,b)
+      (_interleave_bits_16b(a >> 16,    b >> 16) << 32) |
+      (_interleave_bits_16b(a & 0xffff, b & 0xffff))
+    end
+
+
+    def _deinterleave_bits(z)
+      x_hi, y_hi = _deinterleave_bits_16b(z >> 32)
+      x_lo, y_lo = _deinterleave_bits_16b(z & 0xFFFFFFFF)
+
+      [((x_hi << 16) | x_lo), ((y_hi << 16) | y_lo)]
+    end
+
+
+    Magic = [0x55555555, 0x33333333, 0x0F0F0F0F, 0x00FF00FF]
+
+    # Interleave lower 16 bits of x and y, so the bits of x
+    # are in the even positions and bits from y in the odd;
+    # z gets the resulting 32-bit Morton Number.  
+    # x and y must initially be less than 65536.
+    # Rubyfied from http://graphics.stanford.edu/~seander/bithacks.html
+    def _interleave_bits_16b(x,y)
+      x = (x | (x << 8)) & Magic[3]
+      x = (x | (x << 4)) & Magic[2]
+      x = (x | (x << 2)) & Magic[1]
+      x = (x | (x << 1)) & Magic[0]
+      y = (y | (y << 8)) & Magic[3]
+      y = (y | (y << 4)) & Magic[2]
+      y = (y | (y << 2)) & Magic[1]
+      y = (y | (y << 1)) & Magic[0]
+      z = x | (y << 1)
+    end
+
+    # Deinterleave even bits and odd bits (resp.) to a 2-tuple.
+    # Rubyfied from http://fgiesen.wordpress.com/2009/12/13/decoding-morton-codes/
+    def _deinterleave_bits_16b(z)
+      [_even_bits(z), _even_bits(z >> 1)]
+    end
+
+
+    def _even_bits(z)
+      x = z & 0x55555555
+      x = (x ^ (x >>  1)) & 0x33333333
+      x = (x ^ (x >>  2)) & 0x0f0f0f0f
+      x = (x ^ (x >>  4)) & 0x00ff00ff
+      x = (x ^ (x >>  8)) & 0x0000ffff
+    end
+  end
+
+
+  class Point
+    module Fields
+      attr_accessor :lat, :lng, :geohash
+    end
+    include Fields
+    include PointTrait
+
+    def initialize(geohash: nil, lat: nil, lng: nil)
+      if geohash
+        self.geohash = geohash
+      else
+        self.lat = lat
+        self.lng = lng
+      end
+    end
+  end
+end

data/lib/tsuga/model/record.rb

@@ -0,0 +1,20 @@
+require 'tsuga'
+require 'tsuga/model/point'
+
+module Tsuga::Model
+  # Concretions have the following accessors:
+  # (same as Point)
+  # 
+  # And respond to class methods:
+  # - :find(id)
+  # - :collect_ids (returns a Set)
+  # 
+  module Record
+    include Tsuga::Model::PointTrait
+
+    def update_geohash
+      self.geohash
+    end
+
+  end
+end

data/lib/tsuga/model/tile.rb

@@ -0,0 +1,136 @@
+require 'tsuga'
+require 'tsuga/model/point'
+
+module Tsuga::Model
+  class Tile
+    # corner points
+    attr_reader :southwest, :northeast
+
+    # level in the tile tree, also number of relevant high bits
+    # in the geohash.
+    attr_reader :depth
+
+    # geohash prefix
+    attr_reader :prefix
+
+    WIGGLE_FACTOR = 1e-4
+
+    def initialize(prefix:nil)
+      raise ArgumentError, 'bad prefix' if prefix !~ /^[0-3]{1,32}$/
+      @prefix    = prefix
+      @depth     = prefix.length
+      @southwest = Point.new(geohash: prefix.ljust(32, '0'))
+      @northeast = Point.new(geohash: prefix.ljust(32, '3'))
+    end
+
+    def contains?(point)
+      point.geohash.start_with?(@prefix)
+    end
+
+    def dlat(count = 1)
+      (northeast.lat - southwest.lat) * (count + WIGGLE_FACTOR)
+    end
+
+    def dlng(count = 1)
+      (northeast.lng - southwest.lng) * (count + WIGGLE_FACTOR)
+    end
+
+    # return the 4 children of this tile
+    def children
+      %w(0 1 2 3).map { |quadrant|
+        self.class.new(prefix: @prefix + quadrant)
+      }
+    end
+
+    # return a neighouring tile offset in tile increments
+    # TODO: this could be implemented using bit logic
+    def neighbour(lat:0, lng:0)
+      new_point = Point.new(
+        lat: southwest.lat + dlat(lat),
+        lng: southwest.lng + dlng(lng))
+      Tile.including(new_point, depth: depth)
+    end
+
+    # return neighbouring tiles to the north, northeast, and east
+    def neighbours
+      offsets = (-1..1).to_a.product((-1..1).to_a)
+      offsets.map do |lat, lng|
+        begin 
+          neighbour(lat:lat, lng:lng)
+        rescue ArgumentError
+          nil # occurs on world boundaries
+        end
+      end.compact
+    end
+
+    def inspect
+      "<%s depth:%d prefix:%s>" % [
+        (self.class.name || 'Tile'),
+        depth, prefix
+      ]
+    end
+
+    module ClassMethods
+      # Returns a Tile instance.
+      # +point+ should respond to +geohash+.
+      # Options:
+      # - :depth
+      def including(point, options={})
+        depth = options[:depth]
+        raise ArgumentError, 'bad depth' unless (0..31).include?(depth)
+
+        new(prefix: point.prefix(depth))
+      end
+
+      # Return an array of Tile instances that encloses both corner points
+      # FIXME: this is untested
+      def enclosing_viewport(point_ne:nil, point_sw:nil, depth:nil)
+        # $stderr.puts "aiming to enclose:"
+        # $stderr.puts "%.2f %.2f -> %.2f %.2f" % [point_ne.lat, point_ne.lng, point_sw.lat, point_sw.lng]
+        # $stderr.flush
+
+        tiles = []
+        first_tile = including(point_sw, depth:depth)
+
+        offset_lat = 0
+        loop do
+          offset_lng = 0
+          loop do 
+            # $stderr.puts("offset: #{offset_lat} #{offset_lng}")
+            # $stderr.flush
+            new_tile = first_tile.neighbour(lat:offset_lat, lng:offset_lng)
+            tiles << new_tile
+
+            # $stderr.puts "%.2f %.2f -> %.2f %.2f" % [new_tile.southwest.lat, new_tile.southwest.lng, new_tile.northeast.lat, new_tile.northeast.lng]
+            # $stderr.flush
+
+            offset_lng += 1
+            break if tiles.last.northeast.lng >= point_ne.lng
+          end
+          break if tiles.last.northeast.lat >= point_ne.lat
+          offset_lat += 1
+          offset_lng = 0
+        end
+
+        return tiles
+      end
+    end
+    extend ClassMethods
+  end
+end
+
+__END__
+
+load 'lib/tsuga/model/tile.rb'
+
+# {"n"=>"41.41169761785169", "e"=>"2.2055472226562642", "s"=>"41.33015287320352", "w"=>"2.107700237792983", "z"=>"3"
+  
+
+sw = Tsuga::Model::Point.new(lat: 41.33015287320352, lng: 2.107700237792983)
+ne = Tsuga::Model::Point.new(lat: 41.41169761785169, lng: 2.2055472226562642)
+
+Tsuga::Model::Tile.including(sw, depth: 7)
+Tsuga::Model::Tile.including(ne, depth: 7)
+
+Tsuga::Model::Tile.enclosing_viewport(point_sw:sw, point_ne:ne, depth:7).length
+

data/lib/tsuga/service/aggregator.rb

@@ -0,0 +1,175 @@
+require 'set'
+require 'tsuga/model/point'
+require 'tsuga/model/tile'
+
+module Tsuga::Service
+
+  # Aggregates clusters together until no two clusters are closer than
+  # a given minimum distance.
+  class Aggregator
+    # - clusters (Array): list of points to aggregate
+    # - fence (Tile): clusters outside this will not be aggregated
+    # - ratio (0..1): minimum distance between clusters after aggregation,
+    #   as a ratio of the tile diagonal
+    def initialize(clusters:nil, ratio:nil, fence:nil)
+      @_clusters          = clusters
+      @_fence             = fence || _default_fence
+      @min_distance_ratio = ratio # fraction of tile diagonal
+      @_dropped_clusters   = IdSet.new
+      @_updated_clusters   = IdSet.new
+    end
+
+    def run
+      return if _clusters.empty?
+      warn "warning: running aggregation on many clusters (#{_clusters.size})" if _clusters.size > 100
+
+      if DENSITY_BIAS_FACTOR
+        @min_density, @max_density = _clusters.collect(&:density).minmax
+      end
+
+      # build the set of pairs (n²/2)
+      pairs  = []
+      source = _clusters.dup
+      while left = source.pop
+        source.each do |right| 
+          pairs << _build_pair(left, right, _fence)
+        end
+      end
+
+      # pop & merge
+      while pairs.any?
+        best_pair = pairs.min
+        break if best_pair.distance > min_distance
+
+        # remove the closest pair
+        left, right = best_pair.values
+        left_id  = left.id
+        right_id = right.id
+
+        # remove pairs containing one of the items
+        pairs.delete_if { |p| p.has?(left) || p.has?(right) }
+
+        # merge clusters
+        left.merge(right)
+        _clusters.delete_if { |c| c.id == right_id }
+        _updated_clusters.remove right
+        _dropped_clusters.add    right
+        _updated_clusters.add    left
+
+        # create new pairs
+        _clusters.each do |cluster|
+          next if cluster.id == left_id
+          pairs << _build_pair(left, cluster, _fence)
+        end
+      end
+      nil
+    end
+
+    # after #run, this contains the clusters that were merged into other clusters
+    def dropped_clusters
+      _dropped_clusters.to_a
+    end
+
+    # after #run, this contains the clusters that were modified and need to be persisted
+    def updated_clusters
+      _updated_clusters.to_a
+    end
+
+    # fraction of the diagonal of the fence tile
+    def min_distance
+      @min_distance ||= (_fence.southwest & _fence.northeast) * @min_distance_ratio
+    end
+
+    private
+
+    # FIXME: a sensible value would be ~0.4 in theory, but this
+    # biasing seems to have little impact. remove?
+    DENSITY_BIAS_FACTOR = nil
+
+    attr_reader :_clusters, :_fence, :_dropped_clusters, :_updated_clusters
+
+    # factory for pairs, switches between fenced/unfenced
+    # and conditionnaly adds density bias
+    def _build_pair(c1, c2, fence)
+      pair = fence.nil? ? Pair.new(c1, c2) : FencedPair.new(c1, c2, fence)
+
+      if DENSITY_BIAS_FACTOR && (@max_density != @min_density)
+        # the least dense cluster pairs have a density_bias value close to 0, the densest closer to 1
+        density_bias = (c1.density + c2.density - 2 * @min_density) / (2 * (@max_density - @min_density))
+        # this makes dense clusters appear closer, and vice-versa
+        pair.distance = pair.distance * (1 + DENSITY_BIAS_FACTOR * (1 - density_bias) - 0.5 * DENSITY_BIAS_FACTOR)
+      end
+      pair
+    end
+
+    def _default_fence
+      return if _clusters.empty?
+      Tsuga::Model::Tile.including(_clusters.first, depth:_clusters.first.depth)
+    end
+
+    # model a pair of clusters such as [a,b] == [b,a]
+    # and comparison is based on distance
+    class Pair
+      include Comparable
+      attr_accessor :distance
+
+      def initialize(c1, c2)
+        @left     = c1
+        @right    = c2
+        @left_id  = c1.id
+        @right_id = c2.id
+        @distance = (@left & @right)
+
+        raise ArgumentError, 'pair elements must be distinct' if @left_id == @right_id
+      end
+
+      def <=>(other)
+        self.distance <=> other.distance
+      end
+
+      # def ==(other)
+      #   (self.left_id == other.left_id) && (self.right_id == other.right_id)
+      # end
+
+      def values
+        [@left, @right]
+      end
+
+      def has?(c)
+        c_id = c.id
+        (@left_id == c_id) || (@right_id == c_id)
+      end
+    end
+
+    # pairs where both points fall outside the fence are considered "at horizon"
+    # i.e. their distance infinite. the point is to never aggregate them.
+    class FencedPair < Pair
+      def initialize(c1, c2, fence)
+        super(c1, c2)
+        @outside = !fence.contains?(c1) && !fence.contains?(c2)
+      end
+
+      def distance
+        @outside ? Float::MAX : super
+      end
+    end
+
+    class IdSet
+      def initialize
+        @data = {}
+      end
+
+      def add(item)
+        @data[item.id] = item
+      end
+
+      def remove(item)
+        @data.delete(item.id)
+      end
+
+      def to_a
+        @data.values
+      end
+    end
+  end
+end