whispr 0.0.1

data/README.md

@@ -0,0 +1,14 @@
+# Whispr
+
+Whispr is a port of the [Graphite](http://graphite.wikidot.com/) [Whisper](https://github.com/graphite-project/) library in Ruby.
+
+It supports the basic create, update and read (fetch,dump) operations. 
+
+
+## TODO
+
+- unit tests
+- whispr-resize
+- whispr-merge
+- whispr-set-aggregation-method
+- rrd2whispr

data/bin/whispr-create

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require "whispr"
+require "optparse"
+
+options    = {:xff => 0.5, :overwrite => false, :aggregationMethod => :average}
+opt_parser = OptionParser.new do |opts|
+  opts.on("--xFilesFactor Float", Float, "default 0.5") { |x| options[:xff] = x }
+  opts.on("--aggregationMethod String", String, "function to use when aggregating values #{Whispr::AGGR_TYPES[1..-1].map(&:to_s).inspect}") do |aggr|
+    aggr = aggr.intern
+    unless Whispr::AGGR_TYPES[1..-1].include?(aggr)
+      $stderr.puts "aggregationMethod must be one of: #{Whispr::AGGR_TYPES[1..-1]}"
+      exit 1
+    end
+    options[:aggregationMethod] = aggr
+  end
+  opts.on("--overwrite") {|o| options[:overwrite] = o }
+  opts.banner += " path timePerPoint:timeToStore [timePerPoint:timeToStore]*"
+end
+opt_parser.parse!
+
+if ARGV.length < 2
+  $stderr.puts opt_parser
+  exit 1
+end
+
+path = ARGV.shift
+unless File.exists?(File.dirname(path))
+  $stderr.puts "#{File.dirname(path)} does not exists"
+  exit 1
+end
+File.unlink if options[:overwrite] && File.exists?(path)
+
+Whispr.create(path, ARGV.map{|a| Whispr.parse_retention_def(a) }, options)

data/bin/whispr-dump

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require "whispr"
+require "stringio"
+
+if ARGV[0].nil? || ARGV[0] == "-"
+  data = StringIO.new(STDIN.read)
+else
+  abort("data file #{ARGV[0]} does not exist") unless File.exist?(ARGV[0])
+  data = File.open(ARGV[0], "r")
+end 
+
+whisper = Whispr.new(data)
+
+puts "Meta data:"
+puts "  aggregation method: #{whisper.info[:aggregationMethod]}"
+puts "  max retention: #{whisper.info[:maxRetention]}"
+puts "  xFilesFactor: #{whisper.info[:xFilesFactor]}"
+
+whisper.archives.each.with_index do |archive, i|
+  puts "\nArchive #{i} info:"
+  puts "  offset #{archive.offset}"
+  puts "  seconds per point #{archive.spp}"
+  puts "  points #{archive.points}"
+  puts "  retention #{archive.retention}"
+  puts "  size #{archive.size}"
+end
+
+whisper.archives.each.with_index do |archive, i|
+  puts "\nArchive #{i} data: "
+  archive.to_enum.each do |point, timestamp, value|
+    puts sprintf("#{point}, #{timestamp}, %10.35g", value)
+  end
+end

data/bin/whispr-fetch

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require "whispr"
+require "stringio"
+require "optparse"
+require "json"
+
+options = {:from => Time.new - 86400, :until => Time.new}
+OptionParser.new do |opts|
+  opts.on("--from Integer", "Unix epoch time of the beginning of your requested interval (default: 24 hours ago") do |f|
+    if ["n", "N"].include?(f[0])
+      f = Time.new + f[1..-1].to_i
+    end
+    options[:from] = Time.at(f.to_i)
+  end
+  opts.on("--until Integer", "Unix epoch time of the end of your requested interval (default: now)") do |u|
+    if ["n", "N"].include?(u[0])
+      u = Time.new + u[1..-1].to_i
+    end
+    options[:until] = Time.at(u.to_i)
+  end
+  opts.on("--pretty", "Show human-readable timestamps instead of unix times") { options[:pretty] = true }
+  opts.on("--json", "Output results in JSON form") { options[:json] = true }
+end.parse!
+
+if ARGV[0].nil? || ARGV[0] == "-"
+  data = StringIO.new(STDIN.read)
+else
+  abort("data file #{ARGV[0]} does not exist") unless File.exist?(ARGV[0])
+  data = File.open(ARGV[0], "r")
+end 
+
+
+(start_t, end_t, step), values = Whispr.new(data).fetch(options[:from], options[:until])
+if options[:json]
+  puts JSON.dump({:start => start_t, :end => end_t, :step => step, :values => values })
+else
+  t = start_t
+  values.each do |value|
+    time = options[:pretty] ? Time.at(t) : t
+    value = value ? sprintf("%f", value) : 'None'
+    puts "#{time}\t#{value}"
+    t += step
+  end
+end

data/bin/whispr-info

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require "whispr"
+require "stringio"
+
+if ARGV[0].nil? || ARGV[0] == "-"
+  data = StringIO.new(STDIN.read)
+else
+  abort("data file #{ARGV[0]} does not exist") unless File.exist?(ARGV[0])
+  data = File.open(ARGV[0], "r")
+end 
+
+info = Whispr.new(data).info
+info[:fileSize] = data.size
+
+unless (fields = Array(ARGV[1..-1])).empty?
+  fields.each do |field|
+    unless info.include?(field.to_sym)
+      puts "Unknown field '#{field}'. Valid fields are #{info.keys.join(", ")}"
+      exit 1
+    end
+    puts info[field]
+  end
+  exit 0
+end
+
+archives = info.delete(:archives)
+
+info.each { |k,v| puts "#{k}: #{v}" }
+
+archives.each_index do |i|
+  puts "\nArchive #{i}"
+  archives[i].each { |k,v| puts "#{k}: #{v}" }
+end

data/bin/whispr-update

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require "whispr"
+require "stringio"
+
+points = ARGV.select{|e| e.include?(":") }
+if points.empty?
+  puts "#{File.basename($0)} path timestamp:value [timestamp:value]*"
+  exit -1
+end
+
+file   = ARGV - points
+if file.empty? || file == "-"
+  data = StringIO.new(STDIN.read, "r+")
+else
+  abort("data file #{file[0]} does not exist") unless File.exist?(file[0])
+  data = File.open(file[0], "r+")
+end 
+
+now    = Time.now.to_i
+points = points.map{|p| p.split(":") }.map{|t,v| [t == 'N' ? now : t.to_i, v.to_f] }
+
+Whispr.new(data).update(*points)

data/lib/whispr.rb

@@ -0,0 +1,456 @@
+require 'whispr/version'
+require 'whispr/archive'
+require 'stringio'
+
+class Whispr
+  module Error; end
+  class WhisprError < StandardError
+    include Error
+    def self.exception(e)
+      return e if e.nil? || e == self
+      ne = new(e.to_s)
+      ne.set_backtrace e.backtrace if e.respond_to?(:backtrace)
+      ne
+    end
+  end
+  class CorruptWhisprFile < WhisprError; end
+  class InvalidTimeInterval < WhisprError; end
+  class TimestampNotCovered < WhisprError; end
+  class InvalidAggregationMethod < WhisprError; end
+  class ArchiveBoundaryExceeded < WhisprError; end
+  class ValueError < WhisprError; end
+  class InvalidConfiguration < WhisprError; end
+
+  LONG_FMT          = "N"
+  METADATA_FMT      = "#{LONG_FMT*2}g#{LONG_FMT}"
+  METADATA_SIZE     = 16
+  ARCHIVE_INFO_FMT  = LONG_FMT * 3
+  ARCHIVE_INFO_SIZE =  12
+  POINT_FMT         = "#{LONG_FMT}G"
+  POINT_SIZE        = 12
+  CHUNK_SIZE        = 16384
+
+  AGGR_TYPES = [
+    :_,
+    :average,
+    :sum,
+    :last,
+    :max,
+    :min
+  ].freeze
+
+  class << self
+
+    def unitMultipliers
+      @unitMultipliers ||= {
+        's' => 1,
+        'm' => 60,
+        'h' => 3600,
+        'd' => 86400,
+        'w' => 86400 * 7,
+        'y' => 86400 * 365
+      }
+    end
+
+    def parse_retention_def(rdef)
+      raise ArgumentError.new("precision and points must be separated by a ':'") unless rdef && rdef.include?(":")
+      (precision, points) = rdef.strip.split(':')
+      if precision.to_i.to_s == precision
+        precision = precision.to_i * unitMultipliers['s']
+      else
+        _, precision, unit = precision.split(/([\d]+)/)
+        unit = 's' unless unit
+        raise ValueError.new("Invalid precision specification unit #{unit}") unless unitMultipliers[unit[0]]
+        precision = precision.to_i * unitMultipliers[unit[0]]
+      end
+
+      if points.to_i.to_s == points
+        points = points.to_i
+      else
+        _, points, unit = points.split(/([\d]+)/)
+        raise ValueError.new("Invalid retention specification unit #{unit}") unless unitMultipliers[unit[0]]
+        points = points.to_i * unitMultipliers[unit[0]] / precision
+      end
+
+      [precision, points]
+    end
+
+    # Create whipser file.
+    # @param [String] path
+    # @param [Array] archiveList each archive is an array with two elements: [secondsPerPoint,numberOfPoints]
+    # @param [Hash] opts
+    # @option opts [Float] :xff the fraction of data points in a propagation interval that must have known values for a propagation to occur
+    # @option opts [Symbol] :aggregationMethod the function to use when propogating data; must be one of AGGR_TYPES[1..-1]
+    # @option opts [Boolean] :overwrite (false)
+    # @raise [InvalidConfiguration] if the archiveList is inavlid, or if 'path' exists and :overwrite is not true
+    # @see Whsipr.validateArchiveList
+    def create(path, archiveList, opts = {})
+      opts = {:xff => 0.5, :aggregationMethod => :average, :sparse => false, :overwrite => false}.merge(opts)
+      unless AGGR_TYPES[1..-1].include?(opts[:aggregationMethod])
+        raise InvalidConfiguration.new("aggregationMethod must be one of #{AGGR_TYPES[1..-1]}")
+      end
+
+      validateArchiveList!(archiveList)
+      raise InvalidConfiguration.new("File #{path} already exists!") if File.exists?(path) && !opts[:overwrite]
+
+      # if file exists it will be truncated
+      File.open(path, "wb")  do |fh|
+        fh.flock(File::LOCK_EX)
+        aggregationType = AGGR_TYPES.index(opts[:aggregationMethod])
+        oldest         = archiveList.map{|spp, points| spp * points }.sort.last
+        packedMetadata = [aggregationType, oldest, opts[:xff], archiveList.length].pack(METADATA_FMT)
+        fh.write(packedMetadata)
+        headerSize            = METADATA_SIZE + (ARCHIVE_INFO_SIZE * archiveList.length)
+        archiveOffsetPointer = headerSize
+        archiveList.each do |spp, points|
+          archiveInfo = [archiveOffsetPointer, spp, points].pack(ARCHIVE_INFO_FMT)
+          fh.write(archiveInfo)
+          archiveOffsetPointer += (points * POINT_SIZE)
+        end
+
+        if opts[:sparse]
+          fh.seek(archiveOffsetPointer - headerSize - 1)
+          fh.write("\0")
+        else
+          remaining = archiveOffsetPointer - headerSize
+          zeroes = "\x00" * CHUNK_SIZE
+          while remaining > CHUNK_SIZE
+            fh.write(zeroes)
+            remaining -= CHUNK_SIZE
+          end
+          fh.write(zeroes[0..remaining])
+        end
+
+        fh.flush
+        fh.fsync rescue nil
+      end
+
+      new(path)
+    end
+
+    # Is the provided archive list valid?
+    # @return [Boolean] true, false
+    def validArchiveList?(archiveList)
+      !(!!(validateArchiveList!(archiveList) rescue true))
+    end
+
+    # Validate an archive list without raising an exception
+    # @return [NilClass, InvalidConfiguration]
+    def validateArchiveList(archiveList)
+      validateArchiveList!(archiveList) rescue $!
+    end
+
+    # Validate an archive list
+    # An ArchiveList must:
+    # 1. Have at least one archive config. Example: [60, 86400]
+    # 2. No archive may be a duplicate of another.
+    # 3. Higher precision archives' precision must evenly divide all lower precision archives' precision.
+    # 4. Lower precision archives must cover larger time intervals than higher precision archives.
+    # 5. Each archive must have at least enough points to consolidate to the next archive
+    # @raise [InvalidConfiguration]
+    # @return [nil]
+    def validateArchiveList!(archiveList)
+      raise InvalidConfiguration.new("you must specify at least on archive configuration") if Array(archiveList).empty?
+      archiveList = archiveList.sort{|a,b| a[0] <=> b[0] }
+      archiveList[0..-2].each_with_index do |archive, i|
+        nextArchive = archiveList[i+1]
+        unless archive[0] < nextArchive[0]
+          raise InvalidConfiguration.new("A Whipser database may not be configured " +
+            "having two archives with the same precision " +
+            "(archive#{i}: #{archive}, archive#{i+1}: #{nextArchive})")
+        end
+        unless nextArchive[0] % archive[0] == 0
+          raise InvalidConfiguration.new("Higher precision archives' precision must " +
+            "evenly divide all lower precision archives' precision " +
+            "(archive#{i}: #{archive}, archive#{i+1}: #{nextArchive})")
+        end
+
+        retention = archive[0] * archive[1]
+        nextRetention = nextArchive[0] * nextArchive[1]
+        unless nextRetention > retention
+          raise InvalidConfiguration.new("Lower precision archives must cover larger " +
+            "time intervals than higher precision archives " +
+            "(archive#{i}: #{archive[1]}, archive#{i + 1}:, #{nextArchive[1]})")
+        end
+
+        archivePoints = archive[1]
+        pointsPerConsolidation = nextArchive[0] / archive[0]
+        unless archivePoints >= pointsPerConsolidation
+          raise InvalidConfiguration.new("Each archive must have at least enough points " +
+            "to consolidate to the next archive (archive#{i+1} consolidates #{pointsPerConsolidation} of " +
+            "archive#{i}'s points but it has only #{archivePoints} total points)")
+        end
+      end
+      nil
+    end
+  end
+
+  # @return [File, StringIO] file handle of the whisper file
+  attr_reader :fh
+
+  attr_accessor :auto_flush
+  alias :auto_flush? :auto_flush
+
+  def initialize(file, auto_flush = true)
+    @fh = file.is_a?(File) || file.is_a?(StringIO) ? file : File.open(file, 'r+')
+    @fh.binmode
+    @auto_flush = auto_flush
+  end
+
+  # @return [Hash]
+  def header
+    @header ||= read_header
+  end
+  alias :info :header
+
+  # @return [Array] Archives
+  # @see Whispr::Archive
+  def archives
+    @archives ||= info[:archives].map { |a| Archive.new(self, a) }
+  end
+
+
+  # Retrieve values from a whisper file within the given time window.
+  #
+  # The most appropriate archive within the whisper file will be chosen. The
+  # return value will be a two element Array.  The first element will be a
+  # three element array containing the start time, end time and step. The
+  # second element will be a N element array containing each value at each
+  # step period.
+  #
+  # @see Archive#fetch
+  def fetch(fromTime, untilTime = Time.new)
+    fromTime  = fromTime.to_i
+    untilTime = untilTime.to_i
+    now       = Time.now.to_i
+    oldest    = header[:maxRetention]
+    fromTime  = oldest if fromTime < oldest
+    raise InvalidTimeInterval.new("Invalid time interval") unless fromTime < untilTime
+    untilTime = now if untilTime > now || untilTime < fromTime
+
+    diff    = now - fromTime
+    archive = archives.find{|a| a.retention >= diff }
+    return archive.fetch(fromTime, untilTime)
+  end
+
+  # Update one or many points
+  # Each element of the points list should be a two dimensional Array where
+  # the first element is a timestamp and the second element is a value.
+  def update(*points)
+    return if points.empty?
+    # TODO lock the file
+    if points.length == 1
+      update_one(points[0][1], points[0][0])
+    else
+      update_many(points)
+    end
+  end
+
+private
+
+
+  def read_header
+    o_pos = @fh.pos
+
+    begin
+      @fh.pos = 0
+      metadata = @fh.read(METADATA_SIZE)
+      aggr_type, max_retention, xff, arch_count = metadata.unpack(METADATA_FMT)
+      archives = arch_count.times.map do |i|
+        arch_info = @fh.read(ARCHIVE_INFO_SIZE)
+        offset, s_per_pnt, points = arch_info.unpack(ARCHIVE_INFO_FMT)
+        { :retention => s_per_pnt * points,
+          :secondsPerPoint => s_per_pnt,
+          :points => points,
+          :size => points * POINT_SIZE,
+          :offset => offset
+        }
+      end
+    rescue => e
+      raise CorruptWhisprFile.exception(e)
+    ensure
+      @fh.pos = o_pos
+    end
+
+    { :maxRetention      => max_retention,
+      :xFilesFactor      => xff,
+      :aggregationMethod => AGGR_TYPES[aggr_type],
+      :archives          => archives
+    }
+  end
+
+  def update_one(value, timestamp = nil)
+    now       = Time.new.to_i
+    timestamp = now if timestamp.nil?
+    diff      = now - timestamp
+    if !(diff < header[:maxRetention] && diff >= 0)
+      raise TimestampNotCovered, "Timestamp (#{timestamp}) not covered by any archives in this database"
+    end
+
+    aidx = (0 ... archives.length).find { |i| archives[i].retention > diff }
+    archive       = archives[aidx]
+    lowerArchives = archives[aidx + 1 .. - 1]
+
+    myInterval    = timestamp - (timestamp % archive.spp)
+    myPackedPoint = [myInterval, value].pack(POINT_FMT)
+    @fh.seek(archive.offset)
+    baseInterval, baseValue = @fh.read(POINT_SIZE).unpack(POINT_FMT)
+
+    if baseInterval == 0
+      # this file's first update
+      @fh.seek(archive.offset)
+      @fh.write(myPackedPoint)
+      baseInterval, baseValue = myInterval, value
+    else
+      timeDistance  = myInterval - baseInterval
+      pointDistance = timeDistance / archive.spp
+      byteDistance  = pointDistance * POINT_SIZE
+      myOffset      = archive.offset + (byteDistance % archive.size)
+      @fh.seek(myOffset)
+      @fh.write(myPackedPoint)
+    end
+
+    higher = archive
+    lowerArchives.each do |lower|
+      break unless propagate(myInterval, higher, lower)
+      higher = lower
+    end
+
+    @fh.flush if auto_flush?
+  end
+
+  def update_many(points)
+    # order points by timestamp, newest first
+    points   = points.map{|ts, v| [ts.to_i, v.to_f ] }.sort {|b,a| a[0] <=> b[0] }
+    now            = Time.new.to_i
+    archives       = self.archives.to_enum
+    currentArchive = archives.next
+    currentPoints  = []
+    points.each do |point|
+      age = now - point[0]
+      while currentArchive.retention < age
+        unless currentPoints.empty?
+          currentPoints.reverse! # put points in chronological order
+          currentArchive.update_many(currentPoints)
+          currentPoints = []
+        end
+        begin
+          currentArchive = archives.next
+        rescue StopIteration
+          currentArchive = nil
+          break
+        end
+      end
+      # drop remaining points that don't fit in the database
+      break unless currentArchive
+
+      currentPoints << point
+    end
+
+    if currentArchive && !currentPoints.empty?
+      # don't forget to commit after we've checked all the archives
+      currentPoints.reverse!
+      currentArchive.update_many(currentPoints)
+    end
+
+    @fh.flush if auto_flush?
+  end
+
+  def propagate(timestamp, higher, lower)
+    aggregationMethod = header[:aggregationMethod]
+    xff               = header[:xFilesFactor]
+
+    lowerIntervalStart = timestamp - (timestamp % lower.spp)
+    lowerIntervalEnd   = lowerIntervalStart + lower.spp
+    @fh.seek(higher.offset)
+    higherBaseInterval, higherBaseValue = @fh.read(POINT_SIZE).unpack(POINT_FMT)
+
+    if higherBaseInterval == 0
+      higherFirstOffset = higher.offset
+     else
+       timeDistance = lowerIntervalStart - higherBaseInterval
+       pointDistance = timeDistance / higher.spp
+       byteDistance  = pointDistance * POINT_SIZE
+       higherFirstOffset = higher.offset + (byteDistance % higher.size)
+    end
+
+    higherPoints        = lower.spp / higher.spp
+    higherSize          = higherPoints * POINT_SIZE
+    relativeFirstOffset = higherFirstOffset - higher.offset
+    relativeLastOffset  = (relativeFirstOffset + higherSize) % higher.size
+    higherLastOffset    = relativeLastOffset + higher.offset
+    @fh.seek(higherFirstOffset)
+
+    if higherFirstOffset < higherLastOffset
+      # don't wrap the archive
+      seriesString = @fh.read(higherLastOffset - higherFirstOffset)
+    else
+      # wrap the archive
+      higherEnd    = higher.offset + higher.size
+      seriesString = @fh.read(higherEnd - higherFirstOffset)
+      @fh.seek(higher.offset)
+      seriesString += @fh.read(higherLastOffset - higher.offset)
+    end
+
+    points         = seriesString.length / POINT_SIZE
+    unpackedSeries = seriesString.unpack(POINT_FMT * points)
+
+    # construct a list of values
+    neighborValues  = points.times.map{}
+    currentInterval = lowerIntervalStart
+    step            = higher.spp
+    (0..unpackedSeries.length).step(2) do |i|
+      pointTime           = unpackedSeries[i]
+      neighborValues[i/2] = unpackedSeries[i+1] if pointTime == currentInterval
+      currentInterval    += step
+    end
+
+    knownValues = neighborValues.select { |v| !v.nil? }
+    return false if knownValues.empty?
+    if (knownValues.length / neighborValues.length).to_f < header[:xFilesFactor]
+      return false
+    end
+
+    # we have enough data to propagate a value
+    aggregateValue = aggregate(aggregationMethod, knownValues)
+    myPackedPoint  = [lowerIntervalStart, aggregateValue].pack(POINT_FMT)
+    @fh.seek(lower.offset)
+    lowerBaseInterval, lowerBaseValue = @fh.read(POINT_SIZE).unpack(POINT_FMT)
+
+    if lowerBaseInterval == 0
+      # first propagated update to this lower archive
+      @fh.seek(lower.offset)
+      @fh.write(myPackedPoint)
+    else
+      timeDistance  = lowerIntervalStart - lowerBaseInterval
+      pointDistance = timeDistance / lower.spp
+      byteDistance  = pointDistance * POINT_SIZE
+      lowerOffset   = lower.offset + (byteDistance % lower.size)
+      @fh.seek(lowerOffset)
+      @fh.write(myPackedPoint)
+    end
+    true
+  end
+
+  def aggregate(aggregationMethod, knownValues)
+    case aggregationMethod
+    when :average
+      (knownValues.inject(0){|sum, i| sum + i } / knownValues.length).to_f
+    when :sum
+      knownValues.inject(0){|sum, i| sum + i }
+    when :last
+      knownValues[-1]
+    when :max
+      v  = knownValues[0]
+      knownValues[1..-1].each { |k| v = k if k > v }
+      v
+    when :min
+      v  = knownValues[0]
+      knownValues[1..-1].each { |k| v = k if k < v }
+      v
+    else
+      raise InvalidAggregationMethod, "Unrecognized aggregation method #{aggregationMethod}"
+    end
+  end
+
+end

data/lib/whispr/archive.rb

@@ -0,0 +1,220 @@
+class Whispr
+  class Archive
+    include Enumerable
+
+    # @return [Hash] the archive header
+    attr_reader :header
+    # @return [Fixnum] the start location in the whisper file of this Archive
+    attr_reader :offset
+    # @return [Fixnum] the number of points in this archive
+    attr_reader :points
+    # @return [Fixnum] the total size of this archive (points * POINT_SIZE)
+    attr_reader :size
+    # @return [Fixnum] number of seconds worth of data retained by this archive
+    attr_reader :retention
+    # @return [Fixnum] seconds per point
+    attr_reader :spp
+    # @return [Whispr} the Whisper that contains this Archive
+    attr_reader :whisper
+
+    def initialize(whisper, header)
+      @whisper   = whisper
+      @header    = header
+      @offset    = @header[:offset]
+      @points    = @header[:points]
+      @size      = @header[:size]
+      @retention = @header[:retention]
+      @spp       = @header[:secondsPerPoint]
+      @eoa       = @size * @points + @offset
+    end
+
+    # Retrieve each point from the archive.
+    #
+    # If a block is provided each point is read directly from
+    # the whisper file one at a time and yielded. If a block
+    # is not provided, all points are read from the file and
+    # returned as an enum.
+    #
+    # Each point is represented as a three element Array. The first
+    # element is the index of the point. The second element is the
+    # timestamp of the point and the third element is the value of
+    # the point.
+    def each(&blk)
+      return slurp.to_enum unless block_given?
+      o_pos = @whisper.fh.pos
+      begin
+        @whisper.fh.pos = @offset
+        points.times {|i| yield(i, *next_point) }
+      ensure
+        @whisper.fh.pos = o_pos
+      end
+    end
+
+    # Has the end of the archive been reached?
+    def eoa?
+      @whisper.fh.pos >= @eoa
+    end
+
+    def to_enum
+      slurp.to_enum
+    end
+
+    # Retrieve the next point from the whisper file.
+    # @api private
+    def next_point
+      return nil if @whisper.fh.pos >= @eoa || @whisper.fh.pos < @offset
+      @whisper.fh.read(POINT_SIZE).unpack(POINT_FMT)
+    end
+
+    # Retrieve all points for this archive from the whisper file.
+    #
+    # Each point is represented as a three element Array. The first
+    # element is the index of the point. The second element is the
+    # timestamp of the point and the third element is the value of
+    # the point.
+    #
+    # @return [Array]
+    def slurp
+      o_pos = @whisper.fh.pos
+      @whisper.fh.pos = @offset
+      data = @whisper.fh.read(@size).unpack(POINT_FMT * @points)
+      @points.times.map { |i| [i, data.shift, data.shift] }
+    ensure
+      @whisper.fh.pos = o_pos
+    end
+
+    # Retrieve values for a time period from an archive within a whisper file
+    #
+    # The return value will be a two element Array.  The first element will be
+    # a three element array containing the start time, end time and step. The
+    # second element will be a N element array containing each value at each
+    # step period.
+    #
+    # @see Whispr#fetch
+    def fetch(fromTime, untilTime)
+      fromInterval  = (fromTime - (fromTime % spp)) + spp
+      untilInterval = (untilTime - (untilTime % spp)) + spp
+      o_pos         = @whisper.fh.pos
+      begin
+        @whisper.fh.seek(offset)
+        baseInterval, baseValue = @whisper.fh.read(POINT_SIZE).unpack(POINT_FMT)
+        if baseInterval == 0
+          step     = spp
+          points   = (untilInterval - fromInterval) / step
+          timeInfo = [fromInterval, untilInterval, step]
+          return [timeInfo, points.times.map{}]
+        end
+
+        # Determine fromOffset
+        timeDistance  = fromInterval - baseInterval
+        pointDistance = timeDistance / spp
+        byteDistance  = pointDistance * POINT_SIZE
+        fromOffset    = offset + (byteDistance % size)
+
+        # Determine untilOffset
+        timeDistance  = untilInterval - baseInterval
+        pointDistance = timeDistance / spp
+        byteDistance  = pointDistance * POINT_SIZE
+        untilOffset   = offset + (byteDistance % size)
+
+        # Reall all the points in the interval
+        @whisper.fh.seek(fromOffset)
+        if fromOffset < untilOffset
+          # we don't wrap around the archive
+          series = @whisper.fh.read(untilOffset - fromOffset)
+        else
+          # we wrap around the archive, so we need two reads
+          archiveEnd  = offset + size
+          series      = @whisper.fh.read(archiveEnd - fromOffset)
+          @whisper.fh.seek(offset)
+          series     += @whisper.fh.read(untilOffset - offset)
+        end
+
+        points          = series.length / POINT_SIZE
+        series          = series.unpack(POINT_FMT * points)
+        currentInterval = fromInterval
+        step            = spp
+        valueList       = points.times.map{}
+        (0..series.length).step(2) do |i|
+          pointTime = series[i]
+          if pointTime == currentInterval
+            pointValue     = series[i+1]
+            valueList[i/2] = pointValue
+          end
+          currentInterval += step
+        end
+
+        timeInfo = [fromInterval, untilInterval, step]
+      ensure
+        @whisper.fh.pos = o_pos
+      end
+      [timeInfo, valueList]
+    end
+
+    def update_many(points)
+      step = spp
+      alignedPoints = points.map { |ts, v| [(ts - (ts % step)), v] }
+      # Create a packed string for each contiguous sequence of points
+      packedStrings    = []
+      previousInterval = nil
+      currentString    = ''
+      alignedPoints.each do |interval, value|
+        next if interval == previousInterval
+        if previousInterval.nil? || (interval == previousInterval + step)
+          currentString   += [interval, value].pack(POINT_FMT)
+        else
+          numberOfPoints = currentString.length / POINT_SIZE
+          startInterval  = previousInterval - (step * (numberOfPoints - 1))
+          packedStrings << [startInterval, currentString]
+          currentString  = [interval, value].pack(POINT_FMT)
+        end
+        previousInterval = interval
+      end
+      if !currentString.empty?
+        numberOfPoints = currentString.length / POINT_SIZE
+        startInterval = previousInterval - (step * (numberOfPoints - 1))
+        packedStrings << [startInterval, currentString]
+      end
+
+      # Read base point and determine where our writes will start
+      @whisper.fh.seek(offset)
+      baseInterval, baseValue = @whisper.fh.read(POINT_SIZE).unpack(POINT_FMT)
+      baseInterval = packedStrings[0][0] if baseInterval == 0
+      packedStrings.each do |interval, packedString|
+        timeDistance = interval - baseInterval
+        pointDistance = timeDistance / step
+        byteDistance = pointDistance * POINT_SIZE
+        myOffset = offset + (byteDistance % size)
+        @whisper.fh.seek(myOffset)
+        archiveEnd = offset + size
+        bytesBeyond = (myOffset + packedString.length) - archiveEnd
+
+        if bytesBeyond > 0
+          @whisper.fh.write(packedString[0..-bytesBeyond])
+          if(@whisper.fh.pos != archiveEnd)
+            raise ArchiveBoundaryExceeded.new("archiveEnd=#{archiveEnd} pos=#{@whisper.fh.pos} bytesBeyond=#{bytesBeyond} len(packedString)=#{packedString.length}")
+          end
+          @whisper.fh.seek(offset)
+          @whisper.fh.write(packedString[-bytesBeyond..-1])
+        else
+          @whisper.fh.write(packedString)
+        end
+      end # interval, packedString|
+
+      # Now we propagate the updates to the lower-precision archives
+      higher = self
+      @whisper.archives.select{|a| a.spp > spp }.each do |lower|
+        lowerIntervals = alignedPoints.map{|p| p[0] - (p[0] % lower.spp) }
+        propagateFurther = false
+        lowerIntervals.uniq.each do |interval|
+          propagateFuther = @whisper.send(:propagate, interval, higher, lower)
+        end
+        break unless propagateFurther
+        higher = lower
+      end
+    end
+
+
+
+  end
+end

data/lib/whispr/version.rb

@@ -0,0 +1,3 @@
+class Whispr
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: whispr
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Caleb Crane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-14 00:00:00.000000000 Z
+dependencies: []
+description: ''
+email: whispr@simulacre.org
+executables:
+- whispr-create
+- whispr-dump
+- whispr-fetch
+- whispr-info
+- whispr-update
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/whispr/archive.rb
+- lib/whispr/version.rb
+- lib/whispr.rb
+- bin/whispr-create
+- bin/whispr-dump
+- bin/whispr-fetch
+- bin/whispr-info
+- bin/whispr-update
+- README.md
+homepage: http://github.com/simulacre/whispr
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Read and write Graphite Whisper round-robin files
+test_files: []
+has_rdoc: