table_setter 0.1.2

data/lib/table_setter/command.rb

@@ -0,0 +1,139 @@
+require 'optparse'
+require 'rack'
+require 'rack/showexceptions'
+require 'rack/commonlogger'
+require 'rack/lint'
+
+module TableSetter
+  class Command
+    BANNER = <<-EOB
+table-setter is a Sinatra application for rendering and processing CSVs from google docs into HTML.
+
+Usage:
+  table-setter COMMAND path/to/table-setter/assets OPTIONS
+  
+commands:
+  start    run the development server, for deployment use config.ru
+  install  copy the table-setter assets into the the directory
+  export   statically build tables in the ./out/
+  
+options:
+    EOB
+    
+    
+    def initialize
+      @prefix = ""
+      parse_options
+      command = ARGV.shift
+      @directory = ARGV.shift || '.'
+      TableSetter.configure @directory
+      case command
+      when 'start' then start_server
+      when 'install' then install_assets
+      when 'build' then build_out
+      else puts BANNER
+      end
+    end
+    
+    def start_server
+      app = build_rack
+      Rack::Handler::Thin.run app, :Port => "3000"
+    end
+    
+    def install_assets
+      FileUtils.mkdir_p @directory unless File.exists? @directory
+      puts "\nInstalling TableSetter files...\n\n"
+      base_files.each do |path|
+        copy_file path, File.join(TableSetter.config_path, path.gsub(ROOT + "/template/", "/"))
+      end
+    end
+    
+
+    
+    def build_out
+      @out_dir = File.join(TableSetter.config_path, 'out', @prefix)
+      puts "\nBuilding your TableSetter files...\n\n"
+      app = build_rack
+      request = Rack::MockRequest.new(app)
+      build_index request
+      build_assets
+      build_tables request
+    end
+    
+    private
+    # Option parsing
+    def parse_options
+      @options = {}
+      @option_parser = OptionParser.new do |opts|
+        opts.on "-p", "--prefix PREFIX", "url prefix for the export command" do |prefix|
+          @prefix = "#{prefix}"
+        end
+      end
+      @option_parser.banner = BANNER
+      @option_parser.parse! ARGV
+    end
+    
+    def build_rack
+      prefix = @prefix
+      Rack::Builder.app do
+        map "/#{prefix}" do
+          use Rack::CommonLogger, STDERR
+          use Rack::ShowExceptions
+          use Rack::Lint
+          run TableSetter::App
+        end
+      end
+    end
+    
+    def build_index(request)
+      install_file(request.request("GET", "/#{@prefix}/").body,
+                      File.join(@out_dir, "index.html"))
+    end
+    
+    def build_assets
+      Dir[ROOT + "/template/public/**/*"].each do |path|
+        copy_file path, File.join(path.gsub(ROOT + "/template/public/", "#{@out_dir}/"))
+      end
+    end
+    
+    def build_tables(request)
+      TableSetter::Table.all.each do |table|
+        puts "Building #{table.slug}"
+        install_file(request.request("GET", "/#{@prefix}/#{table.slug}/").body,
+                    File.join(@out_dir, table.slug, "index.html"))
+        if table.hard_paginate?
+          table.load
+          (1..table.total_pages).each do |page|
+            puts "Building #{table.slug} #{page} of #{table.total_pages}"
+            install_file(request.request("GET", "/#{@prefix}/#{table.slug}/#{page}/").body,
+                File.join(@out_dir, table.slug, page.to_s, "index.html"))
+          end
+        end
+      end
+    end
+    
+    def base_files
+      Dir[ROOT + "/template/**/*"]
+    end
+    
+    def copy_file(source, dest)
+      ensure_directory dest
+      exists = File.exists? dest
+      FileUtils.cp_r(source, dest) unless exists
+      puts "#{exists ? "exists" : "created"}\t#{dest}"
+    end
+    
+    def ensure_directory(dest)
+      expanded_path = File.dirname dest
+      FileUtils.mkdir_p(expanded_path) unless File.exists?(expanded_path)
+    end
+    
+    def install_file(body, dest)
+      ensure_directory dest
+      File.open(dest, "w") do |file|
+        file.write(body)
+      end
+    end
+    
+  end
+end  

data/lib/table_setter/table.rb

@@ -0,0 +1,226 @@
+require 'curb'
+require 'fastercsv'
+require 'table_fu'
+require 'net/http'
+
+module TableSetter
+  class Table
+    # The +Table+ class handles processing the yaml processing and csv loading,
+    # through table fu
+    attr_reader :data, :table_opts, :facets, :prev_page, :next_page, :page
+    
+    # A new Table should accept a slug, mapped to a yaml in the tables directory,
+    # optionally you can defer loading of the table until you're ready to render it.
+    def initialize(slug, opts={:defer => false})
+      options = indifferent_access YAML.load_file(Table.table_path(slug))
+      @table_opts = options[:table]
+      @table_opts[:slug] = slug
+      @deferred = opts[:defer]
+      if !@deferred
+        self.load
+      end
+    end
+    
+    # The load method handles the actual request either to the file system or remote url.
+    # It performs the requested data manipulations form the yml file after the data has been loaded.
+    # We're keeping this explicit to control against unnecessary http requests.
+    def load
+      csv = csv_data
+      @data = TableFu.new(csv_data, @table_opts[:column_options] || {})
+      if @table_opts[:faceting]
+        @data.col_opts[:ignored] = [@table_opts[:faceting][:facet_by]]
+        @facets = @data.faceted_by @table_opts[:faceting][:facet_by]
+      end
+      @data.delete_rows! @table_opts[:dead_rows] if @table_opts[:dead_rows]
+    end
+    
+    # The csv_data for the table fu instance is loaded either from the remote source or from a local
+    # file, depending on the keys present in the yaml file.
+    def csv_data
+      case
+      when google_key || url then Curl::Easy.perform(uri).body_str
+      when file then File.open(uri).read
+      end
+    end
+    
+    # Returns a usable uri based on what sort of input we have.
+    def uri
+      case 
+      when google_key then "http://spreadsheets.google.com/pub?key=#{google_key}&output=csv"
+      when url then url
+      when file then File.expand_path("#{TableSetter.table_path}#{file}")
+      end
+    end
+    
+    # The real +updated_at+ of a Table instance is the newer modification time of the csv file or
+    # the yaml file. Updates to either resource should break the cache.
+    def updated_at
+      csv_time = google_key.nil? ? modification_time(uri) : google_modification_time
+      (csv_time > yaml_time ? csv_time : yaml_time).to_s
+    end
+  
+    def faceted?
+      !@facets.nil?
+    end
+    
+    # A table isn't sortable by tablesorter if it's either faceted or multi-page paginated.
+    def sortable?
+      !faceted? && !hard_paginate?
+    end
+    
+    # hard_paginate instructs the app to render batches of a table.
+    def hard_paginate?
+      @table_opts[:hard_paginate] == true
+    end
+    
+    # The number of rows per page. Defaults to 20 
+    def per_page
+      @table_opts[:per_page] || 20
+    end
+    
+    # paginate uses TableFu's only! method to batch the table. It also computes the page attributes
+    # which are nil and meaningless otherwise.
+    def paginate!(curr_page)
+      return if !hard_paginate?
+      @page = curr_page.to_i
+      raise ArgumentError if @page < 1 || @page > total_pages
+      adj_page = @page - 1 > 0 ? @page - 1 : 0 
+      @prev_page = adj_page > 0 ? adj_page : nil
+      @next_page = page < total_pages ? (@page + 1) : nil
+      @data.only!(adj_page * per_page..(@page * per_page - 1))
+    end
+    
+    
+    # The total pages we'll have. We need to calculate it before paginate, so that we still have the
+    # full @data.rows.length
+    def total_pages
+      @total_pages ||= (@data.rows.length / per_page.to_f).ceil
+    end
+    
+    # A convienence method to return the sort array for table setter.
+    def sort_array
+      @data.sorted_by.inject([]) do |memo, (key, value)|
+        memo << [@data.columns.index(key), value == 'descending' ? 0 : 1]
+      end
+    end
+    
+    # We magically need access to the top level keys like google_key, or uri for the other methods.
+    # It's a bit dangerous because everything returns nil otherwise. At some point we should eval
+    # and create methods at boot time.  
+    def method_missing(method)
+      if @table_opts[method]
+        @table_opts[method]
+      end
+    end
+
+  private
+
+    # Returns the google modification time of the spreadsheet. The public urls don't set the
+    # last-modified header on anything, so we have to do a little dance to find out when exactly
+    # the spreadsheet was last modified. The od[0-9] part of the feed url changes at whim, so we'll 
+    # need to keep an eye on it. Another propblem is that curb doesn't feel like parsing headers, so
+    # since a head request from google is pretty lightweight we can get away with using Net:HTTP.
+    # If for whatever reason the google modification time is busted we'll return the epoch,
+    # and rely on the yaml modified time.
+    def google_modification_time
+      local_url = URI.parse "http://spreadsheets.google.com/feeds/list/#{google_key}/od7/public/basic"
+      web_modification_time local_url
+    end
+
+    # Returns the last-modified time from the remote server. Assumes the remote server knows how to
+    # do this. Returns the epoch if the remote is dense.
+    def web_modification_time(local_url)
+      resp = nil
+      Net::HTTP.start(local_url.host, 80) do |http|
+        resp = http.head(local_url.path)
+      end
+      resp['Last-Modified'].nil? ? Time.at(0) : Time.parse(resp['Last-Modified'])
+    end
+    
+    # Dispatches to web_modification_time if we're dealing with a url, otherwise just stats the
+    # local file.
+    def modification_time(path)
+      is_uri = URI.parse(path)
+      if !is_uri.host.nil?
+        return web_modification_time is_uri
+      end
+      File.new(path).mtime
+    end
+
+    # The modification time of this Table's yaml file.
+    def yaml_time
+      modification_time(Table.table_path(slug))
+    end
+
+    # Enable string or symbol key access to col_opts
+    # from sinatra.
+    def indifferent_access(params)
+      params = indifferent_hash.merge(params)
+      params.each do |key, value|
+        next unless value.is_a?(Hash)
+        params[key] = indifferent_access(value)
+      end
+    end
+
+    # Duplicate a hash's keys and convert them into symbols.
+    def indifferent_hash
+      Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
+    end
+
+  public
+
+    class << self
+      
+      # Returns all the tables in the table directory. Each table is deferred so accessing the @data
+      # attribute will throw and error.
+      def all
+        tables=[] 
+        Dir.glob("#{TableSetter.table_path}/*.yml").each do |file|
+          table = new(File.basename(file, ".yml"), :defer => true)
+          tables << table if table.live
+        end
+        tables
+      end
+
+      # +fresh_yaml_time+ checks each file in the tables directory and returns the newest file's
+      # modification time -- there's probably a more unix-y way to do this but for now this is
+      # plenty speedy.
+      def fresh_yaml_time
+        newest_file = Dir["#{TableSetter.table_path}/*.yml"].inject do |memo, obj|
+          memo_time = File.new(File.expand_path memo).mtime
+          obj_time = File.new(File.expand_path obj).mtime
+          if memo_time > obj_time
+            memo
+          else 
+            obj
+          end
+        end
+        File.new(newest_file).mtime
+      end
+
+      # Convenience method for looking up by slug.
+      def table_path(slug)
+        "#{TableSetter.table_path}#{slug}.yml"
+      end
+
+      # Does a table with this slug exist?
+      def exists?(slug)
+        File.exists? table_path(slug)
+      end
+    end
+
+  end
+end
+
+class TableFu::Formatting
+  class << self
+    # In order to show a sideways bar chart, we're extending the builtin TableFu formatters.
+    def bar(percent)
+      percent = percent.to_f
+      if percent < 1
+        percent = percent * 100
+      end
+      "<div class=\"bar\" style=\"width:#{percent}%\">#{percent}%</div>"
+    end
+  end
+end

data/lib/table_setter.rb

@@ -0,0 +1,32 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'table_fu'
+require 'yaml'
+
+autoload :Sinatra,   'sinatra/base'
+autoload :Thin,      'thin'
+autoload :ERB,       'erb'
+autoload :FasterCSV, 'FasterCSV'
+autoload :Curb,      'curb'
+
+module TableSetter
+  # autoload internals
+  autoload :App,     'table_setter/app'
+  autoload :Command, 'table_setter/command'
+  autoload :Table,   'table_setter/table'
+  
+  ROOT = File.expand_path(File.dirname(__FILE__) + "/..") unless defined? ROOT
+  
+  class << self
+    attr_reader :config_path
+    
+    def configure(path)
+      @config_path = File.expand_path(path)
+    end
+    
+    def table_path
+      @config_path + "/tables/"
+    end
+    
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rubygems'
+require 'table_setter'
+require 'spec'
+require 'rack/test'
+require 'spec/autorun'
+TableSetter.configure(File.join(File.dirname(__FILE__), "..", "template"))
+Spec::Runner.configure do |config|
+  
+end

data/spec/table-setter-app_spec.rb

@@ -0,0 +1,24 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+
+describe TableSetter::App, "in the application" do
+  include Rack::Test::Methods
+  
+  def app
+    TableSetter::App
+  end
+
+  
+  it "should render the homepage" do
+    get '/'
+    last_response.body.include?("All Tables").should be_true
+  end
+  
+  
+  it "should render a table" do
+    get '/example/'
+    last_response.ok?.should be_true
+    last_response.body.include?("Failed Banks List").should be_true
+  end
+  
+end

data/spec/table-setter_spec.rb

@@ -0,0 +1,179 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+
+describe TableSetter::Table do
+  it 'should return the latest yaml modification time' do
+    `touch #{TableSetter::Table.table_path('example')}`
+    TableSetter::Table.fresh_yaml_time.should eql(
+        File.new(TableSetter::Table.table_path('example')).mtime)
+  end
+end
+
+describe TableSetter::Table do
+  before :all do
+    @table = TableSetter::Table.new("example_local")
+  end
+  
+  it "should load from a google key, and defer loading when asked" do
+    table = TableSetter::Table.new("example", :defer => true)
+    table.data.should be_nil
+    table.load
+    table.data.should_not be_nil
+    table.data.headers.should_not be_nil
+  end
+  
+  it 'should be able to find out if a given table exists' do
+    TableSetter::Table.exists?("non_existent_table").should be_false
+    TableSetter::Table.exists?("example").should be_true
+  end
+  
+  it "should have a slug" do
+    @table.slug.should eql "example_local"
+  end
+  
+  it "should have a deck" do
+    @table.deck.should_not be_nil
+  end
+  
+  it "should have 250 items per page" do
+    @table.per_page.should eql 250
+  end
+  
+  it "should be sortable" do
+    @table.sortable?.should be_false
+  end
+  
+  it "should have a footer" do
+    @table.footer.should_not be_nil
+  end
+  
+  it "should have a title" do
+    @table.title.should_not be_nil
+  end
+  
+  it "should have 5805 rows" do
+    @table.data.rows.length.should eql 5805
+  end
+  
+  it "should be stylish" do
+    @table.data.rows[1].column_for('State').style.should eql 'text-align:left;'
+    @table.data.rows[1].column_for('Project Description').style.should eql 'text-align:right;'
+  end
+  
+  it "should have stylish headers" do
+    @table.data.headers[0].style.should eql 'text-align:left;'
+    @table.data.headers[4].style.length.should eql 0
+  end
+  
+  it "should be formatted" do
+    @table.data.rows[1].column_for('ARRA Funds Obligated').to_s.should eql '$154,446'
+  end
+
+end
+
+
+describe TableSetter::Table, "with hard pagination" do
+  
+  before :each do
+    @data = TableSetter::Table.new("example_local")
+  end
+  
+  it "should not be sortable" do
+    @data.sortable?.should eql false
+  end
+  
+  it "should be paginated" do
+    @data.hard_paginate?.should eql true
+  end
+  
+  it "should paginate based on a page" do
+    @data.paginate! 3
+    @data.page.should eql 3
+    @data.prev_page.should eql 2
+    @data.next_page.should eql 4
+    @data.data.rows.length.should eql @data.per_page
+    @data.data.rows[0].column_for('State').to_s.should eql 'CALIFORNIA'
+  end
+  
+  it 'should not paginate when given a bad value' do
+    lambda {@data.paginate!(-1)}.should raise_exception(ArgumentError)
+    lambda {@data.paginate!(10000000)}.should raise_exception(ArgumentError)
+  end
+  
+  it 'should handle first page' do
+    @data.paginate! 1
+    @data.page.should eql 1
+    @data.prev_page.should eql nil
+  end
+  
+  it 'should handle last page' do
+    @data.paginate! @data.total_pages
+    @data.page.should eql @data.total_pages
+    @data.next_page.should eql nil
+  end
+  
+end
+
+
+describe TableSetter::Table, "with faceting and macros" do
+  
+  before :all do
+    @data = TableSetter::Table.new("example_faceted")
+    @tables = @data.facets
+  end
+  
+  it 'should load a faceted_table' do
+    data = TableSetter::Table.new("example_faceted", :defer=> true)
+    data.facets.should be_nil
+    data.load
+    tables = data.facets
+    tables.length.should eql 56 
+  end
+  
+  it "should be faceted" do
+    @data.faceted?.should be_true
+  end
+  
+  it "should not be sortable" do
+    @data.sortable?.should be_false
+  end
+  
+  
+  it "should have 3 tables" do
+    @tables.length.should eql 56
+  end
+  
+  it "should have $212,774,529 for Alabama" do
+    @tables[0].total_for('Total Appropriation').to_s.should eql '$212,774,529'
+  end
+  
+  it "should have $416,075,044 for North Carolina with dead row" do
+    @tables[35].total_for('Total Appropriation').to_s.should eql '$423,318,645'
+  end
+  
+
+  
+end
+
+describe TableSetter::Table, "group fetchers" do
+  it "should return live tables" do
+    TableSetter::Table.all.length.should eql 4
+  end
+end
+
+describe TableSetter::Table, "with urls and google bars" do
+  before :each do
+    @table = TableSetter::Table.new("example_formatted")
+  end
+  
+  it "should have a link row" do
+    @table.data.rows[1].column_for('Agency Webpage').to_s.should eql "<a href='http://www.hhs.gov/recovery/' title='Health and Human Services'>Health and Human Services</a>"
+  end
+  
+  it 'should show a bar' do
+    @table.data.rows[1].column_for('Spent (%)').to_s.should eql "<div class=\"bar\" style=\"width:42.0%\">42.0%</div>"
+  end
+end
+
+
+