rflare 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3d3aaf86caee415439c6b079238eb08f64f99361
+  data.tar.gz: 8577586e4feef9089da739591cc7ba6fe0ac13cf
+SHA512:
+  metadata.gz: e322a3d7229063a594df73674c2fbb149fcc7a1b8a0a3041cdfcfc2511982ef9c91faf6d08b115949e3c958c069bd6c91cc7513f70a589a98e715c82d20f9197
+  data.tar.gz: eda58c277896406c7d880bc77684b412b19820c27448e8186d75281b280c967aff40faf0db1a74d084b8f4d7c09f3f9c80c566033e393082c41c69be75f4f474

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/bin/rflare

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+require 'optparse'
+require 'tgf'
+require 'rflare'
+
+csv_opts = {}
+nodes, edges = [], []
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: rflare [options]"
+
+  opts.on("-F", "--field-separator [FS]") do |fs|
+    csv_opts[:col_sep] = fs
+  end
+
+  opts.on("-Q", "--quote-char [QC]") do |qc|
+    csv_opts[:quote_char] = qc
+  end
+
+  opts.on("-f", "--file [file]") do |file|
+    nodes, edges = TGF.parse(File(q))
+  end
+
+  opts.on("-e", "--evaluate [query]") do |query|
+    nodes, edges = TGF.parse(query, ';')
+  end
+end.parse!
+
+def rows_and_cols str
+  str.nil? ? [nil, nil] : str.split(/\s+/, 2)
+end
+
+out = CSV.new $stdout, csv_opts
+ARGV.each do |csv_file|
+  ss = RFlare::Spreadsheet.new(CSV.read(csv_file, csv_opts))
+  my_nodes = nodes.map { |node|
+    if node.label.nil?
+      RFlare::Node.new node.id, nil, nil, nil, ss.row_bounds, ss.col_bounds
+    else
+      m =  /^\/((?:.*\\\/)*.*)\/\s*(.+)?/.match node.label
+      rows, columns = rows_and_cols m[2]
+      RFlare::Node.new node.id, m[1], rows, columns, ss.row_bounds, ss.col_bounds
+    end
+  }
+  my_edges = edges.map { |edge|
+    rows, columns = rows_and_cols edge.label
+    RFlare::Edge.new edge.from, edge.to, rows, columns
+  }
+  root = my_nodes[0]
+ 
+  RFlare::Results.new(my_edges, my_nodes, ss, root).each { |match|
+    n = my_nodes.select {|node| node.id[0] != '_'}
+    out << n.map {|node| match[node.id] || ''}
+  }
+end
+out.flush
+

data/lib/rflare.rb

@@ -0,0 +1,181 @@
+
+class Array
+  def bounds; 0 ... size; end
+end
+
+class Range
+  def cap num
+    num < min ? min : (num > max ? max : num)
+  end
+end
+
+module RFlare
+
+  class Square
+    def initialize rows, columns
+      @rows, @columns = rows, columns
+    end
+
+    attr_reader :rows, :columns
+    include Enumerable
+
+    def each
+      @rows.each {|row|
+        @columns.each {|col|
+          yield row, col
+        }
+      }
+    end
+
+    def include? row, col
+      @rows.include? row and @columns.include? col
+    end
+
+    def == other
+      @rows == other.rows and @columns == other.columns
+    end
+  end
+
+  class Node
+    def initialize id, match, rows, columns, row_bounds, col_bounds
+      @id = id || '_'
+      @match = Regexp.new(match || '.*')
+      @valid = Square.new(
+        Spec.new(rows || '0:*').range(0, row_bounds),
+        Spec.new(columns || '0:*').range(0, col_bounds))
+    end
+
+    attr_reader :id, :match, :valid
+
+    def matches ss, row, col
+      @valid.include? row, col and (ss[row,col] || '').to_s =~ @match
+    end
+  end
+
+  class Spec
+    def initialize spec
+      @spec = spec.to_s
+      raise "invalid spec '#{@spec}'" if @spec !~ /[+-]?([0-9]+|\*)(:([+-]?([0-9]+|\*))?)?/
+    end
+
+    # + or - means relative to num, otherwise absolute
+    def range num, bounds
+      bits = @spec.split ":", 2
+      s = bits.size == 1 ? @spec : bits[0]
+      e = bits.size == 1 ? @spec : bits[1]
+      range_start(num, bounds, s) .. range_end(num, bounds, e)
+    end
+
+  private
+
+    def range_start num, bounds, spec
+      if spec == '+*' or spec == '*'
+        num + 1
+      elsif spec == '-*' 
+        bounds.min
+      elsif spec[0] == '+' or spec[0] == '-'
+        offset = spec[1, spec.length - 1].to_i
+        spec[0] == '+' ? (num + offset) : (num - offset)
+      elsif spec == ''
+        bounds.min
+      else
+        spec.to_i
+      end
+    end
+
+    def range_end num, bounds, spec
+      if spec == '+*' or spec == '*' or spec == ""
+        bounds.max
+      elsif spec == '-*' 
+        num - 1
+      elsif spec[0] == '+' or spec[0] == '-'
+        offset = spec[1, spec.length - 1].to_i
+        spec[0] == '+' ? (num + offset) : (num - offset)
+      else
+        spec.to_i
+      end
+    end
+  end
+
+  class Edge
+    def initialize from, to, vert, horiz
+      @from, @to = from, to
+      @vert = Spec.new(vert || '+0')
+      @horiz = Spec.new(horiz || '+0')
+    end
+
+    attr_reader :from, :to
+
+    def get_square row, col, row_bounds, col_bounds
+       Square.new(
+        @vert.range(row, row_bounds),
+        @horiz.range(col, col_bounds))
+    end
+  end
+
+  class Spreadsheet
+    def initialize arr_of_rows
+      @data = arr_of_rows
+      @row_bounds = @data.bounds
+      @col_bounds = @data.empty? ? (0...0) : @data[0].bounds
+    end
+
+    def [] row, col
+      if @row_bounds.include? row and @col_bounds.include? col
+        @data[row][col]
+      else
+        nil
+      end
+    end
+
+    attr_reader :row_bounds, :col_bounds
+  end
+
+  class Results
+    def initialize edges, nodes, ss, root
+      @ss, @root = ss, root
+      @edges_byfrom = Hash.new {|h,k| h[k] = [] }
+      edges.each {|edge| @edges_byfrom[edge.from] << edge }
+      @nodes_byid = Hash.new
+      nodes.each {|node| @nodes_byid[node.id] = node}
+    end
+
+    include Enumerable
+
+    # start at root, looking for trees
+    def each
+      @root.valid.each {|row, col|
+        matches(row, col, @root).each { |match| yield match }
+      }
+    end
+
+  private
+
+    def matches row, col, node
+      return [] if !node.matches(@ss, row, col)
+
+      me = {node.id => @ss[row,col]}
+      edges = @edges_byfrom[node.id]
+      return [me] if edges.empty?
+
+      # get array with dims:
+      # (1) each edge from this
+      # -- we flatten the square-for-each-edge dim
+      # (2) each match (a Hash) from edge
+      edge_matches = edges.map do |edge|
+        to = @nodes_byid[edge.to]
+        sq = edge.get_square(row, col, @ss.row_bounds, @ss.col_bounds)
+        paths = sq.flat_map do |sq_row, sq_col| 
+          matches sq_row, sq_col, to
+        end
+        paths.select {|a| !a.empty?}
+      end
+
+      [me].product(*edge_matches).map do |assignments_arr|
+        assignments_arr.inject Hash.new, :merge
+      end
+    end
+
+  end
+end
+

data/test/test_rflare.rb

@@ -0,0 +1,290 @@
+require 'minitest/autorun'
+require 'rflare'
+
+class Hash
+  def <=> other
+    k = keys.sort <=> other.keys.sort
+    return k if k != 0
+    keys.sort.each {|key|
+      k = self[key] <=> other[key]
+      return k if k != 0
+    }
+    0
+  end
+end
+
+class FlashRangeTest < Minitest::Test
+  def test_in
+    assert_equal 4, (2..5).cap(4)
+  end
+
+  def test_after
+    assert_equal 5, (2..5).cap(7)
+  end
+
+  def test_before
+    assert_equal 2, (2..5).cap(1)
+  end
+end
+
+class FlashSquareTest < Minitest::Test
+  def test_include_in
+    assert RFlare::Square.new(2..4, 3..5).include? 3, 4
+  end
+
+  def test_include_out
+    refute RFlare::Square.new(2..4, 3..5).include? 1, 4
+  end
+
+  def test_enumerate
+    assert_equal [[2,3], [2,4], [2,5], [3,3], [3,4], [3,5]], RFlare::Square.new(2..3, 3..5).to_a
+  end
+
+  def test_equals
+    assert_equal RFlare::Square.new(2..3, 4..5), RFlare::Square.new(2..3, 4..5)
+  end
+end
+
+class FlashSpecTest < Minitest::Test
+  def setup
+    @bounds = 0..10
+  end
+
+  def test_single_relativeconst
+    assert_equal (4..4), RFlare::Spec.new('+1').range(3, @bounds)
+  end 
+
+  def test_single_plusstar
+    assert_equal (4..@bounds.max), RFlare::Spec.new('+*').range(3, @bounds)
+  end 
+
+  def test_single_plusstar2
+    assert_equal (4..@bounds.max), RFlare::Spec.new('*').range(3, @bounds)
+  end 
+
+  def test_minusone
+    assert_equal (2..2), RFlare::Spec.new('-1').range(3, @bounds)
+  end 
+
+  def test_minusstar
+    assert_equal (@bounds.min..2), RFlare::Spec.new('-*').range(3, @bounds)
+  end 
+
+  def test_abs
+    assert_equal (1..1), RFlare::Spec.new('1').range(3, @bounds)
+  end 
+
+  def test_range_relativeconst
+    assert_equal (4..6), RFlare::Spec.new('+1:+3').range(3, @bounds)
+  end 
+
+  def test_range_star
+    assert_equal (5..@bounds.max), RFlare::Spec.new('+2:+*').range(3, @bounds)
+  end 
+
+  def test_invalid
+    assert_raises(RuntimeError) { RFlare::Spec.new 'hat' }
+  end 
+
+  def test_range_blank_start
+    assert_equal (@bounds.min..4), RFlare::Spec.new(':4').range(3, @bounds)
+  end 
+
+  def test_range_blank_end
+    assert_equal (1..@bounds.max), RFlare::Spec.new('1:').range(3, @bounds)
+  end 
+
+end
+
+class FlashNodeTest < Minitest::Test
+  def setup
+    @ss = RFlare::Spreadsheet.new [[0,1,2],['a','b','c']]
+  end
+
+  def test_match
+    n = RFlare::Node.new nil, nil, 0, nil, @ss.row_bounds, @ss.col_bounds
+    assert_equal RFlare::Square.new(0..0, 0..2), n.valid
+    assert n.matches @ss, 0, 0
+    assert n.matches @ss, 0, 1
+    assert n.matches @ss, 0, 2
+    refute n.matches @ss, 1, 0
+    refute n.matches @ss, 1, 1
+    refute n.matches @ss, 1, 2
+  end
+end
+
+class FlashSpreadsheetTest < Minitest::Test
+  def test_empty
+    ss = RFlare::Spreadsheet.new []
+    assert_equal 0...0, ss.row_bounds
+    assert_equal 0...0, ss.col_bounds
+    assert_nil ss[0,0]
+  end
+
+  def test_full
+    ss = RFlare::Spreadsheet.new [[1,2,3],['a','b','c']]
+    assert_equal 0...2, ss.row_bounds
+    assert_equal 0...3, ss.col_bounds
+    assert_equal 1, ss[0,0]
+    assert_equal 'b', ss[1,1]
+  end
+end
+
+class FlashResultsTest < Minitest::Test
+  def setup
+    @ss = RFlare::Spreadsheet.new [
+      [      nil, 'value', 'year', 'value', 'year', 'Comments'],
+      [ 'Albania',  1000,   1950,     930,   1981,     'FRA 1'],
+      [ 'Austria',  3139,   1951,    3177,   1955,     'FRA 3'],
+      [ 'Belgium',   541,   1947,     601,   1950,        nil ],
+      ['Bulgaria',  2964,   1947,    3259,   1958,     'FRA 1'],
+      [   'Czech',  2416,   1950,    2503,   1960,        'NC']
+    ] 
+  end
+
+  def node id, match, rows = nil, columns = nil
+    RFlare::Node.new id, match, rows, columns, @ss.row_bounds, @ss.col_bounds
+  end
+
+  def edge from, to, vert, horiz
+    RFlare::Edge.new from, to, vert, horiz
+  end
+
+  def assert_matches edges, nodes, expected
+    actual = RFlare::Results.new(edges, nodes, @ss, nodes[0]).to_a
+    assert_equal expected.sort, actual.sort
+  end
+
+  # the tests:
+
+  def test_simplecell
+    nodes = [node(3, '^value$')]
+    edges = []
+    expected = [{3 => 'value'}, {3 => 'value'}]
+    assert_matches edges, nodes, expected
+  end
+
+  def test_header
+    nodes = [node(3, '^value$', 0, nil)]
+    edges = []
+    expected = [{3 => 'value'}, {3 => 'value'}]
+    assert_matches edges, nodes, expected
+  end
+
+  def test_oneedge
+    nodes = [node(4,'^[0-9]+$'), node(3, '^value$')]
+    edges = [edge(4, 3, '-*', nil)]
+    expected = [
+      {3 => 'value', 4 => 1000},
+      {3 => 'value', 4 => 3139},
+      {3 => 'value', 4 =>  541},
+      {3 => 'value', 4 => 2964},
+      {3 => 'value', 4 => 2416},
+      {3 => 'value', 4 =>  930},
+      {3 => 'value', 4 => 3177},
+      {3 => 'value', 4 =>  601},
+      {3 => 'value', 4 => 3259},
+      {3 => 'value', 4 => 2503}
+    ]
+    assert_matches edges, nodes, expected
+  end
+
+  def test_twoedges
+    nodes = [
+      node('v','^[0-9]+$'), 
+      node('vcol', '^value$'),
+      node('y','^19[0-9]{2}$'), 
+    ]
+    edges = [
+      edge('v', 'vcol', '-*', nil),
+      edge('v', 'y', nil, '+1')
+    ]
+    expected = [
+      {'vcol' => 'value', 'v' => 1000, 'y' => 1950},
+      {'vcol' => 'value', 'v' => 3139, 'y' => 1951},
+      {'vcol' => 'value', 'v' =>  541, 'y' => 1947},
+      {'vcol' => 'value', 'v' => 2964, 'y' => 1947},
+      {'vcol' => 'value', 'v' => 2416, 'y' => 1950},
+      {'vcol' => 'value', 'v' =>  930, 'y' => 1981},
+      {'vcol' => 'value', 'v' => 3177, 'y' => 1955},
+      {'vcol' => 'value', 'v' =>  601, 'y' => 1950},
+      {'vcol' => 'value', 'v' => 3259, 'y' => 1958},
+      {'vcol' => 'value', 'v' => 2503, 'y' => 1960}
+    ]
+    assert_matches edges, nodes, expected
+  end
+
+  def test_twolevelrecursion
+    nodes = [
+      node('v','^[0-9]+$'), 
+      node('vcol', '^value$'),
+      node('y','^19[0-9]{2}$'), 
+      node('ycol', '^year$'),
+    ]
+    edges = [
+      edge('v', 'vcol', '-*', nil),
+      edge('y', 'ycol', '-*', nil),
+      edge('v', 'y', nil, '+1'),
+    ]
+    expected = [
+      {'vcol' => 'value', 'v' => 1000, 'y' => 1950, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' => 3139, 'y' => 1951, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' =>  541, 'y' => 1947, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' => 2964, 'y' => 1947, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' => 2416, 'y' => 1950, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' =>  930, 'y' => 1981, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' => 3177, 'y' => 1955, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' =>  601, 'y' => 1950, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' => 3259, 'y' => 1958, 'ycol' => 'year'},
+      {'vcol' => 'value', 'v' => 2503, 'y' => 1960, 'ycol' => 'year'},
+    ]
+    assert_matches edges, nodes, expected
+  end
+
+  def test_withrowlimit
+    nodes = [node(3, '^[A-Za-z]+$', '1:*', 0)]
+    edges = []
+    expected = [
+      {3 => 'Albania'},
+      {3 => 'Austria'},
+      {3 => 'Belgium'},
+      {3 => 'Bulgaria'},
+      {3 => 'Czech'},
+    ]
+    assert_matches edges, nodes, expected
+  end
+
+  def test_frompaper
+    nodes = [
+      node('c', '^[A-Za-z]+$', '1:*', 0),
+      node('v','^[0-9]+$', '1:*', nil), 
+      node('vcol', '^value$', 0, nil),
+      node('y','^19[0-9]{2}$', '1:*', nil), 
+      node('ycol', '^year$', 0, nil),
+      node('m','^[A-Za-z 0-9]*$', '1:*', nil), 
+      node('mcol', '^Comments$', 0, nil),
+    ]
+    edges = [
+      edge('v', 'vcol', '-*', nil),
+      edge('y', 'ycol', '-*', nil),
+      edge('m', 'mcol', '-*', nil),
+      edge('c', 'v', nil, '+*'),
+      edge('v', 'y', nil, '+1'),
+      edge('y', 'm', nil, '+*'),
+    ]
+    expected = [
+      {'vcol' => 'value', 'v' => 1000, 'y' => 1950, 'ycol' => 'year', 'c' => 'Albania', 'mcol' => 'Comments', 'm' => 'FRA 1'},
+      {'vcol' => 'value', 'v' => 3139, 'y' => 1951, 'ycol' => 'year', 'c' => 'Austria', 'mcol' => 'Comments', 'm' => 'FRA 3'},
+      {'vcol' => 'value', 'v' =>  541, 'y' => 1947, 'ycol' => 'year', 'c' => 'Belgium', 'mcol' => 'Comments', 'm' => nil},
+      {'vcol' => 'value', 'v' => 2964, 'y' => 1947, 'ycol' => 'year', 'c' => 'Bulgaria', 'mcol' => 'Comments', 'm' => 'FRA 1'},
+      {'vcol' => 'value', 'v' => 2416, 'y' => 1950, 'ycol' => 'year', 'c' => 'Czech', 'mcol' => 'Comments', 'm' => 'NC'},
+      {'vcol' => 'value', 'v' =>  930, 'y' => 1981, 'ycol' => 'year', 'c' => 'Albania', 'mcol' => 'Comments', 'm' => 'FRA 1'},
+      {'vcol' => 'value', 'v' => 3177, 'y' => 1955, 'ycol' => 'year', 'c' => 'Austria', 'mcol' => 'Comments', 'm' => 'FRA 3'},
+      {'vcol' => 'value', 'v' =>  601, 'y' => 1950, 'ycol' => 'year', 'c' => 'Belgium', 'mcol' => 'Comments', 'm' => nil},
+      {'vcol' => 'value', 'v' => 3259, 'y' => 1958, 'ycol' => 'year', 'c' => 'Bulgaria', 'mcol' => 'Comments', 'm' => 'FRA 1'},
+      {'vcol' => 'value', 'v' => 2503, 'y' => 1960, 'ycol' => 'year', 'c' => 'Czech', 'mcol' => 'Comments', 'm' => 'NC'},
+    ]
+    assert_matches edges, nodes, expected
+  end
+end
+

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: rflare
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Remis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tgf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+description: A command-line tool to extract relational data from semi-structured CSV
+  files
+email: remis.thoughts@gmail.com
+executables:
+- rflare
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gemtest"
+- Rakefile
+- bin/rflare
+- lib/rflare.rb
+- test/test_rflare.rb
+homepage: https://research.microsoft.com/pubs/214302/flashrelate-tech-report-April2014.pdf
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Ruby version of Flare
+test_files:
+- test/test_rflare.rb