ul-wukong 4.1.0

data/examples/graph/minimum_spanning_tree/airfares_graphviz.rb

@@ -0,0 +1,73 @@
+# require Pathname.path_to(:examples, 'graph/union_find')
+# require 'gorillib/model/serialization'
+
+# class Airfare
+#   include Gorillib::Model
+
+#   field :from,  String
+#   field :into,  String
+#   field :price, Integer
+#   field :from_name, String
+#   field :into_name, String
+
+#   def undirected_edge
+#     [from, into].sort
+#   end
+# end
+
+
+# airfares = File.open(Pathname.path_to(:data, 'graph/airfares.tsv')).
+#   readlines.
+#   map{|line| line.split("\t").map(&:strip) }.
+#   map{|vals| Airfare.from_tuple(*vals) }
+
+# # airfares.sort_by(&:undirected_edge).each do |airfare|
+# #   puts "%-7s\t%-7s\t%-7s\t%-7s\t%d" % [*airfare.undirected_edge, airfare.from, airfare.into, airfare.price]
+# # end
+
+# cities = airfares.map(&:from).uniq | airfares.map(&:into)
+
+# edges   = {}
+# airfares.each do |airfare|
+#   edge = airfare.undirected_edge
+#   edges[edge] = airfare.price unless edges.include?(edge) && edges[edge] <= airfare.price
+# end
+
+# mst = Hash.new{|h,k| h[k] = {} }
+# forest = Wukong::Widget::DisjointForest.new
+
+# edges.sort_by(&:last).each do |(city_a, city_b), price|
+#   forest.add(city_a) if not forest.include?(city_a)
+#   forest.add(city_b) if not forest.include?(city_b)
+#   next if forest.find(city_a) == forest.find(city_b)
+#   forest.union(city_a, city_b)
+#   mst[city_a][city_b] = price
+# end
+
+# $mst_both = Hash.new{|h,k| h[k] = {} }
+# mst.each{|city_a, hsh| hsh.each{|city_b, price|
+#     $mst_both[city_a][city_b] = price
+#     $mst_both[city_b][city_a] = price
+# }}
+# def dfs(city, seen)
+#   seen << city
+#   children = $mst_both[city].keys - seen
+#   # [children, children.map{|child| dfs(child, seen) } ]
+#   children.map{|child| [ child, dfs(child, seen) ] }
+# end
+# sorted_cities = ['LAS', dfs('LAS', [])]
+
+# gv_filename = Pathname.path_to(:tmp, 'airfares_mst')
+# File.open("#{gv_filename}.dot", 'w') do |gv_file|
+#   gv_file.puts "graph AirfareMST {\n  label=\"#{Time.now}\"; height= 800; labelloc = t; mindist = 1.5 ; "
+#   # gv_file.puts "mode = hier;"
+#   gv_file.puts 'node [ shape = "plaintext" ]; '
+#   sorted_cities.flatten.each{|city| gv_file.puts "  #{city};" }
+#   mst.sort_by(&:first).each do |from, hsh|
+#     hsh.sort_by(&:first).each do |into, price|
+#       gv_file.puts " %-7s -- %-7s [ label = \"%d\" ];" % [from, into, price]
+#     end
+#   end
+#   gv_file.puts "}"
+# end
+# `neato -Tpng #{gv_filename}.dot -o #{gv_filename}.png`

data/examples/improver/tweet_summary.rb

@@ -0,0 +1,73 @@
+
+#
+# Stupidly simple class for holding a twitter user's tweet summary
+#
+class UserTweetSummary  
+  attr_accessor :fields
+
+  def initialize fields
+    @fields = fields    
+  end
+    
+  def update tweets
+    deletes_this_batch = 0
+    tweets_this_batch  = 0
+    tweets.each do |tweet|
+      deletes_this_batch += (tweet.is_deleted ? 1 : 0)
+      tweets_this_batch  += 1
+    end
+    fields['deletes'] += deletes_this_batch
+    fields['tweets']  += (tweets_this_batch - deletes_this_batch)
+    self
+  end
+
+  def to_json *args
+    fields.to_json     
+  end  
+end
+
+#
+# Holds a tweet and whether
+# or not it's been deleted
+#
+class Tweet
+  attr_accessor :is_deleted
+  def initialize hash
+    @is_deleted = hash.keys.include?("delete")
+  end
+end
+
+
+#
+# Improver processor
+#
+class TweetSummarizer < Wukong::Processor::Improver
+  attr_accessor :user_id
+  
+  def zero
+    super
+    {
+      'tweets'  => 0,
+      'deletes' => 0
+    }
+  end
+
+  def accumulate record
+    @user_id = record[0]
+    json     = record[1]
+    self.group << Tweet.new(JSON.parse(json))
+  end
+
+  def improve summary, deltas    
+    UserTweetSummary.new(JSON.parse(summary)).update(deltas)
+  end  
+end
+
+#
+# Is this necessary?
+#
+TweetSummarizer.register(:tweet_summarizer)
+
+Wukong.dataflow(:summarize_tweets) do
+  tweet_summarizer | to_json
+end

data/examples/loadable.rb

@@ -0,0 +1,2 @@
+# Just a file to test that we can correctly load Ruby files from the
+# command-line.

data/examples/munging/airline_flights/airline_flights.rake

@@ -0,0 +1,83 @@
+require_relative('../../rake_helper')
+require_relative('./models')
+
+Pathname.register_paths(
+  af_data:                  [:data, 'airline_flights'],
+  af_work:                  [:work, 'airline_flights'],
+  af_code:                  File.dirname(__FILE__),
+  #
+  openflights_raw_airports: [:af_data, "openflights_airports-raw#{Settings[:mini_slug]}.csv"   ],
+  openflights_raw_airlines: [:af_data, "openflights_airlines-raw.csv"   ],
+  dataexpo_raw_airports:    [:af_data, "dataexpo_airports-raw#{Settings[:mini_slug]}.csv"      ],
+  wikipedia_icao:           [:af_data, "wikipedia_icao.tsv" ],
+  wikipedia_iata:           [:af_data, "wikipedia_iata.tsv" ],
+  wikipedia_us_abroad:      [:af_data, "wikipedia_us_abroad.tsv" ],
+  #
+  openflights_airports:     [:af_work, "openflights_airports-parsed#{Settings[:mini_slug]}.tsv"],
+  openflights_airlines:     [:af_work, "openflights_airlines-parsed#{Settings[:mini_slug]}.tsv"],
+  dataexpo_airports:        [:af_work, "dataexpo_airports-parsed#{Settings[:mini_slug]}.tsv"   ],
+  airport_identifiers:      [:af_work, "airport_identifiers.tsv"   ],
+  airport_identifiers_mini: [:af_work, "airport_identifiers-sample.tsv"   ],
+  # helpers
+  country_name_lookup:       [:work, 'geo', "country_name_lookup.tsv"],
+  )
+
+chain :airline_flights do
+  code_files = FileList[Pathname.of(:af_code, '*.rb').to_s]
+  chain(:parse) do
+
+    # desc 'parse the dataexpo airports'
+    # create_file(:dataexpo_airports, after: code_files) do |dest|
+    #   RawDataexpoAirport.load_airports(:dataexpo_raw_airports) do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+
+    desc 'parse the openflights airports'
+    create_file(:openflights_airports, after: [code_files, :force]) do |dest|
+      require_relative('../geo/geo_models')
+      Geo::CountryNameLookup.load
+      RawOpenflightAirport.load_airports(:openflights_raw_airports) do |airport|
+        dest << airport.to_tsv << "\n"
+        # puts airport.country
+      end
+    end
+
+    # task :reconcile_airports => [:dataexpo_airports, :openflights_airports] do
+    #   require_relative 'reconcile_airports'
+    #   Airport::IdReconciler.load_all
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.airports.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers_mini, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.exemplars.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'parse the openflights airlines'
+    # create_file(:openflights_airlines, after: code_files) do |dest|
+    #   RawOpenflightAirline.load_airlines(:openflights_raw_airlines) do |airline|
+    #     dest << airline.to_tsv << "\n"
+    #     puts airline.to_tsv
+    #   end
+    # end
+
+  end
+end
+
+task :default => [
+  'airline_flights',
+  # 'airline_flights:parse:dataexpo_airports',
+  # 'airline_flights:parse:openflights_airports',
+  # 'airline_flights:parse:airport_identifiers',
+  # 'airline_flights:parse:airport_identifiers_mini',
+  # 'airline_flights:parse:openflights_airlines',
+]

data/examples/munging/airline_flights/airport_id_unification.rb

@@ -0,0 +1,129 @@
+class Airport
+
+  # [Hash] all options passed to the field not recognized by one of its own current fields
+  attr_reader :_extra_attributes
+
+    # # Airports whose IATA and FAA codes differ; all are in the US, so their ICAO is "K"+the FAA id
+    # FAA_ICAO_FIXUP = {
+    #   "GRM" => "CKC", "CLD" => "CRQ", "SDX" => "SEZ", "AZA" => "IWA", "SCE" => "UNV", "BLD" => "BVU",
+    #   "LKE" => "W55", "HSH" => "HND", "BKG" => "BBG", "UST" => "SGJ", "LYU" => "ELO", "WFK" => "FVE",
+    #   "FRD" => "FHR", "ESD" => "ORS", "RKH" => "UZA", "NZC" => "VQQ", "SCF" => "SDL", "JCI" => "IXD",
+    #   "AVW" => "AVQ", "UTM" => "UTA", "ONP" => "NOP", }
+    #
+    # [:iata, :icao, :latitude, :longitude, :country, :city, :name].each do |attr|
+    #   define_method("of_#{attr}"){ @_extra_attributes[:"of_#{attr}"] }
+    #   define_method("de_#{attr}"){ @_extra_attributes[:"de_#{attr}"] }
+    # end
+    #
+    # def lint_differences
+    #   errors = {}
+    #   return errors unless de_name.present? && of_name.present?
+    #   [
+    #     [:iata, of_iata, de_iata], [:icao, of_icao, de_icao], [:country, of_country, de_country],
+    #     [:city, of_city, de_city],
+    #     [:name, of_name, de_name],
+    #   ].each{|attr, of, de| next unless of && de ; errors[attr] = [of, de] if of != de }
+    #
+    #   if (of_latitude && of_longitude && de_latitude && de_longitude)
+    #     lat_diff = (of_latitude  - de_latitude ).abs
+    #     lng_diff = (of_longitude - de_longitude).abs
+    #     unless (lat_diff < 0.015) && (lng_diff < 0.015)
+    #       msg = [of_latitude, de_latitude, of_longitude, de_longitude, lat_diff, lng_diff].map{|val| "%9.4f" % val }.join(" ")
+    #       errors["distance"] = ([msg, of_city, de_city, of_name, de_name])
+    #     end
+    #   end
+    #
+    #   errors
+    # end
+    #
+    # AIRPORTS      = Hash.new # unless defined?(AIRPORTS)
+    # def self.load(of_filename, de_filename)
+    #   RawOpenflightAirport.load_csv(of_filename) do |raw_airport|
+    #     airport = raw_airport.to_airport
+    #     AIRPORTS[airport.iata_icao] = airport
+    #   end
+    #   RawDataexpoAirport.load_csv(de_filename) do |raw_airport|
+    #     airport = (AIRPORTS[raw_airport.iata_icao] ||= self.new)
+    #     if airport.de_name
+    #       warn "duplicate data for #{[iata, de_iata, icao, de_icao]}: #{raw_airport.to_tsv} #{airport.to_tsv}"
+    #     end
+    #     airport.receive!(raw_airport.airport_attrs)
+    #   end
+    #   AIRPORTS
+    # end
+
+    def self.load(dirname)
+      load_csv(File.join(dirname, 'wikipedia_icao.tsv')) do |id_mapping|
+        [:icao, :iata, :faa ].each do |attr|
+          val = id_mapping.read_attribute(attr) or next
+          next if (val == '.') || (val == '_')
+          if that = ID_MAPPINGS[attr][val]
+            lint = that.disagreements(id_mapping)
+            puts [attr, val, "%-25s" % lint.inspect, id_mapping, that, "%-60s" % id_mapping.name, "%-25s" % that.name].join("\t") if lint.present?
+          else
+            ID_MAPPINGS[attr][val] = id_mapping
+          end
+        end
+        # [:icao, :iata, :faa ].each do |attr|
+        #   val = id_mapping.read_attribute(attr)
+        #   ID_MAPPINGS[attr][val] = id_mapping
+        # end
+      end
+      load_csv(File.join(dirname, 'wikipedia_iata.tsv')) do |id_mapping|
+        # if not ID_MAPPINGS[:icao].has_key?(id_mapping.icao)
+        #   puts [:badicao, "%-25s" % "", id_mapping, " "*24, "%-60s" % id_mapping.name].join("\t")
+        # end
+        [:icao, :iata, :faa ].each do |attr|
+          val = id_mapping.read_attribute(attr) or next
+          next if (val == '.') || (val == '_')
+          if that = ID_MAPPINGS[attr][val]
+            lint = that.disagreements(id_mapping)
+            puts [attr, val, "%-25s" % lint.inspect, id_mapping, that, "%-60s" % id_mapping.name, "%-25s" % that.name].join("\t") if lint.present?
+          else
+            ID_MAPPINGS[attr][val] = id_mapping
+          end
+        end
+      end
+
+      # def adopt_field(that, attr)
+      #   this_val = self.read_attribute(attr)
+      #   that_val = that.read_attribute(attr)
+      #   if name =~ /Bogus|Austin/i
+      #     puts [attr, this_val, that_val, attribute_set?(attr), that.attribute_set?(attr), to_tsv, that.to_tsv].join("\t")
+      #   end
+      #   if    this_val && that_val
+      #     if (this_val != that_val) then warn [attr, this_val, that_val, name].join("\t") ; end
+      #   elsif that_val
+      #     write_attribute(that_val)
+      #   end
+      # end
+
+      def to_s
+        attributes.values[0..2].join("\t")
+      end
+
+      def disagreements(that)
+        errors = {}
+        [:icao, :iata, :faa ].each do |attr|
+          this_val = self.read_attribute(attr) or next
+          that_val = that.read_attribute(attr) or next
+          next if that_val == '.' || that_val == '_'
+          errors[attr] = [this_val, that_val] if this_val != that_val
+        end
+        errors
+      end
+
+    def self.dump_ids(ids)
+      "%s\t%s\t%s" % [icao, iata, faa]
+    end
+    def self.dump_mapping
+      [:icao, :iata, :faa].map do |attr|
+        "%-50s" % ID_MAP[attr].to_a.sort.map{|id, val| "#{id}:#{val.icao||'    '}|#{val.iata||'   '}|#{val.faa||'   '}"}.join(";")
+      end
+    end
+
+    def self.dump_info(kind, ids, reconciler, existing, *args)
+      ex_str = [existing.map{|el| dump_ids(el.ids) }, "\t\t","\t\t","\t\t"].flatten[0..2]
+      puts [kind, dump_ids(ids), dump_ids(reconciler.ids), ex_str, *args, dump_mapping.join("//") ].flatten.join("\t| ")
+    end
+end

data/examples/munging/airline_flights/airport_ok_chars.rb

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+
+OK_CHARS_RE      = /[^a-zA-Z0-9\ \/\.\,\-\(\)\'ÁÂÄÅÇÉÍÎÑÖØÜÞàáâãäåæçèéêëìíîïðñóôõöøúüýāăčėęěğİıŁłńōőřŞşŠšţťūźŽžơț]/

data/examples/munging/airline_flights/indexable.rb

@@ -0,0 +1,75 @@
+module Gorillib
+  module Model
+
+    #
+    # @example
+    #   class Airport
+    #     include Gorillib::Model::Indexable
+    #     # ... define model
+    #     index_on :icao_code, :city
+    #   end
+    #   Airport.for_icao_code('KAUS') #=> #<Airport icao_code="KAUS" ... >
+    #   Airport.for_city('Austin')    #=> #<Airport icao_code="KAUS" ... >
+    #
+    # You must implement a `load` method
+    #
+    module Indexable
+      extend Gorillib::Concern
+
+      included do |base|
+        base.class_attribute :lookups, instance_writer: false
+        self.lookups ||= []
+      end
+
+      module ClassMethods
+
+        def values
+          @values ||= Array.new
+        end
+
+        def flush_lookups
+          lookups.each{|idx| remove_instance_variable("@#{idx}") if instance_variable_defined?("@#{idx}") }
+        end
+
+        #
+        # @example
+        #   class Airport
+        #     index_on :icao_code, [:city, :cities]
+        #   end
+        #   Airport.for_icao_code('KAUS') #=> #<Airport icao_code="KAUS" ... >
+        #   Airport.for_city('Austin')    #=> #<Airport icao_code="KAUS" ... >
+        #
+        # NOTE: `.#{key_name}_index` method is NOT part of the framework interface.
+        # only the `.for_#{keyname}` method is suported.
+        def index_on(*key_names)
+          self.lookups += key_names
+          self.lookups.uniq!
+          #
+          key_names.each do |key_name, index_name|
+            index_name ||= "#{key_name}_index"
+            class_eval <<-EOV, __FILE__, __LINE__ + 1
+              class << self
+                def #{index_name}                                # def name_index
+                  @#{index_name} ||=                             #   @name_index ||=
+                    Hash[values.map{|el| [el.#{key_name}, el] }] #     Hash[values.map{|el| [el.name, el]
+                end                                              # end
+                protected(:#{index_name})                        # protected :name_index
+              end
+            EOV
+
+            instance_eval <<-EOV, __FILE__, __LINE__ + 1
+              def for_#{key_name}(*args, &block)               # def for_name(*args, &block)
+                #{index_name}.fetch(*args, &block)             #   name_index.fetch(*args, &block)
+              end                                              # end
+            EOV
+          end
+        end
+
+
+
+      end
+
+    end
+
+  end
+end