swineherd 0.0.1

data/bin/hdp-tree

@@ -0,0 +1,26 @@
+#!/usr/bin/env jruby
+
+require 'swineherd'
+
+#
+# Creates a 'tree' view of an hdfs path. It's not as pretty as the
+# unix tree command but that's only because I'm not smart enough to
+# print the hierarchy properly.
+#
+
+FS   = Swineherd::FileSystem.get(:hdfs)
+path = ARGV[0]
+
+# Recursively list paths
+def lr path
+  paths = FS.entries(path)
+  if (paths && !paths.empty?)
+    paths.map{|e| {File.basename(path) => lr(e)}}.flatten
+  else
+    File.basename(path)
+  end
+end
+
+
+tree = {File.dirname(path) => lr(path)}.to_yaml
+puts tree

data/examples/pagerank/data/seinfeld_network.tsv

@@ -0,0 +1,429 @@
+jerry	costanza81
+jerry	ElaineBenes
+jerry	kramer
+jerry	NewmanUSPS
+jerry	THE_REAL_PUDDY
+jerry	JPeterman
+jerry	FRANKCOSTANZA
+costanza81	jerry
+costanza81	ElaineBenes
+costanza81	kramer
+costanza81	NewmanUSPS
+costanza81	THE_REAL_PUDDY
+costanza81	JPeterman
+costanza81	FRANKCOSTANZA
+ElaineBenes	jerry
+ElaineBenes	costanza81
+ElaineBenes	kramer
+ElaineBenes	THE_REAL_PUDDY
+ElaineBenes	JPeterman
+kramer	jerry
+kramer	costanza81
+kramer	ElaineBenes
+kramer	NewmanUSPS
+kramer	THE_REAL_PUDDY
+kramer	JPeterman
+kramer	FRANKCOSTANZA
+NewmanUSPS	jerry
+NewmanUSPS	costanza81
+NewmanUSPS	ElaineBenes
+NewmanUSPS	kramer
+NewmanUSPS	THE_REAL_PUDDY
+NewmanUSPS	JPeterman
+NewmanUSPS	FRANKCOSTANZA
+THE_REAL_PUDDY	jerry
+THE_REAL_PUDDY	costanza81
+THE_REAL_PUDDY	ElaineBenes
+THE_REAL_PUDDY	kramer
+THE_REAL_PUDDY	NewmanUSPS
+THE_REAL_PUDDY	JPeterman
+THE_REAL_PUDDY	FRANKCOSTANZA
+THE_REAL_PUDDY	Vegetable_Lasagna
+FRANKCOSTANZA	jerry
+FRANKCOSTANZA	costanza81
+FRANKCOSTANZA	kramer
+jerry	MortySeinfeld
+jerry	HelenSeinfeld
+jerry	Izzy_Mandelbaum
+jerry	UncleLEO
+jerry	Artie_Levine
+MortySeinfeld	UncleLEO
+MortySeinfeld	HelenSeinfeld
+MortySeinfeld	Cousin_Jeffrey
+Izzy_Mandelbaum	jerry
+UncleLEO	jerry
+UncleLEO	MortySeinfeld
+UncleLEO	HelenSeinfeld
+UncleLEO	Cousin_Jeffrey
+UncleLEO	Babs_Kramer
+Babs_Kramer	UncleLEO
+Cousin_Jeffrey	jerry
+Cousin_Jeffrey	MortySeinfeld
+Cousin_Jeffrey	UncleLEO
+jerry	Nana
+Nana	MortySeinfeld
+MortySeinfeld	Nana
+Cousin_Jeffrey	Nana
+UncleLEO	Nana
+Nana	UncleLEO
+JackKlompus	MortySeinfeld
+MortySeinfeld	JackKlompus
+Dolores	jerry
+MarlatheVirgin	jerry
+TiaVanCamp	jerry
+Rachel_Goldstein	jerry
+meryl	jerry
+MissRhodeIsland	jerry
+Pam	jerry
+Sheila	jerry
+MelissaFlyingFree	jerry
+jerry	Dolores
+jerry	MarlatheVirgin
+jerry	TiaVanCamp
+jerry	Rachel_Goldstein
+jerry	meryl
+jerry	MissRhodeIsland
+jerry	Pam
+jerry	Sheila
+jerry	MelissaFlyingFree
+jerry	Laura
+jerry	Sandy
+Laura	Sandy
+Laura	jerry
+Sandy	jerry
+kramer	MissRhodeIsland
+kramer	Pam
+jerry	Jenna
+Jenna	jerry
+jerry	bania
+costanza81	bania
+bania	jerry
+bania	The_Soup_Nazi
+bania	Poppie
+bania	Jenna
+jerry	Noreen
+jerry	JackKlompus
+jerry	Milos
+jerry	JeanPaul_JeanPaul
+jerry	FusilliJerry
+FusilliJerry	jerry
+FusilliJerry	kramer
+kramer	FusilliJerry
+FusilliJerry	FRANKCOSTANZA
+jerry	pez
+jerry	superman
+jerry	bigstein
+Milos	jerry
+jerry	Roy_the_Dentist
+jerry	BabuBhatt
+kramer	BabuBhatt
+BabuBhatt	kramer
+kramer	Poppie
+Poppie	kramer
+Poppie	ElaineBenes
+ElaineBenes	Poppie
+jerry	Poppie
+jerry	Shaky_the_Mohel
+jerry	bubble_boy
+bubble_boy	jerry
+MatthewSeinfeldFan	jerry
+FragileFrankieMerman	jerry
+jerry	FragileFrankieMerman
+costanza81	FragileFrankieMerman
+costanza81	EstelleC
+FRANKCOSTANZA	EstelleC
+EstelleC	costanza81
+EstelleC	FRANKCOSTANZA
+FRANKCOSTANZA	Lloyd_Braun
+EstelleC	Lloyd_Braun
+Lloyd_Braun	FRANKCOSTANZA
+Lloyd_Braun	costanza81
+Lloyd_Braun	EstelleC
+kramer	MrWilhelm
+costanza81	MrWilhelm
+costanza81	Allison
+Allison	costanza81
+costanza81	LindsayEnright
+LindsayEnright	costanza81
+costanza81	marisa_tomei
+costanza81	SusanRoss
+SusanRoss	MrandMrsRoss
+MrandMrsRoss	SusanRoss
+SusanRoss	jerry
+jerry	SusanRoss
+SusanRoss	ElaineBenes
+kramer	SusanRoss
+SusanRoss	Russell_Dalrymple
+Russell_Dalrymple	SusanRoss
+Russell_Dalrymple	ElaineBenes
+SallyWeaver	SusanRoss
+SusanRoss	SallyWeaver
+SallyWeaver	MrandMrsRoss
+SallyWeaver	jerry
+WyckThayer	MrandMrsRoss
+MrandMrsRoss	WyckThayer
+SusanRoss	WyckThayer
+WyckThayer	SusanRoss
+costanza81	MrKruger
+MrKruger	costanza81
+ElaineBenes	MrKruger
+costanza81	guitarbern
+costanza81	intangibles
+costanza81	cushman
+cushman	bigstein
+bigstein	cushman
+costanza81	JonVoight
+costanza81	bubble_boy
+costanza81	Pastrami
+costanza81	bigstein
+Victoria	bigstein
+bigstein	Victoria
+cushman	Victoria
+Victoria	cushman
+bigstein	intangibles
+bigstein	guitarbern
+guitarbern	intangibles
+guitarbern	bigstein
+intangibles	Victoria
+intangibles	bigstein
+bubble_boy	trivial_pursuit
+costanza81	StankyHanke
+ElaineBenes	MrLippman
+ElaineBenes	MrPitt
+ElaineBenes	Jack_The_Wiz
+Jack_The_Wiz	ElaineBenes
+TheSuzie	ElaineBenes
+jerry	TheSuzie
+TheSuzie	Peggy
+Peggy	TheSuzie
+Peggy	JPeterman
+JPeterman	kramer
+JPeterman	TheSuzie
+ElaineBenes	RobertKennedyJr
+MarlatheVirgin	RobertKennedyJr
+RobertKennedyJr	MarlatheVirgin
+Jackie_Chiles	MarlatheVirgin
+Sue_Ellen_Mischke	RobertKennedyJr
+JPeterman	RobertKennedyJr
+RobertKennedyJr	JPeterman
+MrPitt	RobertKennedyJr
+RobertKennedyJr	MrPitt
+kramer	TinaRobbins
+TinaRobbins	kramer
+ElaineBenes	TinaRobbins
+TinaRobbins	ElaineBenes
+Jake_Jarmel	ElaineBenes
+Noreen
+ElaineBenes	HalKitzmiller
+HalKitzmiller	ElaineBenes
+kramer	HalKitzmiller
+HalKitzmiller	kramer
+Joel_Rifkin	ElaineBenes
+Darryl	ElaineBenes
+NedIsakoff	ElaineBenes
+ElaineBenes	NedIsakoff
+ElaineBenes	Carl_Farbman
+Carl_Farbman	JPeterman
+JPeterman	Carl_Farbman
+CrazyJoeDavola	jerry
+CrazyJoeDavola	costanza81
+CrazyJoeDavola	ElaineBenes
+CrazyJoeDavola	kramer
+CrazyJoeDavola	NewmanUSPS
+costanza81	CrazyJoeDavola
+ElaineBenes	CrazyJoeDavola
+kramer	CrazyJoeDavola
+NewmanUSPS	CrazyJoeDavola
+jerry	DrTimWhatley
+costanza81	DrTimWhatley
+ElaineBenes	DrTimWhatley
+kramer	DrTimWhatley
+NewmanUSPS	DrTimWhatley
+DrTimWhatley	jerry
+DrTimWhatley	costanza81
+DrTimWhatley	ElaineBenes
+DrTimWhatley	kramer
+jerry	TheDrake
+costanza81	TheDrake
+ElaineBenes	TheDrake
+kramer	TheDrake
+NewmanUSPS	TheDrake
+TheDrake	jerry
+TheDrake	costanza81
+TheDrake	ElaineBenes
+TheDrake	kramer
+jerry	JoeMayo
+ElaineBenes	JoeMayo
+costanza81	JoeMayo
+JoeMayo	jerry
+JoeMayo	ElaineBenes
+JoeMayo	costanza81
+jerry	Alec_Berg
+Alec_Berg	jerry
+ElaineBenes	Sue_Ellen_Mischke
+Sue_Ellen_Mischke	ElaineBenes
+Sue_Ellen_Mischke	jerry
+jerry	Sue_Ellen_Mischke
+kramer	Sue_Ellen_Mischke
+kramer	Mickey_Abbott
+jerry	Mickey_Abbott
+costanza81	Mickey_Abbott
+NewmanUSPS	Mickey_Abbott
+Mickey_Abbott	kramer
+Mickey_Abbott	jerry
+Mickey_Abbott	costanza81
+Mickey_Abbott	NewmanUSPS
+Babs_Kramer	NewmanUSPS
+NewmanUSPS	Babs_Kramer
+kramer	Bob_Sacamano
+kramer	Lomez
+kramer	JayRiemenschneider
+kramer	CorkyRamirez
+kramer	LenNicademo
+kramer	Specter
+kramer	Brody
+Bob_Sacamano	kramer
+Lomez	kramer
+JayRiemenschneider	kramer
+CorkyRamirez	kramer
+LenNicademo	kramer
+Specter	kramer
+Brody	kramer
+Bob_Sacamano	Lomez
+Bob_Sacamano	JayRiemenschneider
+Bob_Sacamano	CorkyRamirez
+Bob_Sacamano	LenNicademo
+Bob_Sacamano	Specter
+Bob_Sacamano	Brody
+Bob_Sacamano	jerry
+Brody	Bob_Sacamano
+Brody	Lomez
+Brody	JayRiemenschneider
+Brody	CorkyRamirez
+Brody	LenNicademo
+Brody	Specter
+CorkyRamirez	Bob_Sacamano
+CorkyRamirez	Lomez
+CorkyRamirez	JayRiemenschneider
+CorkyRamirez	Specter
+CorkyRamirez	Brody
+JayRiemenschneider	Bob_Sacamano
+JayRiemenschneider	Lomez
+JayRiemenschneider	CorkyRamirez
+JayRiemenschneider	LenNicademo
+JayRiemenschneider	Brody
+LenNicademo	Bob_Sacamano
+LenNicademo	Lomez
+LenNicademo	JayRiemenschneider
+LenNicademo	CorkyRamirez
+LenNicademo	Brody
+Lomez	Bob_Sacamano
+Lomez	JayRiemenschneider
+Lomez	CorkyRamirez
+Lomez	LenNicademo
+Lomez	Brody
+Specter	Bob_Sacamano
+Specter	Lomez
+Specter	CorkyRamirez
+kramer	FranklinDelanoRomanowski
+kramer	SalBass
+kramer	EstelleC
+kramer	Vegetable_Lasagna
+kramer	MortySeinfeld
+kramer	Noreen
+kramer	Babs_Kramer
+kramer	Shaky_the_Mohel
+kramer	assman
+assman	DrTimWhatley
+DrTimWhatley	assman
+kramer	Stan_the_Caddy
+Stan_the_Caddy	Jackie_Chiles
+NewmanUSPS	Jackie_Chiles
+Jackie_Chiles	NewmanUSPS
+kramer	Jackie_Chiles
+Jackie_Chiles	kramer
+EstelleC	kramer
+Vegetable_Lasagna	kramer
+MortySeinfeld	kramer
+Noreen	kramer
+Babs_Kramer	kramer
+Shaky_the_Mohel	kramer
+Bob_Cobb	kramer
+kramer	Bob_Cobb
+Bob_Cobb	FRANKCOSTANZA
+FRANKCOSTANZA	Bob_Cobb
+kramer	Earl_Haffler
+Earl_Haffler	kramer
+kramer	MikeMoffit
+MikeMoffit	kramer
+MikeMoffit	jerry
+NewmanUSPS	Henry_Atkins
+AvisRental	jerry
+jerry	The_Soup_Nazi
+costanza81	The_Soup_Nazi
+ElaineBenes	The_Soup_Nazi
+kramer	The_Soup_Nazi
+NewmanUSPS	The_Soup_Nazi
+FRANKCOSTANZA	The_Soup_Nazi
+THE_REAL_PUDDY	The_Soup_Nazi
+jerry	ArtVandelay
+costanza81	ArtVandelay
+ElaineBenes	ArtVandelay
+kramer	ArtVandelay
+jerry	Kel_Varnsen
+costanza81	Kel_Varnsen
+ElaineBenes	Kel_Varnsen
+kramer	Kel_Varnsen
+jerry	HEPennypacker
+costanza81	HEPennypacker
+ElaineBenes	HEPennypacker
+kramer	HEPennypacker
+jerry	MartinvanNostrand
+costanza81	MartinvanNostrand
+ElaineBenes	MartinvanNostrand
+kramer	MartinvanNostrand
+jerry	WandaPepper
+costanza81	WandaPepper
+ElaineBenes	WandaPepper
+kramer	WandaPepper
+ArtVandelay	ArtVandelay
+ArtVandelay	Kel_Varnsen
+ArtVandelay	HEPennypacker
+ArtVandelay	MartinvanNostrand
+ArtVandelay	WandaPepper
+Kel_Varnsen	ArtVandelay
+Kel_Varnsen	HEPennypacker
+Kel_Varnsen	MartinvanNostrand
+Kel_Varnsen	WandaPepper
+HEPennypacker	ArtVandelay
+HEPennypacker	Kel_Varnsen
+HEPennypacker	MartinvanNostrand
+HEPennypacker	WandaPepper
+MartinvanNostrand	ArtVandelay
+MartinvanNostrand	Kel_Varnsen
+MartinvanNostrand	HEPennypacker
+MartinvanNostrand	WandaPepper
+WandaPepper	ArtVandelay
+WandaPepper	Kel_Varnsen
+WandaPepper	HEPennypacker
+WandaPepper	MartinvanNostrand
+Kevin	Gene
+Kevin	Feldman
+Kevin	Vargas
+Kevin	ElaineBenes
+Gene	Kevin
+Gene	Feldman
+Gene	Vargas
+Gene	ElaineBenes
+Feldman	Kevin
+Feldman	Gene
+Feldman	Vargas
+Feldman	ElaineBenes
+Vargas	Kevin
+Vargas	Gene
+Vargas	Feldman
+Vargas	ElaineBenes
+ElaineBenes	Kevin
+ElaineBenes	Gene
+ElaineBenes	Feldman
+ElaineBenes	Vargas