RubyGems - swineherd - Versions diffs - 0.0.1 - Mend

swineherd 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

data/LICENSE +201 -0
data/README.textile +207 -0
data/Rakefile +30 -0
data/VERSION +1 -0
data/bin/hadoop-stream +35 -0
data/bin/hdp-tree +26 -0
data/examples/pagerank/data/seinfeld_network.tsv +429 -0
data/examples/pagerank/pagerank.rb +99 -0
data/examples/pagerank/scripts/cut_off_list.rb +16 -0
data/examples/pagerank/scripts/histogram.R +5 -0
data/examples/pagerank/scripts/pagerank.pig +20 -0
data/examples/pagerank/scripts/pagerank_initialize.pig +24 -0
data/lib/swineherd.rb +11 -0
data/lib/swineherd/filesystem.rb +26 -0
data/lib/swineherd/filesystem/README_filesystem.textile +47 -0
data/lib/swineherd/filesystem/basefilesystem.rb +125 -0
data/lib/swineherd/filesystem/filesystems.rb +103 -0
data/lib/swineherd/filesystem/hadoopfilesystem.rb +263 -0
data/lib/swineherd/filesystem/localfilesystem.rb +83 -0
data/lib/swineherd/filesystem/localfs.rb +11 -0
data/lib/swineherd/filesystem/s3filesystem.rb +249 -0
data/lib/swineherd/script.rb +74 -0
data/lib/swineherd/script/hadoop_script.rb +59 -0
data/lib/swineherd/script/pig_script.rb +46 -0
data/lib/swineherd/script/r_script.rb +14 -0
data/lib/swineherd/script/wukong_script.rb +31 -0
data/lib/swineherd/template.rb +45 -0
data/lib/swineherd/workflow.rb +53 -0
data/lib/swineherd/workflow/job.rb +60 -0
data/notes.txt +20 -0
data/swineherd.gemspec +97 -0
data/tests/test_filesystem.rb +105 -0
data/tests/test_s3_filesystem.rb +132 -0
data/tests/testcfg.yaml +7 -0
metadata +204 -0

data/bin/hdp-tree ADDED

@@ -0,0 +1,26 @@
+#!/usr/bin/env jruby
+require 'swineherd'
+#
+# Creates a 'tree' view of an hdfs path. It's not as pretty as the
+# unix tree command but that's only because I'm not smart enough to
+# print the hierarchy properly.
+#
+FS   = Swineherd::FileSystem.get(:hdfs)
+path = ARGV[0]
+# Recursively list paths
+def lr path
+  paths = FS.entries(path)
+  if (paths && !paths.empty?)
+    paths.map{|e| {File.basename(path) => lr(e)}}.flatten
+  else
+    File.basename(path)
+  end
+end
+tree = {File.dirname(path) => lr(path)}.to_yaml
+puts tree

data/examples/pagerank/data/seinfeld_network.tsv ADDED

@@ -0,0 +1,429 @@
+jerry	costanza81
+jerry	ElaineBenes
+jerry	kramer
+jerry	NewmanUSPS
+jerry	THE_REAL_PUDDY
+jerry	JPeterman
+jerry	FRANKCOSTANZA
+costanza81	jerry
+costanza81	ElaineBenes
+costanza81	kramer
+costanza81	NewmanUSPS
+costanza81	THE_REAL_PUDDY
+costanza81	JPeterman
+costanza81	FRANKCOSTANZA
+ElaineBenes	jerry
+ElaineBenes	costanza81
+ElaineBenes	kramer
+ElaineBenes	THE_REAL_PUDDY
+ElaineBenes	JPeterman
+kramer	jerry
+kramer	costanza81
+kramer	ElaineBenes
+kramer	NewmanUSPS
+kramer	THE_REAL_PUDDY
+kramer	JPeterman
+kramer	FRANKCOSTANZA
+NewmanUSPS	jerry
+NewmanUSPS	costanza81
+NewmanUSPS	ElaineBenes
+NewmanUSPS	kramer
+NewmanUSPS	THE_REAL_PUDDY
+NewmanUSPS	JPeterman
+NewmanUSPS	FRANKCOSTANZA
+THE_REAL_PUDDY	jerry
+THE_REAL_PUDDY	costanza81
+THE_REAL_PUDDY	ElaineBenes
+THE_REAL_PUDDY	kramer
+THE_REAL_PUDDY	NewmanUSPS
+THE_REAL_PUDDY	JPeterman
+THE_REAL_PUDDY	FRANKCOSTANZA
+THE_REAL_PUDDY	Vegetable_Lasagna
+FRANKCOSTANZA	jerry
+FRANKCOSTANZA	costanza81
+FRANKCOSTANZA	kramer
+jerry	MortySeinfeld
+jerry	HelenSeinfeld
+jerry	Izzy_Mandelbaum
+jerry	UncleLEO
+jerry	Artie_Levine
+MortySeinfeld	UncleLEO
+MortySeinfeld	HelenSeinfeld
+MortySeinfeld	Cousin_Jeffrey
+Izzy_Mandelbaum	jerry
+UncleLEO	jerry
+UncleLEO	MortySeinfeld
+UncleLEO	HelenSeinfeld
+UncleLEO	Cousin_Jeffrey
+UncleLEO	Babs_Kramer
+Babs_Kramer	UncleLEO
+Cousin_Jeffrey	jerry
+Cousin_Jeffrey	MortySeinfeld
+Cousin_Jeffrey	UncleLEO
+jerry	Nana
+Nana	MortySeinfeld
+MortySeinfeld	Nana
+Cousin_Jeffrey	Nana
+UncleLEO	Nana
+Nana	UncleLEO
+JackKlompus	MortySeinfeld
+MortySeinfeld	JackKlompus
+Dolores	jerry
+MarlatheVirgin	jerry
+TiaVanCamp	jerry
+Rachel_Goldstein	jerry
+meryl	jerry
+MissRhodeIsland	jerry
+Pam	jerry
+Sheila	jerry
+MelissaFlyingFree	jerry
+jerry	Dolores
+jerry	MarlatheVirgin
+jerry	TiaVanCamp
+jerry	Rachel_Goldstein
+jerry	meryl
+jerry	MissRhodeIsland
+jerry	Pam
+jerry	Sheila
+jerry	MelissaFlyingFree
+jerry	Laura
+jerry	Sandy
+Laura	Sandy
+Laura	jerry
+Sandy	jerry
+kramer	MissRhodeIsland
+kramer	Pam
+jerry	Jenna
+Jenna	jerry
+jerry	bania
+costanza81	bania
+bania	jerry
+bania	The_Soup_Nazi
+bania	Poppie
+bania	Jenna
+jerry	Noreen
+jerry	JackKlompus
+jerry	Milos
+jerry	JeanPaul_JeanPaul
+jerry	FusilliJerry
+FusilliJerry	jerry
+FusilliJerry	kramer
+kramer	FusilliJerry
+FusilliJerry	FRANKCOSTANZA
+jerry	pez
+jerry	superman
+jerry	bigstein
+Milos	jerry
+jerry	Roy_the_Dentist
+jerry	BabuBhatt
+kramer	BabuBhatt
+BabuBhatt	kramer
+kramer	Poppie
+Poppie	kramer
+Poppie	ElaineBenes
+ElaineBenes	Poppie
+jerry	Poppie
+jerry	Shaky_the_Mohel
+jerry	bubble_boy
+bubble_boy	jerry
+MatthewSeinfeldFan	jerry
+FragileFrankieMerman	jerry
+jerry	FragileFrankieMerman
+costanza81	FragileFrankieMerman
+costanza81	EstelleC
+FRANKCOSTANZA	EstelleC
+EstelleC	costanza81
+EstelleC	FRANKCOSTANZA
+FRANKCOSTANZA	Lloyd_Braun
+EstelleC	Lloyd_Braun
+Lloyd_Braun	FRANKCOSTANZA
+Lloyd_Braun	costanza81
+Lloyd_Braun	EstelleC
+kramer	MrWilhelm
+costanza81	MrWilhelm
+costanza81	Allison
+Allison	costanza81
+costanza81	LindsayEnright
+LindsayEnright	costanza81
+costanza81	marisa_tomei
+costanza81	SusanRoss
+SusanRoss	MrandMrsRoss
+MrandMrsRoss	SusanRoss
+SusanRoss	jerry
+jerry	SusanRoss
+SusanRoss	ElaineBenes
+kramer	SusanRoss
+SusanRoss	Russell_Dalrymple
+Russell_Dalrymple	SusanRoss
+Russell_Dalrymple	ElaineBenes
+SallyWeaver	SusanRoss
+SusanRoss	SallyWeaver
+SallyWeaver	MrandMrsRoss
+SallyWeaver	jerry
+WyckThayer	MrandMrsRoss
+MrandMrsRoss	WyckThayer
+SusanRoss	WyckThayer
+WyckThayer	SusanRoss
+costanza81	MrKruger
+MrKruger	costanza81
+ElaineBenes	MrKruger
+costanza81	guitarbern
+costanza81	intangibles
+costanza81	cushman
+cushman	bigstein
+bigstein	cushman
+costanza81	JonVoight
+costanza81	bubble_boy
+costanza81	Pastrami
+costanza81	bigstein
+Victoria	bigstein
+bigstein	Victoria
+cushman	Victoria
+Victoria	cushman
+bigstein	intangibles
+bigstein	guitarbern
+guitarbern	intangibles
+guitarbern	bigstein
+intangibles	Victoria
+intangibles	bigstein
+bubble_boy	trivial_pursuit
+costanza81	StankyHanke
+ElaineBenes	MrLippman
+ElaineBenes	MrPitt
+ElaineBenes	Jack_The_Wiz
+Jack_The_Wiz	ElaineBenes
+TheSuzie	ElaineBenes
+jerry	TheSuzie
+TheSuzie	Peggy
+Peggy	TheSuzie
+Peggy	JPeterman
+JPeterman	kramer
+JPeterman	TheSuzie
+ElaineBenes	RobertKennedyJr
+MarlatheVirgin	RobertKennedyJr
+RobertKennedyJr	MarlatheVirgin
+Jackie_Chiles	MarlatheVirgin
+Sue_Ellen_Mischke	RobertKennedyJr
+JPeterman	RobertKennedyJr
+RobertKennedyJr	JPeterman
+MrPitt	RobertKennedyJr
+RobertKennedyJr	MrPitt
+kramer	TinaRobbins
+TinaRobbins	kramer
+ElaineBenes	TinaRobbins
+TinaRobbins	ElaineBenes
+Jake_Jarmel	ElaineBenes
+Noreen
+ElaineBenes	HalKitzmiller
+HalKitzmiller	ElaineBenes
+kramer	HalKitzmiller
+HalKitzmiller	kramer
+Joel_Rifkin	ElaineBenes
+Darryl	ElaineBenes
+NedIsakoff	ElaineBenes
+ElaineBenes	NedIsakoff
+ElaineBenes	Carl_Farbman
+Carl_Farbman	JPeterman
+JPeterman	Carl_Farbman
+CrazyJoeDavola	jerry
+CrazyJoeDavola	costanza81
+CrazyJoeDavola	ElaineBenes
+CrazyJoeDavola	kramer
+CrazyJoeDavola	NewmanUSPS
+costanza81	CrazyJoeDavola
+ElaineBenes	CrazyJoeDavola
+kramer	CrazyJoeDavola
+NewmanUSPS	CrazyJoeDavola
+jerry	DrTimWhatley
+costanza81	DrTimWhatley
+ElaineBenes	DrTimWhatley
+kramer	DrTimWhatley
+NewmanUSPS	DrTimWhatley
+DrTimWhatley	jerry
+DrTimWhatley	costanza81
+DrTimWhatley	ElaineBenes
+DrTimWhatley	kramer
+jerry	TheDrake
+costanza81	TheDrake
+ElaineBenes	TheDrake
+kramer	TheDrake
+NewmanUSPS	TheDrake
+TheDrake	jerry
+TheDrake	costanza81
+TheDrake	ElaineBenes
+TheDrake	kramer
+jerry	JoeMayo
+ElaineBenes	JoeMayo
+costanza81	JoeMayo
+JoeMayo	jerry
+JoeMayo	ElaineBenes
+JoeMayo	costanza81
+jerry	Alec_Berg
+Alec_Berg	jerry
+ElaineBenes	Sue_Ellen_Mischke
+Sue_Ellen_Mischke	ElaineBenes
+Sue_Ellen_Mischke	jerry
+jerry	Sue_Ellen_Mischke
+kramer	Sue_Ellen_Mischke
+kramer	Mickey_Abbott
+jerry	Mickey_Abbott
+costanza81	Mickey_Abbott
+NewmanUSPS	Mickey_Abbott
+Mickey_Abbott	kramer
+Mickey_Abbott	jerry
+Mickey_Abbott	costanza81
+Mickey_Abbott	NewmanUSPS
+Babs_Kramer	NewmanUSPS
+NewmanUSPS	Babs_Kramer
+kramer	Bob_Sacamano
+kramer	Lomez
+kramer	JayRiemenschneider
+kramer	CorkyRamirez
+kramer	LenNicademo
+kramer	Specter
+kramer	Brody
+Bob_Sacamano	kramer
+Lomez	kramer
+JayRiemenschneider	kramer
+CorkyRamirez	kramer
+LenNicademo	kramer
+Specter	kramer
+Brody	kramer
+Bob_Sacamano	Lomez
+Bob_Sacamano	JayRiemenschneider
+Bob_Sacamano	CorkyRamirez
+Bob_Sacamano	LenNicademo
+Bob_Sacamano	Specter
+Bob_Sacamano	Brody
+Bob_Sacamano	jerry
+Brody	Bob_Sacamano
+Brody	Lomez
+Brody	JayRiemenschneider
+Brody	CorkyRamirez
+Brody	LenNicademo
+Brody	Specter
+CorkyRamirez	Bob_Sacamano
+CorkyRamirez	Lomez
+CorkyRamirez	JayRiemenschneider
+CorkyRamirez	Specter
+CorkyRamirez	Brody
+JayRiemenschneider	Bob_Sacamano
+JayRiemenschneider	Lomez
+JayRiemenschneider	CorkyRamirez
+JayRiemenschneider	LenNicademo
+JayRiemenschneider	Brody
+LenNicademo	Bob_Sacamano
+LenNicademo	Lomez
+LenNicademo	JayRiemenschneider
+LenNicademo	CorkyRamirez
+LenNicademo	Brody
+Lomez	Bob_Sacamano
+Lomez	JayRiemenschneider
+Lomez	CorkyRamirez
+Lomez	LenNicademo
+Lomez	Brody
+Specter	Bob_Sacamano
+Specter	Lomez
+Specter	CorkyRamirez
+kramer	FranklinDelanoRomanowski
+kramer	SalBass
+kramer	EstelleC
+kramer	Vegetable_Lasagna
+kramer	MortySeinfeld
+kramer	Noreen
+kramer	Babs_Kramer
+kramer	Shaky_the_Mohel
+kramer	assman
+assman	DrTimWhatley
+DrTimWhatley	assman
+kramer	Stan_the_Caddy
+Stan_the_Caddy	Jackie_Chiles
+NewmanUSPS	Jackie_Chiles
+Jackie_Chiles	NewmanUSPS
+kramer	Jackie_Chiles
+Jackie_Chiles	kramer
+EstelleC	kramer
+Vegetable_Lasagna	kramer
+MortySeinfeld	kramer
+Noreen	kramer
+Babs_Kramer	kramer
+Shaky_the_Mohel	kramer
+Bob_Cobb	kramer
+kramer	Bob_Cobb
+Bob_Cobb	FRANKCOSTANZA
+FRANKCOSTANZA	Bob_Cobb
+kramer	Earl_Haffler
+Earl_Haffler	kramer
+kramer	MikeMoffit
+MikeMoffit	kramer
+MikeMoffit	jerry
+NewmanUSPS	Henry_Atkins
+AvisRental	jerry
+jerry	The_Soup_Nazi
+costanza81	The_Soup_Nazi
+ElaineBenes	The_Soup_Nazi
+kramer	The_Soup_Nazi
+NewmanUSPS	The_Soup_Nazi
+FRANKCOSTANZA	The_Soup_Nazi
+THE_REAL_PUDDY	The_Soup_Nazi
+jerry	ArtVandelay
+costanza81	ArtVandelay
+ElaineBenes	ArtVandelay
+kramer	ArtVandelay
+jerry	Kel_Varnsen
+costanza81	Kel_Varnsen
+ElaineBenes	Kel_Varnsen
+kramer	Kel_Varnsen
+jerry	HEPennypacker
+costanza81	HEPennypacker
+ElaineBenes	HEPennypacker
+kramer	HEPennypacker
+jerry	MartinvanNostrand
+costanza81	MartinvanNostrand
+ElaineBenes	MartinvanNostrand
+kramer	MartinvanNostrand
+jerry	WandaPepper
+costanza81	WandaPepper
+ElaineBenes	WandaPepper
+kramer	WandaPepper
+ArtVandelay	ArtVandelay
+ArtVandelay	Kel_Varnsen
+ArtVandelay	HEPennypacker
+ArtVandelay	MartinvanNostrand
+ArtVandelay	WandaPepper
+Kel_Varnsen	ArtVandelay
+Kel_Varnsen	HEPennypacker
+Kel_Varnsen	MartinvanNostrand
+Kel_Varnsen	WandaPepper
+HEPennypacker	ArtVandelay
+HEPennypacker	Kel_Varnsen
+HEPennypacker	MartinvanNostrand
+HEPennypacker	WandaPepper
+MartinvanNostrand	ArtVandelay
+MartinvanNostrand	Kel_Varnsen
+MartinvanNostrand	HEPennypacker
+MartinvanNostrand	WandaPepper
+WandaPepper	ArtVandelay
+WandaPepper	Kel_Varnsen
+WandaPepper	HEPennypacker
+WandaPepper	MartinvanNostrand
+Kevin	Gene
+Kevin	Feldman
+Kevin	Vargas
+Kevin	ElaineBenes
+Gene	Kevin
+Gene	Feldman
+Gene	Vargas
+Gene	ElaineBenes
+Feldman	Kevin
+Feldman	Gene
+Feldman	Vargas
+Feldman	ElaineBenes
+Vargas	Kevin
+Vargas	Gene
+Vargas	Feldman
+Vargas	ElaineBenes
+ElaineBenes	Kevin
+ElaineBenes	Gene
+ElaineBenes	Feldman
+ElaineBenes	Vargas