wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong/widget/gibberish.rb

@@ -1,123 +0,0 @@
-module Wukong
-  module Widget
-    #
-    # Uses bigram frequencies taken from the British National Corpus
-    #
-    class Gibberish < Wukong::Source
-      register_source
-      include CappedGenerator
-
-      field :min_wordlen, Integer, :default =>  3, :doc => "Shortest word length to generate"
-      field :max_wordlen, Integer, :default => 10, :doc => "Largest word length to generate"
-      field :max_linelen, Integer, :default => 80, :doc => "Max line length to generate; might be shorter by up to `min_wordlen` chars."
-      attr_accessor :rng
-
-      def setup
-        self.rng = Random.new
-        super
-      end
-
-      def next_item
-        funny_line
-      end
-
-      def random_from_cdf cdf, except=[]
-        offset = rng.rand
-        cdf.find{|freq, c| (offset < freq) && (not except.include?(c)) }.last
-      end
-
-      def funny_charseq(len = 5)
-        char = random_from_cdf(BIGRAM_CDF_HSH['^'])
-        word = [char]
-        len.times do
-          next_char = random_from_cdf(BIGRAM_CDF_HSH[char], [])
-          break if next_char == '$'
-          word << next_char
-        end
-        word.join
-      end
-
-      def funny_word(cap)
-        cap = [max_wordlen, cap].min
-        word = ''
-        100.times do
-          word = funny_charseq(cap)
-          return word if word.length >= min_wordlen && word.length <= cap
-        end
-        word
-      end
-
-      def funny_line
-        min_wordlen = 3
-        min_linelen = max_linelen - min_wordlen
-        line = ""
-        max_linelen.times do
-          line << funny_word(max_linelen-line.length) << " "
-          return line.strip if line.strip.length > min_linelen
-        end
-      end
-
-      UNIGRAM_CDF = [
-        [0.1083564980, "^"],
-        [0.2034715900, "e"],
-        [0.2836505016, "a"],
-        [0.3608587709, "i"],
-        [0.4254609572, "r"],
-        [0.4878711345, "n"],
-        [0.5491446929, "o"],
-        [0.6081589386, "t"],
-        [0.6604080108, "s"],
-        [0.7104964048, "l"],
-        [0.7482015140, "c"],
-        [0.7807004566, "u"],
-        [0.8107323736, "d"],
-        [0.8387591070, "m"],
-        [0.8645706232, "h"],
-        [0.8900984199, "p"],
-        [0.9124467010, "g"],
-        [0.9302838994, "y"],
-        [0.9480503129, "b"],
-        [0.9604887171, "f"],
-        [0.9713402644, "k"],
-        [0.9806072759, "v"],
-        [0.9890852104, "w"],
-        [0.9930607653, "z"],
-        [0.9962141717, "x"],
-        [0.9983324949, "j"],
-        [1.0000000000, "q"],
-      ] # UNIGRAM_CDF
-
-      BIGRAM_CDF_HSH = {
-        '^' => [ [0.1036218381, "s"], [0.1906712502, "c"], [0.2682537643, "p"], [0.3317216047, "a"], [0.3936580351, "m"], [0.4519587055, "b"], [0.5062006040, "d"], [0.5582364154, "t"], [0.6060742359, "r"], [0.6465762814, "h"], [0.6851613871, "f"], [0.7236340466, "e"], [0.7599541648, "g"], [0.7954175502, "i"], [0.8298153741, "l"], [0.8625318598, "u"], [0.8880517895, "o"], [0.9135503009, "w"], [0.9378761593, "n"], [0.9575114053, "k"], [0.9749726916, "v"], [0.9844556534, "j"], [0.9891783932, "y"], [0.9938957784, "q"], [0.9981580244, "z"], [1.0000000000, "x"],  ],
-        'a' => [ [0.1475494063, "n"], [0.2731292197, "t"], [0.3885347092, "l"], [0.5028764536, "r"], [0.5712094131, "$"], [0.6238394686, "s"], [0.6733868342, "c"], [0.7157919112, "m"], [0.7521546266, "b"], [0.7858181792, "d"], [0.8182153686, "i"], [0.8462128503, "p"], [0.8732913142, "g"], [0.8945879254, "u"], [0.9098711204, "y"], [0.9241484612, "k"], [0.9383968565, "v"], [0.9508578706, "e"], [0.9609453582, "f"], [0.9688908829, "w"], [0.9766844440, "h"], [0.9836313508, "z"], [0.9897822579, "a"], [0.9943628746, "x"], [0.9964758921, "o"], [0.9984297096, "j"], [1.0000000000, "q"],  ],
-        'b' => [ [0.1548283857, "a"], [0.3006433493, "l"], [0.4444335587, "e"], [0.5720257340, "o"], [0.6942947650, "i"], [0.7909604520, "r"], [0.8741386630, "u"], [0.8985663434, "$"], [0.9228307371, "b"], [0.9419352732, "s"], [0.9592109990, "y"], [0.9649586885, "t"], [0.9702165181, "c"], [0.9754416903, "h"], [0.9804382613, "d"], [0.9838672806, "m"], [0.9871983279, "j"], [0.9895496555, "p"], [0.9918683257, "n"], [0.9939257372, "f"], [0.9954606316, "w"], [0.9969628686, "v"], [0.9981385324, "g"], [0.9991182522, "k"], [0.9996081121, "z"], [0.9998367134, "x"], [1.0000000000, "q"],  ],
-        'c' => [ [0.1561567107, "o"], [0.3072354045, "h"], [0.4498199612, "a"], [0.5342381436, "e"], [0.6138245160, "$"], [0.6810697689, "i"], [0.7428984704, "t"], [0.8037423445, "k"], [0.8614778568, "r"], [0.9075339304, "u"], [0.9451574185, "l"], [0.9638537531, "y"], [0.9812728895, "c"], [0.9879666390, "s"], [0.9894131044, "n"], [0.9908287939, "m"], [0.9920906041, "q"], [0.9933062506, "d"], [0.9944757332, "p"], [0.9955067245, "z"], [0.9964300003, "g"], [0.9972763364, "f"], [0.9980303450, "x"], [0.9987535777, "b"], [0.9993998707, "v"], [0.9998307328, "w"], [1.0000000000, "j"],  ],
-        'd' => [ [0.2735843589, "$"], [0.4794729623, "e"], [0.6243696992, "i"], [0.6957748112, "a"], [0.7647456579, "o"], [0.8067270725, "r"], [0.8421205154, "u"], [0.8667529607, "l"], [0.8871737408, "d"], [0.9052375341, "y"], [0.9209829795, "s"], [0.9341589227, "g"], [0.9428334074, "n"], [0.9506578312, "w"], [0.9579219876, "m"], [0.9650122679, "h"], [0.9720445896, "b"], [0.9768938004, "f"], [0.9816657329, "c"], [0.9861478719, "t"], [0.9899731458, "v"], [0.9937597805, "p"], [0.9968702305, "j"], [0.9984351152, "z"], [0.9994590522, "k"], [0.9997295261, "q"], [1.0000000000, "x"],  ],
-        'e' => [ [0.1940659046, "$"], [0.3647626484, "r"], [0.4678651164, "n"], [0.5406383057, "d"], [0.6084582820, "s"], [0.6653470299, "l"], [0.7148364586, "t"], [0.7581282711, "a"], [0.7938804641, "c"], [0.8234045409, "m"], [0.8492808081, "e"], [0.8678553564, "p"], [0.8835079971, "i"], [0.8977820342, "x"], [0.9112447692, "g"], [0.9229689997, "u"], [0.9342845291, "v"], [0.9454719582, "f"], [0.9560981859, "y"], [0.9666695133, "o"], [0.9757646370, "b"], [0.9844937597, "w"], [0.9899593739, "h"], [0.9938206839, "k"], [0.9962789903, "q"], [0.9987189967, "z"], [1.0000000000, "j"],  ],
-        'f' => [ [0.1662002052, "i"], [0.2924246665, "o"], [0.4152439593, "e"], [0.5190316261, "a"], [0.6096650807, "l"], [0.6926485680, "f"], [0.7724134714, "r"], [0.8517119134, "u"], [0.9112790372, "$"], [0.9520011195, "t"], [0.9698199459, "y"], [0.9751842523, "s"], [0.9794290512, "c"], [0.9824610505, "m"], [0.9849332960, "n"], [0.9873588954, "b"], [0.9897378487, "w"], [0.9918835712, "h"], [0.9938893554, "d"], [0.9958484933, "p"], [0.9973878160, "g"], [0.9983673850, "k"], [0.9990670772, "j"], [0.9994868924, "v"], [0.9997667693, "x"], [0.9999067077, "z"], [1.0000000000, "q"],  ],
-        'g' => [ [0.2294771276, "$"], [0.3781089361, "e"], [0.4743756166, "a"], [0.5630354639, "i"], [0.6496183602, "r"], [0.7118749675, "o"], [0.7740277273, "l"], [0.8293005867, "h"], [0.8777714315, "u"], [0.9066929747, "g"], [0.9340827665, "n"], [0.9539176489, "y"], [0.9633677761, "s"], [0.9717534659, "m"], [0.9797237655, "t"], [0.9840853627, "w"], [0.9880834934, "b"], [0.9911469962, "c"], [0.9936912612, "d"], [0.9960278311, "f"], [0.9975855444, "p"], [0.9983644011, "k"], [0.9988836388, "j"], [0.9993769147, "v"], [0.9996365336, "x"], [0.9998701906, "z"], [1.0000000000, "q"],  ],
-        'h' => [ [0.1921235417, "e"], [0.3644435453, "a"], [0.5135657608, "o"], [0.6515836087, "i"], [0.7645154764, "$"], [0.8146424798, "y"], [0.8527884551, "u"], [0.8857868591, "r"], [0.9185379999, "t"], [0.9357115562, "l"], [0.9473778857, "n"], [0.9580551622, "m"], [0.9658776721, "w"], [0.9721491672, "s"], [0.9781284420, "h"], [0.9828714006, "b"], [0.9868051341, "c"], [0.9895699867, "f"], [0.9922674040, "p"], [0.9947400364, "d"], [0.9965832715, "k"], [0.9982691573, "g"], [0.9991233394, "v"], [0.9994829950, "j"], [0.9997752152, "q"], [0.9999550430, "z"], [1.0000000000, "x"],  ],
-        'i' => [ [0.1973908665, "n"], [0.3137122288, "s"], [0.4121934907, "c"], [0.4902570808, "t"], [0.5606405603, "o"], [0.6174823966, "a"], [0.6742490851, "l"], [0.7181354315, "$"], [0.7553711928, "d"], [0.7895559513, "e"], [0.8198179919, "m"], [0.8494187314, "r"], [0.8767049169, "g"], [0.9008874961, "v"], [0.9198247552, "p"], [0.9372139685, "z"], [0.9544904600, "f"], [0.9682274876, "b"], [0.9765388402, "k"], [0.9841588325, "u"], [0.9891110760, "i"], [0.9927707765, "x"], [0.9952957444, "q"], [0.9969114232, "h"], [0.9981513628, "j"], [0.9992109475, "y"], [1.0000000000, "w"],  ],
-        'j' => [ [0.2434949329, "a"], [0.4396055875, "u"], [0.6343467543, "o"], [0.8184059162, "e"], [0.9104354971, "i"], [0.9367296631, "$"], [0.9457682827, "k"], [0.9520679266, "m"], [0.9569980827, "n"], [0.9619282388, "d"], [0.9668583950, "l"], [0.9709668584, "j"], [0.9748014243, "t"], [0.9786359901, "c"], [0.9821966584, "h"], [0.9857573268, "s"], [0.9887701999, "r"], [0.9912352780, "b"], [0.9934264585, "p"], [0.9953437414, "f"], [0.9972610244, "y"], [0.9983566146, "v"], [0.9989044098, "w"], [0.9994522049, "g"], [1.0000000000, "z"],  ],
-        'k' => [ [0.2145110410, "e"], [0.4128214725, "$"], [0.5650430412, "i"], [0.6673260974, "a"], [0.7285462225, "o"], [0.7652782976, "l"], [0.7947388120, "u"], [0.8235042507, "y"], [0.8500775277, "s"], [0.8754745228, "n"], [0.9003903117, "r"], [0.9246644923, "h"], [0.9366411806, "t"], [0.9481366626, "w"], [0.9578142544, "b"], [0.9671175747, "m"], [0.9750307437, "k"], [0.9813933594, "f"], [0.9869539646, "p"], [0.9911778859, "c"], [0.9941185906, "d"], [0.9963642196, "g"], [0.9978613057, "v"], [0.9990375876, "j"], [0.9994653264, "x"], [0.9997861306, "z"], [1.0000000000, "q"],  ],
-        'l' => [ [0.1721321920, "e"], [0.3312328418, "i"], [0.4596138030, "a"], [0.5644569032, "l"], [0.6646897335, "$"], [0.7533389707, "o"], [0.8401464166, "y"], [0.8755574604, "u"], [0.8986783120, "d"], [0.9217875801, "t"], [0.9320853942, "s"], [0.9413986030, "m"], [0.9491480267, "f"], [0.9568163653, "c"], [0.9643341172, "k"], [0.9708440964, "p"], [0.9770413187, "v"], [0.9830300363, "b"], [0.9868757891, "g"], [0.9903392834, "n"], [0.9929340083, "w"], [0.9953318120, "h"], [0.9976137798, "r"], [0.9984130478, "z"], [0.9990848962, "j"], [0.9997104102, "x"], [1.0000000000, "q"],  ],
-        'm' => [ [0.2069558017, "a"], [0.3746403064, "e"], [0.5182279267, "i"], [0.6378221716, "$"], [0.7489907877, "o"], [0.8112410724, "p"], [0.8594348411, "u"], [0.8979194700, "m"], [0.9363212918, "b"], [0.9540420246, "y"], [0.9636269537, "s"], [0.9718662664, "n"], [0.9770417141, "l"], [0.9810371597, "t"], [0.9841010247, "f"], [0.9867922575, "c"], [0.9893385778, "r"], [0.9917192837, "d"], [0.9936445503, "h"], [0.9955698168, "w"], [0.9970810475, "g"], [0.9981368388, "k"], [0.9987371908, "v"], [0.9993168409, "j"], [0.9996273678, "q"], [0.9998757893, "z"], [1.0000000000, "x"],  ],
-        'n' => [ [0.1731959913, "$"], [0.2872468996, "g"], [0.4009352398, "e"], [0.5103657289, "t"], [0.5939609170, "i"], [0.6679155123, "a"], [0.7369707900, "d"], [0.7893386386, "o"], [0.8332651581, "c"], [0.8768384062, "s"], [0.8977279066, "n"], [0.9132160720, "u"], [0.9263893795, "k"], [0.9386981016, "f"], [0.9482736181, "y"], [0.9546882844, "l"], [0.9609170184, "v"], [0.9668947437, "h"], [0.9727330197, "b"], [0.9780785750, "r"], [0.9831359351, "p"], [0.9879051001, "m"], [0.9919026458, "w"], [0.9952773181, "z"], [0.9978431846, "j"], [0.9994514995, "q"], [1.0000000000, "x"],  ],
-        'o' => [ [0.1792873578, "n"], [0.3023095060, "r"], [0.3778063954, "l"], [0.4461351994, "u"], [0.5056388308, "m"], [0.5562320680, "$"], [0.6058215838, "s"], [0.6533752497, "t"], [0.6988551895, "p"], [0.7370818222, "c"], [0.7749770375, "o"], [0.8078347079, "g"], [0.8369805032, "d"], [0.8623197326, "w"], [0.8856041740, "v"], [0.9054797504, "i"], [0.9235656727, "b"], [0.9396914977, "a"], [0.9530334163, "f"], [0.9646140881, "e"], [0.9753425436, "k"], [0.9832965618, "x"], [0.9897734052, "y"], [0.9942806822, "h"], [0.9976422017, "z"], [0.9989678714, "j"], [1.0000000000, "q"],  ],
-        'p' => [ [0.1630755943, "e"], [0.2878539934, "a"], [0.4087231238, "o"], [0.5266603027, "r"], [0.6227101232, "h"], [0.7102822856, "i"], [0.7778535388, "l"], [0.8271512341, "$"], [0.8667893995, "p"], [0.9040865494, "u"], [0.9379289968, "t"], [0.9606573026, "s"], [0.9756579845, "y"], [0.9790217737, "m"], [0.9823628347, "f"], [0.9856584390, "n"], [0.9888858584, "c"], [0.9914996136, "w"], [0.9938633574, "b"], [0.9958407200, "d"], [0.9972498750, "g"], [0.9985453884, "k"], [0.9992499659, "v"], [0.9995227056, "x"], [0.9997499886, "z"], [0.9999545434, "j"], [1.0000000000, "q"],  ],
-        'q' => [ [0.8851774530, "u"], [0.9144050104, "$"], [0.9345859429, "a"], [0.9512874043, "i"], [0.9592901879, "e"], [0.9641614475, "l"], [0.9686847599, "v"], [0.9718162839, "r"], [0.9745998608, "b"], [0.9773834377, "f"], [0.9801670146, "m"], [0.9829505915, "o"], [0.9853862213, "s"], [0.9878218511, "t"], [0.9899095338, "k"], [0.9919972164, "q"], [0.9933890049, "p"], [0.9947807933, "n"], [0.9961725818, "w"], [0.9975643702, "d"], [0.9986082116, "c"], [0.9993041058, "h"], [0.9996520529, "g"], [1.0000000000, "j"],  ],
-        'r' => [ [0.1577633281, "e"], [0.2944208938, "a"], [0.4238126886, "i"], [0.5479055899, "$"], [0.6580327633, "o"], [0.6988881305, "t"], [0.7281847248, "u"], [0.7566550510, "y"], [0.7843889208, "d"], [0.8109282943, "s"], [0.8362103032, "r"], [0.8607468746, "m"], [0.8826160368, "n"], [0.9026081334, "c"], [0.9184778704, "g"], [0.9321651818, "b"], [0.9458165685, "l"], [0.9588213105, "k"], [0.9713769938, "p"], [0.9787864636, "v"], [0.9857827993, "f"], [0.9924647938, "h"], [0.9966230780, "w"], [0.9981229343, "z"], [0.9990839201, "j"], [0.9997575083, "q"], [1.0000000000, "x"],  ],
-        's' => [ [0.1670682820, "t"], [0.2939047006, "$"], [0.4058387838, "e"], [0.4933650184, "s"], [0.5800473055, "i"], [0.6458085794, "h"], [0.7020976536, "a"], [0.7513464293, "c"], [0.7976858072, "o"], [0.8403273628, "u"], [0.8806036445, "p"], [0.9077099042, "m"], [0.9273649962, "l"], [0.9428558738, "k"], [0.9567810067, "y"], [0.9666307619, "n"], [0.9762029027, "w"], [0.9812332737, "q"], [0.9854863247, "f"], [0.9895728071, "b"], [0.9922823226, "d"], [0.9949141061, "r"], [0.9973237982, "g"], [0.9985119874, "v"], [0.9992337845, "z"], [0.9999222680, "j"], [1.0000000000, "x"],  ],
-        't' => [ [0.1842635651, "i"], [0.3614680523, "e"], [0.4876171188, "$"], [0.5830228191, "a"], [0.6670435441, "r"], [0.7509364585, "o"], [0.8272885472, "h"], [0.8631934954, "u"], [0.8966897053, "t"], [0.9280622929, "y"], [0.9435961971, "l"], [0.9553842675, "s"], [0.9665037901, "c"], [0.9732777521, "w"], [0.9779870813, "m"], [0.9817034204, "f"], [0.9851739699, "z"], [0.9884970456, "b"], [0.9917218055, "n"], [0.9942976807, "p"], [0.9964016399, "g"], [0.9975912617, "d"], [0.9984662727, "k"], [0.9993117891, "v"], [0.9998033683, "j"], [0.9999213473, "x"], [1.0000000000, "q"],  ],
-        'u' => [ [0.1659793262, "n"], [0.3036437970, "s"], [0.4266152500, "r"], [0.5220573795, "l"], [0.5933801082, "m"], [0.6608108832, "t"], [0.6990520058, "c"], [0.7350257976, "e"], [0.7697320265, "i"], [0.8040097834, "a"], [0.8368414475, "p"], [0.8691910806, "b"], [0.8962205202, "d"], [0.9214825130, "$"], [0.9456197668, "g"], [0.9569207148, "f"], [0.9658829200, "o"], [0.9740595933, "k"], [0.9797189938, "v"], [0.9851641583, "x"], [0.9894667309, "z"], [0.9922696517, "h"], [0.9948047775, "y"], [0.9966793423, "u"], [0.9984289361, "j"], [0.9997143520, "w"], [1.0000000000, "q"],  ],
-        'v' => [ [0.4405209116, "e"], [0.6701101928, "i"], [0.8340846481, "a"], [0.9149135988, "o"], [0.9356999750, "$"], [0.9479714500, "u"], [0.9574254946, "r"], [0.9664412722, "y"], [0.9723891811, "s"], [0.9777109942, "l"], [0.9827197596, "v"], [0.9859754570, "n"], [0.9884172302, "c"], [0.9902955172, "t"], [0.9919859755, "d"], [0.9936138242, "m"], [0.9946781868, "g"], [0.9956799399, "k"], [0.9966190834, "p"], [0.9973703982, "b"], [0.9979964939, "f"], [0.9984973704, "h"], [0.9989356374, "j"], [0.9992486852, "q"], [0.9995617330, "x"], [0.9998121713, "w"], [1.0000000000, "z"],  ],
-        'w' => [ [0.2197508897, "a"], [0.3764713934, "e"], [0.5284697509, "i"], [0.6595264166, "o"], [0.7374076102, "$"], [0.8001642486, "h"], [0.8445113605, "n"], [0.8720914317, "r"], [0.8938543663, "l"], [0.9145223104, "s"], [0.9261565836, "y"], [0.9366274295, "d"], [0.9460717219, "t"], [0.9542157131, "b"], [0.9619490829, "u"], [0.9690665207, "k"], [0.9750889680, "f"], [0.9811114153, "c"], [0.9867916781, "m"], [0.9911716397, "p"], [0.9946619217, "w"], [0.9976731454, "g"], [0.9985628251, "j"], [0.9993840679, "z"], [0.9998631262, "q"], [1.0000000000, "v"],  ],
-        'x' => [ [0.2546458142, "$"], [0.4077276909, "i"], [0.5054277829, "t"], [0.5863845446, "p"], [0.6559337626, "e"], [0.7238270469, "a"], [0.7696412144, "o"], [0.8147194112, "y"], [0.8594296228, "c"], [0.8885004600, "x"], [0.9138914443, "u"], [0.9306347746, "h"], [0.9425942962, "l"], [0.9521619135, "s"], [0.9613615455, "v"], [0.9703771849, "m"], [0.9781048758, "f"], [0.9849126035, "b"], [0.9889604416, "w"], [0.9913523459, "r"], [0.9933762649, "d"], [0.9954001840, "n"], [0.9974241030, "g"], [0.9992640294, "q"], [0.9996320147, "j"], [0.9998160074, "k"], [1.0000000000, "z"],  ],
-        'y' => [ [0.5555410988, "$"], [0.5980873695, "l"], [0.6377386722, "s"], [0.6736492860, "a"], [0.7088768175, "e"], [0.7438441271, "p"], [0.7767621898, "n"], [0.8066226458, "m"], [0.8352470481, "c"], [0.8630907849, "t"], [0.8858276681, "o"], [0.9072309144, "r"], [0.9262921641, "d"], [0.9444426373, "i"], [0.9558923983, "b"], [0.9643496080, "g"], [0.9714081254, "u"], [0.9783690596, "w"], [0.9846143838, "f"], [0.9887779332, "h"], [0.9924860944, "k"], [0.9945678691, "v"], [0.9964219497, "z"], [0.9980158085, "x"], [0.9992518622, "y"], [0.9998048336, "j"], [1.0000000000, "q"],  ],
-        'z' => [ [0.2946584939, "e"], [0.4734384121, "a"], [0.6180677175, "i"], [0.7089900759, "o"], [0.7905720957, "$"], [0.8610624635, "z"], [0.8884997081, "y"], [0.9098073555, "u"], [0.9306771745, "l"], [0.9455633392, "h"], [0.9519848219, "b"], [0.9576765908, "m"], [0.9627845884, "k"], [0.9676007005, "w"], [0.9724168126, "n"], [0.9767950963, "s"], [0.9810274372, "t"], [0.9852597782, "d"], [0.9886164623, "g"], [0.9910974898, "v"], [0.9934325744, "c"], [0.9957676591, "r"], [0.9976649154, "p"], [0.9988324577, "f"], [0.9995621716, "j"], [0.9998540572, "q"], [1.0000000000, "x"],  ],
-      } # BIGRAM_CDF_HSH
-
-    end
-
-  end
-end

data/lib/wukong/widget/monitor.rb

@@ -1,26 +0,0 @@
-module Wukong
-  module Widget
-
-    class Monitor < AsIs
-      include CountingProcessor
-      register_processor
-
-      field :every, Integer, :default => 1000, :doc => "How often to announce progress"
-
-      def process(rec)
-        super(rec)
-        $stderr.puts("%-7d\t%s\t%s" % [count, report, rec.inspect[0..1000]]) if ready?
-      end
-
-      def ready?
-        (count % every) == 0
-      end
-    end
-
-    class DumpSystemConfig < Monitor
-      def setup ; require 'rbconfig' ;  end
-      def report() super.merge({ :rbconfig => RbConfig::CONFIG })  end
-    end
-
-  end
-end

data/lib/wukong/widget/reducer.rb

@@ -1,66 +0,0 @@
-module Wukong
-  class Counter < Wukong::Processor
-    field :count,       Integer, :doc => 'count of records this run'
-
-    def setup
-      super
-      reset!
-    end
-
-    def reset!
-      self.count = 0
-    end
-
-    def beg_group(*args)
-      reset!
-    end
-
-    def end_group(key)
-      emit( [key, count] )
-    end
-
-    def process(record)
-      @count += 1
-    end
-  end
-
-  class GroupArrays < Wukong::Processor
-    def beg_group
-      @records = []
-    end
-
-    def end_group(key)
-      emit(key, @records)
-    end
-
-    def process(record)
-      @records << record
-    end
-  end
-
-  class Group < Wukong::Processor
-    def start(key, *vals)
-      @key = key
-      next_stage.tell(:beg_group, @key)
-    end
-
-    def end_group
-      next_stage.tell(:end_group, @key)
-    end
-
-    def process( (key, *vals) )
-      start(key, *vals) unless defined?(@key)
-      if key != @key
-        end_group
-        start(key, *vals)
-      end
-      emit( [key, *vals] )
-    end
-
-    def finally
-      end_group
-      super()
-    end
-  end
-
-end

data/lib/wukong/widget/stringifier.rb

@@ -1,50 +0,0 @@
-module Wukong
-  module Widget
-
-    class Stringifier < Processor
-    end
-
-    class ToJson < Stringifier
-      def process(record)
-        emit record.to_json
-        # emit MultiJson.dump(record)
-      end
-      register_processor
-    end
-
-    class FromJson < Stringifier
-      # FIXME some of this belongs in gorillib factories...
-      def process(record)
-        obj = MultiJson.load(record)
-        if obj.respond_to?(:has_key?) && obj.has_key?("_metadata")
-          metadata_hash = obj.delete("_metadata")
-          obj.define_singleton_method(:_metadata) do
-            metadata_hash
-          end
-        end
-        if obj.respond_to?(:has_key?) && obj.has_key?("_type")
-          klass = Gorillib::Factory(obj.delete("_type"))
-          obj = klass.receive(obj)
-        end
-        emit obj
-      end
-      register_processor
-    end
-
-    class ToTsv < Stringifier
-      def process(record)
-        emit record.join("\t")
-      end
-      register_processor
-    end
-
-    class FromTsv < Stringifier
-      def process(record)
-        emit record.chomp.split(/\t/)
-      end
-      register_processor
-    end
-
-  end
-
-end

data/lib/wukong/workflow.rb

@@ -1,22 +0,0 @@
-module Wukong
-  class WorkflowGraph < Hanuman::Graph
-  end
-
-  class Workflow < WorkflowGraph
-    include Hanuman::IsOwnInputSlot
-    include Hanuman::IsOwnOutputSlot
-
-    #
-    # lifecycle
-    #
-
-    def setup
-      stages.each_value{|stage| stage.setup}
-    end
-
-    def stop
-      stages.each_value{|stage| stage.stop}
-    end
-
-  end
-end

data/lib/wukong/workflow/command.rb

@@ -1,42 +0,0 @@
-module Wukong
-  class Workflow < Hanuman::Graph
-
-    class ActionWithInputs < Hanuman::Action
-      include Hanuman::Slottable
-      include Hanuman::SplatInputs
-      include Hanuman::SplatOutputs
-
-      def self.make(workflow, *input_stages, &block)
-        options  = input_stages.extract_options!
-        stage    = new
-        workflow.add_stage stage
-        input_stages.map do |input|
-          workflow.connect(input, stage)
-        end
-        stage.receive!(options, &block)
-        stage
-      end
-    end
-
-    #
-    # A command is a workflow action that runs a type of command on many inputs,
-    # under a given configuration, into named outputs.
-    #
-    # @example
-    #   bash 'create_archive.sh', filenames, :compression_level => 9 > 'archive.tar.gz'
-    #
-    class Command < ActionWithInputs
-
-      def self.make(workflow, stage_name, *input_stages, &block)
-        options  = input_stages.extract_options!
-        super(workflow, *input_stages, options.merge(:name => stage_name, :script => stage_name), &block)
-      end
-    end
-
-    class Shell < Command
-      field :script, String
-      register_action
-    end
-
-  end
-end

data/old/config/emr-example.yaml

@@ -1,48 +0,0 @@
-#
-# Elastic MapReduce config in wukong
-#
-
-#
-# Infrastructure options
-#
-
-# == Fill all your information into yet another file with your amazon key
-:emr_credentials_file:          ~/.wukong/credentials.json
-#
-# == Use the credentials file, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars, or enter them here:
-# :access_key:                  ASDFAHKHASDF
-# :secret_access_key:           ADSGHASDFJASDFASDF
-#
-# == Path to your keypair file
-# :key_pair_file:               ~/.wukong/keypairs/gibbon.pem
-# == Keypair will be named after your file, or force the name:
-# :key_pair:                    ~
-
-# == Path to the Amazon elastic-mapreduce runner. Get a copy from
-#    http://elasticmapreduce.s3.amazonaws.com/elastic-mapreduce-ruby.zip
-:emr_runner:                    ~/ics/hadoop/elastic-mapreduce/elastic-mapreduce
-
-#
-# Cluster Config
-#
-:num_instances:                 1
-:instance_type:                 m2.xlarge
-:master_instance_type:          ~
-:hadoop_version:                '0.20'
-# :availability_zone:           us-east-1b
-
-#
-# Running and reporting options
-#
-:alive:                         true
-:enable_debugging:              true
-:emr_runner_verbose:            true
-:emr_runner_debug:              ~
-:step_action:                   CANCEL_AND_WAIT         # CANCEL_AND_WAIT, TERMINATE_JOB_FLOW or CONTINUE
-
-#
-# Remote Paths
-#
-:emr_root:                      s3n://emr.infinitemonkeys.info
-
-

data/old/examples/README.txt

@@ -1,17 +0,0 @@
-Examples:
-
-
-* sample_records -- extract a random sample from a collection of data
-
-* word_count
-
-* apache_log_parser -- example for parsing standard apache webserver log files.
-
-* wordchains -- solving a word puzzle using breadth-first search of a graph
-
-* graph -- some generic graph
-
-* pagerank -- use the pagerank algorithm to find the most 'interesting'
-  (central) nodes of a network graph
-
-  

data/old/examples/contrib/jeans/README.markdown

@@ -1,165 +0,0 @@
-Notes and scripts from a talk by Fredrik Möllerstrand, (@lenbust / http://fredrikmollerstrand.se) with some modifications by @mrflip
-
-See http://bit.ly/6ItaHI and forks of that gist!
-
-Wukong who?
------------
-
-Here be some notes from my talk on [Wukong](http://github.com/mrflip/wukong) at the January meetup of [Got.rb](http://www.meetup.com/got-rb/).
-
-Wukong is a framework for writing Hadoop jobs in Ruby. Other such frameworks are [MRToolkit](http://code.google.com/p/mrtoolkit/) (which is also written in Ruby and which I have not tried it) and [Dumbo](http://github.com/klbostee/dumbo) (which is written in Python and which I love dearly). You could also write your jobs in Java(!) or as bare scripts hooked into [Hadoop Streaming](http://hadoop.apache.org/common/docs/current/streaming.html), but that would be nuts.
-
-Wukong gives you the option of treating your data as a stream of lines or as a stream of fields or lightweight objects. My forenoon's experience of Wukong covers only the basic text streaming so we'll skip the structured data and interpret the data as dumb chunks of text.
-
-The data, which I regrettably can not share with you on the wider interwebs, are records of jeans ordered by stores from a central jean distributor. There is one line per order, and each order is for a specific market and for a specified number of jeans for each size. There are 13 sizes in total.
-
-An example of which:
-
-	SS10	vaxjobutiken	2010-01-10 10:45:54	sweden	Storgatan 64 	VÄXJÖ	352 30	SWEDEN	retailer	120664	L34	0	0	0	0	0	1	1	1	2	2	1	1	0
-	SS10	vaxjobutiken	2009-01-10 10:45:54	sweden	Storgatan 64 	VÄXJÖ	352 30	SWEDEN	retailer	120721	L32	0	0	0	0	0	1	2	2	2	1	1	1	0
-	SS09	kubic	2010-01-10 13:33:37	spain	NULL	NULL	NULL	NULL	retailer	120571	L34	0	0	0	0	0	0	0	1	1	1	1	1	0
-
-The integers at the end there describe how many of each jean size was ordered.
-
-We'll use Wukong to summarize the orders for each country. The job is run locally (as oppposed to on Hadoop) to avoid any startup overhead.
-
-sizes.rb
---------
-        $> ruby sizes.rb --run=local data/orders.tsv data/sizes
-
-        require 'rubygems'
-        require 'wukong'
-        module JeanSizes
-          class Mapper < Wukong::Streamer::RecordStreamer
-            def process(code,model,time,country,j1,j2,j3, n1,n2,c1, venue,n3,n4, *sizes)
-              yield [country, *sizes] if sizes.length == 13
-            end
-          end
-
-          class JeansListReducer < Wukong::Streamer::ListReducer
-            def finalize
-              return if values.empty?
-              sums = []; 13.times{ sums << 0 }
-              values.each do |country, *sizes|
-                sizes.map!(&:to_i)
-                sums = sums.zip(sizes).map{|sum, val| sum + val }
-              end
-              yield [key, *sums]
-            end
-          end
-	end
-
-	Wukong::Script.new(JeanSizes::Mapper, JeanSizes::JeansListReducer).run
-
-*JeanSizes::Mapper#process*, being a RecordStreamer, is given one set of input fields to work with at a time. It picks out the good parts, namely the country at index 3 and the integers at index 11 through 23. The rest of the fields are unimportant and just given placeholder names.
-
-The country is promoted to key and the sizes array is value. These are yielded as a list – since the reducer is a list reducer!
-
-*JeanSizes::Reducer#finalize* is given the key ('sweden' for example) and a list of lists of integers. These are (over)cleverly summarized into one list, *sums*.
-
-The output of these two steps is a much smaller data set containing the number of jeans of each size purchased, broken down by market.
-
-An example of which:
-
-	sweden  807     1443    2215    2460    2316    2077    2392    2563 3068    2356    2051    1016    255
-	switzerland     90      201     731     886     585     325     404 624     770     721     635     295     41
-	unitedstates    446     1103    2007    2442    2863    2879    3920 3687    5588    4256    5299    3777    1842
-
-
-That's all peachy, but what if I'd like to compare the relative amount of large jeans bought in Sweden with those bought in the US? A working hypothesis might be that swedes wear smaller jean sizes than do americans. Well, let's normalize the data and see what we can make of it.
-
-normalize.rb
-------------
-        $> ruby normalize.rb --run=local data/sizes.tsv data/normalized_sizes.tsv
-
-        require 'rubygems'
-        require 'wukong'
-        require 'active_support/core_ext/enumerable' # for array#sum
-
-	module Normalize
-	  class Mapper < Wukong::Streamer::RecordStreamer
-	    def process(country, *sizes)
-	      sizes.map!(&:to_i)
-	      sum = sizes.sum.to_f
-	      normalized = sizes.map{|x| 100 * x/sum }
-	      s = normalized.join(",")
-	      yield [country, s]
-	    end
-	  end
-	end
-
-	Wukong::Script.new(Normalize::Mapper, nil).run
-
-Again we're dealing with a line streamer. The normalization divides each jean size by the total number of jeans sold in that country and scales it up by 100 to make the figures into proper percentages.
-
-You will also notice that I join the list of normalized values with a comma. Why in the name of Buddha would I do that? Bear with me.
-
-Parts of the output looks like so:
-
-	sweden	1.01922538870458,4.06091370558376,8.19776969503178,9.41684319916863,12.2626803629242,10.2442143970582,9.56073384227987,8.30169071505656,9.25696470682282,9.83252727926776,8.85327151364963,5.76761661137536,3.22554858307686
-	switzerland	0.64996829422955,4.67660114140774,10.0665821179455,11.429930247305,12.2067216233354,9.89220038046925,6.40456563094483,5.15218769816107,9.27393785668992,14.0456563094483,11.5884590995561,3.1864299302473,1.42675967025999
-	unitedstates	4.59248547707497,9.41683911341594,13.2114986661348,10.6110847939365,13.9320352040689,9.19245057219078,9.77336757336259,7.17794011319155,7.13804881697375,6.08840908524271,5.0038644693211,2.75000623301503,1.11196988207136
-
-
-Data visualized
----------------
-
-In the words of the late great R. Dingly, '*data not visualized is not data*'.
-
-Let's put our hypothesis to work and graph this. The quickest path to a graph just happens to be Google Charts, and here's
-[a graph comparing jeans sizes bought in Sweden and the US ](http://chart.apis.google.com/chart?cht=bvg&chd=t:1.01922538870458,4.06091370558376,8.19776969503178,9.41684319916863,12.2626803629242,10.2442143970582,9.56073384227987,8.30169071505656,9.25696470682282,9.83252727926776,8.85327151364963,5.76761661137536,3.22554858307686|4.59248547707497,9.41683911341594,13.2114986661348,10.6110847939365,13.9320352040689,9.19245057219078,9.77336757336259,7.17794011319155,7.13804881697375,6.08840908524271,5.0038644693211,2.75000623301503,1.11196988207136&chds=0,20&chs=800x375&chdl=Sweden|USA&chco=eecc00,00eedd).
-
-It seems that americans buy smaller sizes while swedes go for larger breeches, which is quite the opposite of what we thought.
-
-As someone in the crowd pointed out during the meet, this might be due to the fact that the particular brand of jeans under scrutiny here has become mainstream in Sweden while it is still mostly worn by thin punk-rockers in the states. Again the the words of Mr. Dingly echo so true: '*data is nothing without interpretation*'.
-
-Chaining Runs
--------------
-
-**Local Mode**: If you're running in local mode, chaining is straightforward: just use a single dash '-' as the output file, and wukong will leave its output on STDOUT rather than dumping to a file on disk. (NOTE: this is only true as of wukong v1.4.5.) (OTHER NOTE: if you've already written a file named '-' to disk, use "rm -- -" to remove it).
-
-       ./sizes.rb --run=local data/orders.tsv -  | ./normalize.rb --run=local - data/normalized_sizes
-
-For anything fancier (or for earlier versions of wukong): all the local runner does is to take your script and run
-
-       cat input.tsv | myscript.rb --map [..args..] | sort | myscript.rb --reduce [..args..] > output.tsv
-
-You can do the chaining by hand:
-
-       cat input.tsv | myscript.rb --map [..args..] | sort | myscript.rb --reduce [..args..] | whatever | ./anotherscript.rb --map | sort | ./anotherscript --reduce > output.tsv
-
-**Hadoop Mode**: Wukong doesn't let you chain jobs or define workflows.  There are tools out there that enable this, but there are no mature solutions that I [@mrflip] know of.
-
-In closing
-----------
-
-This has been a quick rundown of Wukong as I know it after a few hours of use. Improvements can certainly be made and I welcome any and all comments. Please consider amending the code in this presentation by forking [this gist](http://gist.github.com/278043).
-
-Also, you should follow me on twitter: [@lenbust](http://twitter.com/lenbust).
-
-Postscript
-__________
-
-The LineReducer used in sizes.rb is perfect for a small, local run such as this one.  However, for large amounts of data it's best to avoid the ListReducer, as it collects every single record in memory before finalizing.
-
-You can instead use an Accumulating reducer directly. Compare this class with the JeansListReducer above and you'll see we are applying the same basic workflow, just in explicitly separated steps.
-
-          class JeansAccumulatingReducer < Wukong::Streamer::AccumulatingReducer
-            attr_accessor :sums
-
-            # start the sum with 0 for each size
-            def start! *_
-              self.sums = []; 13.times{ self.sums << 0 }
-            end
-
-            # accumulate each size count into the sizes_sum
-            def accumulate country, *sizes
-              sizes.map!(&:to_i)
-	      self.sums = self.sums.zip(sizes).map{|sum, val| sum + val }
-	    end
-
-	    # emit [country, size_0_sum, size_1_sum, ...]
-	    def finalize
-	      yield [key, *sums]
-	    end
-	  end