wuclan 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (111) hide show
  1. data/LICENSE.textile +20 -0
  2. data/README.textile +28 -0
  3. data/examples/analyze/strong_links/gen_multi_edge.rb +103 -0
  4. data/examples/analyze/strong_links/main.rb +51 -0
  5. data/examples/analyze/word_count/dump_schema.rb +13 -0
  6. data/examples/analyze/word_count/freq_user.rb +31 -0
  7. data/examples/analyze/word_count/freq_whole_corpus.rb +27 -0
  8. data/examples/analyze/word_count/word_count.pig +43 -0
  9. data/examples/analyze/word_count/word_count.rb +34 -0
  10. data/examples/lastfm/scrape/load_lastfm.rb +31 -0
  11. data/examples/lastfm/scrape/scrape_lastfm.rb +47 -0
  12. data/examples/lastfm/scrape/seed.tsv +147 -0
  13. data/examples/twitter/old/load_twitter_search_jobs.rb +157 -0
  14. data/examples/twitter/old/scrape_twitter_api.rb +104 -0
  15. data/examples/twitter/old/scrape_twitter_search.rb +57 -0
  16. data/examples/twitter/old/scrape_twitter_trending.rb +73 -0
  17. data/examples/twitter/parse/parse_twitter_requests.rb +81 -0
  18. data/examples/twitter/parse/parse_twitter_search_requests.rb +28 -0
  19. data/examples/twitter/scrape_twitter_api/scrape_twitter_api.rb +61 -0
  20. data/examples/twitter/scrape_twitter_api/seed.tsv +4 -0
  21. data/examples/twitter/scrape_twitter_api/start_cache_twitter.sh +2 -0
  22. data/examples/twitter/scrape_twitter_api/support/make_request_stats.rb +291 -0
  23. data/examples/twitter/scrape_twitter_api/support/make_requests_by_id_and_date_1.rb +98 -0
  24. data/examples/twitter/scrape_twitter_api/support/make_requests_by_id_and_date_2.pig +4 -0
  25. data/examples/twitter/scrape_twitter_api/support/twitter_search_jobs.tsv +6 -0
  26. data/examples/twitter/scrape_twitter_api/support/twitter_trending_seed.tsv +725 -0
  27. data/examples/twitter/scrape_twitter_hosebird/edamame-killall +4 -0
  28. data/examples/twitter/scrape_twitter_hosebird/foo.rb +19 -0
  29. data/examples/twitter/scrape_twitter_hosebird/ps_emulation.rb +111 -0
  30. data/examples/twitter/scrape_twitter_hosebird/scrape_twitter_hosebird.rb +110 -0
  31. data/examples/twitter/scrape_twitter_hosebird/test_spewer.rb +20 -0
  32. data/examples/twitter/scrape_twitter_hosebird/twitter_hosebird_god.yaml +10 -0
  33. data/examples/twitter/scrape_twitter_search/dump_twitter_search_jobs.rb +38 -0
  34. data/examples/twitter/scrape_twitter_search/load_twitter_search_jobs.rb +63 -0
  35. data/examples/twitter/scrape_twitter_search/scrape_twitter_search.rb +44 -0
  36. data/examples/twitter/scrape_twitter_search/twitter_search_daemons.god +25 -0
  37. data/lib/old/twitter_api.rb +88 -0
  38. data/lib/wuclan/delicious/delicious_html_request.rb +31 -0
  39. data/lib/wuclan/delicious/delicious_models.rb +26 -0
  40. data/lib/wuclan/delicious/delicious_request.rb +65 -0
  41. data/lib/wuclan/friendfeed/scrape/friendfeed_search_request.rb +60 -0
  42. data/lib/wuclan/friendster.rb +7 -0
  43. data/lib/wuclan/lastfm/model/base.rb +49 -0
  44. data/lib/wuclan/lastfm/model/sample_responses.txt +16 -0
  45. data/lib/wuclan/lastfm/scrape/base.rb +195 -0
  46. data/lib/wuclan/lastfm/scrape/concrete.rb +143 -0
  47. data/lib/wuclan/lastfm/scrape/lastfm_job.rb +12 -0
  48. data/lib/wuclan/lastfm/scrape/lastfm_request_stream.rb +17 -0
  49. data/lib/wuclan/lastfm/scrape/recursive_requests.rb +154 -0
  50. data/lib/wuclan/lastfm/scrape.rb +12 -0
  51. data/lib/wuclan/lastfm.rb +7 -0
  52. data/lib/wuclan/metrics/user_graph_metrics.rb +99 -0
  53. data/lib/wuclan/metrics/user_metrics.rb +443 -0
  54. data/lib/wuclan/metrics/user_metrics_basic.rb +277 -0
  55. data/lib/wuclan/metrics/user_scraping_metrics.rb +64 -0
  56. data/lib/wuclan/metrics.rb +0 -0
  57. data/lib/wuclan/myspace.rb +21 -0
  58. data/lib/wuclan/open_social/model/base.rb +0 -0
  59. data/lib/wuclan/open_social/scrape/base.rb +111 -0
  60. data/lib/wuclan/open_social/scrape_request.rb +6 -0
  61. data/lib/wuclan/open_social.rb +0 -0
  62. data/lib/wuclan/rdf_output/relationship_rdf.rb +47 -0
  63. data/lib/wuclan/rdf_output/text_element_rdf.rb +64 -0
  64. data/lib/wuclan/rdf_output/tweet_rdf.rb +10 -0
  65. data/lib/wuclan/rdf_output/twitter_rdf.rb +84 -0
  66. data/lib/wuclan/rdf_output/twitter_user_rdf.rb +12 -0
  67. data/lib/wuclan/shorturl/shorturl_request.rb +271 -0
  68. data/lib/wuclan/twitter/api_response_examples.textile +300 -0
  69. data/lib/wuclan/twitter/model/base.rb +72 -0
  70. data/lib/wuclan/twitter/model/multi_edge.rb +31 -0
  71. data/lib/wuclan/twitter/model/relationship.rb +176 -0
  72. data/lib/wuclan/twitter/model/text_element/extract_info_tests.rb +83 -0
  73. data/lib/wuclan/twitter/model/text_element/grok_tweets.rb +96 -0
  74. data/lib/wuclan/twitter/model/text_element/more_regexes.rb +370 -0
  75. data/lib/wuclan/twitter/model/text_element.rb +38 -0
  76. data/lib/wuclan/twitter/model/tweet/tokenize.rb +38 -0
  77. data/lib/wuclan/twitter/model/tweet/tweet_regexes.rb +202 -0
  78. data/lib/wuclan/twitter/model/tweet/tweet_token.rb +79 -0
  79. data/lib/wuclan/twitter/model/tweet.rb +74 -0
  80. data/lib/wuclan/twitter/model/twitter_user/style/color_to_hsv.rb +57 -0
  81. data/lib/wuclan/twitter/model/twitter_user.rb +145 -0
  82. data/lib/wuclan/twitter/model.rb +21 -0
  83. data/lib/wuclan/twitter/parse/ff_ids_parser.rb +27 -0
  84. data/lib/wuclan/twitter/parse/friends_followers_parser.rb +52 -0
  85. data/lib/wuclan/twitter/parse/generic_json_parser.rb +26 -0
  86. data/lib/wuclan/twitter/parse/json_tweet.rb +63 -0
  87. data/lib/wuclan/twitter/parse/json_twitter_user.rb +122 -0
  88. data/lib/wuclan/twitter/parse/public_timeline_parser.rb +54 -0
  89. data/lib/wuclan/twitter/parse/twitter_search_parse.rb +60 -0
  90. data/lib/wuclan/twitter/parse/user_parser.rb +30 -0
  91. data/lib/wuclan/twitter/scrape/base.rb +97 -0
  92. data/lib/wuclan/twitter/scrape/old_skool_request_classes.rb +40 -0
  93. data/lib/wuclan/twitter/scrape/twitter_fake_fetcher.rb +31 -0
  94. data/lib/wuclan/twitter/scrape/twitter_ff_ids_request.rb +75 -0
  95. data/lib/wuclan/twitter/scrape/twitter_followers_request.rb +135 -0
  96. data/lib/wuclan/twitter/scrape/twitter_json_response.rb +124 -0
  97. data/lib/wuclan/twitter/scrape/twitter_request_stream.rb +44 -0
  98. data/lib/wuclan/twitter/scrape/twitter_search_fake_fetcher.rb +44 -0
  99. data/lib/wuclan/twitter/scrape/twitter_search_flat_stream.rb +30 -0
  100. data/lib/wuclan/twitter/scrape/twitter_search_job.rb +25 -0
  101. data/lib/wuclan/twitter/scrape/twitter_search_request.rb +70 -0
  102. data/lib/wuclan/twitter/scrape/twitter_search_request_stream.rb +19 -0
  103. data/lib/wuclan/twitter/scrape/twitter_timeline_request.rb +72 -0
  104. data/lib/wuclan/twitter/scrape/twitter_user_request.rb +64 -0
  105. data/lib/wuclan/twitter/scrape.rb +27 -0
  106. data/lib/wuclan/twitter.rb +7 -0
  107. data/lib/wuclan.rb +1 -0
  108. data/spec/spec_helper.rb +9 -0
  109. data/spec/wuclan_spec.rb +7 -0
  110. data/wuclan.gemspec +184 -0
  111. metadata +219 -0
metadata ADDED
@@ -0,0 +1,219 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: wuclan
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Philip (flip) Kromer
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-10-12 00:00:00 -05:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: wukong
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: "0"
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: monkeyshines
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: "0"
34
+ version:
35
+ - !ruby/object:Gem::Dependency
36
+ name: edamame
37
+ type: :runtime
38
+ version_requirement:
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: "0"
44
+ version:
45
+ description: Massive-scale social network analysis. Nothing to f with.
46
+ email: flip@infochimps.org
47
+ executables: []
48
+
49
+ extensions: []
50
+
51
+ extra_rdoc_files:
52
+ - LICENSE.textile
53
+ - README.textile
54
+ files:
55
+ - LICENSE.textile
56
+ - README.textile
57
+ - examples/analyze/strong_links/gen_multi_edge.rb
58
+ - examples/analyze/strong_links/main.rb
59
+ - examples/analyze/word_count/dump_schema.rb
60
+ - examples/analyze/word_count/freq_user.rb
61
+ - examples/analyze/word_count/freq_whole_corpus.rb
62
+ - examples/analyze/word_count/word_count.pig
63
+ - examples/analyze/word_count/word_count.rb
64
+ - examples/lastfm/scrape/load_lastfm.rb
65
+ - examples/lastfm/scrape/scrape_lastfm.rb
66
+ - examples/lastfm/scrape/seed.tsv
67
+ - examples/twitter/old/load_twitter_search_jobs.rb
68
+ - examples/twitter/old/scrape_twitter_api.rb
69
+ - examples/twitter/old/scrape_twitter_search.rb
70
+ - examples/twitter/old/scrape_twitter_trending.rb
71
+ - examples/twitter/parse/parse_twitter_requests.rb
72
+ - examples/twitter/parse/parse_twitter_search_requests.rb
73
+ - examples/twitter/scrape_twitter_api/scrape_twitter_api.rb
74
+ - examples/twitter/scrape_twitter_api/seed.tsv
75
+ - examples/twitter/scrape_twitter_api/start_cache_twitter.sh
76
+ - examples/twitter/scrape_twitter_api/support/make_request_stats.rb
77
+ - examples/twitter/scrape_twitter_api/support/make_requests_by_id_and_date_1.rb
78
+ - examples/twitter/scrape_twitter_api/support/make_requests_by_id_and_date_2.pig
79
+ - examples/twitter/scrape_twitter_api/support/twitter_search_jobs.tsv
80
+ - examples/twitter/scrape_twitter_api/support/twitter_trending_seed.tsv
81
+ - examples/twitter/scrape_twitter_hosebird/edamame-killall
82
+ - examples/twitter/scrape_twitter_hosebird/foo.rb
83
+ - examples/twitter/scrape_twitter_hosebird/ps_emulation.rb
84
+ - examples/twitter/scrape_twitter_hosebird/scrape_twitter_hosebird.rb
85
+ - examples/twitter/scrape_twitter_hosebird/test_spewer.rb
86
+ - examples/twitter/scrape_twitter_hosebird/twitter_hosebird_god.yaml
87
+ - examples/twitter/scrape_twitter_search/dump_twitter_search_jobs.rb
88
+ - examples/twitter/scrape_twitter_search/load_twitter_search_jobs.rb
89
+ - examples/twitter/scrape_twitter_search/scrape_twitter_search.rb
90
+ - examples/twitter/scrape_twitter_search/twitter_search_daemons.god
91
+ - lib/old/twitter_api.rb
92
+ - lib/wuclan.rb
93
+ - lib/wuclan/delicious/delicious_html_request.rb
94
+ - lib/wuclan/delicious/delicious_models.rb
95
+ - lib/wuclan/delicious/delicious_request.rb
96
+ - lib/wuclan/friendfeed/scrape/friendfeed_search_request.rb
97
+ - lib/wuclan/friendster.rb
98
+ - lib/wuclan/lastfm.rb
99
+ - lib/wuclan/lastfm/model/base.rb
100
+ - lib/wuclan/lastfm/model/sample_responses.txt
101
+ - lib/wuclan/lastfm/scrape.rb
102
+ - lib/wuclan/lastfm/scrape/base.rb
103
+ - lib/wuclan/lastfm/scrape/concrete.rb
104
+ - lib/wuclan/lastfm/scrape/lastfm_job.rb
105
+ - lib/wuclan/lastfm/scrape/lastfm_request_stream.rb
106
+ - lib/wuclan/lastfm/scrape/recursive_requests.rb
107
+ - lib/wuclan/metrics.rb
108
+ - lib/wuclan/metrics/user_graph_metrics.rb
109
+ - lib/wuclan/metrics/user_metrics.rb
110
+ - lib/wuclan/metrics/user_metrics_basic.rb
111
+ - lib/wuclan/metrics/user_scraping_metrics.rb
112
+ - lib/wuclan/myspace.rb
113
+ - lib/wuclan/open_social.rb
114
+ - lib/wuclan/open_social/model/base.rb
115
+ - lib/wuclan/open_social/scrape/base.rb
116
+ - lib/wuclan/open_social/scrape_request.rb
117
+ - lib/wuclan/rdf_output/relationship_rdf.rb
118
+ - lib/wuclan/rdf_output/text_element_rdf.rb
119
+ - lib/wuclan/rdf_output/tweet_rdf.rb
120
+ - lib/wuclan/rdf_output/twitter_rdf.rb
121
+ - lib/wuclan/rdf_output/twitter_user_rdf.rb
122
+ - lib/wuclan/shorturl/shorturl_request.rb
123
+ - lib/wuclan/twitter.rb
124
+ - lib/wuclan/twitter/api_response_examples.textile
125
+ - lib/wuclan/twitter/model.rb
126
+ - lib/wuclan/twitter/model/base.rb
127
+ - lib/wuclan/twitter/model/multi_edge.rb
128
+ - lib/wuclan/twitter/model/relationship.rb
129
+ - lib/wuclan/twitter/model/text_element.rb
130
+ - lib/wuclan/twitter/model/text_element/extract_info_tests.rb
131
+ - lib/wuclan/twitter/model/text_element/grok_tweets.rb
132
+ - lib/wuclan/twitter/model/text_element/more_regexes.rb
133
+ - lib/wuclan/twitter/model/tweet.rb
134
+ - lib/wuclan/twitter/model/tweet/tokenize.rb
135
+ - lib/wuclan/twitter/model/tweet/tweet_regexes.rb
136
+ - lib/wuclan/twitter/model/tweet/tweet_token.rb
137
+ - lib/wuclan/twitter/model/twitter_user.rb
138
+ - lib/wuclan/twitter/model/twitter_user/style/color_to_hsv.rb
139
+ - lib/wuclan/twitter/parse/ff_ids_parser.rb
140
+ - lib/wuclan/twitter/parse/friends_followers_parser.rb
141
+ - lib/wuclan/twitter/parse/generic_json_parser.rb
142
+ - lib/wuclan/twitter/parse/json_tweet.rb
143
+ - lib/wuclan/twitter/parse/json_twitter_user.rb
144
+ - lib/wuclan/twitter/parse/public_timeline_parser.rb
145
+ - lib/wuclan/twitter/parse/twitter_search_parse.rb
146
+ - lib/wuclan/twitter/parse/user_parser.rb
147
+ - lib/wuclan/twitter/scrape.rb
148
+ - lib/wuclan/twitter/scrape/base.rb
149
+ - lib/wuclan/twitter/scrape/old_skool_request_classes.rb
150
+ - lib/wuclan/twitter/scrape/twitter_fake_fetcher.rb
151
+ - lib/wuclan/twitter/scrape/twitter_ff_ids_request.rb
152
+ - lib/wuclan/twitter/scrape/twitter_followers_request.rb
153
+ - lib/wuclan/twitter/scrape/twitter_json_response.rb
154
+ - lib/wuclan/twitter/scrape/twitter_request_stream.rb
155
+ - lib/wuclan/twitter/scrape/twitter_search_fake_fetcher.rb
156
+ - lib/wuclan/twitter/scrape/twitter_search_flat_stream.rb
157
+ - lib/wuclan/twitter/scrape/twitter_search_job.rb
158
+ - lib/wuclan/twitter/scrape/twitter_search_request.rb
159
+ - lib/wuclan/twitter/scrape/twitter_search_request_stream.rb
160
+ - lib/wuclan/twitter/scrape/twitter_timeline_request.rb
161
+ - lib/wuclan/twitter/scrape/twitter_user_request.rb
162
+ - spec/spec_helper.rb
163
+ - spec/wuclan_spec.rb
164
+ - wuclan.gemspec
165
+ has_rdoc: true
166
+ homepage: http://github.com/mrflip/wuclan
167
+ licenses: []
168
+
169
+ post_install_message:
170
+ rdoc_options:
171
+ - --charset=UTF-8
172
+ require_paths:
173
+ - lib
174
+ required_ruby_version: !ruby/object:Gem::Requirement
175
+ requirements:
176
+ - - ">="
177
+ - !ruby/object:Gem::Version
178
+ version: "0"
179
+ version:
180
+ required_rubygems_version: !ruby/object:Gem::Requirement
181
+ requirements:
182
+ - - ">="
183
+ - !ruby/object:Gem::Version
184
+ version: "0"
185
+ version:
186
+ requirements: []
187
+
188
+ rubyforge_project:
189
+ rubygems_version: 1.3.5
190
+ signing_key:
191
+ specification_version: 3
192
+ summary: Massive-scale social network analysis. Nothing to f with.
193
+ test_files:
194
+ - spec/spec_helper.rb
195
+ - spec/wuclan_spec.rb
196
+ - examples/analyze/strong_links/gen_multi_edge.rb
197
+ - examples/analyze/strong_links/main.rb
198
+ - examples/analyze/word_count/dump_schema.rb
199
+ - examples/analyze/word_count/freq_user.rb
200
+ - examples/analyze/word_count/freq_whole_corpus.rb
201
+ - examples/analyze/word_count/word_count.rb
202
+ - examples/lastfm/scrape/load_lastfm.rb
203
+ - examples/lastfm/scrape/scrape_lastfm.rb
204
+ - examples/twitter/old/load_twitter_search_jobs.rb
205
+ - examples/twitter/old/scrape_twitter_api.rb
206
+ - examples/twitter/old/scrape_twitter_search.rb
207
+ - examples/twitter/old/scrape_twitter_trending.rb
208
+ - examples/twitter/parse/parse_twitter_requests.rb
209
+ - examples/twitter/parse/parse_twitter_search_requests.rb
210
+ - examples/twitter/scrape_twitter_api/scrape_twitter_api.rb
211
+ - examples/twitter/scrape_twitter_api/support/make_request_stats.rb
212
+ - examples/twitter/scrape_twitter_api/support/make_requests_by_id_and_date_1.rb
213
+ - examples/twitter/scrape_twitter_hosebird/foo.rb
214
+ - examples/twitter/scrape_twitter_hosebird/ps_emulation.rb
215
+ - examples/twitter/scrape_twitter_hosebird/scrape_twitter_hosebird.rb
216
+ - examples/twitter/scrape_twitter_hosebird/test_spewer.rb
217
+ - examples/twitter/scrape_twitter_search/dump_twitter_search_jobs.rb
218
+ - examples/twitter/scrape_twitter_search/load_twitter_search_jobs.rb
219
+ - examples/twitter/scrape_twitter_search/scrape_twitter_search.rb