twitter_ebooks_poll 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +7 -0
  2. data/.gitattributes +2 -0
  3. data/.gitignore +6 -0
  4. data/.rspec +1 -0
  5. data/.travis.yml +8 -0
  6. data/Gemfile +4 -0
  7. data/LICENSE +22 -0
  8. data/README.md +167 -0
  9. data/Rakefile +2 -0
  10. data/bin/ebooks +449 -0
  11. data/data/adjectives.txt +1466 -0
  12. data/data/nouns.txt +2193 -0
  13. data/lib/twitter_ebooks/archive.rb +116 -0
  14. data/lib/twitter_ebooks/bot.rb +521 -0
  15. data/lib/twitter_ebooks/model.rb +336 -0
  16. data/lib/twitter_ebooks/nlp.rb +195 -0
  17. data/lib/twitter_ebooks/suffix.rb +104 -0
  18. data/lib/twitter_ebooks/sync.rb +52 -0
  19. data/lib/twitter_ebooks/version.rb +3 -0
  20. data/lib/twitter_ebooks.rb +22 -0
  21. data/skeleton/Gemfile +4 -0
  22. data/skeleton/Procfile +1 -0
  23. data/skeleton/bots.rb +65 -0
  24. data/skeleton/corpus/.gitignore +0 -0
  25. data/skeleton/gitignore +1 -0
  26. data/skeleton/image/.gitignore +0 -0
  27. data/skeleton/model/.gitignore +0 -0
  28. data/skeleton/stopwords.txt +843 -0
  29. data/spec/bot_spec.rb +216 -0
  30. data/spec/data/0xabad1dea.json +203945 -0
  31. data/spec/data/0xabad1dea.model +6158 -1
  32. data/spec/memprof.rb +37 -0
  33. data/spec/model_spec.rb +88 -0
  34. data/spec/spec_helper.rb +6 -0
  35. data/twitter_ebooks.gemspec +37 -0
  36. metadata +309 -0
data/spec/memprof.rb ADDED
@@ -0,0 +1,37 @@
1
+ require 'objspace'
2
+
3
+ module MemoryUsage
4
+ MemoryReport = Struct.new(:total_memsize)
5
+
6
+ def self.full_gc
7
+ GC.start(full_mark: true)
8
+ end
9
+
10
+ def self.report(&block)
11
+ rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
12
+
13
+ full_gc
14
+ GC.disable
15
+
16
+ total_memsize = 0
17
+
18
+ generation = nil
19
+ ObjectSpace.trace_object_allocations do
20
+ generation = GC.count
21
+ block.call
22
+ end
23
+
24
+ ObjectSpace.each_object do |obj|
25
+ next unless generation == ObjectSpace.allocation_generation(obj)
26
+ memsize = ObjectSpace.memsize_of(obj) + rvalue_size
27
+ # compensate for API bug
28
+ memsize = rvalue_size if memsize > 100_000_000_000
29
+ total_memsize += memsize
30
+ end
31
+
32
+ GC.enable
33
+ full_gc
34
+
35
+ return MemoryReport.new(total_memsize)
36
+ end
37
+ end
data/spec/model_spec.rb ADDED
@@ -0,0 +1,88 @@
1
+ require 'spec_helper'
2
+ require 'memory_profiler'
3
+ require 'tempfile'
4
+
5
+ def Process.rss; `ps -o rss= -p #{Process.pid}`.chomp.to_i; end
6
+
7
+ describe Ebooks::Model do
8
+ describe 'making tweets' do
9
+ before(:all) { @model = Ebooks::Model.consume(path("data/0xabad1dea.json")) }
10
+
11
+ it "generates a tweet" do
12
+ s = @model.make_statement
13
+ expect(s.length).to be <= 140
14
+ puts s
15
+ end
16
+
17
+ it "generates an appropriate response" do
18
+ s = @model.make_response("hi")
19
+ expect(s.length).to be <= 140
20
+ expect(s.downcase).to include("hi")
21
+ puts s
22
+ end
23
+ end
24
+
25
+ it "consumes, saves and loads models correctly" do
26
+ model = nil
27
+
28
+ report = MemoryUsage.report do
29
+ model = Ebooks::Model.consume(path("data/0xabad1dea.json"))
30
+ end
31
+ expect(report.total_memsize).to be < 200000000
32
+
33
+ file = Tempfile.new("0xabad1dea")
34
+ model.save(file.path)
35
+
36
+ report2 = MemoryUsage.report do
37
+ model = Ebooks::Model.load(file.path)
38
+ end
39
+ expect(report2.total_memsize).to be < 4000000
40
+
41
+ expect(model.tokens[0]).to be_a String
42
+ expect(model.sentences[0][0]).to be_a Fixnum
43
+ expect(model.mentions[0][0]).to be_a Fixnum
44
+ expect(model.keywords[0]).to be_a String
45
+
46
+ puts "0xabad1dea.model uses #{report2.total_memsize} bytes in memory"
47
+ end
48
+
49
+ describe '.consume' do
50
+ it 'interprets lines with @ as mentions' do
51
+ file = Tempfile.new('mentions')
52
+ file.write('@m1spy hello!')
53
+ file.close
54
+
55
+ model = Ebooks::Model.consume(file.path)
56
+ expect(model.sentences.count).to eq 0
57
+ expect(model.mentions.count).to eq 1
58
+
59
+ file.unlink
60
+ end
61
+
62
+ it 'interprets lines without @ as statements' do
63
+ file = Tempfile.new('statements')
64
+ file.write('hello!')
65
+ file.close
66
+
67
+ model = Ebooks::Model.consume(file.path)
68
+ expect(model.mentions.count).to eq 0
69
+ expect(model.sentences.count).to eq 1
70
+
71
+ file.unlink
72
+ end
73
+
74
+ it 'handles strange unicode edge-cases' do
75
+ file = Tempfile.new('unicode')
76
+ file.write("💞\n💞")
77
+ file.close
78
+
79
+ model = Ebooks::Model.consume(file.path)
80
+ expect(model.mentions.count).to eq 0
81
+ expect(model.sentences.count).to eq 2
82
+
83
+ file.unlink
84
+
85
+ p model.make_statement
86
+ end
87
+ end
88
+ end
data/spec/spec_helper.rb ADDED
@@ -0,0 +1,6 @@
1
+ require 'twitter_ebooks'
2
+ require_relative 'memprof'
3
+
4
+ def path(relpath)
5
+ File.join(File.dirname(__FILE__), relpath)
6
+ end
data/twitter_ebooks.gemspec ADDED
@@ -0,0 +1,37 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/twitter_ebooks/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.required_ruby_version = '~> 2.1'
6
+
7
+ gem.authors = ["Magnus Niemann"]
8
+ gem.email = ["ebooks@zwergenmaschine.de"]
9
+ gem.description = %q{Markov chains for all your friends~}
10
+ gem.summary = %q{Markov chains for all your friends~}
11
+ gem.homepage = ""
12
+
13
+ gem.files = `git ls-files`.split($\)
14
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
15
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
+ gem.name = "twitter_ebooks_poll"
17
+ gem.require_paths = ["lib"]
18
+ gem.version = Ebooks::VERSION
19
+
20
+ gem.add_development_dependency 'rspec'
21
+ gem.add_development_dependency 'rspec-mocks'
22
+ gem.add_development_dependency 'memory_profiler'
23
+ gem.add_development_dependency 'timecop'
24
+ gem.add_development_dependency 'pry-byebug'
25
+ gem.add_development_dependency 'yard'
26
+
27
+ gem.add_runtime_dependency 'twitter', '~> 5.15'
28
+ gem.add_runtime_dependency 'rufus-scheduler'
29
+ gem.add_runtime_dependency 'gingerice'
30
+ gem.add_runtime_dependency 'htmlentities'
31
+ gem.add_runtime_dependency 'engtagger'
32
+ gem.add_runtime_dependency 'fast-stemmer'
33
+ gem.add_runtime_dependency 'highscore'
34
+ gem.add_runtime_dependency 'pry'
35
+ gem.add_runtime_dependency 'oauth'
36
+ gem.add_runtime_dependency 'mini_magick'
37
+ end
metadata ADDED
@@ -0,0 +1,309 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: twitter_ebooks_poll
3
+ version: !ruby/object:Gem::Version
4
+ version: 3.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Magnus Niemann
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-09-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rspec
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec-mocks
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: memory_profiler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: timecop
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pry-byebug
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: yard
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: twitter
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '5.15'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '5.15'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rufus-scheduler
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: gingerice
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: htmlentities
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: engtagger
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: fast-stemmer
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: '0'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: '0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: highscore
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: '0'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: pry
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: '0'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: '0'
209
+ - !ruby/object:Gem::Dependency
210
+ name: oauth
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - ">="
214
+ - !ruby/object:Gem::Version
215
+ version: '0'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - ">="
221
+ - !ruby/object:Gem::Version
222
+ version: '0'
223
+ - !ruby/object:Gem::Dependency
224
+ name: mini_magick
225
+ requirement: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - ">="
228
+ - !ruby/object:Gem::Version
229
+ version: '0'
230
+ type: :runtime
231
+ prerelease: false
232
+ version_requirements: !ruby/object:Gem::Requirement
233
+ requirements:
234
+ - - ">="
235
+ - !ruby/object:Gem::Version
236
+ version: '0'
237
+ description: Markov chains for all your friends~
238
+ email:
239
+ - ebooks@zwergenmaschine.de
240
+ executables:
241
+ - ebooks
242
+ extensions: []
243
+ extra_rdoc_files: []
244
+ files:
245
+ - ".gitattributes"
246
+ - ".gitignore"
247
+ - ".rspec"
248
+ - ".travis.yml"
249
+ - Gemfile
250
+ - LICENSE
251
+ - README.md
252
+ - Rakefile
253
+ - bin/ebooks
254
+ - data/adjectives.txt
255
+ - data/nouns.txt
256
+ - lib/twitter_ebooks.rb
257
+ - lib/twitter_ebooks/archive.rb
258
+ - lib/twitter_ebooks/bot.rb
259
+ - lib/twitter_ebooks/model.rb
260
+ - lib/twitter_ebooks/nlp.rb
261
+ - lib/twitter_ebooks/suffix.rb
262
+ - lib/twitter_ebooks/sync.rb
263
+ - lib/twitter_ebooks/version.rb
264
+ - skeleton/Gemfile
265
+ - skeleton/Procfile
266
+ - skeleton/bots.rb
267
+ - skeleton/corpus/.gitignore
268
+ - skeleton/gitignore
269
+ - skeleton/image/.gitignore
270
+ - skeleton/model/.gitignore
271
+ - skeleton/stopwords.txt
272
+ - spec/bot_spec.rb
273
+ - spec/data/0xabad1dea.json
274
+ - spec/data/0xabad1dea.model
275
+ - spec/memprof.rb
276
+ - spec/model_spec.rb
277
+ - spec/spec_helper.rb
278
+ - twitter_ebooks.gemspec
279
+ homepage: ''
280
+ licenses: []
281
+ metadata: {}
282
+ post_install_message:
283
+ rdoc_options: []
284
+ require_paths:
285
+ - lib
286
+ required_ruby_version: !ruby/object:Gem::Requirement
287
+ requirements:
288
+ - - "~>"
289
+ - !ruby/object:Gem::Version
290
+ version: '2.1'
291
+ required_rubygems_version: !ruby/object:Gem::Requirement
292
+ requirements:
293
+ - - ">="
294
+ - !ruby/object:Gem::Version
295
+ version: '0'
296
+ requirements: []
297
+ rubyforge_project:
298
+ rubygems_version: 2.6.11
299
+ signing_key:
300
+ specification_version: 4
301
+ summary: Markov chains for all your friends~
302
+ test_files:
303
+ - spec/bot_spec.rb
304
+ - spec/data/0xabad1dea.json
305
+ - spec/data/0xabad1dea.model
306
+ - spec/memprof.rb
307
+ - spec/model_spec.rb
308
+ - spec/spec_helper.rb
309
+ has_rdoc: