twitterscraper-ruby 0.9.0 → 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +1 -0
- data/lib/twitterscraper.rb +1 -0
- data/lib/twitterscraper/cli.rb +16 -2
- data/lib/twitterscraper/template.rb +48 -0
- data/lib/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c2429cf6172b5f19caede64ac35f5c796a7c8a67e76fff8dd2f08677fb15406b
|
|
4
|
+
data.tar.gz: 0f32ca6b559a18c4e3aac3205f6503149e372d4d7d1976b1e83db26036d9ff17
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a36ce6c91a363b64b36deeb3abbaaaebb725f3449f280b70be92532497a94dc5915ba449926acfacfc0d852d52471d258d41140a8891e64b6040bf262d0c347f
|
|
7
|
+
data.tar.gz: a737c7db151190a1493b1a2a92bea304cfcf7512b2ee03fc13c6f25794f5dc727fe548e52cb39eccc2a63261fee0d58fc005920a0e7cd7650d20600e184d79cb
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -143,6 +143,7 @@ $ cat tweets.json | jq . | less
|
|
|
143
143
|
| `--limit` | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
|
|
144
144
|
| `--threads` | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
|
|
145
145
|
| `--proxy` | Scrape https://twitter.com/search via proxies. | false |
|
|
146
|
+
| `--format` | The format of the output. | json |
|
|
146
147
|
| `--output` | The name of the output file. | tweets.json |
|
|
147
148
|
|
|
148
149
|
|
data/lib/twitterscraper.rb
CHANGED
data/lib/twitterscraper/cli.rb
CHANGED
|
@@ -25,7 +25,19 @@ module Twitterscraper
|
|
|
25
25
|
}
|
|
26
26
|
client = Twitterscraper::Client.new
|
|
27
27
|
tweets = client.query_tweets(options['query'], query_options)
|
|
28
|
-
|
|
28
|
+
export(tweets) unless tweets.empty?
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def export(tweets)
|
|
32
|
+
write_json = lambda { File.write(options['output'], generate_json(tweets)) }
|
|
33
|
+
|
|
34
|
+
if options['format'] == 'json'
|
|
35
|
+
write_json.call
|
|
36
|
+
elsif options['format'] == 'html'
|
|
37
|
+
File.write('tweets.html', Template.tweets_embedded_html(tweets))
|
|
38
|
+
else
|
|
39
|
+
write_json.call
|
|
40
|
+
end
|
|
29
41
|
end
|
|
30
42
|
|
|
31
43
|
def generate_json(tweets)
|
|
@@ -53,6 +65,7 @@ module Twitterscraper
|
|
|
53
65
|
'limit:',
|
|
54
66
|
'threads:',
|
|
55
67
|
'output:',
|
|
68
|
+
'format:',
|
|
56
69
|
'proxy',
|
|
57
70
|
'pretty',
|
|
58
71
|
'verbose',
|
|
@@ -61,7 +74,8 @@ module Twitterscraper
|
|
|
61
74
|
options['lang'] ||= ''
|
|
62
75
|
options['limit'] = (options['limit'] || 100).to_i
|
|
63
76
|
options['threads'] = (options['threads'] || 2).to_i
|
|
64
|
-
options['
|
|
77
|
+
options['format'] ||= 'json'
|
|
78
|
+
options['output'] ||= "tweets.#{options['format']}"
|
|
65
79
|
|
|
66
80
|
options
|
|
67
81
|
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
module Twitterscraper
|
|
2
|
+
module Template
|
|
3
|
+
module_function
|
|
4
|
+
|
|
5
|
+
def tweets_embedded_html(tweets)
|
|
6
|
+
tweets_html = tweets.map { |t| EMBED_TWEET_HTML.sub('__TWEET_URL__', t.tweet_url) }
|
|
7
|
+
EMBED_TWEETS_HTML.sub('__TWEETS__', tweets_html.join)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
EMBED_TWEET_HTML = <<~'HTML'
|
|
11
|
+
<blockquote class="twitter-tweet">
|
|
12
|
+
<a href="__TWEET_URL__"></a>
|
|
13
|
+
</blockquote>
|
|
14
|
+
HTML
|
|
15
|
+
|
|
16
|
+
EMBED_TWEETS_HTML = <<~'HTML'
|
|
17
|
+
<html>
|
|
18
|
+
<head>
|
|
19
|
+
<style type=text/css>
|
|
20
|
+
.twitter-tweet {
|
|
21
|
+
margin: 30px auto 0 auto !important;
|
|
22
|
+
}
|
|
23
|
+
</style>
|
|
24
|
+
<script>
|
|
25
|
+
window.twttr = (function(d, s, id) {
|
|
26
|
+
var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
|
|
27
|
+
if (d.getElementById(id)) return t;
|
|
28
|
+
js = d.createElement(s);
|
|
29
|
+
js.id = id;
|
|
30
|
+
js.src = "https://platform.twitter.com/widgets.js";
|
|
31
|
+
fjs.parentNode.insertBefore(js, fjs);
|
|
32
|
+
|
|
33
|
+
t._e = [];
|
|
34
|
+
t.ready = function(f) {
|
|
35
|
+
t._e.push(f);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
return t;
|
|
39
|
+
}(document, "script", "twitter-wjs"));
|
|
40
|
+
</script>
|
|
41
|
+
</head>
|
|
42
|
+
<body>
|
|
43
|
+
__TWEETS__
|
|
44
|
+
</body>
|
|
45
|
+
</html>
|
|
46
|
+
HTML
|
|
47
|
+
end
|
|
48
|
+
end
|
data/lib/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: twitterscraper-ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.10.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- ts-3156
|
|
@@ -68,6 +68,7 @@ files:
|
|
|
68
68
|
- lib/twitterscraper/logger.rb
|
|
69
69
|
- lib/twitterscraper/proxy.rb
|
|
70
70
|
- lib/twitterscraper/query.rb
|
|
71
|
+
- lib/twitterscraper/template.rb
|
|
71
72
|
- lib/twitterscraper/tweet.rb
|
|
72
73
|
- lib/version.rb
|
|
73
74
|
- twitterscraper-ruby.gemspec
|