grailbird_updater 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +50 -0
- data/bin/grailbird_updater +4 -2
- data/grailbird_updater.gemspec +1 -0
- data/lib/grailbird_updater/version.rb +1 -1
- data/lib/grailbird_updater.rb +100 -7
- metadata +18 -2
data/README.md
CHANGED
@@ -8,6 +8,56 @@ Turns out the contents in the archive are partial/trimmed API responses from
|
|
8
8
|
the Twitter API, so it is actually possible to drop a whole API response in
|
9
9
|
there, do some sorting and update the archive.
|
10
10
|
|
11
|
+
This currently does not work with a private/protected Twitter account, see
|
12
|
+
[this issue](https://github.com/DeMarko/grailbird_updater/issues/6)
|
13
|
+
|
14
|
+
## How do I know if I have a Twitter archive?
|
15
|
+
|
16
|
+
Hopefully, you downloaded it from Twitter once the feature was made available
|
17
|
+
to you and have their web application which can consume it. The file structure
|
18
|
+
looks somewhat like this (as of 19.12.12):
|
19
|
+
|
20
|
+
```
|
21
|
+
tweets
|
22
|
+
├── README.txt
|
23
|
+
├── css
|
24
|
+
│ └─ ... // provided by Twitter
|
25
|
+
├── data
|
26
|
+
│ ├── csv
|
27
|
+
│ │ ├── 2007_03.csv
|
28
|
+
│ │ ├── 2007_04.csv
|
29
|
+
│ │ ├── 2007_05.csv
|
30
|
+
│ │ ├─ ...
|
31
|
+
│ │ ├── 2012_10.csv
|
32
|
+
│ │ ├── 2012_11.csv
|
33
|
+
│ │ └── 2012_12.csv
|
34
|
+
│ └── js
|
35
|
+
│ ├── payload_details.js
|
36
|
+
│ ├── tweet_index.js
|
37
|
+
│ ├── tweets
|
38
|
+
│ │ ├── 2007_03.js
|
39
|
+
│ │ ├── 2007_04.js
|
40
|
+
│ │ ├── 2007_05.js
|
41
|
+
│ │ ├─ ... // you get the idea, I've been on Twitter a while
|
42
|
+
│ │ ├── 2012_10.js
|
43
|
+
│ │ ├── 2012_11.js
|
44
|
+
│ │ └── 2012_12.js
|
45
|
+
│ └── user_details.js
|
46
|
+
├── img
|
47
|
+
│ └─ ... // provided by Twitter
|
48
|
+
├── index.html
|
49
|
+
├── js
|
50
|
+
│ └─ ... // provided by Twitter
|
51
|
+
└── lib
|
52
|
+
└─ ... // provided by Twitter
|
53
|
+
```
|
54
|
+
|
55
|
+
This gem only modifies what's in the data directory for a given archive,
|
56
|
+
the rest of the files are provided by Twitter. To check if you can download
|
57
|
+
a copy of your Twitter archive, go to your [Account Settings](https://twitter.com/settings/account)
|
58
|
+
and scroll all the way to the bottom. If the feature is enabled for you, you should
|
59
|
+
see a section labeled "Your Twitter Archive".
|
60
|
+
|
11
61
|
## Installation
|
12
62
|
|
13
63
|
Add this line to your application's Gemfile:
|
data/bin/grailbird_updater
CHANGED
data/grailbird_updater.gemspec
CHANGED
data/lib/grailbird_updater.rb
CHANGED
@@ -5,8 +5,8 @@ class GrailbirdUpdater
|
|
5
5
|
KEEP_FIELDS = {'user' => ['name', 'screen_name', 'protected', 'id_str', 'profile_image_url_https', 'id', 'verified']}
|
6
6
|
|
7
7
|
def initialize(dir, count, verbose, prune)
|
8
|
+
@base_dir = dir
|
8
9
|
data_path = dir + "/data"
|
9
|
-
|
10
10
|
@js_path = data_path + "/js"
|
11
11
|
@csv_path = data_path + "/csv"
|
12
12
|
|
@@ -37,10 +37,7 @@ class GrailbirdUpdater
|
|
37
37
|
|
38
38
|
vputs "Last tweet in archive is\n\t" + display_tweet(last_tweet)
|
39
39
|
|
40
|
-
|
41
|
-
twitter_url = "http://api.twitter.com/1/statuses/user_timeline.json?count=#{@count}&user_id=#{user_id}&since_id=#{last_tweet_id}&include_rts=true&include_entities=true"
|
42
|
-
vputs "Making request to #{twitter_url}"
|
43
|
-
tweets = JSON.parse(open(twitter_url).read)
|
40
|
+
tweets = JSON.parse(get_twitter_user_timeline_response(screen_name, user_id, last_tweet_id, @count))
|
44
41
|
|
45
42
|
vputs "There have been #{tweets.length} tweets since the archive" + (archive_details.has_key?('updated_at') ? " was last updated on #{archive_details['updated_at']}" : " was created")
|
46
43
|
|
@@ -87,9 +84,105 @@ class GrailbirdUpdater
|
|
87
84
|
def read_twitter_js_file(file_path)
|
88
85
|
file_contents = open(file_path).read.force_encoding("UTF-8").split("\n").join(" ")
|
89
86
|
json_file_contents = file_contents.gsub(/^((var)?\s*(.+?)\s+=\s+)/m, '')
|
90
|
-
|
87
|
+
return JSON.parse(json_file_contents)
|
88
|
+
end
|
89
|
+
|
90
|
+
def get_twitter_user_timeline_response(screen_name, user_id, last_tweet_id, count)
|
91
|
+
twitter_url = "http://api.twitter.com/1/statuses/user_timeline.json"
|
92
|
+
twitter_uri = URI(twitter_url)
|
93
|
+
params = {
|
94
|
+
:count => count,
|
95
|
+
:user_id => user_id,
|
96
|
+
:since_id => last_tweet_id,
|
97
|
+
:include_rts => true,
|
98
|
+
:include_entities => true}
|
99
|
+
twitter_uri.query = URI.encode_www_form(params)
|
100
|
+
|
101
|
+
vputs "Making request to #{twitter_uri}"
|
102
|
+
response = Net::HTTP.get_response(twitter_uri)
|
103
|
+
|
104
|
+
if response.is_a?(Net::HTTPUnauthorized)
|
105
|
+
access_token = do_oauth_dance(screen_name)
|
106
|
+
response = access_token.request(:get, twitter_uri.to_s)
|
107
|
+
end
|
108
|
+
|
109
|
+
return response.body
|
110
|
+
end
|
111
|
+
|
112
|
+
def do_oauth_dance(screen_name)
|
113
|
+
puts "It seems " + "@#{screen_name}".blue + " has a protected account."
|
114
|
+
key_file_path = "#{@base_dir}/#{screen_name}_keys.yaml"
|
115
|
+
if File.exists?(key_file_path)
|
116
|
+
keys = YAML.load_file(key_file_path)
|
117
|
+
consumer_key = keys['consumer_key']
|
118
|
+
consumer_secret = keys['consumer_secret']
|
119
|
+
token = keys['token']
|
120
|
+
token_secret = keys['secret']
|
121
|
+
else
|
122
|
+
puts <<-EOS
|
123
|
+
To be able to retrieve your protected tweets, you will need a consumer key/secret
|
124
|
+
|
125
|
+
Please follow these steps to authorize grailbird_updater to download tweets:
|
126
|
+
1. Go to https://dev.twitter.com/apps/new
|
127
|
+
2. Give it a name (I recommend #{screen_name}_grailbird), description and URL
|
128
|
+
3. Create application
|
129
|
+
4. Go to your application page, you should see a "Consumer key" and a "Consumer secret"
|
130
|
+
|
131
|
+
Note: you will only need to create this application once!
|
132
|
+
|
133
|
+
So you don't have to enter these again, we'll save a copy of your keys in a file called #{screen_name}_keys.yaml
|
134
|
+
|
135
|
+
IMPORTANT: Do NOT store the folder of your tweets on a public server. If someone gets access to #{screen_name}_keys.yaml they can access your entire account!
|
136
|
+
Did I mention that was IMPORTANT? BECAUSE IT IS.
|
137
|
+
EOS
|
138
|
+
|
139
|
+
puts "Enter your 'Consumer key'"
|
140
|
+
consumer_key = STDIN.gets.chomp
|
141
|
+
puts "Enter your 'Consumer secret'"
|
142
|
+
consumer_secret = STDIN.gets.chomp
|
143
|
+
consumer = OAuth::Consumer.new(
|
144
|
+
consumer_key,
|
145
|
+
consumer_secret,
|
146
|
+
{ :site => 'http://api.twitter.com/',
|
147
|
+
:request_token_path => '/oauth/request_token',
|
148
|
+
:access_token_path => '/oauth/access_token',
|
149
|
+
:authorize_path => '/oauth/authorize' }
|
150
|
+
)
|
151
|
+
request_token = consumer.get_request_token
|
152
|
+
puts "Go to this URL: #{request_token.authorize_url()}"
|
153
|
+
puts "Authorize the application and you will receive a PIN"
|
154
|
+
puts "Enter the PIN here:"
|
155
|
+
pin = STDIN.gets.chomp
|
156
|
+
access_token = request_token.get_access_token(:oauth_verifier => pin)
|
157
|
+
|
158
|
+
token = access_token.token
|
159
|
+
token_secret = access_token.secret
|
160
|
+
tokens = {
|
161
|
+
'consumer_key' => consumer_key,
|
162
|
+
'consumer_secret' => consumer_secret,
|
163
|
+
'token' => token,
|
164
|
+
'secret' => token_secret}
|
165
|
+
File.open(key_file_path, 'w+') {|f| f.write(tokens.to_yaml) }
|
166
|
+
end
|
167
|
+
|
168
|
+
# Exchange our oauth_token and oauth_token secret for the AccessToken instance.
|
169
|
+
access_token = prepare_access_token(consumer_key, consumer_secret, token, token_secret)
|
91
170
|
end
|
92
171
|
|
172
|
+
def prepare_access_token(consumer_key, consumer_secret, oauth_token, oauth_token_secret)
|
173
|
+
consumer = OAuth::Consumer.new(consumer_key, consumer_secret,
|
174
|
+
{ :site => "http://api.twitter.com",
|
175
|
+
:scheme => :header
|
176
|
+
})
|
177
|
+
# now create the access token object from passed values
|
178
|
+
token_hash = { :oauth_token => oauth_token,
|
179
|
+
:oauth_token_secret => oauth_token_secret
|
180
|
+
}
|
181
|
+
access_token = OAuth::AccessToken.from_hash(consumer, token_hash )
|
182
|
+
return access_token
|
183
|
+
end
|
184
|
+
|
185
|
+
|
93
186
|
def prune_tweet(tweet)
|
94
187
|
KEEP_FIELDS.each do |parent_field, field_names|
|
95
188
|
tweet[parent_field].delete_if { |key, value| !field_names.include?(key) }
|
@@ -124,7 +217,7 @@ class GrailbirdUpdater
|
|
124
217
|
"tweet_count" => count,
|
125
218
|
"month" => month
|
126
219
|
}
|
127
|
-
|
220
|
+
return tweet_index.unshift(new_month).sort_by {|m| [-m['year'], -m['month']]}
|
128
221
|
end
|
129
222
|
|
130
223
|
def write_twitter_js_to_path_with_heading(contents, path, heading)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: grailbird_updater
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,8 +9,24 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-12-
|
12
|
+
date: 2012-12-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: oauth
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
14
30
|
- !ruby/object:Gem::Dependency
|
15
31
|
name: trollop
|
16
32
|
requirement: !ruby/object:Gem::Requirement
|