wikidata-fetcher 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +27 -0
  3. data/.rubocop.yml +21 -0
  4. data/.rubocop_todo.yml +24 -0
  5. data/.travis.yml +10 -0
  6. data/CHANGES +67 -0
  7. data/Gemfile +6 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.md +68 -0
  10. data/Rakefile +14 -0
  11. data/lib/wikidata.rb +140 -0
  12. data/lib/wikidata/category.rb +62 -0
  13. data/lib/wikidata/fetcher.rb +117 -0
  14. data/lib/wikidata/fetcher/version.rb +5 -0
  15. data/lookup.json +356 -0
  16. data/t/category.rb +40 -0
  17. data/t/data.rb +139 -0
  18. data/t/multi.rb +18 -0
  19. data/t/test_helper.rb +12 -0
  20. data/test/vcr_cassettes/Bierasniewa.yml +589 -0
  21. data/test/vcr_cassettes/Eesmaa.yml +855 -0
  22. data/test/vcr_cassettes/Eriksson.yml +1051 -0
  23. data/test/vcr_cassettes/Lupu.yml +2080 -0
  24. data/test/vcr_cassettes/Parts.yml +2025 -0
  25. data/test/vcr_cassettes/R_ivas.yml +2874 -0
  26. data/test/vcr_cassettes/Simpson.yml +750 -0
  27. data/test/vcr_cassettes/broken.yml +2826 -0
  28. data/test/vcr_cassettes/multi-ee.yml +2474 -0
  29. data/test/vcr_cassettes/noclaims.yml +1568 -0
  30. data/test/vcr_cassettes/nonhuman.yml +577 -0
  31. data/test/vcr_cassettes/riigikogu_13.yml +926 -0
  32. data/test/vcr_cassettes/ukmps.yml +2003 -0
  33. data/wikidata-fetcher.gemspec +40 -0
  34. metadata +341 -0
data/lib/wikidata/fetcher.rb ADDED
@@ -0,0 +1,117 @@
1
+ require 'json'
2
+ require 'open-uri'
3
+ require 'require_all'
4
+ require 'wikisnakker'
5
+
6
+ require_rel '..'
7
+
8
+ class WikiData
9
+ class Fetcher < WikiData
10
+ LOOKUP_FILE = 'https://raw.githubusercontent.com/everypolitician/wikidata-fetcher/master/lookup.json'.freeze
11
+
12
+ def self.find(ids)
13
+ Hash[Wikisnakker::Item.find(ids).map { |wditem| [wditem.id, new(item: wditem)] }]
14
+ end
15
+
16
+ def self.wikidata_properties
17
+ @wikidata_properties ||= JSON.parse(open(LOOKUP_FILE).read, symbolize_names: true)
18
+ end
19
+
20
+ def initialize(h)
21
+ if h[:id]
22
+ @item = Wikisnakker::Item.find(h[:id]) or raise "No such item #{h[:id]}"
23
+ @id = @item.id or raise "No ID for #{h[:id]} = #{@item}"
24
+ warn "Different ID (#{@id}) for #{h[:id]}" if @id != h[:id]
25
+ elsif h[:item]
26
+ # Already have a Wikisnakker::Item, eg from a bulk lookup
27
+ @item = h[:item]
28
+ @id = @item.id or raise "No ID for #{h[:id]} = #{@item}"
29
+ else
30
+ raise 'No id'
31
+ end
32
+ end
33
+
34
+ def data(*lang)
35
+ return unless item
36
+
37
+ data = {
38
+ id: id,
39
+ name: first_label_used(lang | ['en']),
40
+ }.merge(labels).merge(wikipedia_links)
41
+
42
+ # Short-circuit if there are no claims
43
+ return data if item.properties.empty?
44
+
45
+ # Short-circuit if this is not a human
46
+ unless human?
47
+ warn "‼ #{id} is_instance_of #{type.join(' & ')}. Skipping"
48
+ return nil
49
+ end
50
+
51
+ unknown_properties.each do |p|
52
+ puts "⁇ Unknown property: https://www.wikidata.org/wiki/Property:#{p} for #{id}"
53
+ end
54
+
55
+ wanted_properties.each do |p|
56
+ val = property_value(p)
57
+ next warn "Unknown value for #{p} for #{id}" unless val
58
+ data[want[p].to_sym] = val
59
+ end
60
+
61
+ data
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :item, :id
67
+
68
+ def skip
69
+ @skip ||= self.class.wikidata_properties[:skip]
70
+ end
71
+
72
+ def want
73
+ @want ||= self.class.wikidata_properties[:want]
74
+ end
75
+
76
+ def type
77
+ item.P31s.map { |p| p.value.label('en') }
78
+ end
79
+
80
+ def human?
81
+ type.include? 'human'
82
+ end
83
+
84
+ def unknown_properties
85
+ item.properties.reject { |c| skip[c] || want[c] }
86
+ end
87
+
88
+ def wanted_properties
89
+ item.properties & want.keys
90
+ end
91
+
92
+ def labels
93
+ # remove any bracketed element at the end
94
+ Hash[item.labels.map do |k, v|
95
+ ["name__#{k.to_s.tr('-', '_')}".to_sym, v[:value].sub(/ \(.*?\)$/, '')]
96
+ end]
97
+ end
98
+
99
+ def wikipedia_links
100
+ Hash[item.sitelinks.map do |k, v|
101
+ ["wikipedia__#{k.to_s.sub(/wiki$/, '')}".to_sym, v.title]
102
+ end]
103
+ end
104
+
105
+ def property_value(property)
106
+ val = item[property].value rescue nil or return
107
+ return val unless val.respond_to?(:label)
108
+ return unless val.labels
109
+ val.label('en')
110
+ end
111
+
112
+ def first_label_used(language_codes)
113
+ prefered = (item.labels.keys & language_codes.map(&:to_sym)).first or return
114
+ item.labels[prefered][:value]
115
+ end
116
+ end
117
+ end
data/lib/wikidata/fetcher/version.rb ADDED
@@ -0,0 +1,5 @@
1
+ module Wikidata
2
+ module Fetcher
3
+ VERSION = '0.19.1'.freeze
4
+ end
5
+ end
data/lookup.json ADDED
@@ -0,0 +1,356 @@
1
+ {
2
+ "skip": {
3
+ "P7": "Brother",
4
+ "P9": "Sister",
5
+ "P10": "video",
6
+ "P17": "Country",
7
+ "P19": "Place of Birth",
8
+ "P20": "Place of Death",
9
+ "P22": "Father",
10
+ "P25": "Mother",
11
+ "P26": "Spouse",
12
+ "P27": "Country of Citizenship",
13
+ "P31": "Instance of",
14
+ "P39": "Position Held",
15
+ "P40": "Child",
16
+ "P43": "Stepfather",
17
+ "P51": "audio",
18
+ "P53": "noble family",
19
+ "P54": "Member of sports team",
20
+ "P66": "Ancestral home",
21
+ "P69": "Educated at",
22
+ "P91": "Sexual orientation",
23
+ "P94": "coat of arms image",
24
+ "P101": "Field of Work",
25
+ "P103": "Native language",
26
+ "P102": "Party",
27
+ "P108": "Employer",
28
+ "P109": "Signature",
29
+ "P119": "Place of burial",
30
+ "P135": "movement",
31
+ "P136": "genre",
32
+ "P138": "named after",
33
+ "P140": "Religion",
34
+ "P155": "follows",
35
+ "P156": "followed by",
36
+ "P157": "killed by",
37
+ "P166": "Award received",
38
+ "P172": "Ethnic group",
39
+ "P184": "Doctoral advisor",
40
+ "P241": "Military branch",
41
+ "P361": "party of",
42
+ "P373": "Commons category",
43
+ "P410": "Military rank",
44
+ "P412": "voice type",
45
+ "P413": "position on team",
46
+ "P425": "field of this profession",
47
+ "P428": "Botanist author",
48
+ "P443": "Pronunciation audio",
49
+ "P450": "Astronaut mission",
50
+ "P451": "Cohabitant",
51
+ "P463": "Member of",
52
+ "P485": "Archives at",
53
+ "P488": "Chairperson",
54
+ "P495": "Country of origin",
55
+ "P509": "Cause of death",
56
+ "P512": "Academic degree",
57
+ "P535": "Find a Grave",
58
+ "P551": "Residence",
59
+ "P552": "handedness",
60
+ "P555": "Tennis doubles record",
61
+ "P564": "Tennis singles record",
62
+ "P598": "Commander of",
63
+ "P607": "Conflicts",
64
+ "P641": "Sport",
65
+ "P647": "drafted by",
66
+ "P650": "RKDartists",
67
+ "P737": "influenced by",
68
+ "P741": "tennis playing hand",
69
+ "P793": "significant event",
70
+ "P800": "Notable work",
71
+ "P802": "student",
72
+ "P812": "Academic major",
73
+ "P866": "Perlentaucher ID",
74
+ "P898": "IPA",
75
+ "P900": "<deleted>",
76
+ "P910": "Main category",
77
+ "P935": "Commons gallery",
78
+ "P937": "Work location",
79
+ "P941": "Inspired by",
80
+ "P948": "Wikivoyage banner",
81
+ "P990": "voice recording",
82
+ "P1019": "feed URL",
83
+ "P1026": "doctoral thesis",
84
+ "P1038": "Relative",
85
+ "P1050": "Medical condition",
86
+ "P1066": "Student of",
87
+ "P1087": "Elo rating",
88
+ "P1151": "Wikimedia portal",
89
+ "P1185": "Rodovid ID",
90
+ "P1196": "Manner of death",
91
+ "P1220": "Broadway Database ID",
92
+ "P1233": "Speculative fiction DB",
93
+ "P1317": "floruit",
94
+ "P1303": "instrument played",
95
+ "P1321": "Place of Origin (Swiss)",
96
+ "P1343": "Described by source",
97
+ "P1344": "Participant in",
98
+ "P1399": "Convicted of",
99
+ "P1409": "Cycling Archives ID",
100
+ "P1411": "nominated for",
101
+ "P1412": "Languages",
102
+ "P1416": "affiliation",
103
+ "P1429": "pet",
104
+ "P1440": "Identifier FIDE",
105
+ "P1442": "Image of grave",
106
+ "P1447": "SportsReference ID",
107
+ "P1448": "Official name",
108
+ "P1449": "nickname",
109
+ "P1472": "Commons Creator page",
110
+ "P1477": "birth_name",
111
+ "P1532": "Country for sport",
112
+ "P1559": "Name in native language",
113
+ "P1563": "MacTutor id",
114
+ "P1576": "lifestyle",
115
+ "P1618": "sport number",
116
+ "P1665": "Chess Games ID",
117
+ "P1683": "quote",
118
+ "P1728": "AllMusic ID",
119
+ "P1801": "commemorative plaque",
120
+ "P1819": "genealogics ID",
121
+ "P1825": "Baseball-Reference.com ID",
122
+ "P1826": "Baseball-Reference.com ID",
123
+ "P1853": "blood type",
124
+ "P1889": "different from",
125
+ "P1967": "boxrec ID",
126
+ "P1971": "Number of children",
127
+ "P2020": "worldfootball.net",
128
+ "P2021": "Erdős number",
129
+ "P2042": "Artsy ID",
130
+ "P2048": "height",
131
+ "P2067": "mass",
132
+ "P2091": "FISA ID",
133
+ "P2174": "MOMA ID",
134
+ "P2193": "Soccerbase ID",
135
+ "P2276": "UEFA ID",
136
+ "P2350": "Speedskatingbase.eu ID",
137
+ "P2416": "sports discipline",
138
+ "P2423": "FIE fencer ID",
139
+ "P2446": "transfermarkt player ID",
140
+ "P2447": "transfermarkt manager ID",
141
+ "P2448": "Turkish Football Federation player ID",
142
+ "P2449": "Turkish Football Federation manager ID",
143
+ "P2458": "Mackolik ID",
144
+ "P2574": "NFT ID",
145
+ "P2593": "Latvian Olympic athlete ID",
146
+ "P2632": "place of detention",
147
+ "P2640": "SwimRankings ID",
148
+ "P2696": "FIG gymnast ID",
149
+ "P2727": "United World Wrestling ID",
150
+ "P2728": "CageMatch ID",
151
+ "P2767": "JudoInside ID",
152
+ "P2773": "FIS cross-country skier ID",
153
+ "P2779": "IAT weightlifter ID",
154
+ "P2796": "FIG gymnast ID",
155
+ "P2818": "Sherdog ID",
156
+ "P3065": "identifier__rero",
157
+ "P3172": "World Bridge Federation ID",
158
+ "P3373": "sibling"
159
+ },
160
+ "want": {
161
+ "P18": "image",
162
+ "P21": "gender",
163
+ "P97": "noble_title",
164
+ "P106": "occupation",
165
+ "P213": "identifier__ISNI",
166
+ "P214": "identifier__VIAF",
167
+ "P227": "identifier__GND",
168
+ "P244": "identifier__LCAuth",
169
+ "P245": "identifier__ULAN",
170
+ "P268": "identifier__BNF",
171
+ "P269": "identifier__SUDOC",
172
+ "P271": "identifier__CiNii",
173
+ "P345": "identifier__IMDB",
174
+ "P349": "identifier__NDL",
175
+ "P396": "identifier__SBN_it",
176
+ "P409": "identifier__NLA",
177
+ "P434": "identifier__MusicBrainz",
178
+ "P496": "identifier__ORCID",
179
+ "P511": "honorific_prefix",
180
+ "P513": "birth_name",
181
+ "P536": "identifier__ATP",
182
+ "P549": "identifier__MGP",
183
+ "P553": "website",
184
+ "P569": "birth_date",
185
+ "P570": "death_date",
186
+ "P599": "identifier__ITF",
187
+ "P640": "identifier__leonore",
188
+ "P646": "identifier__freebase",
189
+ "P648": "identifier__OLID",
190
+ "P651": "identifier__BPN",
191
+ "P691": "identifier__NKC",
192
+ "P723": "identifier__DBNL",
193
+ "P734": "family_name",
194
+ "P735": "given_name",
195
+ "P742": "pseudonym",
196
+ "P768": "electoral_district",
197
+ "P856": "website",
198
+ "P865": "identifier__BMLO",
199
+ "P902": "identifier__HDS",
200
+ "P906": "identifier__SELIBR",
201
+ "P947": "identifier__RSL",
202
+ "P949": "identifier__NLI",
203
+ "P950": "identifier__BNE",
204
+ "P951": "identifier__NSZL",
205
+ "P968": "email",
206
+ "P998": "identifier__dmoz",
207
+ "P1005": "identifier__PTBNP",
208
+ "P1006": "identifier__NTA",
209
+ "P1015": "identifier__BIBSYS",
210
+ "P1025": "identifier__SUDOC",
211
+ "P1017": "identifier__BAV",
212
+ "P1035": "honorific_suffix",
213
+ "P1045": "identifier__sycomore",
214
+ "P1047": "identifier__catholic_hierarchy",
215
+ "P1048": "identifier__NCL",
216
+ "P1146": "identifier__IIAF",
217
+ "P1157": "identifier__UScongress",
218
+ "P1186": "identifier__EuroparlMEP",
219
+ "P1207": "identifier__NUKAT",
220
+ "P1213": "identifier__NLC",
221
+ "P1214": "identifier__Riksdagen",
222
+ "P1229": "identifier__openpolis",
223
+ "P1258": "identifier__rotten_tomatoes",
224
+ "P1263": "identifier__NNDB",
225
+ "P1266": "identifier__AlloCine",
226
+ "P1273": "identifier__CANTIC",
227
+ "P1280": "identifier__CONOR",
228
+ "P1284": "identifier__Munzinger",
229
+ "P1285": "identifier__Munzinger",
230
+ "P1288": "identifier__Munzinger",
231
+ "P1289": "identifier__KLfG",
232
+ "P1291": "identifier__ADS",
233
+ "P1296": "identifier__GNC",
234
+ "P1307": "identifier__parlamentDOTch",
235
+ "P1309": "identifier__EGAXA",
236
+ "P1315": "identifier__NLAtrove",
237
+ "P1331": "identifier__PACE",
238
+ "P1341": "identifier__italian_cod",
239
+ "P1368": "identifier__LNB",
240
+ "P1375": "identifier__NSK",
241
+ "P1387": "political_alignment",
242
+ "P1415": "identifier__Oxforddnb",
243
+ "P1417": "identifier__Britannica",
244
+ "P1430": "identifier__OpenPlaques",
245
+ "P1438": "identifier__JewishEnc",
246
+ "P1469": "identifier__FIFA",
247
+ "P1556": "identifier__zbMATH",
248
+ "P1581": "weblog",
249
+ "P1607": "identifier__dialnet",
250
+ "P1615": "identifier__CLARA",
251
+ "P1617": "identifier__BBC_things",
252
+ "P1631": "identifier__ChinaVitae",
253
+ "P1648": "identifier__WelshBiography",
254
+ "P1649": "identifier__KMDb",
255
+ "P1650": "identifier__BBF",
256
+ "P1670": "identifier__LAC",
257
+ "P1695": "identifier__NLP",
258
+ "P1710": "identifier__saebi",
259
+ "P1711": "identifier__britishmuseum",
260
+ "P1713": "identifier__bundestag",
261
+ "P1714": "identifier__journalisted",
262
+ "P1741": "identifier__GTAA",
263
+ "P1749": "identifier__parlement",
264
+ "P1808": "identifier__senatDOTfr",
265
+ "P1814": "name__kana",
266
+ "P1816": "identifier__NPG",
267
+ "P1839": "identifier__FEC",
268
+ "P1871": "identifier__CERL",
269
+ "P1883": "identifier__declarator",
270
+ "P1890": "identifier__BNC",
271
+ "P1907": "identifier__AuDB",
272
+ "P1938": "identifier__Gutenberg",
273
+ "P1946": "identifier__N6I",
274
+ "P1969": "identifier__moviemeter",
275
+ "P1950": "second_surname",
276
+ "P1953": "identifier__discogs",
277
+ "P1959": "identifier__senate_nl",
278
+ "P1960": "identifier__google_scholar",
279
+ "P1980": "identifier__polsys",
280
+ "P1982": "identifier__anime_news",
281
+ "P1986": "identifier__treccani",
282
+ "P1996": "identifier__parliamentDOTuk",
283
+ "P2002": "twitter",
284
+ "P2003": "instagram",
285
+ "P2005": "identifier__halensis",
286
+ "P2013": "facebook",
287
+ "P2015": "identifier__hansard",
288
+ "P2019": "identifier__teuchos",
289
+ "P2019": "identifier__allmovie",
290
+ "P2029": "identifier__DoUB",
291
+ "P2035": "linkedin",
292
+ "P2163": "identifier__FAST",
293
+ "P2168": "identifier__SFDb",
294
+ "P2169": "identifier__publicwhip",
295
+ "P2170": "identifier__current_hansard",
296
+ "P2171": "identifier__TWFY",
297
+ "P2172": "identifier__parliamentary_record",
298
+ "P2173": "identifier__bbc_democracy_live",
299
+ "P2180": "identifier__kansallisbiografia",
300
+ "P2181": "identifier__eduskunta",
301
+ "P2182": "identifier__valtioneuvostosta",
302
+ "P2188": "identifier__BiblioNet",
303
+ "P2190": "identifier__CSPAN",
304
+ "P2255": "identifier__debretts_today",
305
+ "P2267": "identifier__politifact",
306
+ "P2273": "identifier__heidelberg",
307
+ "P2277": "identifier__magdeburg",
308
+ "P2278": "identifier__hellenic_parliament",
309
+ "P2280": "identifier__parlaments_at",
310
+ "P2342": "identifier__AGORHA",
311
+ "P2372": "identifier__ODIS",
312
+ "P2383": "identifier__CTHS",
313
+ "P2387": "identifier__elonet",
314
+ "P2390": "identifier__ballotpedia",
315
+ "P2397": "identifier__youtube",
316
+ "P2435": "identifier__PORT",
317
+ "P2487": "identifier__google_plus",
318
+ "P2492": "identifier__MTMT",
319
+ "P2519": "identifier__SCOPE",
320
+ "P2558": "identifier__autores_uy",
321
+ "P2562": "married_name",
322
+ "P2604": "identifier__kinopoisk",
323
+ "P2605": "identifier__CSFD",
324
+ "P2611": "identifier__TED",
325
+ "P2626": "identifier__DNF",
326
+ "P2639": "identifier__filmportal_de",
327
+ "P2671": "identifier__google_knowledge",
328
+ "P2686": "identifier__opensecrets",
329
+ "P2732": "identifier__persee",
330
+ "P2736": "identifier__FJC_bio",
331
+ "P2799": "identifier__BVMC",
332
+ "P2847": "identifier__google_plus",
333
+ "P2924": "identifier__bigenc_ru",
334
+ "P2949": "identifier__wikitree",
335
+ "P2953": "identifier__etis_ee",
336
+ "P2963": "identifier__goodreads",
337
+ "P3029": "identifier__national_archives",
338
+ "P3072": "identifier__storting",
339
+ "P3106": "identifier__guardian_topic",
340
+ "P3124": "identifier__polish_scientist",
341
+ "P3185": "identifier__vkontakte",
342
+ "P3188": "identifier__nobel_prize",
343
+ "P3221": "identifier__nyt_topic",
344
+ "P3222": "identifier__ne_se",
345
+ "P3258": "identifier__livejournal",
346
+ "P3267": "identifier__flickr",
347
+ "P3297": "identifier__flemish_parliament",
348
+ "P3298": "identifier__belgian_senate",
349
+ "P3344": "identifier__votesmart",
350
+ "P3348": "identifier__nlg",
351
+ "P3391": "identifier__verkhovna_rada",
352
+ "P3417": "identifier__quora",
353
+ "P3430": "identifier__snac",
354
+ "P3479": "identifier__omni"
355
+ }
356
+ }