forki 0.2.8 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +10 -8
  3. data/lib/forki/scrapers/post_scraper.rb +13 -3
  4. data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb +23 -7
  5. data/lib/forki/version.rb +1 -1
  6. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b8c9afa0643b490f011539868cf5c6fa72a816e047e259ecb11dab00016d1ee4
4
- data.tar.gz: f24cbd85f1e68dc1bbb8222b6e1c8730a0c101d2d0ee2eb0fd1748fbd276850b
3
+ metadata.gz: f761b7a50faba5ef8fc0b701031e2a06cbbed257ff1b572d466dd9c31282287d
4
+ data.tar.gz: 7e469c3f8ccdc65bb34f1e57c1c20bcce62cd8853b5136a0c50a9e4a71cf99e7
5
5
  SHA512:
6
- metadata.gz: '0579059d7a8756652705ceea98d34bac0b0217e0a0898ca891040bcb123306b9b5e9e0ce438566d6d51be9a1eb2bf21b1a12c9a369c8c5d67cb158d57067d8d0'
7
- data.tar.gz: d32fc074ee987b6479c5b6076f9b1ae3d69b8fde25beaa895efefed550efb23ce33376be6945f428d0f1fd2524f9914b6d73bd44143b14474cdac77fbb89f10c
6
+ metadata.gz: 9d9dfc35ff002c90d4a1821a9e00cbd1b4672142f2e199a5e2417da53ef5c4b11b5fdd2c0cca93515baa79ab56c699681f8879ae80f918bd962344f3ae5053d8
7
+ data.tar.gz: bf34e313bcf938328d3d94dabe5c7dcc6a855543c75fef2a04afc006c9b642097f3ead0680d6dea0eb0b22626745c89bc271b357170da68b627d4c0fce07a5ac
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- forki (0.2.5)
4
+ forki (0.2.8)
5
5
  apparition
6
6
  capybara
7
7
  oj
@@ -35,14 +35,15 @@ GEM
35
35
  capybara (~> 3.13, < 4)
36
36
  websocket-driver (>= 0.6.5)
37
37
  ast (2.4.2)
38
- bigdecimal (3.1.5)
38
+ base64 (0.2.0)
39
+ bigdecimal (3.1.8)
39
40
  builder (3.2.4)
40
41
  byebug (11.1.3)
41
- capybara (3.39.2)
42
+ capybara (3.40.0)
42
43
  addressable
43
44
  matrix
44
45
  mini_mime (>= 0.1.3)
45
- nokogiri (~> 1.8)
46
+ nokogiri (~> 1.11)
46
47
  rack (>= 1.6.0)
47
48
  rack-test (>= 0.6.3)
48
49
  regexp_parser (>= 1.5, < 3.0)
@@ -54,7 +55,7 @@ GEM
54
55
  erubi (1.12.0)
55
56
  ethon (0.16.0)
56
57
  ffi (>= 1.15.0)
57
- ffi (1.16.3)
58
+ ffi (1.17.0-arm64-darwin)
58
59
  i18n (1.13.0)
59
60
  concurrent-ruby (~> 1.0)
60
61
  json (2.6.3)
@@ -67,12 +68,12 @@ GEM
67
68
  minitest (5.18.0)
68
69
  nokogiri (1.15.1-arm64-darwin)
69
70
  racc (~> 1.4)
70
- oj (3.16.3)
71
+ oj (3.16.4)
71
72
  bigdecimal (>= 3.0)
72
73
  parallel (1.23.0)
73
74
  parser (3.2.2.1)
74
75
  ast (~> 2.4.1)
75
- public_suffix (5.0.4)
76
+ public_suffix (5.1.1)
76
77
  racc (1.6.2)
77
78
  rack (2.2.4)
78
79
  rack-test (2.1.0)
@@ -129,7 +130,8 @@ GEM
129
130
  rubocop-rails (~> 2.0)
130
131
  ruby-progressbar (1.13.0)
131
132
  rubyzip (2.3.2)
132
- selenium-webdriver (4.16.0)
133
+ selenium-webdriver (4.21.1)
134
+ base64 (~> 0.2)
133
135
  rexml (~> 3.2, >= 3.2.5)
134
136
  rubyzip (>= 1.2.2, < 3.0)
135
137
  websocket (~> 1.0)
data/lib/forki/scrapers/post_scraper.rb CHANGED
@@ -204,7 +204,11 @@ module Forki
204
204
  begin
205
205
  feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]
206
206
  rescue NoMethodError
207
- feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
207
+ begin
208
+ feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
209
+ rescue NoMethodError
210
+ feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
211
+ end
208
212
  end
209
213
 
210
214
  if feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"].key?("cannot_see_top_custom_reactions")
@@ -315,8 +319,14 @@ module Forki
315
319
  # TODO: These two branches are *super* similar, probably a lot of overlap
316
320
  attachments = graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"]
317
321
 
318
- if graphql_object["node"]["comet_sections"]["feedback"]["story"].key?("feedback_context")
319
- feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
322
+ if graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("story_ufi_container")
323
+ feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
324
+ elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].dig("feedback_context")
325
+ begin
326
+ feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
327
+ rescue NoMethodError
328
+ debugger
329
+ end
320
330
  elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("comet_feed_ufi_container")
321
331
  feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
322
332
  else
data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb CHANGED
@@ -41,8 +41,15 @@ class VideoSieveWatchTab < VideoSieve
41
41
  def self.sieve(graphql_objects)
42
42
  video_object = self.extractor(graphql_objects)
43
43
 
44
- video_url = video_object["attachments"].first["media"]["browser_native_sd_url"]
45
- video_preview_image_url = video_object["attachments"].first["media"]["preferred_thumbnail"]["image"]["uri"]
44
+ video_url = video_object["attachments"]&.first.dig("media", "browser_native_sd_url")
45
+
46
+ video_url = video_object.dig("short_form_video_context", "playback_video", "browser_native_hd_url") if video_url.nil?
47
+ video_url = video_object.dig("short_form_video_context", "playback_video", "browser_native_sd_url") if video_url.nil?
48
+
49
+ debugger if video_url.nil?
50
+
51
+ video_preview_image_url = video_object["attachments"]&.first.dig("media", "preferred_thumbnail", "image", "uri")
52
+ video_preview_image_url = video_object["short_form_video_context"]["video"]["first_frame_thumbnail"] if video_preview_image_url.nil?
46
53
 
47
54
  if !video_object["feedback_context"].nil?
48
55
  feedback_object = video_object["feedback_context"]["feedback_target_with_context"]
@@ -51,16 +58,25 @@ class VideoSieveWatchTab < VideoSieve
51
58
  feedback_object = feedback_object["feedback"] if feedback_object.has_key?("feedback")
52
59
  end
53
60
 
54
- profile_link = video_object["attachments"].first["media"]["owner"]["url"]
61
+ begin
62
+ profile_link = video_object["attachments"].first["media"]["owner"]["url"]
63
+ rescue StandardError => e
64
+ profile_link = video_object["short_form_video_context"]["video_owner"]["url"]
65
+ end
66
+
55
67
  if profile_link.nil?
56
68
  filtered_json = graphql_objects.find { |go| go.has_key? "attachments" }
57
69
  profile_link = filtered_json["attachments"].first["media"]["creation_story"]["comet_sections"]["title"]["story"]["actors"].first["url"]
58
70
  end
59
71
 
60
- if feedback_object.key?("cannot_see_top_custom_reactions")
61
- reactions = feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
62
- else
63
- reactions = feedback_object["top_reactions"]["edges"]
72
+ begin
73
+ if feedback_object.key?("cannot_see_top_custom_reactions")
74
+ reactions = feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
75
+ else
76
+ reactions = feedback_object["top_reactions"]["edges"]
77
+ end
78
+ rescue StandardError => e
79
+ reactions = feedback_object["unified_reactors"]["count"]
64
80
  end
65
81
 
66
82
  post_details = {
data/lib/forki/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Forki
4
- VERSION = "0.2.8"
4
+ VERSION = "0.2.10"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: forki
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.8
4
+ version: 0.2.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - ''
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-05-30 00:00:00.000000000 Z
11
+ date: 2024-09-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: capybara