fbcrawl-colly 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 383de209c502265a8533af201e3b9538edffc115af1575bf9662508abb142e00
-  data.tar.gz: 631c53a8c82e063df262ccac4f6c74f4591296b700ffa627110a9bf0dce40358
+  metadata.gz: 6e43a203c1c38131f0d9eb32b67a1859bd1530aac9249751ff0329a5901388f6
+  data.tar.gz: f3520762f00d2b1a2d475a9b97d2141fe8cd98faf718ccf3d8a797e61489f976
 SHA512:
-  metadata.gz: b8c1505f7183c0a30a6d6679319a947be589828771e3bc231915cb841aad5e79b7221091862f6df537a701eaa2bf21bb996ba689e301d00a79964fd47e45f258
-  data.tar.gz: 2ce2e0870f9455cdaeab48c840e1c9f5fb35afc340e0c56830afca90eb7bcebcf120430b7c3d043d58274d545ab0c7bb99d4062b36885b1e832bb540b7bf5080
+  metadata.gz: 2560aa6a3239671a14f82a95034bbad03684dd44b0fc32cb2092b404678b2ec0e124a33db64a92f5c22f4cd3d9d94a42ccc40a6ab2e1e362088d0f02f05d22f3
+  data.tar.gz: 305f3d463d75bc3edcfcd1b5a985d9a9882e990e850786132eb1b6d4e93f4e3e31a2c0e9aeb23141cc0633cd1d5ad8e1d88ec4c0e3e4d2d201a6298a7c8699c6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.2)
+    fbcrawl-colly (0.2.3)
       ffi
       google-protobuf
 

data/fbcolly/fbcolly.go

@@ -201,16 +201,52 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
 		}
 	})
+	collector.OnHTML("div[role=\"article\"]", func(element *colly.HTMLElement) {
+		dataElement := element
+		post := &fbcrawl.FacebookPost{}
+		var fbDataFt FbDataFt
+		jsonData := dataElement.Attr("data-ft")
+
+		logger.Info(jsonData)
+		err = json.Unmarshal([]byte(jsonData), &fbDataFt)
+		if err != nil {
+			logger.Error(err)
+			return
+		}
+		logger.Info("Post ", fbDataFt)
+		post.Id = fbDataFt.TopLevelPostId
+		post.Group = &fbcrawl.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		post.User = &fbcrawl.FacebookUser{
+			Id:   fbDataFt.ContentOwnerIdNew,
+			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
+		}
+		post.CreatedAt = fbDataFt.PageInsights[strconv.FormatInt(fbDataFt.PageId, 10)].PublishTime
+		//Content
 
-	collector.OnXML("//a[text()=\"Full Story\"]", func(element *colly.XMLElement) {
-		logger.Info("Post found at", element.Attr("href"))
-		u, _ := url.Parse("http://mbasic.facebook.com" + element.Attr("href"))
-		postId, _ := strconv.ParseInt(u.Query().Get("id"), 10, 64)
-		result = append(result, &fbcrawl.FacebookPost{
-			Id:    postId,
-			Group: &fbcrawl.FacebookGroup{Id: groupId},
-		})
-		//f.detailCollector.Visit(url)
+		//NO BACKGROUND TEXT ONLY
+		post.Content = strings.Join(dataElement.DOM.Find("p").Map(func(i int, selection *goquery.Selection) string {
+			return selection.Text()
+		}), "\n")
+
+		if len(post.Content) == 0 {
+			// TEXT WITH BACKGROUND
+			post.Content = dataElement.DOM.Find("div[style*=\"background-image:url\"]").Text()
+		}
+
+		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
+		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
+		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+			i, _ := strconv.ParseInt(id, 10, 64)
+			return &fbcrawl.FacebookImage{
+				Id: i,
+			}
+		})).([]*fbcrawl.FacebookImage)
+
+		if fbDataFt.PhotoId > 0 {
+			post.ContentImage = &fbcrawl.FacebookImage{Id: fbDataFt.PhotoId}
+		}
+		result = append(result, post)
 	})
 
 	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
@@ -300,7 +336,7 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 
 				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
-				post.ReactionCount = getReactionFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
+				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
 				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
 					i, _ := strconv.ParseInt(id, 10, 64)
 					return &fbcrawl.FacebookImage{
@@ -311,10 +347,6 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				if result.PhotoId > 0 {
 					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
 				}
-
-				logger.Info("content", strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
-					return selection.Text()
-				}), "\n"))
 			}
 
 			//Comment
@@ -375,11 +407,18 @@ func getImageIdFromHref(href string) int64 {
 	return i
 }
 
-func getReactionFromText(text string) int64 {
+func getNumberFromText(text string) int64 {
 	logger.Error("reaction", text)
 	if len(text) > 0 {
-		id, _ := strconv.ParseInt(regexp.MustCompile("(\\d+)").FindStringSubmatch(text)[1], 10, 64)
-		return id
+		match := regexp.MustCompile("(\\d*)\\s?([km]?)").FindStringSubmatch(text)
+		count, _ := strconv.ParseInt(match[1], 10, 64)
+		switch match[2] {
+		case "k":
+			count *= 1000
+		case "m":
+			count *= 1000000
+		}
+		return count
 	}
 	return 0
 }

data/fbcrawl.proto

@@ -25,6 +25,7 @@ message FacebookPost {
   repeated FacebookComment comments = 5;
   int64 created_at = 9;
   int64 reaction_count = 10;
+  int64 comment_count = 11;
 }
 
 message FacebookImage {

data/lib/fbcrawl_colly/version.rb

@@ -1,3 +1,3 @@
 module FbcrawlColly
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fbcrawl-colly
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Duy Le
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi