fbcrawl-colly 0.2.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64e7674aa28cfc9c5c6817afef3c3b4ed7106d1620e6da97c3fc7362ac76dc1e
-  data.tar.gz: 440cb83ecb7bcb9c461b4c5f9d9e5b3de0f6300384074adee9c08f3be1a68e8b
+  metadata.gz: 00d52fcc50bd651adaa80cf6102d7cc3997a07605ebb91fd33006b6e535ed6fc
+  data.tar.gz: 24388fa7c98d6dfbe58ba195c330ea3c363f6ef8e586873dd9e485ad29907a6c
 SHA512:
-  metadata.gz: 1fa921c01c8b0381cf85ce4b1544d59912e1997f47fab12f8e29636de10305f410fff12a2137f7370a9749cfd2f28cd2a3ef5953de703056bf63a49770af3daf
-  data.tar.gz: 5696f7fe083b55ed24a5cd128dc71a8c7ebe0a7c7a0dda8811fe2e5796a023635c44d3712584b2c98937bfc1a170e7639b6af250c287bca7cbee36211f10669a
+  metadata.gz: 15622279b09030f929218482add87878b58fdcb1e89e1ab8a1124531435707dbe26367fee75204a55fc88f8486c47cbb6ddfdb86035aa24694b1b7d6639aaecc
+  data.tar.gz: df50b30ad0234c824505cca54fdf33d0be54c4725c684f0317b1eba81f89d22d7b434bec4e9373b3d43dd830dc17693ac2fb785e75cefc71c6cbeefc44da2616

data/.gitignore

@@ -12,8 +12,6 @@
 /parse.log
 last.html
 *.db
-/fbcrawl/fbcrawl.pb.go
-/lib/fbcrawl_pb.rb
 
 mkmf.log
 .rakeTasks

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.14-alpine
+RUN apk add --no-cache git build-base tzdata
+
+RUN mkdir -p /app
+WORKDIR /app
+ADD ./go.mod /app
+ADD ./go.sum /app
+ADD ./ /app
+RUN go get
+
+
+ENV PORT 3000
+RUN go build -o server qnetwork.net/fbcrawl
+ENTRYPOINT ["./server"]

data/Gemfile.lock

@@ -1,19 +1,21 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.5)
-      ffi
+    fbcrawl-colly (1.0.0)
       google-protobuf
+      grpc
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ffi (1.13.1)
-    google-protobuf (3.13.0)
+    google-protobuf (3.13.0-universal-darwin)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    grpc (1.30.2-universal-darwin)
+      google-protobuf (~> 3.12)
+      googleapis-common-protos-types (~> 1.0)
     minitest (5.14.1)
     rake (12.3.3)
-    rake-compiler (1.1.1)
-      rake
 
 PLATFORMS
   ruby
@@ -22,7 +24,6 @@ DEPENDENCIES
   fbcrawl-colly!
   minitest (~> 5.0)
   rake (~> 12.0)
-  rake-compiler
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -2,7 +2,7 @@
 
 This project is to crawl mbasic.facebook.com using GO Colly. 
 
-Ruby gem will available to use
+Ruby gem is only client for GRPC colly service
 
 ## Installation
 

data/fbcrawl-colly.gemspec

@@ -25,12 +25,8 @@ Gem::Specification.new do |spec|
   end
   # spec.bindir        = "exe"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.extensions = [
-      'ext/fbcrawl_colly/extconf.rb'
-  ]
-  spec.require_paths = ["lib"]
+  spec.require_paths = %w[lib lib/pb]
 
-  spec.add_runtime_dependency 'ffi'
   spec.add_runtime_dependency 'google-protobuf'
-  spec.add_development_dependency 'rake-compiler'
+  spec.add_runtime_dependency 'grpc'
 end

data/fbcrawl.proto

@@ -1,7 +1,73 @@
 syntax = "proto3";
 
 package fbcrawl_colly;
-option go_package = "./fbcrawl;fbcrawl";
+option go_package = "./fbcrawl/pb;pb";
+
+service Grpc {
+  // Sends a greeting
+  rpc Init (Empty) returns (Pointer) {}
+  rpc FreeColly (Pointer) returns (Empty) {}
+  rpc Login  (LoginRequest) returns (LoginResponse) {}
+  rpc LoginWithCookies  (LoginWithCookiesRequest) returns (Empty) {}
+  rpc FetchGroupInfo  (FetchGroupInfoRequest) returns (FacebookGroup) {}
+  rpc FetchGroupFeed  (FetchGroupFeedRequest) returns (FacebookPostList) {}
+  rpc FetchPost  (FetchPostRequest) returns (FacebookPost) {}
+  rpc FetchContentImages  (FetchContentImagesRequest) returns (FacebookImageList) {}
+  rpc FetchImageUrl  (FetchImageUrlRequest) returns (FacebookImage) {}
+}
+
+message Empty {
+
+}
+
+message Pointer {
+  int64 address = 1;
+}
+
+message LoginRequest {
+  Pointer pointer = 1;
+  string email = 2;
+  string password = 3;
+  string totp_secret = 4;
+}
+
+message LoginResponse {
+  string cookies = 1;
+}
+
+message LoginWithCookiesRequest {
+  Pointer pointer = 1;
+  string cookies = 2;
+}
+
+message FetchGroupInfoRequest {
+  Pointer pointer = 1;
+  string group_username = 2;
+}
+
+message FetchGroupFeedRequest {
+  Pointer pointer = 1;
+  int64 group_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchPostRequest {
+  Pointer pointer = 1;
+  int64 group_id = 2;
+  int64 post_id = 3;
+  string comment_next_cursor = 4;
+}
+
+message FetchContentImagesRequest {
+  Pointer pointer = 1;
+  int64 post_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchImageUrlRequest {
+  Pointer pointer = 1;
+  int64 image_id = 2;
+}
 
 // The request message containing the user's name.
 message FacebookGroup {
@@ -20,15 +86,20 @@ message FacebookPost {
   FacebookGroup group = 2;
   FacebookUser user = 3;
   string content = 4;
+  CommentList comments = 5;
   string content_link = 6;
-  FacebookImage content_image = 8;
   repeated FacebookImage content_images = 7;
-  repeated FacebookComment comments = 5;
+  FacebookImage content_image = 8;
   int64 created_at = 9;
   int64 reaction_count = 10;
   int64 comment_count = 11;
 }
 
+message CommentList {
+  repeated FacebookComment comments = 5;
+  string next_cursor = 12;
+}
+
 message FacebookImage {
   int64 id = 1;
   string url = 2;
@@ -44,8 +115,10 @@ message FacebookComment {
 
 message FacebookPostList {
   repeated FacebookPost posts = 1;
+  string next_cursor = 2;
 }
 
 message FacebookImageList {
   repeated FacebookImage images = 1;
+  string next_cursor = 2;
 }

data/{fbcolly → fbcrawl}/fbcolly.go

@@ -15,8 +15,9 @@ import (
 	"github.com/olebedev/when/rules/common"
 	"github.com/olebedev/when/rules/en"
 	"github.com/thoas/go-funk"
+	"github.com/xlzd/gotp"
 	"net/url"
-	"qnetwork.net/fbcrawl/fbcrawl"
+	"qnetwork.net/fbcrawl/fbcrawl/pb"
 	"regexp"
 	"strconv"
 	"strings"
@@ -108,7 +109,7 @@ func New() *Fbcolly {
 	return &f
 }
 
-func (f *Fbcolly) Login(email string, password string, otp string) (string, error) {
+func (f *Fbcolly) Login(email string, password string, totpSecret string) (string, error) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
 
@@ -157,9 +158,12 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 			//logger.Info("Please input OTP")
 			//reader := bufio.NewReader(os.Stdin)
 			//code, _ := reader.ReadString('\n')
-			code := otp[0:6]
-			reqMap["approvals_code"] = code
-			shouldSubmit = true
+			if len(totpSecret) > 0 {
+				code := gotp.NewDefaultTOTP(totpSecret).Now()
+				reqMap["approvals_code"] = code
+				shouldSubmit = true
+			}
+
 		} else {
 			logger.Info("OnHTML Only Continue checkpoint")
 
@@ -193,22 +197,17 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 
 }
 
-func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostList) {
+func (f *Fbcolly) FetchGroupFeed(groupId int64, nextCursor string) (error, *pb.FacebookPostList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookPost
+	result := pb.FacebookPostList{Posts: []*pb.FacebookPost{}}
 
 	collector.OnHTML("#m_group_stories_container > :last-child a", func(element *colly.HTMLElement) {
-		currentPage++
-		if currentPage < 3 {
-			logger.Info("Will fetch page", currentPage)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-		}
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
 	collector.OnHTML("div[role=\"article\"]", func(element *colly.HTMLElement) {
 		dataElement := element
-		post := &fbcrawl.FacebookPost{}
+		post := &pb.FacebookPost{}
 		var fbDataFt FbDataFt
 		jsonData := dataElement.Attr("data-ft")
 
@@ -220,8 +219,8 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 		}
 		logger.Info("Post ", fbDataFt)
 		post.Id = fbDataFt.TopLevelPostId
-		post.Group = &fbcrawl.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
-		post.User = &fbcrawl.FacebookUser{
+		post.Group = &pb.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		post.User = &pb.FacebookUser{
 			Id:   fbDataFt.ContentOwnerIdNew,
 			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
 		}
@@ -241,30 +240,34 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
 		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
 		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
-		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 			i, _ := strconv.ParseInt(id, 10, 64)
-			return &fbcrawl.FacebookImage{
+			return &pb.FacebookImage{
 				Id: i,
 			}
-		})).([]*fbcrawl.FacebookImage)
+		})).([]*pb.FacebookImage)
 
 		if fbDataFt.PhotoId > 0 {
-			post.ContentImage = &fbcrawl.FacebookImage{Id: fbDataFt.PhotoId}
+			post.ContentImage = &pb.FacebookImage{Id: fbDataFt.PhotoId}
 		}
-		result = append(result, post)
+		result.Posts = append(result.Posts, post)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookPostList{Posts: result}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *fbcrawl.FacebookGroup) {
+func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *pb.FacebookGroup) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result := &fbcrawl.FacebookGroup{}
+	result := &pb.FacebookGroup{}
 
 	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
 		result.Name = element.Text
@@ -281,36 +284,37 @@ func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *fbcrawl.Face
 	return err, result
 }
 
-func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
+func (f *Fbcolly) FetchContentImages(postId int64, nextCursor string) (error, *pb.FacebookImageList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookImage
+	result := pb.FacebookImageList{Images: []*pb.FacebookImage{}}
 
 	collector.OnHTML("a[href*=\"/media/set/\"]", func(element *colly.HTMLElement) {
-		currentPage++
-		logger.Info("Will fetch page", currentPage)
-		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
 
 	collector.OnHTML("a[href*=\"/photo.php\"]", func(element *colly.HTMLElement) {
-		result = append(result, &fbcrawl.FacebookImage{
+		result.Images = append(result.Images, &pb.FacebookImage{
 			Id: getImageIdFromHref(element.Attr("href")),
 		})
 		//f.detailCollector.Visit(url)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookImageList{Images: result}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
+func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *pb.FacebookImage) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result := fbcrawl.FacebookImage{Id: imageId}
+	result := pb.FacebookImage{Id: imageId}
 
 	collector.OnHTML("a", func(element *colly.HTMLElement) {
 		result.Url = element.Attr("href")
@@ -323,11 +327,10 @@ func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
 	return err, &result
 }
 
-func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.FacebookPost) {
+func (f *Fbcolly) FetchPost(groupId int64, postId int64, commentNextCursor string) (error, *pb.FacebookPost) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	post := &fbcrawl.FacebookPost{Comments: []*fbcrawl.FacebookComment{}}
-	commentPaging := 0
+	post := &pb.FacebookPost{Comments: &pb.CommentList{Comments: []*pb.FacebookComment{}}}
 	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
 		dataElement := element.DOM.Find("div[data-ft]")
 		if dataElement.Length() > 0 {
@@ -342,8 +345,8 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 				logger.Info("Post ", result)
 				post.Id = result.TopLevelPostId
-				post.Group = &fbcrawl.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
-				post.User = &fbcrawl.FacebookUser{
+				post.Group = &pb.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
+				post.User = &pb.FacebookUser{
 					Id:   result.ContentOwnerIdNew,
 					Name: dataElement.Find("h3 strong:first-child a").Text(),
 				}
@@ -362,15 +365,15 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 
 				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
 				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
-				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 					i, _ := strconv.ParseInt(id, 10, 64)
-					return &fbcrawl.FacebookImage{
+					return &pb.FacebookImage{
 						Id: i,
 					}
-				})).([]*fbcrawl.FacebookImage)
+				})).([]*pb.FacebookImage)
 
 				if result.PhotoId > 0 {
-					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
+					post.ContentImage = &pb.FacebookImage{Id: result.PhotoId}
 				}
 			}
 
@@ -380,10 +383,10 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				commentId, _ := strconv.ParseInt(selection.AttrOr("id", ""), 10, 64)
 				logger.Info("comment", commentId)
 				createdAtWhenResult, _ := f.w.Parse(selection.Find("abbr").Text(), time.Now())
-				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
+				post.Comments.Comments = append(post.Comments.Comments, &pb.FacebookComment{
 					Id:   commentId,
-					Post: &fbcrawl.FacebookPost{Id: post.Id},
-					User: &fbcrawl.FacebookUser{
+					Post: &pb.FacebookPost{Id: post.Id},
+					User: &pb.FacebookUser{
 						Id:   getUserIdFromCommentHref(selection.Find("a[href*=\"#comment_form_\"]").AttrOr("href", "")),
 						Name: selection.Find("h3 > a").Text(),
 					},
@@ -396,14 +399,14 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 	})
 
 	collector.OnHTML("div[id*=\"see_prev_\"] > a", func(element *colly.HTMLElement) {
-		if commentPaging < 3 {
-			logger.Info("Comment paging", commentPaging)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-			commentPaging = commentPaging + 1
-		}
+		post.Comments.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
+	if len(commentNextCursor) > 0 {
+		err = collector.Visit(commentNextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
 	return err, post
 }
 
@@ -417,8 +420,12 @@ func (f *Fbcolly) LoginWithCookies(cookies string) error {
 //}
 
 func getUserIdFromCommentHref(href string) int64 {
-	id, _ := strconv.ParseInt(regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)[1], 10, 64)
-	return id
+	match := regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)
+	if len(match) > 0 {
+		id, _ := strconv.ParseInt(match[1], 10, 64)
+		return id
+	}
+	return 0
 }
 
 func getUrlFromRedirectHref(href string) string {