fbcrawl-colly 0.2.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e43a203c1c38131f0d9eb32b67a1859bd1530aac9249751ff0329a5901388f6
-  data.tar.gz: f3520762f00d2b1a2d475a9b97d2141fe8cd98faf718ccf3d8a797e61489f976
+  metadata.gz: 138016b11ed36541b96961dbfad0627b5d8bccd26a7638f95e5a8806d0fadf31
+  data.tar.gz: 376c58bc102d6ece54631f0151889d7546af436e9f40f64d994ac44b7110e29e
 SHA512:
-  metadata.gz: 2560aa6a3239671a14f82a95034bbad03684dd44b0fc32cb2092b404678b2ec0e124a33db64a92f5c22f4cd3d9d94a42ccc40a6ab2e1e362088d0f02f05d22f3
-  data.tar.gz: 305f3d463d75bc3edcfcd1b5a985d9a9882e990e850786132eb1b6d4e93f4e3e31a2c0e9aeb23141cc0633cd1d5ad8e1d88ec4c0e3e4d2d201a6298a7c8699c6
+  metadata.gz: d8580cf0cf980989aab899d4f1185a33dff13743cfedea5487e59f8a990cd23d8c9c4751e18c2192b396c27c4ef5d5e95919404522fc4399b9754c1c0753f63a
+  data.tar.gz: 84b182c8dcfd4a3b5aa7297c6cd3bca485a4f372d65579f03220c32bcf2fcda1d95be02cff4caf5c0bb88a997daa53684b6f25fc8c088be4e7e31406c4f1f4ba

data/.gitignore

@@ -12,8 +12,6 @@
 /parse.log
 last.html
 *.db
-/fbcrawl/fbcrawl.pb.go
-/lib/fbcrawl_pb.rb
 
 mkmf.log
 .rakeTasks

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.14-alpine
+RUN apk add --no-cache git build-base tzdata
+
+RUN mkdir -p /app
+WORKDIR /app
+ADD ./go.mod /app
+ADD ./go.sum /app
+ADD ./ /app
+RUN go get
+
+
+ENV PORT 3000
+RUN go build -o server qnetwork.net/fbcrawl
+ENTRYPOINT ["./server"]

data/Gemfile.lock

@@ -1,19 +1,21 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.3)
-      ffi
+    fbcrawl-colly (1.1.0)
       google-protobuf
+      grpc
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ffi (1.13.1)
-    google-protobuf (3.12.4)
+    google-protobuf (3.13.0)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    grpc (1.31.1)
+      google-protobuf (~> 3.12)
+      googleapis-common-protos-types (~> 1.0)
     minitest (5.14.1)
     rake (12.3.3)
-    rake-compiler (1.1.1)
-      rake
 
 PLATFORMS
   ruby
@@ -22,7 +24,6 @@ DEPENDENCIES
   fbcrawl-colly!
   minitest (~> 5.0)
   rake (~> 12.0)
-  rake-compiler
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -2,7 +2,7 @@
 
 This project is to crawl mbasic.facebook.com using GO Colly. 
 
-Ruby gem will available to use
+Ruby gem is only client for GRPC colly service
 
 ## Installation
 

data/fbcrawl-colly.gemspec

@@ -25,12 +25,8 @@ Gem::Specification.new do |spec|
   end
   # spec.bindir        = "exe"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.extensions = [
-      'ext/fbcrawl_colly/extconf.rb'
-  ]
-  spec.require_paths = ["lib"]
+  spec.require_paths = %w[lib lib/pb]
 
-  spec.add_runtime_dependency 'ffi'
   spec.add_runtime_dependency 'google-protobuf'
-  spec.add_development_dependency 'rake-compiler'
+  spec.add_runtime_dependency 'grpc'
 end

data/fbcrawl.proto

@@ -1,17 +1,87 @@
 syntax = "proto3";
 
 package fbcrawl_colly;
-option go_package = "./fbcrawl;fbcrawl";
+option go_package = "./fbcrawl/pb;pb";
+
+service Grpc {
+  // Sends a greeting
+  rpc Login  (LoginRequest) returns (LoginResponse) {}
+  rpc FetchGroupInfo  (FetchGroupInfoRequest) returns (FacebookGroup) {}
+  rpc FetchUserInfo  (FetchUserInfoRequest) returns (FacebookUser) {}
+  rpc FetchGroupFeed  (FetchGroupFeedRequest) returns (FacebookPostList) {}
+  rpc FetchPost  (FetchPostRequest) returns (FacebookPost) {}
+  rpc FetchContentImages  (FetchContentImagesRequest) returns (FacebookImageList) {}
+  rpc FetchImageUrl  (FetchImageUrlRequest) returns (FacebookImage) {}
+}
+
+message Empty {
+
+}
+
+message Context {
+  string cookies = 1;
+}
+
+message LoginRequest {
+  string email = 2;
+  string password = 3;
+  string totp_secret = 4;
+}
+
+message LoginResponse {
+  string cookies = 1;
+}
+
+message LoginWithCookiesRequest {
+  string cookies = 1;
+}
+
+message FetchGroupInfoRequest {
+  Context context = 1;
+  string group_username = 2;
+}
+
+message FetchUserInfoRequest {
+  Context context = 1;
+  string username = 2;
+}
+
+message FetchGroupFeedRequest {
+  Context context = 1;
+  int64 group_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchPostRequest {
+  Context context = 1;
+  int64 group_id = 2;
+  int64 post_id = 3;
+  string comment_next_cursor = 4;
+}
+
+message FetchContentImagesRequest {
+  Context context = 1;
+  int64 post_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchImageUrlRequest {
+  Context context = 1;
+  int64 image_id = 2;
+}
 
 // The request message containing the user's name.
 message FacebookGroup {
   int64 id = 1;
   string name = 2;
+  int64 member_count = 3;
 }
 
 message FacebookUser {
   int64 id = 1;
   string name = 2;
+  string username = 3;
+  int64 friend_count = 4;
 }
 
 message FacebookPost {
@@ -19,15 +89,20 @@ message FacebookPost {
   FacebookGroup group = 2;
   FacebookUser user = 3;
   string content = 4;
+  CommentList comments = 5;
   string content_link = 6;
-  FacebookImage content_image = 8;
   repeated FacebookImage content_images = 7;
-  repeated FacebookComment comments = 5;
+  FacebookImage content_image = 8;
   int64 created_at = 9;
   int64 reaction_count = 10;
   int64 comment_count = 11;
 }
 
+message CommentList {
+  repeated FacebookComment comments = 5;
+  string next_cursor = 12;
+}
+
 message FacebookImage {
   int64 id = 1;
   string url = 2;
@@ -43,8 +118,10 @@ message FacebookComment {
 
 message FacebookPostList {
   repeated FacebookPost posts = 1;
+  string next_cursor = 2;
 }
 
 message FacebookImageList {
   repeated FacebookImage images = 1;
+  string next_cursor = 2;
 }

data/{fbcolly → fbcrawl}/fbcolly.go

@@ -6,16 +6,17 @@ import (
 	"errors"
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
-	"github.com/gocolly/colly"
-	"github.com/gocolly/colly/extensions"
-	"github.com/gocolly/colly/storage"
+	"github.com/gocolly/colly/v2"
+	"github.com/gocolly/colly/v2/extensions"
+	"github.com/gocolly/colly/v2/storage"
 	"github.com/google/logger"
 	"github.com/olebedev/when"
 	"github.com/olebedev/when/rules/common"
 	"github.com/olebedev/when/rules/en"
 	"github.com/thoas/go-funk"
+	"github.com/xlzd/gotp"
 	"net/url"
-	"qnetwork.net/fbcrawl/fbcrawl"
+	"qnetwork.net/fbcrawl/fbcrawl/pb"
 	"regexp"
 	"strconv"
 	"strings"
@@ -33,7 +34,7 @@ type FbDataInsight struct {
 	FbDataPostContext `json:"post_context"`
 }
 type FbDataFt struct {
-	ContentOwnerIdNew    int64                    `json:"content_owner_id_new"`
+	ContentOwnerIdNew    json.Number              `json:"content_owner_id_new"`
 	PhotoAttachmentsList []string                 `json:"photo_attachments_list"`
 	PhotoId              int64                    `json:"photo_id,string"`
 	PageId               int64                    `json:"page_id,string"`
@@ -42,7 +43,7 @@ type FbDataFt struct {
 }
 
 func sharedOnRequest(request *colly.Request) {
-	logger.Info("OnRequest")
+	logger.Info("OnRequest ", request.URL)
 	//request.Headers.Set("Host", "facebook.com")
 	request.Headers.Set("Accept-Language", "en-US,en;q=0.9")
 	request.Headers.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
@@ -106,26 +107,29 @@ func New() *Fbcolly {
 	return &f
 }
 
-func (f *Fbcolly) Login(email string, password string, otp string) (string, error) {
+func (f *Fbcolly) Login(email string, password string, totpSecret string) (string, error) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
 
 	logger.Info("Login using email", email)
 	loggedIn := false
-
+	firstLogin := true
 	collector.OnHTML("#login_form", func(element *colly.HTMLElement) {
-		logger.Info("OnHTML login_form")
-		loginURL, err, reqMap := getForm(element, err)
-		if err != nil {
-			logger.Error(err)
-			return
-		}
-		reqMap["email"] = email
-		reqMap["pass"] = password
-		logger.Info("req map:", reqMap)
-		err = collector.Post(loginURL, reqMap)
-		if err != nil {
-			logger.Error("post err:", err)
+		if firstLogin {
+			firstLogin = false
+			logger.Info("OnHTML login_form")
+			loginURL, err, reqMap := getForm(element, err)
+			if err != nil {
+				logger.Error(err)
+				return
+			}
+			reqMap["email"] = email
+			reqMap["pass"] = password
+			logger.Info("req map:", reqMap)
+			err = collector.Post(loginURL, reqMap)
+			if err != nil {
+				logger.Error("post err:", err)
+			}
 		}
 	})
 
@@ -152,9 +156,12 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 			//logger.Info("Please input OTP")
 			//reader := bufio.NewReader(os.Stdin)
 			//code, _ := reader.ReadString('\n')
-			code := otp[0:6]
-			reqMap["approvals_code"] = code
-			shouldSubmit = true
+			if len(totpSecret) > 0 {
+				code := gotp.NewDefaultTOTP(totpSecret).Now()
+				reqMap["approvals_code"] = code
+				shouldSubmit = true
+			}
+
 		} else {
 			logger.Info("OnHTML Only Continue checkpoint")
 
@@ -188,22 +195,17 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 
 }
 
-func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostList) {
+func (f *Fbcolly) FetchGroupFeed(groupId int64, nextCursor string) (error, *pb.FacebookPostList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookPost
+	result := pb.FacebookPostList{Posts: []*pb.FacebookPost{}}
 
 	collector.OnHTML("#m_group_stories_container > :last-child a", func(element *colly.HTMLElement) {
-		currentPage++
-		if currentPage < 3 {
-			logger.Info("Will fetch page", currentPage)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-		}
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
-	collector.OnHTML("div[role=\"article\"]", func(element *colly.HTMLElement) {
+	collector.OnHTML("#m_group_stories_container div[role=\"article\"]", func(element *colly.HTMLElement) {
 		dataElement := element
-		post := &fbcrawl.FacebookPost{}
+		post := &pb.FacebookPost{}
 		var fbDataFt FbDataFt
 		jsonData := dataElement.Attr("data-ft")
 
@@ -215,9 +217,10 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 		}
 		logger.Info("Post ", fbDataFt)
 		post.Id = fbDataFt.TopLevelPostId
-		post.Group = &fbcrawl.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
-		post.User = &fbcrawl.FacebookUser{
-			Id:   fbDataFt.ContentOwnerIdNew,
+		post.Group = &pb.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		userId, _ := fbDataFt.ContentOwnerIdNew.Int64()
+		post.User = &pb.FacebookUser{
+			Id:   userId,
 			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
 		}
 		post.CreatedAt = fbDataFt.PageInsights[strconv.FormatInt(fbDataFt.PageId, 10)].PublishTime
@@ -236,56 +239,108 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
 		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
 		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
-		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 			i, _ := strconv.ParseInt(id, 10, 64)
-			return &fbcrawl.FacebookImage{
+			return &pb.FacebookImage{
 				Id: i,
 			}
-		})).([]*fbcrawl.FacebookImage)
+		})).([]*pb.FacebookImage)
 
 		if fbDataFt.PhotoId > 0 {
-			post.ContentImage = &fbcrawl.FacebookImage{Id: fbDataFt.PhotoId}
+			post.ContentImage = &pb.FacebookImage{Id: fbDataFt.PhotoId}
 		}
-		result = append(result, post)
+		result.Posts = append(result.Posts, post)
+	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	}
+
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &result
+}
+
+func (f *Fbcolly) FetchUserInfo(userIdOrUsername string) (error, *pb.FacebookUser) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+
+	result := &pb.FacebookUser{}
+
+	collector.OnHTML("a[href*=\"lst=\"]", func(element *colly.HTMLElement) {
+		parsed, _ := url.Parse(element.Attr("href"))
+		result.Username = strings.Split(parsed.Path[1:], "/")[0]
+		result.Id = getUserIdFromCommentHref(parsed.Query().Get("lst"))
+	})
+
+	collector.OnHTML("a[href*=\"/friends\"]", func(element *colly.HTMLElement) {
+		result.FriendCount = getNumberFromText(element.Text)
 	})
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	collector.OnHTML("#objects_container", func(element *colly.HTMLElement) {
+		result.Name = element.DOM.Find("strong").First().Text()
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/%s", userIdOrUsername))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookPostList{Posts: result}
+	return err, result
 }
 
-func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
+func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *pb.FacebookGroup) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookImage
+	result := &pb.FacebookGroup{}
+
+	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
+		result.Name = element.Text
+	})
+	collector.OnHTML("a[href*=\"view=member\"]", func(element *colly.HTMLElement) {
+		result.Id = getNumberFromText(element.Attr("href"))
+		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(), 10, 64)
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%s?view=info", groupIdOrUsername))
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, result
+}
+
+func (f *Fbcolly) FetchContentImages(postId int64, nextCursor string) (error, *pb.FacebookImageList) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result := pb.FacebookImageList{Images: []*pb.FacebookImage{}}
 
 	collector.OnHTML("a[href*=\"/media/set/\"]", func(element *colly.HTMLElement) {
-		currentPage++
-		logger.Info("Will fetch page", currentPage)
-		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
 
 	collector.OnHTML("a[href*=\"/photo.php\"]", func(element *colly.HTMLElement) {
-		result = append(result, &fbcrawl.FacebookImage{
+		result.Images = append(result.Images, &pb.FacebookImage{
 			Id: getImageIdFromHref(element.Attr("href")),
 		})
 		//f.detailCollector.Visit(url)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookImageList{Images: result}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
+func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *pb.FacebookImage) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result := fbcrawl.FacebookImage{Id: imageId}
+	result := pb.FacebookImage{Id: imageId}
 
 	collector.OnHTML("a", func(element *colly.HTMLElement) {
 		result.Url = element.Attr("href")
@@ -298,11 +353,10 @@ func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
 	return err, &result
 }
 
-func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.FacebookPost) {
+func (f *Fbcolly) FetchPost(groupId int64, postId int64, commentNextCursor string) (error, *pb.FacebookPost) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	post := &fbcrawl.FacebookPost{Comments: []*fbcrawl.FacebookComment{}}
-	commentPaging := 0
+	post := &pb.FacebookPost{Comments: &pb.CommentList{Comments: []*pb.FacebookComment{}}}
 	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
 		dataElement := element.DOM.Find("div[data-ft]")
 		if dataElement.Length() > 0 {
@@ -317,9 +371,10 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 				logger.Info("Post ", result)
 				post.Id = result.TopLevelPostId
-				post.Group = &fbcrawl.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
-				post.User = &fbcrawl.FacebookUser{
-					Id:   result.ContentOwnerIdNew,
+				post.Group = &pb.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
+				userId, _ := result.ContentOwnerIdNew.Int64()
+				post.User = &pb.FacebookUser{
+					Id:   userId,
 					Name: dataElement.Find("h3 strong:first-child a").Text(),
 				}
 				post.CreatedAt = result.PageInsights[strconv.FormatInt(result.PageId, 10)].PublishTime
@@ -337,15 +392,15 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 
 				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
 				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
-				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 					i, _ := strconv.ParseInt(id, 10, 64)
-					return &fbcrawl.FacebookImage{
+					return &pb.FacebookImage{
 						Id: i,
 					}
-				})).([]*fbcrawl.FacebookImage)
+				})).([]*pb.FacebookImage)
 
 				if result.PhotoId > 0 {
-					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
+					post.ContentImage = &pb.FacebookImage{Id: result.PhotoId}
 				}
 			}
 
@@ -355,30 +410,33 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				commentId, _ := strconv.ParseInt(selection.AttrOr("id", ""), 10, 64)
 				logger.Info("comment", commentId)
 				createdAtWhenResult, _ := f.w.Parse(selection.Find("abbr").Text(), time.Now())
-				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
-					Id:   commentId,
-					Post: &fbcrawl.FacebookPost{Id: post.Id},
-					User: &fbcrawl.FacebookUser{
-						Id:   getUserIdFromCommentHref(selection.Find("a[href*=\"#comment_form_\"]").AttrOr("href", "")),
-						Name: selection.Find("h3 > a").Text(),
-					},
-					Content:   selection.Find("h3 + div").Text(),
-					CreatedAt: createdAtWhenResult.Time.Unix(),
-				})
+				parsed, _ := url.Parse(selection.Find("h3 > a").AttrOr("href", ""))
+				if len(parsed.Path) > 1 {
+					post.Comments.Comments = append(post.Comments.Comments, &pb.FacebookComment{
+						Id:   commentId,
+						Post: &pb.FacebookPost{Id: post.Id},
+						User: &pb.FacebookUser{
+							Username: parsed.Path[1:],
+							Name:     selection.Find("h3 > a").Text(),
+						},
+						Content:   selection.Find("h3 + div").Text(),
+						CreatedAt: createdAtWhenResult.Time.Unix(),
+					})
+				}
 			})
 
 		}
 	})
 
 	collector.OnHTML("div[id*=\"see_prev_\"] > a", func(element *colly.HTMLElement) {
-		if commentPaging < 3 {
-			logger.Info("Comment paging", commentPaging)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-			commentPaging = commentPaging + 1
-		}
+		post.Comments.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
+	if len(commentNextCursor) > 0 {
+		err = collector.Visit(commentNextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
 	return err, post
 }
 
@@ -392,8 +450,12 @@ func (f *Fbcolly) LoginWithCookies(cookies string) error {
 //}
 
 func getUserIdFromCommentHref(href string) int64 {
-	id, _ := strconv.ParseInt(regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)[1], 10, 64)
-	return id
+	match := regexp.MustCompile("\\d+:(\\d+)\\d+").FindStringSubmatch(href)
+	if len(match) > 0 {
+		id, _ := strconv.ParseInt(match[1], 10, 64)
+		return id
+	}
+	return 0
 }
 
 func getUrlFromRedirectHref(href string) string {
@@ -410,15 +472,17 @@ func getImageIdFromHref(href string) int64 {
 func getNumberFromText(text string) int64 {
 	logger.Error("reaction", text)
 	if len(text) > 0 {
-		match := regexp.MustCompile("(\\d*)\\s?([km]?)").FindStringSubmatch(text)
-		count, _ := strconv.ParseInt(match[1], 10, 64)
-		switch match[2] {
-		case "k":
-			count *= 1000
-		case "m":
-			count *= 1000000
+		match := regexp.MustCompile("(\\d+)\\s?([km]?)").FindStringSubmatch(text)
+		if len(match) > 0 {
+			count, _ := strconv.ParseInt(match[1], 10, 64)
+			switch match[2] {
+			case "k":
+				count *= 1000
+			case "m":
+				count *= 1000000
+			}
+			return count
 		}
-		return count
 	}
 	return 0
 }