fbcrawl-colly 0.2.2 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 383de209c502265a8533af201e3b9538edffc115af1575bf9662508abb142e00
-  data.tar.gz: 631c53a8c82e063df262ccac4f6c74f4591296b700ffa627110a9bf0dce40358
+  metadata.gz: eed929533973a523ed9cc5ac0cabf35559cb2e50e1c028a7ef2d8b90de15b58b
+  data.tar.gz: 385c7dcd852e31acdc3d591b323ba4eb2b8d979b89c8fe55a55e18b00fbbb691
 SHA512:
-  metadata.gz: b8c1505f7183c0a30a6d6679319a947be589828771e3bc231915cb841aad5e79b7221091862f6df537a701eaa2bf21bb996ba689e301d00a79964fd47e45f258
-  data.tar.gz: 2ce2e0870f9455cdaeab48c840e1c9f5fb35afc340e0c56830afca90eb7bcebcf120430b7c3d043d58274d545ab0c7bb99d4062b36885b1e832bb540b7bf5080
+  metadata.gz: 16777a1a0ae7d48b2ce7ac3bd3573a351903321a009e81d649379aa8f508986ae4511d0ed85101c8605910d383dfa9ba8b7748d759ab05c7a7076e32c5cf49cd
+  data.tar.gz: d902d6226148cadd62d1e0c5a45419cfdd1343831bcfd5dced56bd298982a51d1da1835b048925b347351a01e63e875e8b295e568fa803b512c57183484fd2c6

data/.gitignore

@@ -12,8 +12,6 @@
 /parse.log
 last.html
 *.db
-/fbcrawl/fbcrawl.pb.go
-/lib/fbcrawl_pb.rb
 
 mkmf.log
 .rakeTasks

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.14-alpine
+RUN apk add --no-cache git build-base tzdata
+
+RUN mkdir -p /app
+WORKDIR /app
+ADD ./go.mod /app
+ADD ./go.sum /app
+ADD ./ /app
+RUN go get
+
+
+ENV PORT 3000
+RUN go build -o server qnetwork.net/fbcrawl
+ENTRYPOINT ["./server"]

data/Gemfile.lock

@@ -1,19 +1,21 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.2)
-      ffi
+    fbcrawl-colly (1.0.1)
       google-protobuf
+      grpc
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ffi (1.13.1)
-    google-protobuf (3.12.4)
+    google-protobuf (3.13.0)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    grpc (1.30.2)
+      google-protobuf (~> 3.12)
+      googleapis-common-protos-types (~> 1.0)
     minitest (5.14.1)
     rake (12.3.3)
-    rake-compiler (1.1.1)
-      rake
 
 PLATFORMS
   ruby
@@ -22,7 +24,6 @@ DEPENDENCIES
   fbcrawl-colly!
   minitest (~> 5.0)
   rake (~> 12.0)
-  rake-compiler
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -2,7 +2,7 @@
 
 This project is to crawl mbasic.facebook.com using GO Colly. 
 
-Ruby gem will available to use
+Ruby gem is only client for GRPC colly service
 
 ## Installation
 

data/fbcrawl-colly.gemspec

@@ -25,12 +25,8 @@ Gem::Specification.new do |spec|
   end
   # spec.bindir        = "exe"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.extensions = [
-      'ext/fbcrawl_colly/extconf.rb'
-  ]
-  spec.require_paths = ["lib"]
+  spec.require_paths = %w[lib lib/pb]
 
-  spec.add_runtime_dependency 'ffi'
   spec.add_runtime_dependency 'google-protobuf'
-  spec.add_development_dependency 'rake-compiler'
+  spec.add_runtime_dependency 'grpc'
 end

data/fbcrawl.proto

@@ -1,17 +1,92 @@
 syntax = "proto3";
 
 package fbcrawl_colly;
-option go_package = "./fbcrawl;fbcrawl";
+option go_package = "./fbcrawl/pb;pb";
+
+service Grpc {
+  // Sends a greeting
+  rpc Init (Empty) returns (Pointer) {}
+  rpc FreeColly (Pointer) returns (Empty) {}
+  rpc Login  (LoginRequest) returns (LoginResponse) {}
+  rpc LoginWithCookies  (LoginWithCookiesRequest) returns (Empty) {}
+  rpc FetchGroupInfo  (FetchGroupInfoRequest) returns (FacebookGroup) {}
+  rpc FetchUserInfo  (FetchUserInfoRequest) returns (FacebookUser) {}
+  rpc FetchGroupFeed  (FetchGroupFeedRequest) returns (FacebookPostList) {}
+  rpc FetchPost  (FetchPostRequest) returns (FacebookPost) {}
+  rpc FetchContentImages  (FetchContentImagesRequest) returns (FacebookImageList) {}
+  rpc FetchImageUrl  (FetchImageUrlRequest) returns (FacebookImage) {}
+}
+
+message Empty {
+
+}
+
+message Pointer {
+  int64 address = 1;
+}
+
+message LoginRequest {
+  Pointer pointer = 1;
+  string email = 2;
+  string password = 3;
+  string totp_secret = 4;
+}
+
+message LoginResponse {
+  string cookies = 1;
+}
+
+message LoginWithCookiesRequest {
+  Pointer pointer = 1;
+  string cookies = 2;
+}
+
+message FetchGroupInfoRequest {
+  Pointer pointer = 1;
+  string group_username = 2;
+}
+
+message FetchUserInfoRequest {
+  Pointer pointer = 1;
+  string username = 2;
+}
+
+message FetchGroupFeedRequest {
+  Pointer pointer = 1;
+  int64 group_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchPostRequest {
+  Pointer pointer = 1;
+  int64 group_id = 2;
+  int64 post_id = 3;
+  string comment_next_cursor = 4;
+}
+
+message FetchContentImagesRequest {
+  Pointer pointer = 1;
+  int64 post_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchImageUrlRequest {
+  Pointer pointer = 1;
+  int64 image_id = 2;
+}
 
 // The request message containing the user's name.
 message FacebookGroup {
   int64 id = 1;
   string name = 2;
+  int64 member_count = 3;
 }
 
 message FacebookUser {
   int64 id = 1;
   string name = 2;
+  string username = 3;
+  int64 friend_count =4;
 }
 
 message FacebookPost {
@@ -19,12 +94,18 @@ message FacebookPost {
   FacebookGroup group = 2;
   FacebookUser user = 3;
   string content = 4;
+  CommentList comments = 5;
   string content_link = 6;
-  FacebookImage content_image = 8;
   repeated FacebookImage content_images = 7;
-  repeated FacebookComment comments = 5;
+  FacebookImage content_image = 8;
   int64 created_at = 9;
   int64 reaction_count = 10;
+  int64 comment_count = 11;
+}
+
+message CommentList {
+  repeated FacebookComment comments = 5;
+  string next_cursor = 12;
 }
 
 message FacebookImage {
@@ -42,8 +123,10 @@ message FacebookComment {
 
 message FacebookPostList {
   repeated FacebookPost posts = 1;
+  string next_cursor = 2;
 }
 
 message FacebookImageList {
   repeated FacebookImage images = 1;
+  string next_cursor = 2;
 }

data/{fbcolly → fbcrawl}/fbcolly.go

@@ -6,16 +6,18 @@ import (
 	"errors"
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
-	"github.com/gocolly/colly"
-	"github.com/gocolly/colly/extensions"
-	"github.com/gocolly/colly/storage"
+	"github.com/gocolly/colly/v2"
+	"github.com/gocolly/colly/v2/debug"
+	"github.com/gocolly/colly/v2/extensions"
+	"github.com/gocolly/colly/v2/storage"
 	"github.com/google/logger"
 	"github.com/olebedev/when"
 	"github.com/olebedev/when/rules/common"
 	"github.com/olebedev/when/rules/en"
 	"github.com/thoas/go-funk"
+	"github.com/xlzd/gotp"
 	"net/url"
-	"qnetwork.net/fbcrawl/fbcrawl"
+	"qnetwork.net/fbcrawl/fbcrawl/pb"
 	"regexp"
 	"strconv"
 	"strings"
@@ -33,7 +35,7 @@ type FbDataInsight struct {
 	FbDataPostContext `json:"post_context"`
 }
 type FbDataFt struct {
-	ContentOwnerIdNew    int64                    `json:"content_owner_id_new"`
+	ContentOwnerIdNew    json.Number              `json:"content_owner_id_new"`
 	PhotoAttachmentsList []string                 `json:"photo_attachments_list"`
 	PhotoId              int64                    `json:"photo_id,string"`
 	PageId               int64                    `json:"page_id,string"`
@@ -42,7 +44,7 @@ type FbDataFt struct {
 }
 
 func sharedOnRequest(request *colly.Request) {
-	logger.Info("OnRequest")
+	logger.Info("OnRequest ", request.URL)
 	//request.Headers.Set("Host", "facebook.com")
 	request.Headers.Set("Accept-Language", "en-US,en;q=0.9")
 	request.Headers.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
@@ -65,6 +67,7 @@ func setupSharedCollector(collector *colly.Collector) error {
 	collector.AllowURLRevisit = true
 	collector.OnRequest(sharedOnRequest)
 	collector.OnResponse(sharedOnResponse)
+	collector.SetDebugger(&debug.LogDebugger{})
 	collector.OnError(func(resp *colly.Response, errHttp error) {
 		err = errHttp
 		logger.Error("OnError", err)
@@ -106,26 +109,29 @@ func New() *Fbcolly {
 	return &f
 }
 
-func (f *Fbcolly) Login(email string, password string, otp string) (string, error) {
+func (f *Fbcolly) Login(email string, password string, totpSecret string) (string, error) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
 
 	logger.Info("Login using email", email)
 	loggedIn := false
-
+	firstLogin := true
 	collector.OnHTML("#login_form", func(element *colly.HTMLElement) {
-		logger.Info("OnHTML login_form")
-		loginURL, err, reqMap := getForm(element, err)
-		if err != nil {
-			logger.Error(err)
-			return
-		}
-		reqMap["email"] = email
-		reqMap["pass"] = password
-		logger.Info("req map:", reqMap)
-		err = collector.Post(loginURL, reqMap)
-		if err != nil {
-			logger.Error("post err:", err)
+		if firstLogin {
+			firstLogin = false
+			logger.Info("OnHTML login_form")
+			loginURL, err, reqMap := getForm(element, err)
+			if err != nil {
+				logger.Error(err)
+				return
+			}
+			reqMap["email"] = email
+			reqMap["pass"] = password
+			logger.Info("req map:", reqMap)
+			err = collector.Post(loginURL, reqMap)
+			if err != nil {
+				logger.Error("post err:", err)
+			}
 		}
 	})
 
@@ -152,9 +158,12 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 			//logger.Info("Please input OTP")
 			//reader := bufio.NewReader(os.Stdin)
 			//code, _ := reader.ReadString('\n')
-			code := otp[0:6]
-			reqMap["approvals_code"] = code
-			shouldSubmit = true
+			if len(totpSecret) > 0 {
+				code := gotp.NewDefaultTOTP(totpSecret).Now()
+				reqMap["approvals_code"] = code
+				shouldSubmit = true
+			}
+
 		} else {
 			logger.Info("OnHTML Only Continue checkpoint")
 
@@ -188,68 +197,152 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 
 }
 
-func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostList) {
+func (f *Fbcolly) FetchGroupFeed(groupId int64, nextCursor string) (error, *pb.FacebookPostList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookPost
+	result := pb.FacebookPostList{Posts: []*pb.FacebookPost{}}
 
 	collector.OnHTML("#m_group_stories_container > :last-child a", func(element *colly.HTMLElement) {
-		currentPage++
-		if currentPage < 3 {
-			logger.Info("Will fetch page", currentPage)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
+	})
+	collector.OnHTML("#m_group_stories_container div[role=\"article\"]", func(element *colly.HTMLElement) {
+		dataElement := element
+		post := &pb.FacebookPost{}
+		var fbDataFt FbDataFt
+		jsonData := dataElement.Attr("data-ft")
+
+		logger.Info(jsonData)
+		err = json.Unmarshal([]byte(jsonData), &fbDataFt)
+		if err != nil {
+			logger.Error(err)
+			return
+		}
+		logger.Info("Post ", fbDataFt)
+		post.Id = fbDataFt.TopLevelPostId
+		post.Group = &pb.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		userId, _ := fbDataFt.ContentOwnerIdNew.Int64()
+		post.User = &pb.FacebookUser{
+			Id:   userId,
+			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
 		}
+		post.CreatedAt = fbDataFt.PageInsights[strconv.FormatInt(fbDataFt.PageId, 10)].PublishTime
+		//Content
+
+		//NO BACKGROUND TEXT ONLY
+		post.Content = strings.Join(dataElement.DOM.Find("p").Map(func(i int, selection *goquery.Selection) string {
+			return selection.Text()
+		}), "\n")
+
+		if len(post.Content) == 0 {
+			// TEXT WITH BACKGROUND
+			post.Content = dataElement.DOM.Find("div[style*=\"background-image:url\"]").Text()
+		}
+
+		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
+		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
+		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
+			i, _ := strconv.ParseInt(id, 10, 64)
+			return &pb.FacebookImage{
+				Id: i,
+			}
+		})).([]*pb.FacebookImage)
+
+		if fbDataFt.PhotoId > 0 {
+			post.ContentImage = &pb.FacebookImage{Id: fbDataFt.PhotoId}
+		}
+		result.Posts = append(result.Posts, post)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	}
 
-	collector.OnXML("//a[text()=\"Full Story\"]", func(element *colly.XMLElement) {
-		logger.Info("Post found at", element.Attr("href"))
-		u, _ := url.Parse("http://mbasic.facebook.com" + element.Attr("href"))
-		postId, _ := strconv.ParseInt(u.Query().Get("id"), 10, 64)
-		result = append(result, &fbcrawl.FacebookPost{
-			Id:    postId,
-			Group: &fbcrawl.FacebookGroup{Id: groupId},
-		})
-		//f.detailCollector.Visit(url)
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &result
+}
+
+func (f *Fbcolly) FetchUserInfo(userIdOrUsername string) (error, *pb.FacebookUser) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+
+	result := &pb.FacebookUser{}
+
+	collector.OnHTML("a[href*=\"lst=\"]", func(element *colly.HTMLElement) {
+		parsed, _ := url.Parse(element.Attr("href"))
+		result.Username = strings.Split(parsed.Path[1:], "/")[0]
+		result.Id = getUserIdFromCommentHref(parsed.Query().Get("lst"))
+	})
+
+	collector.OnHTML("a[href*=\"/friends\"]", func(element *colly.HTMLElement) {
+		result.FriendCount = getNumberFromText(element.Text)
+	})
+
+	collector.OnHTML("#objects_container", func(element *colly.HTMLElement) {
+		result.Name = element.DOM.Find("strong").First().Text()
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/%s", userIdOrUsername))
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, result
+}
+
+func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *pb.FacebookGroup) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result := &pb.FacebookGroup{}
+
+	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
+		result.Name = element.Text
+	})
+	collector.OnHTML("a[href*=\"view=member\"]", func(element *colly.HTMLElement) {
+		result.Id = getNumberFromText(element.Attr("href"))
+		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(), 10, 64)
 	})
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%s?view=info", groupIdOrUsername))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookPostList{Posts: result}
+	return err, result
 }
 
-func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
+func (f *Fbcolly) FetchContentImages(postId int64, nextCursor string) (error, *pb.FacebookImageList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookImage
+	result := pb.FacebookImageList{Images: []*pb.FacebookImage{}}
 
 	collector.OnHTML("a[href*=\"/media/set/\"]", func(element *colly.HTMLElement) {
-		currentPage++
-		logger.Info("Will fetch page", currentPage)
-		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
 
 	collector.OnHTML("a[href*=\"/photo.php\"]", func(element *colly.HTMLElement) {
-		result = append(result, &fbcrawl.FacebookImage{
+		result.Images = append(result.Images, &pb.FacebookImage{
 			Id: getImageIdFromHref(element.Attr("href")),
 		})
 		//f.detailCollector.Visit(url)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookImageList{Images: result}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
+func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *pb.FacebookImage) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result := fbcrawl.FacebookImage{Id: imageId}
+	result := pb.FacebookImage{Id: imageId}
 
 	collector.OnHTML("a", func(element *colly.HTMLElement) {
 		result.Url = element.Attr("href")
@@ -262,11 +355,10 @@ func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
 	return err, &result
 }
 
-func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.FacebookPost) {
+func (f *Fbcolly) FetchPost(groupId int64, postId int64, commentNextCursor string) (error, *pb.FacebookPost) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	post := &fbcrawl.FacebookPost{Comments: []*fbcrawl.FacebookComment{}}
-	commentPaging := 0
+	post := &pb.FacebookPost{Comments: &pb.CommentList{Comments: []*pb.FacebookComment{}}}
 	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
 		dataElement := element.DOM.Find("div[data-ft]")
 		if dataElement.Length() > 0 {
@@ -281,9 +373,10 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 				logger.Info("Post ", result)
 				post.Id = result.TopLevelPostId
-				post.Group = &fbcrawl.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
-				post.User = &fbcrawl.FacebookUser{
-					Id:   result.ContentOwnerIdNew,
+				post.Group = &pb.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
+				userId, _ := result.ContentOwnerIdNew.Int64()
+				post.User = &pb.FacebookUser{
+					Id:   userId,
 					Name: dataElement.Find("h3 strong:first-child a").Text(),
 				}
 				post.CreatedAt = result.PageInsights[strconv.FormatInt(result.PageId, 10)].PublishTime
@@ -300,21 +393,17 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 
 				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
-				post.ReactionCount = getReactionFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
-				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
+				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 					i, _ := strconv.ParseInt(id, 10, 64)
-					return &fbcrawl.FacebookImage{
+					return &pb.FacebookImage{
 						Id: i,
 					}
-				})).([]*fbcrawl.FacebookImage)
+				})).([]*pb.FacebookImage)
 
 				if result.PhotoId > 0 {
-					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
+					post.ContentImage = &pb.FacebookImage{Id: result.PhotoId}
 				}
-
-				logger.Info("content", strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
-					return selection.Text()
-				}), "\n"))
 			}
 
 			//Comment
@@ -323,12 +412,13 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				commentId, _ := strconv.ParseInt(selection.AttrOr("id", ""), 10, 64)
 				logger.Info("comment", commentId)
 				createdAtWhenResult, _ := f.w.Parse(selection.Find("abbr").Text(), time.Now())
-				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
+				parsed, _ := url.Parse(selection.Find("h3 > a").AttrOr("href", ""))
+				post.Comments.Comments = append(post.Comments.Comments, &pb.FacebookComment{
 					Id:   commentId,
-					Post: &fbcrawl.FacebookPost{Id: post.Id},
-					User: &fbcrawl.FacebookUser{
-						Id:   getUserIdFromCommentHref(selection.Find("a[href*=\"#comment_form_\"]").AttrOr("href", "")),
-						Name: selection.Find("h3 > a").Text(),
+					Post: &pb.FacebookPost{Id: post.Id},
+					User: &pb.FacebookUser{
+						Username: parsed.Path[1:],
+						Name:     selection.Find("h3 > a").Text(),
 					},
 					Content:   selection.Find("h3 + div").Text(),
 					CreatedAt: createdAtWhenResult.Time.Unix(),
@@ -339,14 +429,14 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 	})
 
 	collector.OnHTML("div[id*=\"see_prev_\"] > a", func(element *colly.HTMLElement) {
-		if commentPaging < 3 {
-			logger.Info("Comment paging", commentPaging)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-			commentPaging = commentPaging + 1
-		}
+		post.Comments.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
+	if len(commentNextCursor) > 0 {
+		err = collector.Visit(commentNextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
 	return err, post
 }
 
@@ -360,8 +450,12 @@ func (f *Fbcolly) LoginWithCookies(cookies string) error {
 //}
 
 func getUserIdFromCommentHref(href string) int64 {
-	id, _ := strconv.ParseInt(regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)[1], 10, 64)
-	return id
+	match := regexp.MustCompile("\\d+:(\\d+)\\d+").FindStringSubmatch(href)
+	if len(match) > 0 {
+		id, _ := strconv.ParseInt(match[1], 10, 64)
+		return id
+	}
+	return 0
 }
 
 func getUrlFromRedirectHref(href string) string {
@@ -375,11 +469,20 @@ func getImageIdFromHref(href string) int64 {
 	return i
 }
 
-func getReactionFromText(text string) int64 {
+func getNumberFromText(text string) int64 {
 	logger.Error("reaction", text)
 	if len(text) > 0 {
-		id, _ := strconv.ParseInt(regexp.MustCompile("(\\d+)").FindStringSubmatch(text)[1], 10, 64)
-		return id
+		match := regexp.MustCompile("(\\d+)\\s?([km]?)").FindStringSubmatch(text)
+		if len(match) > 0 {
+			count, _ := strconv.ParseInt(match[1], 10, 64)
+			switch match[2] {
+			case "k":
+				count *= 1000
+			case "m":
+				count *= 1000000
+			}
+			return count
+		}
 	}
 	return 0
 }