fbcrawl-colly 0.2.4 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fc0e07942d352bb9b49c93a106d138bc9946f8e1943bcd81f6181082e79c413
-  data.tar.gz: 1e31f7fe0bc3bf83c82b90d84080a257734cfe63eaa4d79fdb89aa52204e2eb4
+  metadata.gz: f420f1cc6b4b6260b8106422e80c0488a351fb21fdcaa0ad1af56b0402beb8da
+  data.tar.gz: 86193bbf6bfab5b48f73ad4f2594f5e4ff0761901ec5653af90ea638cbf9def6
 SHA512:
-  metadata.gz: 55d00db7f51b078c1ca7c46a59b06eb4d62b1859fac6e11a2da3c117f7c2dd3958d442559f9b84573f4d4f5e2d02a77539287d5628ff60f421405a3e2910e2b7
-  data.tar.gz: 550b26405d7bbd13356f1ca7ef30edfa682c04364f2f8fc409269c74fe25e8cb0535464d395083109b04e17de129bd851da3f035dd2dec6af2095ed357a1ddd7
+  metadata.gz: c6396d3572f0beaedaf198388e688e4f3bed09e6920c260cfddf61c36267c97e2a6babea3d297e07d1b7a816f6b416ec7c6d1a27775d85bea23dc4b5b07d9a51
+  data.tar.gz: a94d9ed5fc604e6e001abfb7f93ac9a82a075b686ada0214e854761ea0aa84712651b6605a7f885dcadc45be1dcad7d8bf2ac580f0e61dd2d3b086b02849c440

data/.gitignore

@@ -12,8 +12,6 @@
 /parse.log
 last.html
 *.db
-/fbcrawl/fbcrawl.pb.go
-/lib/fbcrawl_pb.rb
 
 mkmf.log
 .rakeTasks

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.14-alpine
+RUN apk add --no-cache git build-base tzdata
+
+RUN mkdir -p /app
+WORKDIR /app
+ADD ./go.mod /app
+ADD ./go.sum /app
+ADD ./ /app
+RUN go get
+
+
+ENV PORT 3000
+RUN go build -o server qnetwork.net/fbcrawl
+ENTRYPOINT ["./server"]

data/Gemfile.lock

@@ -1,19 +1,21 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.4)
-      ffi
+    fbcrawl-colly (1.1.0)
       google-protobuf
+      grpc
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ffi (1.13.1)
     google-protobuf (3.13.0)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    grpc (1.31.1)
+      google-protobuf (~> 3.12)
+      googleapis-common-protos-types (~> 1.0)
     minitest (5.14.1)
     rake (12.3.3)
-    rake-compiler (1.1.1)
-      rake
 
 PLATFORMS
   ruby
@@ -22,7 +24,6 @@ DEPENDENCIES
   fbcrawl-colly!
   minitest (~> 5.0)
   rake (~> 12.0)
-  rake-compiler
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -2,7 +2,7 @@
 
 This project is to crawl mbasic.facebook.com using GO Colly. 
 
-Ruby gem will available to use
+Ruby gem is only client for GRPC colly service
 
 ## Installation
 

data/fbcrawl-colly.gemspec

@@ -25,12 +25,8 @@ Gem::Specification.new do |spec|
   end
   # spec.bindir        = "exe"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.extensions = [
-      'ext/fbcrawl_colly/extconf.rb'
-  ]
-  spec.require_paths = ["lib"]
+  spec.require_paths = %w[lib lib/pb]
 
-  spec.add_runtime_dependency 'ffi'
   spec.add_runtime_dependency 'google-protobuf'
-  spec.add_development_dependency 'rake-compiler'
+  spec.add_runtime_dependency 'grpc'
 end

data/fbcrawl.proto

@@ -1,9 +1,80 @@
 syntax = "proto3";
 
 package fbcrawl_colly;
-option go_package = "./fbcrawl;fbcrawl";
+option go_package = "./fbcrawl/pb;pb";
+
+service Grpc {
+  // Sends a greeting
+  rpc Login  (LoginRequest) returns (LoginResponse) {}
+  rpc FetchMyGroups (FetchMyGroupsRequest) returns (FacebookGroupList) {}
+  rpc FetchGroupInfo  (FetchGroupInfoRequest) returns (FacebookGroup) {}
+  rpc FetchUserInfo  (FetchUserInfoRequest) returns (FacebookUser) {}
+  rpc FetchGroupFeed  (FetchGroupFeedRequest) returns (FacebookPostList) {}
+  rpc FetchPost  (FetchPostRequest) returns (FacebookPost) {}
+  rpc FetchContentImages  (FetchContentImagesRequest) returns (FacebookImageList) {}
+  rpc FetchImageUrl  (FetchImageUrlRequest) returns (FacebookImage) {}
+}
+
+message Context {
+  string cookies = 1;
+}
+
+message LoginRequest {
+  string email = 2;
+  string password = 3;
+  string totp_secret = 4;
+}
+
+message LoginResponse {
+  string cookies = 1;
+}
+
+message LoginWithCookiesRequest {
+  string cookies = 1;
+}
+
+message FetchMyGroupsRequest {
+  Context context = 1;
+}
+
+message FetchGroupInfoRequest {
+  Context context = 1;
+  string group_username = 2;
+}
+
+message FetchUserInfoRequest {
+  Context context = 1;
+  string username = 2;
+}
+
+message FetchGroupFeedRequest {
+  Context context = 1;
+  int64 group_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchPostRequest {
+  Context context = 1;
+  int64 group_id = 2;
+  int64 post_id = 3;
+  string comment_next_cursor = 4;
+}
+
+message FetchContentImagesRequest {
+  Context context = 1;
+  int64 post_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchImageUrlRequest {
+  Context context = 1;
+  int64 image_id = 2;
+}
+
+message FacebookGroupList {
+  repeated FacebookGroup groups = 1;
+}
 
-// The request message containing the user's name.
 message FacebookGroup {
   int64 id = 1;
   string name = 2;
@@ -13,6 +84,8 @@ message FacebookGroup {
 message FacebookUser {
   int64 id = 1;
   string name = 2;
+  string username = 3;
+  int64 friend_count = 4;
 }
 
 message FacebookPost {
@@ -20,15 +93,20 @@ message FacebookPost {
   FacebookGroup group = 2;
   FacebookUser user = 3;
   string content = 4;
+  CommentList comments = 5;
   string content_link = 6;
-  FacebookImage content_image = 8;
   repeated FacebookImage content_images = 7;
-  repeated FacebookComment comments = 5;
+  FacebookImage content_image = 8;
   int64 created_at = 9;
   int64 reaction_count = 10;
   int64 comment_count = 11;
 }
 
+message CommentList {
+  repeated FacebookComment comments = 5;
+  string next_cursor = 12;
+}
+
 message FacebookImage {
   int64 id = 1;
   string url = 2;
@@ -44,8 +122,10 @@ message FacebookComment {
 
 message FacebookPostList {
   repeated FacebookPost posts = 1;
+  string next_cursor = 2;
 }
 
 message FacebookImageList {
   repeated FacebookImage images = 1;
+  string next_cursor = 2;
 }

data/{fbcolly → fbcrawl}/fbcolly.go

@@ -6,16 +6,17 @@ import (
 	"errors"
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
-	"github.com/gocolly/colly"
-	"github.com/gocolly/colly/extensions"
-	"github.com/gocolly/colly/storage"
+	"github.com/gocolly/colly/v2"
+	"github.com/gocolly/colly/v2/extensions"
+	"github.com/gocolly/colly/v2/storage"
 	"github.com/google/logger"
 	"github.com/olebedev/when"
 	"github.com/olebedev/when/rules/common"
 	"github.com/olebedev/when/rules/en"
 	"github.com/thoas/go-funk"
+	"github.com/xlzd/gotp"
 	"net/url"
-	"qnetwork.net/fbcrawl/fbcrawl"
+	"qnetwork.net/fbcrawl/fbcrawl/pb"
 	"regexp"
 	"strconv"
 	"strings"
@@ -33,7 +34,7 @@ type FbDataInsight struct {
 	FbDataPostContext `json:"post_context"`
 }
 type FbDataFt struct {
-	ContentOwnerIdNew    int64                    `json:"content_owner_id_new"`
+	ContentOwnerIdNew    json.Number              `json:"content_owner_id_new"`
 	PhotoAttachmentsList []string                 `json:"photo_attachments_list"`
 	PhotoId              int64                    `json:"photo_id,string"`
 	PageId               int64                    `json:"page_id,string"`
@@ -41,30 +42,38 @@ type FbDataFt struct {
 	PageInsights         map[string]FbDataInsight `json:"page_insights"`
 }
 
-func sharedOnRequest(request *colly.Request) {
-	logger.Info("OnRequest")
-	//request.Headers.Set("Host", "facebook.com")
-	request.Headers.Set("Accept-Language", "en-US,en;q=0.9")
-	request.Headers.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
-	request.Headers.Set("origin", "https://mbasic.facebook.com")
-
-	//logger.Info("Saved referrer is", request.Ctx.Get("_referer"))
-	request.Headers.Set("referer", "https://mbasic.facebook.com/checkpoint/?_rdr")
-	request.Headers.Set("cache-control", "max-age=0")
-	request.Headers.Set("upgrade-insecure-requests", "1")
-	//accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
-	//origin: https://mbasic.facebook.com
-	//referer: https://mbasic.facebook.com/checkpoint/?_rdr
-	request.Headers.Set("User-Agent", "Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Mobile Safari/537.36")
-	request.ResponseCharacterEncoding = "utf-8"
-}
-
 func setupSharedCollector(collector *colly.Collector) error {
 	var err error
 	extensions.Referer(collector)
 	collector.AllowURLRevisit = true
-	collector.OnRequest(sharedOnRequest)
-	collector.OnResponse(sharedOnResponse)
+	var lastUrl string
+	collector.OnRequest(func(request *colly.Request) {
+		lastUrl = request.URL.RawPath
+		logger.Info("OnRequest ", request.URL)
+		//request.Headers.Set("Host", "facebook.com")
+		request.Headers.Set("Accept-Language", "en-US,en;q=0.9")
+		request.Headers.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
+		request.Headers.Set("origin", "https://mbasic.facebook.com")
+
+		//logger.Info("Saved referrer is", request.Ctx.Get("_referer"))
+		request.Headers.Set("referer", "https://mbasic.facebook.com/checkpoint/?_rdr")
+		request.Headers.Set("cache-control", "max-age=0")
+		request.Headers.Set("upgrade-insecure-requests", "1")
+		//accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
+		//origin: https://mbasic.facebook.com
+		//referer: https://mbasic.facebook.com/checkpoint/?_rdr
+		request.Headers.Set("User-Agent", "Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Mobile Safari/537.36")
+		request.ResponseCharacterEncoding = "utf-8"
+	})
+	collector.OnResponse(func(response *colly.Response) {
+		logger.Info("OnResponse ./last.html")
+		_ = response.Save("./last.html")
+		//logger.Info(string(resp.Body))
+	})
+
+	collector.OnHTML("a[href*=\"177066345680802\"", func(element *colly.HTMLElement) {
+		logger.Error("RateLimit reached ", lastUrl)
+	})
 	collector.OnError(func(resp *colly.Response, errHttp error) {
 		err = errHttp
 		logger.Error("OnError", err)
@@ -72,12 +81,6 @@ func setupSharedCollector(collector *colly.Collector) error {
 	return err
 }
 
-func sharedOnResponse(response *colly.Response) {
-	logger.Info("OnResponse ./last.html")
-	_ = response.Save("./last.html")
-	//logger.Info(string(resp.Body))
-}
-
 func getForm(element *colly.HTMLElement, err error) (string, error, map[string]string) {
 	submitUrl, exists := element.DOM.Attr("action")
 	if !exists {
@@ -106,7 +109,7 @@ func New() *Fbcolly {
 	return &f
 }
 
-func (f *Fbcolly) Login(email string, password string, otp string) (string, error) {
+func (f *Fbcolly) Login(email string, password string, totpSecret string) (string, error) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
 
@@ -155,9 +158,12 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 			//logger.Info("Please input OTP")
 			//reader := bufio.NewReader(os.Stdin)
 			//code, _ := reader.ReadString('\n')
-			code := otp[0:6]
-			reqMap["approvals_code"] = code
-			shouldSubmit = true
+			if len(totpSecret) > 0 {
+				code := gotp.NewDefaultTOTP(totpSecret).Now()
+				reqMap["approvals_code"] = code
+				shouldSubmit = true
+			}
+
 		} else {
 			logger.Info("OnHTML Only Continue checkpoint")
 
@@ -191,22 +197,17 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 
 }
 
-func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostList) {
+func (f *Fbcolly) FetchGroupFeed(groupId int64, nextCursor string) (error, *pb.FacebookPostList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookPost
+	result := pb.FacebookPostList{Posts: []*pb.FacebookPost{}}
 
 	collector.OnHTML("#m_group_stories_container > :last-child a", func(element *colly.HTMLElement) {
-		currentPage++
-		if currentPage < 3 {
-			logger.Info("Will fetch page", currentPage)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-		}
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
-	collector.OnHTML("div[role=\"article\"]", func(element *colly.HTMLElement) {
+	collector.OnHTML("#m_group_stories_container div[role=\"article\"]", func(element *colly.HTMLElement) {
 		dataElement := element
-		post := &fbcrawl.FacebookPost{}
+		post := &pb.FacebookPost{}
 		var fbDataFt FbDataFt
 		jsonData := dataElement.Attr("data-ft")
 
@@ -218,9 +219,10 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 		}
 		logger.Info("Post ", fbDataFt)
 		post.Id = fbDataFt.TopLevelPostId
-		post.Group = &fbcrawl.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
-		post.User = &fbcrawl.FacebookUser{
-			Id:   fbDataFt.ContentOwnerIdNew,
+		post.Group = &pb.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		userId, _ := fbDataFt.ContentOwnerIdNew.Int64()
+		post.User = &pb.FacebookUser{
+			Id:   userId,
 			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
 		}
 		post.CreatedAt = fbDataFt.PageInsights[strconv.FormatInt(fbDataFt.PageId, 10)].PublishTime
@@ -239,77 +241,110 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
 		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
 		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
-		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 			i, _ := strconv.ParseInt(id, 10, 64)
-			return &fbcrawl.FacebookImage{
+			return &pb.FacebookImage{
 				Id: i,
 			}
-		})).([]*fbcrawl.FacebookImage)
+		})).([]*pb.FacebookImage)
 
 		if fbDataFt.PhotoId > 0 {
-			post.ContentImage = &fbcrawl.FacebookImage{Id: fbDataFt.PhotoId}
+			post.ContentImage = &pb.FacebookImage{Id: fbDataFt.PhotoId}
 		}
-		result = append(result, post)
+		result.Posts = append(result.Posts, post)
+	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	}
+
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &result
+}
+
+func (f *Fbcolly) FetchUserInfo(userIdOrUsername string) (error, *pb.FacebookUser) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+
+	result := &pb.FacebookUser{}
+
+	collector.OnHTML("a[href*=\"lst=\"]", func(element *colly.HTMLElement) {
+		parsed, _ := url.Parse(element.Attr("href"))
+		result.Username = strings.Split(parsed.Path[1:], "/")[0]
+		result.Id = getUserIdFromCommentHref(parsed.Query().Get("lst"))
+	})
+
+	collector.OnHTML("a[href*=\"/friends\"]", func(element *colly.HTMLElement) {
+		result.FriendCount = getNumberFromText(element.Text)
+	})
+
+	collector.OnHTML("#objects_container", func(element *colly.HTMLElement) {
+		result.Name = element.DOM.Find("strong").First().Text()
 	})
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/%s", userIdOrUsername))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookPostList{Posts: result}
+	return err, result
 }
 
-func (f *Fbcolly) FetchGroupInfo(groupId int64) (error, *fbcrawl.FacebookGroup) {
+func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *pb.FacebookGroup) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result :=  &fbcrawl.FacebookGroup{Id: groupId}
+	result := &pb.FacebookGroup{}
 
 	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
 		result.Name = element.Text
 	})
 	collector.OnHTML("a[href*=\"view=member\"]", func(element *colly.HTMLElement) {
-		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(),10,64)
+		result.Id = getNumberFromText(element.Attr("href"))
+		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(), 10, 64)
 	})
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d?view=info", groupId))
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%s?view=info", groupIdOrUsername))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
 	return err, result
 }
 
-func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
+func (f *Fbcolly) FetchContentImages(postId int64, nextCursor string) (error, *pb.FacebookImageList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookImage
+	result := pb.FacebookImageList{Images: []*pb.FacebookImage{}}
 
 	collector.OnHTML("a[href*=\"/media/set/\"]", func(element *colly.HTMLElement) {
-		currentPage++
-		logger.Info("Will fetch page", currentPage)
-		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
 
 	collector.OnHTML("a[href*=\"/photo.php\"]", func(element *colly.HTMLElement) {
-		result = append(result, &fbcrawl.FacebookImage{
+		result.Images = append(result.Images, &pb.FacebookImage{
 			Id: getImageIdFromHref(element.Attr("href")),
 		})
 		//f.detailCollector.Visit(url)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookImageList{Images: result}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
+func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *pb.FacebookImage) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result := fbcrawl.FacebookImage{Id: imageId}
+	result := pb.FacebookImage{Id: imageId}
 
-	collector.OnHTML("a", func(element *colly.HTMLElement) {
+	collector.OnHTML("a[href*=\"fbcdn\"]", func(element *colly.HTMLElement) {
 		result.Url = element.Attr("href")
 	})
 
@@ -320,11 +355,10 @@ func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
 	return err, &result
 }
 
-func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.FacebookPost) {
+func (f *Fbcolly) FetchPost(groupId int64, postId int64, commentNextCursor string) (error, *pb.FacebookPost) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	post := &fbcrawl.FacebookPost{Comments: []*fbcrawl.FacebookComment{}}
-	commentPaging := 0
+	post := &pb.FacebookPost{Comments: &pb.CommentList{Comments: []*pb.FacebookComment{}}}
 	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
 		dataElement := element.DOM.Find("div[data-ft]")
 		if dataElement.Length() > 0 {
@@ -339,9 +373,10 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 				logger.Info("Post ", result)
 				post.Id = result.TopLevelPostId
-				post.Group = &fbcrawl.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
-				post.User = &fbcrawl.FacebookUser{
-					Id:   result.ContentOwnerIdNew,
+				post.Group = &pb.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
+				userId, _ := result.ContentOwnerIdNew.Int64()
+				post.User = &pb.FacebookUser{
+					Id:   userId,
 					Name: dataElement.Find("h3 strong:first-child a").Text(),
 				}
 				post.CreatedAt = result.PageInsights[strconv.FormatInt(result.PageId, 10)].PublishTime
@@ -359,15 +394,15 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 
 				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
 				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
-				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 					i, _ := strconv.ParseInt(id, 10, 64)
-					return &fbcrawl.FacebookImage{
+					return &pb.FacebookImage{
 						Id: i,
 					}
-				})).([]*fbcrawl.FacebookImage)
+				})).([]*pb.FacebookImage)
 
 				if result.PhotoId > 0 {
-					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
+					post.ContentImage = &pb.FacebookImage{Id: result.PhotoId}
 				}
 			}
 
@@ -375,32 +410,48 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 			element.DOM.Find("h3 + div + div + div").Parent().Parent().Each(func(i int, selection *goquery.Selection) {
 				//author
 				commentId, _ := strconv.ParseInt(selection.AttrOr("id", ""), 10, 64)
-				logger.Info("comment", commentId)
-				createdAtWhenResult, _ := f.w.Parse(selection.Find("abbr").Text(), time.Now())
-				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
-					Id:   commentId,
-					Post: &fbcrawl.FacebookPost{Id: post.Id},
-					User: &fbcrawl.FacebookUser{
-						Id:   getUserIdFromCommentHref(selection.Find("a[href*=\"#comment_form_\"]").AttrOr("href", "")),
-						Name: selection.Find("h3 > a").Text(),
-					},
-					Content:   selection.Find("h3 + div").Text(),
-					CreatedAt: createdAtWhenResult.Time.Unix(),
-				})
+				if commentId > 0 {
+					createdAtWhenResult, err := f.w.Parse(selection.Find("abbr").Text(), time.Now())
+					if err != nil {
+						logger.Error(err)
+						return
+					}
+					parsed, err := url.Parse(selection.Find("h3 > a").AttrOr("href", ""))
+					if err != nil {
+						logger.Error(err)
+						return
+					}
+					if len(parsed.Path) == 0 {
+						logger.Error("Empty path for commentId ", commentId)
+						return
+					}
+					if len(parsed.Path) > 1 {
+						post.Comments.Comments = append(post.Comments.Comments, &pb.FacebookComment{
+							Id:   commentId,
+							Post: &pb.FacebookPost{Id: post.Id},
+							User: &pb.FacebookUser{
+								Username: parsed.Path[1:],
+								Name:     selection.Find("h3 > a").Text(),
+							},
+							Content:   selection.Find("h3 + div").Text(),
+							CreatedAt: createdAtWhenResult.Time.Unix(),
+						})
+					}
+				}
 			})
 
 		}
 	})
 
 	collector.OnHTML("div[id*=\"see_prev_\"] > a", func(element *colly.HTMLElement) {
-		if commentPaging < 3 {
-			logger.Info("Comment paging", commentPaging)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-			commentPaging = commentPaging + 1
-		}
+		post.Comments.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
+	if len(commentNextCursor) > 0 {
+		err = collector.Visit(commentNextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
 	return err, post
 }
 
@@ -409,13 +460,36 @@ func (f *Fbcolly) LoginWithCookies(cookies string) error {
 	return collector.SetCookies("https://mbasic.facebook.com/", storage.UnstringifyCookies(cookies))
 }
 
+func (f *Fbcolly) FetchMyGroups() (error, *pb.FacebookGroupList) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result := &pb.FacebookGroupList{Groups: []*pb.FacebookGroup{}}
+
+	collector.OnHTML("li table a", func(element *colly.HTMLElement) {
+		result.Groups = append(result.Groups, &pb.FacebookGroup{
+			Id:   getNumberFromText(element.Attr("href")),
+			Name: element.Text,
+		})
+	})
+
+	err = collector.Visit("https://mbasic.facebook.com/groups/?seemore")
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, result
+}
+
 //func getUsernameFromHref(href string) string {
 //	return regexp.MustCompile("/([\\d\\w.]+).*").FindStringSubmatch(href)[1]
 //}
 
 func getUserIdFromCommentHref(href string) int64 {
-	id, _ := strconv.ParseInt(regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)[1], 10, 64)
-	return id
+	match := regexp.MustCompile("\\d+:(\\d+)\\d+").FindStringSubmatch(href)
+	if len(match) > 0 {
+		id, _ := strconv.ParseInt(match[1], 10, 64)
+		return id
+	}
+	return 0
 }
 
 func getUrlFromRedirectHref(href string) string {
@@ -430,17 +504,19 @@ func getImageIdFromHref(href string) int64 {
 }
 
 func getNumberFromText(text string) int64 {
-	logger.Error("reaction", text)
+	logger.Info("getNumberFromText ", text)
 	if len(text) > 0 {
-		match := regexp.MustCompile("(\\d*)\\s?([km]?)").FindStringSubmatch(text)
-		count, _ := strconv.ParseInt(match[1], 10, 64)
-		switch match[2] {
-		case "k":
-			count *= 1000
-		case "m":
-			count *= 1000000
+		match := regexp.MustCompile("(\\d+)\\s?([km]?)").FindStringSubmatch(text)
+		if len(match) > 0 {
+			count, _ := strconv.ParseInt(match[1], 10, 64)
+			switch match[2] {
+			case "k":
+				count *= 1000
+			case "m":
+				count *= 1000000
+			}
+			return count
 		}
-		return count
 	}
 	return 0
 }