fbcrawl-colly 0.2.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a3dd903339ecec17525ca24b86324664458ef884f70065f611dc5bc15caf0a6
-  data.tar.gz: 7d7fbfd082240d21fc66f100eb530f81579b62c715181a10b95b1c70f5ad1d98
+  metadata.gz: 00d52fcc50bd651adaa80cf6102d7cc3997a07605ebb91fd33006b6e535ed6fc
+  data.tar.gz: 24388fa7c98d6dfbe58ba195c330ea3c363f6ef8e586873dd9e485ad29907a6c
 SHA512:
-  metadata.gz: c5a3b0dcd89eb3b94adcc797785984e04a122a51f9135a37aed10aee0607dd357fe99ebd69e8f2eb0ebeaf05ff4c1f0bf21c4713d173065f4782dbeb192c365e
-  data.tar.gz: d01e881175776aa5039859b1b11d723c3a577a4a68f54b6bb550d0c8c90c87d89004a94b05f21846763d56687d65bc9132e602f61a94fcebb25e934dca93ab40
+  metadata.gz: 15622279b09030f929218482add87878b58fdcb1e89e1ab8a1124531435707dbe26367fee75204a55fc88f8486c47cbb6ddfdb86035aa24694b1b7d6639aaecc
+  data.tar.gz: df50b30ad0234c824505cca54fdf33d0be54c4725c684f0317b1eba81f89d22d7b434bec4e9373b3d43dd830dc17693ac2fb785e75cefc71c6cbeefc44da2616

data/.gitignore

@@ -12,8 +12,6 @@
 /parse.log
 last.html
 *.db
-/fbcrawl/fbcrawl.pb.go
-/lib/fbcrawl_pb.rb
 
 mkmf.log
 .rakeTasks

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.14-alpine
+RUN apk add --no-cache git build-base tzdata
+
+RUN mkdir -p /app
+WORKDIR /app
+ADD ./go.mod /app
+ADD ./go.sum /app
+ADD ./ /app
+RUN go get
+
+
+ENV PORT 3000
+RUN go build -o server qnetwork.net/fbcrawl
+ENTRYPOINT ["./server"]

data/Gemfile.lock

@@ -1,19 +1,21 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.1)
-      ffi
+    fbcrawl-colly (1.0.0)
       google-protobuf
+      grpc
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ffi (1.13.1)
-    google-protobuf (3.12.4)
+    google-protobuf (3.13.0-universal-darwin)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    grpc (1.30.2-universal-darwin)
+      google-protobuf (~> 3.12)
+      googleapis-common-protos-types (~> 1.0)
     minitest (5.14.1)
     rake (12.3.3)
-    rake-compiler (1.1.1)
-      rake
 
 PLATFORMS
   ruby
@@ -22,7 +24,6 @@ DEPENDENCIES
   fbcrawl-colly!
   minitest (~> 5.0)
   rake (~> 12.0)
-  rake-compiler
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -2,7 +2,7 @@
 
 This project is to crawl mbasic.facebook.com using GO Colly. 
 
-Ruby gem will available to use
+Ruby gem is only client for GRPC colly service
 
 ## Installation
 

data/fbcrawl-colly.gemspec

@@ -25,12 +25,8 @@ Gem::Specification.new do |spec|
   end
   # spec.bindir        = "exe"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.extensions = [
-      'ext/fbcrawl_colly/extconf.rb'
-  ]
-  spec.require_paths = ["lib"]
+  spec.require_paths = %w[lib lib/pb]
 
-  spec.add_runtime_dependency 'ffi'
   spec.add_runtime_dependency 'google-protobuf'
-  spec.add_development_dependency 'rake-compiler'
+  spec.add_runtime_dependency 'grpc'
 end

data/fbcrawl.proto

@@ -1,12 +1,79 @@
 syntax = "proto3";
 
 package fbcrawl_colly;
-option go_package = "./fbcrawl;fbcrawl";
+option go_package = "./fbcrawl/pb;pb";
+
+service Grpc {
+  // Sends a greeting
+  rpc Init (Empty) returns (Pointer) {}
+  rpc FreeColly (Pointer) returns (Empty) {}
+  rpc Login  (LoginRequest) returns (LoginResponse) {}
+  rpc LoginWithCookies  (LoginWithCookiesRequest) returns (Empty) {}
+  rpc FetchGroupInfo  (FetchGroupInfoRequest) returns (FacebookGroup) {}
+  rpc FetchGroupFeed  (FetchGroupFeedRequest) returns (FacebookPostList) {}
+  rpc FetchPost  (FetchPostRequest) returns (FacebookPost) {}
+  rpc FetchContentImages  (FetchContentImagesRequest) returns (FacebookImageList) {}
+  rpc FetchImageUrl  (FetchImageUrlRequest) returns (FacebookImage) {}
+}
+
+message Empty {
+
+}
+
+message Pointer {
+  int64 address = 1;
+}
+
+message LoginRequest {
+  Pointer pointer = 1;
+  string email = 2;
+  string password = 3;
+  string totp_secret = 4;
+}
+
+message LoginResponse {
+  string cookies = 1;
+}
+
+message LoginWithCookiesRequest {
+  Pointer pointer = 1;
+  string cookies = 2;
+}
+
+message FetchGroupInfoRequest {
+  Pointer pointer = 1;
+  string group_username = 2;
+}
+
+message FetchGroupFeedRequest {
+  Pointer pointer = 1;
+  int64 group_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchPostRequest {
+  Pointer pointer = 1;
+  int64 group_id = 2;
+  int64 post_id = 3;
+  string comment_next_cursor = 4;
+}
+
+message FetchContentImagesRequest {
+  Pointer pointer = 1;
+  int64 post_id = 2;
+  string next_cursor = 3;
+}
+
+message FetchImageUrlRequest {
+  Pointer pointer = 1;
+  int64 image_id = 2;
+}
 
 // The request message containing the user's name.
 message FacebookGroup {
   int64 id = 1;
   string name = 2;
+  int64 member_count = 3;
 }
 
 message FacebookUser {
@@ -19,12 +86,18 @@ message FacebookPost {
   FacebookGroup group = 2;
   FacebookUser user = 3;
   string content = 4;
+  CommentList comments = 5;
   string content_link = 6;
-  FacebookImage content_image = 8;
   repeated FacebookImage content_images = 7;
-  repeated FacebookComment comments = 5;
+  FacebookImage content_image = 8;
   int64 created_at = 9;
   int64 reaction_count = 10;
+  int64 comment_count = 11;
+}
+
+message CommentList {
+  repeated FacebookComment comments = 5;
+  string next_cursor = 12;
 }
 
 message FacebookImage {
@@ -42,8 +115,10 @@ message FacebookComment {
 
 message FacebookPostList {
   repeated FacebookPost posts = 1;
+  string next_cursor = 2;
 }
 
 message FacebookImageList {
   repeated FacebookImage images = 1;
+  string next_cursor = 2;
 }

data/{fbcolly → fbcrawl}/fbcolly.go

@@ -6,16 +6,18 @@ import (
 	"errors"
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
-	"github.com/gocolly/colly"
-	"github.com/gocolly/colly/extensions"
-	"github.com/gocolly/colly/storage"
+	"github.com/gocolly/colly/v2"
+	"github.com/gocolly/colly/v2/debug"
+	"github.com/gocolly/colly/v2/extensions"
+	"github.com/gocolly/colly/v2/storage"
 	"github.com/google/logger"
 	"github.com/olebedev/when"
 	"github.com/olebedev/when/rules/common"
 	"github.com/olebedev/when/rules/en"
 	"github.com/thoas/go-funk"
+	"github.com/xlzd/gotp"
 	"net/url"
-	"qnetwork.net/fbcrawl/fbcrawl"
+	"qnetwork.net/fbcrawl/fbcrawl/pb"
 	"regexp"
 	"strconv"
 	"strings"
@@ -42,7 +44,7 @@ type FbDataFt struct {
 }
 
 func sharedOnRequest(request *colly.Request) {
-	logger.Info("OnRequest")
+	logger.Info("OnRequest ", request.URL)
 	//request.Headers.Set("Host", "facebook.com")
 	request.Headers.Set("Accept-Language", "en-US,en;q=0.9")
 	request.Headers.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
@@ -62,9 +64,10 @@ func sharedOnRequest(request *colly.Request) {
 func setupSharedCollector(collector *colly.Collector) error {
 	var err error
 	extensions.Referer(collector)
-
+	collector.AllowURLRevisit = true
 	collector.OnRequest(sharedOnRequest)
 	collector.OnResponse(sharedOnResponse)
+	collector.SetDebugger(&debug.LogDebugger{})
 	collector.OnError(func(resp *colly.Response, errHttp error) {
 		err = errHttp
 		logger.Error("OnError", err)
@@ -106,26 +109,29 @@ func New() *Fbcolly {
 	return &f
 }
 
-func (f *Fbcolly) Login(email string, password string, otp string) (string, error) {
+func (f *Fbcolly) Login(email string, password string, totpSecret string) (string, error) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
 
 	logger.Info("Login using email", email)
 	loggedIn := false
-
+	firstLogin := true
 	collector.OnHTML("#login_form", func(element *colly.HTMLElement) {
-		logger.Info("OnHTML login_form")
-		loginURL, err, reqMap := getForm(element, err)
-		if err != nil {
-			logger.Error(err)
-			return
-		}
-		reqMap["email"] = email
-		reqMap["pass"] = password
-		logger.Info("req map:", reqMap)
-		err = collector.Post(loginURL, reqMap)
-		if err != nil {
-			logger.Error("post err:", err)
+		if firstLogin {
+			firstLogin = false
+			logger.Info("OnHTML login_form")
+			loginURL, err, reqMap := getForm(element, err)
+			if err != nil {
+				logger.Error(err)
+				return
+			}
+			reqMap["email"] = email
+			reqMap["pass"] = password
+			logger.Info("req map:", reqMap)
+			err = collector.Post(loginURL, reqMap)
+			if err != nil {
+				logger.Error("post err:", err)
+			}
 		}
 	})
 
@@ -152,9 +158,12 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 			//logger.Info("Please input OTP")
 			//reader := bufio.NewReader(os.Stdin)
 			//code, _ := reader.ReadString('\n')
-			code := otp[0:6]
-			reqMap["approvals_code"] = code
-			shouldSubmit = true
+			if len(totpSecret) > 0 {
+				code := gotp.NewDefaultTOTP(totpSecret).Now()
+				reqMap["approvals_code"] = code
+				shouldSubmit = true
+			}
+
 		} else {
 			logger.Info("OnHTML Only Continue checkpoint")
 
@@ -188,68 +197,124 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 
 }
 
-func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostList) {
+func (f *Fbcolly) FetchGroupFeed(groupId int64, nextCursor string) (error, *pb.FacebookPostList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookPost
+	result := pb.FacebookPostList{Posts: []*pb.FacebookPost{}}
 
 	collector.OnHTML("#m_group_stories_container > :last-child a", func(element *colly.HTMLElement) {
-		currentPage++
-		if currentPage < 3 {
-			logger.Info("Will fetch page", currentPage)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
+	})
+	collector.OnHTML("div[role=\"article\"]", func(element *colly.HTMLElement) {
+		dataElement := element
+		post := &pb.FacebookPost{}
+		var fbDataFt FbDataFt
+		jsonData := dataElement.Attr("data-ft")
+
+		logger.Info(jsonData)
+		err = json.Unmarshal([]byte(jsonData), &fbDataFt)
+		if err != nil {
+			logger.Error(err)
+			return
+		}
+		logger.Info("Post ", fbDataFt)
+		post.Id = fbDataFt.TopLevelPostId
+		post.Group = &pb.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		post.User = &pb.FacebookUser{
+			Id:   fbDataFt.ContentOwnerIdNew,
+			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
+		}
+		post.CreatedAt = fbDataFt.PageInsights[strconv.FormatInt(fbDataFt.PageId, 10)].PublishTime
+		//Content
+
+		//NO BACKGROUND TEXT ONLY
+		post.Content = strings.Join(dataElement.DOM.Find("p").Map(func(i int, selection *goquery.Selection) string {
+			return selection.Text()
+		}), "\n")
+
+		if len(post.Content) == 0 {
+			// TEXT WITH BACKGROUND
+			post.Content = dataElement.DOM.Find("div[style*=\"background-image:url\"]").Text()
 		}
+
+		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
+		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
+		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
+			i, _ := strconv.ParseInt(id, 10, 64)
+			return &pb.FacebookImage{
+				Id: i,
+			}
+		})).([]*pb.FacebookImage)
+
+		if fbDataFt.PhotoId > 0 {
+			post.ContentImage = &pb.FacebookImage{Id: fbDataFt.PhotoId}
+		}
+		result.Posts = append(result.Posts, post)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	}
 
-	collector.OnXML("//a[text()=\"Full Story\"]", func(element *colly.XMLElement) {
-		logger.Info("Post found at", element.Attr("href"))
-		u, _ := url.Parse("http://mbasic.facebook.com" + element.Attr("href"))
-		postId, _ := strconv.ParseInt(u.Query().Get("id"), 10, 64)
-		result = append(result, &fbcrawl.FacebookPost{
-			Id:    postId,
-			Group: &fbcrawl.FacebookGroup{Id: groupId},
-		})
-		//f.detailCollector.Visit(url)
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &result
+}
+
+func (f *Fbcolly) FetchGroupInfo(groupIdOrUsername string) (error, *pb.FacebookGroup) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result := &pb.FacebookGroup{}
+
+	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
+		result.Name = element.Text
+	})
+	collector.OnHTML("a[href*=\"view=member\"]", func(element *colly.HTMLElement) {
+		result.Id = getNumberFromText(element.Attr("href"))
+		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(), 10, 64)
 	})
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%s?view=info", groupIdOrUsername))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookPostList{Posts: result}
+	return err, result
 }
 
-func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
+func (f *Fbcolly) FetchContentImages(postId int64, nextCursor string) (error, *pb.FacebookImageList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	currentPage := 1
-	var result []*fbcrawl.FacebookImage
+	result := pb.FacebookImageList{Images: []*pb.FacebookImage{}}
 
 	collector.OnHTML("a[href*=\"/media/set/\"]", func(element *colly.HTMLElement) {
-		currentPage++
-		logger.Info("Will fetch page", currentPage)
-		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		result.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
 
 	collector.OnHTML("a[href*=\"/photo.php\"]", func(element *colly.HTMLElement) {
-		result = append(result, &fbcrawl.FacebookImage{
+		result.Images = append(result.Images, &pb.FacebookImage{
 			Id: getImageIdFromHref(element.Attr("href")),
 		})
 		//f.detailCollector.Visit(url)
 	})
+	if len(nextCursor) > 0 {
+		err = collector.Visit(nextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookImageList{Images: result}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
+func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *pb.FacebookImage) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	result := fbcrawl.FacebookImage{Id: imageId}
+	result := pb.FacebookImage{Id: imageId}
 
 	collector.OnHTML("a", func(element *colly.HTMLElement) {
 		result.Url = element.Attr("href")
@@ -262,11 +327,10 @@ func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
 	return err, &result
 }
 
-func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.FacebookPost) {
+func (f *Fbcolly) FetchPost(groupId int64, postId int64, commentNextCursor string) (error, *pb.FacebookPost) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	post := &fbcrawl.FacebookPost{Comments: []*fbcrawl.FacebookComment{}}
-	commentPaging := 0
+	post := &pb.FacebookPost{Comments: &pb.CommentList{Comments: []*pb.FacebookComment{}}}
 	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
 		dataElement := element.DOM.Find("div[data-ft]")
 		if dataElement.Length() > 0 {
@@ -281,8 +345,8 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 				logger.Info("Post ", result)
 				post.Id = result.TopLevelPostId
-				post.Group = &fbcrawl.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
-				post.User = &fbcrawl.FacebookUser{
+				post.Group = &pb.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
+				post.User = &pb.FacebookUser{
 					Id:   result.ContentOwnerIdNew,
 					Name: dataElement.Find("h3 strong:first-child a").Text(),
 				}
@@ -300,21 +364,17 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				}
 
 				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
-				post.ReactionCount = getReactionFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
-				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
+				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *pb.FacebookImage {
 					i, _ := strconv.ParseInt(id, 10, 64)
-					return &fbcrawl.FacebookImage{
+					return &pb.FacebookImage{
 						Id: i,
 					}
-				})).([]*fbcrawl.FacebookImage)
+				})).([]*pb.FacebookImage)
 
 				if result.PhotoId > 0 {
-					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
+					post.ContentImage = &pb.FacebookImage{Id: result.PhotoId}
 				}
-
-				logger.Info("content", strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
-					return selection.Text()
-				}), "\n"))
 			}
 
 			//Comment
@@ -323,10 +383,10 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 				commentId, _ := strconv.ParseInt(selection.AttrOr("id", ""), 10, 64)
 				logger.Info("comment", commentId)
 				createdAtWhenResult, _ := f.w.Parse(selection.Find("abbr").Text(), time.Now())
-				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
+				post.Comments.Comments = append(post.Comments.Comments, &pb.FacebookComment{
 					Id:   commentId,
-					Post: &fbcrawl.FacebookPost{Id: post.Id},
-					User: &fbcrawl.FacebookUser{
+					Post: &pb.FacebookPost{Id: post.Id},
+					User: &pb.FacebookUser{
 						Id:   getUserIdFromCommentHref(selection.Find("a[href*=\"#comment_form_\"]").AttrOr("href", "")),
 						Name: selection.Find("h3 > a").Text(),
 					},
@@ -339,14 +399,14 @@ func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.Facebo
 	})
 
 	collector.OnHTML("div[id*=\"see_prev_\"] > a", func(element *colly.HTMLElement) {
-		if commentPaging < 3 {
-			logger.Info("Comment paging", commentPaging)
-			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
-			commentPaging = commentPaging + 1
-		}
+		post.Comments.NextCursor = "http://mbasic.facebook.com" + element.Attr("href")
 	})
+	if len(commentNextCursor) > 0 {
+		err = collector.Visit(commentNextCursor)
+	} else {
+		err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
+	}
 
-	err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
 	return err, post
 }
 
@@ -360,8 +420,12 @@ func (f *Fbcolly) LoginWithCookies(cookies string) error {
 //}
 
 func getUserIdFromCommentHref(href string) int64 {
-	id, _ := strconv.ParseInt(regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)[1], 10, 64)
-	return id
+	match := regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)
+	if len(match) > 0 {
+		id, _ := strconv.ParseInt(match[1], 10, 64)
+		return id
+	}
+	return 0
 }
 
 func getUrlFromRedirectHref(href string) string {
@@ -375,11 +439,20 @@ func getImageIdFromHref(href string) int64 {
 	return i
 }
 
-func getReactionFromText(text string) int64 {
+func getNumberFromText(text string) int64 {
 	logger.Error("reaction", text)
 	if len(text) > 0 {
-		id, _ := strconv.ParseInt(regexp.MustCompile("(\\d+)").FindStringSubmatch(text)[1], 10, 64)
-		return id
+		match := regexp.MustCompile("(\\d+)\\s?([km]?)").FindStringSubmatch(text)
+		if len(match) > 0 {
+			count, _ := strconv.ParseInt(match[1], 10, 64)
+			switch match[2] {
+			case "k":
+				count *= 1000
+			case "m":
+				count *= 1000000
+			}
+			return count
+		}
 	}
 	return 0
 }