fbcrawl-colly 0.1.0 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96d233862f21d1dff4a447d57f0f4a4ee6311506d7b5cf1990d75bf92043defa
-  data.tar.gz: a238ee4a34eac5f33fa90bba0d5bdb22169bd0b54034682f8fbb25c1ca4de039
+  metadata.gz: 0fc0e07942d352bb9b49c93a106d138bc9946f8e1943bcd81f6181082e79c413
+  data.tar.gz: 1e31f7fe0bc3bf83c82b90d84080a257734cfe63eaa4d79fdb89aa52204e2eb4
 SHA512:
-  metadata.gz: 36d58d4ef38bf94164f94fe0a841171cc5bdd12a4f2503e818858d77cae029e1fc79cddd3213b2bb9e130e70833703df97ad18700d92c2cc3e31401b8e1f1443
-  data.tar.gz: 5c56a623910f87369812c0d74908e618490b4aeead9d22b055f8076ca040ba9fe5fff0bb380c44e7e435e0d83a13823b259b0f6c57233d7fb3ec38ff83de0832
+  metadata.gz: 55d00db7f51b078c1ca7c46a59b06eb4d62b1859fac6e11a2da3c117f7c2dd3958d442559f9b84573f4d4f5e2d02a77539287d5628ff60f421405a3e2910e2b7
+  data.tar.gz: 550b26405d7bbd13356f1ca7ef30edfa682c04364f2f8fc409269c74fe25e8cb0535464d395083109b04e17de129bd851da3f035dd2dec6af2095ed357a1ddd7

data/.gitignore

@@ -14,3 +14,6 @@ last.html
 *.db
 /fbcrawl/fbcrawl.pb.go
 /lib/fbcrawl_pb.rb
+
+mkmf.log
+.rakeTasks

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    fbcrawl-colly (0.2.4)
+      ffi
+      google-protobuf
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ffi (1.13.1)
+    google-protobuf (3.13.0)
+    minitest (5.14.1)
+    rake (12.3.3)
+    rake-compiler (1.1.1)
+      rake
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fbcrawl-colly!
+  minitest (~> 5.0)
+  rake (~> 12.0)
+  rake-compiler
+
+BUNDLED WITH
+   2.1.4

data/Rakefile

@@ -7,4 +7,13 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
+task :fbcrawl_colly do
+  Dir.chdir("./ext/fbcrawl_colly/") do
+    require './extconf'
+    `make`
+  end
+end
+
+task :compile => [:fbcrawl_colly]
+task :test => :compile
 task :default => :test

data/ext/fbcrawl_colly/extconf.rb

@@ -1,5 +1,6 @@
 require 'mkmf'
-MakeMakefile::find_executable 'go'
-MakeMakefile::find_executable 'protoc'
-MakeMakefile::find_executable 'protoc-gen-go'
-$makefile_created = true
+requirement_passed = true
+requirement_passed &&= MakeMakefile::find_executable 'go'
+requirement_passed &&= MakeMakefile::find_executable 'protoc'
+requirement_passed &&= MakeMakefile::find_executable 'protoc-gen-go'
+$makefile_created = requirement_passed

data/fbcolly/fbcolly.go

@@ -10,14 +10,35 @@ import (
 	"github.com/gocolly/colly/extensions"
 	"github.com/gocolly/colly/storage"
 	"github.com/google/logger"
+	"github.com/olebedev/when"
+	"github.com/olebedev/when/rules/common"
+	"github.com/olebedev/when/rules/en"
+	"github.com/thoas/go-funk"
 	"net/url"
 	"qnetwork.net/fbcrawl/fbcrawl"
 	"regexp"
+	"strconv"
 	"strings"
+	"time"
 )
 
 type Fbcolly struct {
 	collector *colly.Collector
+	w         *when.Parser
+}
+type FbDataPostContext struct {
+	PublishTime int64 `json:"publish_time"`
+}
+type FbDataInsight struct {
+	FbDataPostContext `json:"post_context"`
+}
+type FbDataFt struct {
+	ContentOwnerIdNew    int64                    `json:"content_owner_id_new"`
+	PhotoAttachmentsList []string                 `json:"photo_attachments_list"`
+	PhotoId              int64                    `json:"photo_id,string"`
+	PageId               int64                    `json:"page_id,string"`
+	TopLevelPostId       int64                    `json:"top_level_post_id,string"`
+	PageInsights         map[string]FbDataInsight `json:"page_insights"`
 }
 
 func sharedOnRequest(request *colly.Request) {
@@ -41,7 +62,7 @@ func sharedOnRequest(request *colly.Request) {
 func setupSharedCollector(collector *colly.Collector) error {
 	var err error
 	extensions.Referer(collector)
-
+	collector.AllowURLRevisit = true
 	collector.OnRequest(sharedOnRequest)
 	collector.OnResponse(sharedOnResponse)
 	collector.OnError(func(resp *colly.Response, errHttp error) {
@@ -79,39 +100,46 @@ func getForm(element *colly.HTMLElement, err error) (string, error, map[string]s
 func New() *Fbcolly {
 	f := Fbcolly{}
 	f.collector = colly.NewCollector()
+	f.w = when.New(nil)
+	f.w.Add(en.All...)
+	f.w.Add(common.All...)
 	return &f
 }
 
-func (f *Fbcolly) Login(email string, password string, otp string) error {
+func (f *Fbcolly) Login(email string, password string, otp string) (string, error) {
 	collector := f.collector.Clone()
-	setupSharedCollector(collector)
+	err := setupSharedCollector(collector)
 
 	logger.Info("Login using email", email)
-
-	var err error
+	loggedIn := false
+	firstLogin := true
 	collector.OnHTML("#login_form", func(element *colly.HTMLElement) {
-		logger.Info("OnHTML login_form")
-		loginURL, err, reqMap := getForm(element, err)
-		if err != nil {
-			logger.Error(err)
-			return
-		}
-		reqMap["email"] = email
-		reqMap["pass"] = password
-		logger.Info("req map:", reqMap)
-		err = collector.Post(loginURL, reqMap)
-		if err != nil {
-			logger.Error("post err:", err)
+		if firstLogin {
+			firstLogin = false
+			logger.Info("OnHTML login_form")
+			loginURL, err, reqMap := getForm(element, err)
+			if err != nil {
+				logger.Error(err)
+				return
+			}
+			reqMap["email"] = email
+			reqMap["pass"] = password
+			logger.Info("req map:", reqMap)
+			err = collector.Post(loginURL, reqMap)
+			if err != nil {
+				logger.Error("post err:", err)
+			}
 		}
 	})
 
 	collector.OnHTML("a[href=\"/login/save-device/cancel/?flow=interstitial_nux&nux_source=regular_login\"]", func(element *colly.HTMLElement) {
-		collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
 	})
 
 	collector.OnHTML("form[action=\"/login/checkpoint/\"]", func(element *colly.HTMLElement) {
 
 		checkpointUrl, err, reqMap := getForm(element, err)
+		shouldSubmit := false
 		if err != nil {
 			logger.Error(err)
 			return
@@ -121,6 +149,7 @@ func (f *Fbcolly) Login(email string, password string, otp string) error {
 			//Save Device
 			logger.Info("OnHTML Save Device checkpoint")
 			reqMap["name_action_selected"] = "dont_save"
+			shouldSubmit = true
 		} else if element.DOM.Find("input[name=\"approvals_code\"]").Length() > 0 {
 			logger.Info("OnHTML OTP checkpoint")
 			//logger.Info("Please input OTP")
@@ -128,11 +157,15 @@ func (f *Fbcolly) Login(email string, password string, otp string) error {
 			//code, _ := reader.ReadString('\n')
 			code := otp[0:6]
 			reqMap["approvals_code"] = code
+			shouldSubmit = true
 		} else {
 			logger.Info("OnHTML Only Continue checkpoint")
+
+		}
+		if shouldSubmit {
+			logger.Info("req map:", reqMap)
+			err = collector.Post(checkpointUrl, reqMap)
 		}
-		logger.Info("req map:", reqMap)
-		err = collector.Post(checkpointUrl, reqMap)
 		if err != nil {
 			logger.Error("post err:", err)
 		}
@@ -141,19 +174,24 @@ func (f *Fbcolly) Login(email string, password string, otp string) error {
 	collector.OnHTML("form[action=\"/search/\"]", func(element *colly.HTMLElement) {
 		//We're in home
 		logger.Info("I'm IN HOME, navigate to page now")
+		loggedIn = true
 	})
 
 	err = collector.Visit("https://mbasic.facebook.com/")
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	logger.Info(storage.StringifyCookies(collector.Cookies("https://mbasic.facebook.com/")))
-	//return err, storage.StringifyCookies(collector.Cookies("https://mbasic.facebook.com/"))
-	//return err, collector.getS.Cookies("https://mbasic.facebook.com/")
-	return err
+
+	if loggedIn {
+		logger.Info(storage.StringifyCookies(collector.Cookies("https://mbasic.facebook.com/")))
+		return storage.StringifyCookies(collector.Cookies("https://mbasic.facebook.com/")), err
+	} else {
+		return "", err
+	}
+
 }
 
-func (f *Fbcolly) FetchGroupFeed(groupId string) (error, *fbcrawl.FacebookPostList) {
+func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
 	currentPage := 1
@@ -163,72 +201,246 @@ func (f *Fbcolly) FetchGroupFeed(groupId string) (error, *fbcrawl.FacebookPostLi
 		currentPage++
 		if currentPage < 3 {
 			logger.Info("Will fetch page", currentPage)
-			collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		}
+	})
+	collector.OnHTML("div[role=\"article\"]", func(element *colly.HTMLElement) {
+		dataElement := element
+		post := &fbcrawl.FacebookPost{}
+		var fbDataFt FbDataFt
+		jsonData := dataElement.Attr("data-ft")
+
+		logger.Info(jsonData)
+		err = json.Unmarshal([]byte(jsonData), &fbDataFt)
+		if err != nil {
+			logger.Error(err)
+			return
+		}
+		logger.Info("Post ", fbDataFt)
+		post.Id = fbDataFt.TopLevelPostId
+		post.Group = &fbcrawl.FacebookGroup{Id: fbDataFt.PageId, Name: dataElement.DOM.Find("h3 strong:nth-child(2) a").Text()}
+		post.User = &fbcrawl.FacebookUser{
+			Id:   fbDataFt.ContentOwnerIdNew,
+			Name: dataElement.DOM.Find("h3 strong:nth-child(1) a").Text(),
+		}
+		post.CreatedAt = fbDataFt.PageInsights[strconv.FormatInt(fbDataFt.PageId, 10)].PublishTime
+		//Content
+
+		//NO BACKGROUND TEXT ONLY
+		post.Content = strings.Join(dataElement.DOM.Find("p").Map(func(i int, selection *goquery.Selection) string {
+			return selection.Text()
+		}), "\n")
+
+		if len(post.Content) == 0 {
+			// TEXT WITH BACKGROUND
+			post.Content = dataElement.DOM.Find("div[style*=\"background-image:url\"]").Text()
+		}
+
+		post.ContentLink = getUrlFromRedirectHref(dataElement.DOM.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
+		post.ReactionCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"]").Text())
+		post.CommentCount = getNumberFromText(element.DOM.Find("span[id*=\"like_\"] ~ a").Text())
+		post.ContentImages = (funk.Map(fbDataFt.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+			i, _ := strconv.ParseInt(id, 10, 64)
+			return &fbcrawl.FacebookImage{
+				Id: i,
+			}
+		})).([]*fbcrawl.FacebookImage)
+
+		if fbDataFt.PhotoId > 0 {
+			post.ContentImage = &fbcrawl.FacebookImage{Id: fbDataFt.PhotoId}
 		}
+		result = append(result, post)
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d", groupId))
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &fbcrawl.FacebookPostList{Posts: result}
+}
+
+func (f *Fbcolly) FetchGroupInfo(groupId int64) (error, *fbcrawl.FacebookGroup) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result :=  &fbcrawl.FacebookGroup{Id: groupId}
+
+	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
+		result.Name = element.Text
 	})
+	collector.OnHTML("a[href*=\"view=member\"]", func(element *colly.HTMLElement) {
+		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(),10,64)
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d?view=info", groupId))
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, result
+}
 
-	//TODO: May not need this
-	collector.OnXML("//a[text()=\"Full Story\"]", func(element *colly.XMLElement) {
-		u, _ := url.Parse("http://mbasic.facebook.com" + element.Attr("href"))
+func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	currentPage := 1
+	var result []*fbcrawl.FacebookImage
+
+	collector.OnHTML("a[href*=\"/media/set/\"]", func(element *colly.HTMLElement) {
+		currentPage++
+		logger.Info("Will fetch page", currentPage)
+		err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+	})
 
-		result = append(result, &fbcrawl.FacebookPost{
-			Id:    u.Query().Get("id"),
-			Group: &fbcrawl.FacebookGroup{Id: groupId},
+	collector.OnHTML("a[href*=\"/photo.php\"]", func(element *colly.HTMLElement) {
+		result = append(result, &fbcrawl.FacebookImage{
+			Id: getImageIdFromHref(element.Attr("href")),
 		})
 		//f.detailCollector.Visit(url)
 	})
 
-	err = collector.Visit("https://mbasic.facebook.com/groups/" + groupId)
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/media/set/?set=pcb.%d", postId))
 	if err != nil {
 		logger.Error("crawl by colly err:", err)
 	}
-	return err, &fbcrawl.FacebookPostList{Posts: result}
+	return err, &fbcrawl.FacebookImageList{Images: result}
+}
+
+func (f *Fbcolly) FetchImageUrl(imageId int64) (error, *fbcrawl.FacebookImage) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result := fbcrawl.FacebookImage{Id: imageId}
+
+	collector.OnHTML("a", func(element *colly.HTMLElement) {
+		result.Url = element.Attr("href")
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/photo/view_full_size/?fbid=%d", imageId))
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &result
 }
 
-func (f *Fbcolly) FetchPost(groupId string, postId string) (error, *fbcrawl.FacebookPost) {
+func (f *Fbcolly) FetchPost(groupId int64, postId int64) (error, *fbcrawl.FacebookPost) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)
-	post := &fbcrawl.FacebookPost{}
+	post := &fbcrawl.FacebookPost{Comments: []*fbcrawl.FacebookComment{}}
+	commentPaging := 0
 	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
 		dataElement := element.DOM.Find("div[data-ft]")
 		if dataElement.Length() > 0 {
-			var result map[string]string
+			var result FbDataFt
 			jsonData, isExist := dataElement.Attr("data-ft")
 			if isExist {
-				json.Unmarshal([]byte(jsonData), &result)
+				logger.Info(jsonData)
+				err = json.Unmarshal([]byte(jsonData), &result)
+				if err != nil {
+					logger.Error(err)
+					return
+				}
 				logger.Info("Post ", result)
-				post.Id = result["top_level_post_id"]
-				post.Group = &fbcrawl.FacebookGroup{Id: result["page_id"], Name: dataElement.Find("h3 strong:last-child a").Text()}
-				post.User = &fbcrawl.FacebookUser{Id: result["content_owner_id_new"], Name: dataElement.Find("h3 strong:first-child a").Text()}
+				post.Id = result.TopLevelPostId
+				post.Group = &fbcrawl.FacebookGroup{Id: result.PageId, Name: dataElement.Find("h3 strong:last-child a").Text()}
+				post.User = &fbcrawl.FacebookUser{
+					Id:   result.ContentOwnerIdNew,
+					Name: dataElement.Find("h3 strong:first-child a").Text(),
+				}
+				post.CreatedAt = result.PageInsights[strconv.FormatInt(result.PageId, 10)].PublishTime
 				//Content
+
+				//NO BACKGROUND TEXT ONLY
 				post.Content = strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
 					return selection.Text()
 				}), "\n")
 
-				logger.Info("content", strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
-					return selection.Text()
-				}), "\n"))
+				if len(post.Content) == 0 {
+					// TEXT WITH BACKGROUND
+					post.Content = dataElement.Find("div[style*=\"background-image:url\"]").Text()
+				}
+
+				post.ContentLink = getUrlFromRedirectHref(dataElement.Find("a[href*=\"https://lm.facebook.com/l.php\"]").AttrOr("href", ""))
+				post.ReactionCount = getNumberFromText(element.DOM.Find("div[id*=\"sentence_\"]").Text())
+				post.ContentImages = (funk.Map(result.PhotoAttachmentsList, func(id string) *fbcrawl.FacebookImage {
+					i, _ := strconv.ParseInt(id, 10, 64)
+					return &fbcrawl.FacebookImage{
+						Id: i,
+					}
+				})).([]*fbcrawl.FacebookImage)
+
+				if result.PhotoId > 0 {
+					post.ContentImage = &fbcrawl.FacebookImage{Id: result.PhotoId}
+				}
 			}
-			post.Comments = []*fbcrawl.FacebookComment{}
+
 			//Comment
 			element.DOM.Find("h3 + div + div + div").Parent().Parent().Each(func(i int, selection *goquery.Selection) {
 				//author
-				commentId := selection.AttrOr("id", "")
+				commentId, _ := strconv.ParseInt(selection.AttrOr("id", ""), 10, 64)
 				logger.Info("comment", commentId)
-				//idRegex, _ := regexp.Compile("")
-				//idRegex.FindString()
+				createdAtWhenResult, _ := f.w.Parse(selection.Find("abbr").Text(), time.Now())
 				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
 					Id:   commentId,
 					Post: &fbcrawl.FacebookPost{Id: post.Id},
 					User: &fbcrawl.FacebookUser{
-						Id:   regexp.MustCompile("/([\\d\\w.]+)").FindStringSubmatch(selection.Find("h3 > a").AttrOr("href", ""))[1],
+						Id:   getUserIdFromCommentHref(selection.Find("a[href*=\"#comment_form_\"]").AttrOr("href", "")),
 						Name: selection.Find("h3 > a").Text(),
 					},
-					Content: selection.Find("h3 + div").Text(),
+					Content:   selection.Find("h3 + div").Text(),
+					CreatedAt: createdAtWhenResult.Time.Unix(),
 				})
 			})
+
 		}
 	})
-	collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%s?view=permalink&id=%s&_rdr", groupId, postId))
+
+	collector.OnHTML("div[id*=\"see_prev_\"] > a", func(element *colly.HTMLElement) {
+		if commentPaging < 3 {
+			logger.Info("Comment paging", commentPaging)
+			err = collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+			commentPaging = commentPaging + 1
+		}
+	})
+
+	err = collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%d?view=permalink&id=%d&_rdr", groupId, postId))
 	return err, post
 }
+
+func (f *Fbcolly) LoginWithCookies(cookies string) error {
+	collector := f.collector
+	return collector.SetCookies("https://mbasic.facebook.com/", storage.UnstringifyCookies(cookies))
+}
+
+//func getUsernameFromHref(href string) string {
+//	return regexp.MustCompile("/([\\d\\w.]+).*").FindStringSubmatch(href)[1]
+//}
+
+func getUserIdFromCommentHref(href string) int64 {
+	id, _ := strconv.ParseInt(regexp.MustCompile("#comment_form_(\\d+)").FindStringSubmatch(href)[1], 10, 64)
+	return id
+}
+
+func getUrlFromRedirectHref(href string) string {
+	u, _ := url.Parse(href)
+	return u.Query().Get("u")
+}
+
+func getImageIdFromHref(href string) int64 {
+	u, _ := url.Parse(href)
+	i, _ := strconv.ParseInt(u.Query().Get("fbid"), 10, 64)
+	return i
+}
+
+func getNumberFromText(text string) int64 {
+	logger.Error("reaction", text)
+	if len(text) > 0 {
+		match := regexp.MustCompile("(\\d*)\\s?([km]?)").FindStringSubmatch(text)
+		count, _ := strconv.ParseInt(match[1], 10, 64)
+		switch match[2] {
+		case "k":
+			count *= 1000
+		case "m":
+			count *= 1000000
+		}
+		return count
+	}
+	return 0
+}

data/fbcrawl-colly.gemspec

@@ -1,4 +1,4 @@
-require_relative 'lib/fbcrawl-colly/version'
+require_relative 'lib/fbcrawl_colly/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "fbcrawl-colly"
@@ -32,4 +32,5 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'ffi'
   spec.add_runtime_dependency 'google-protobuf'
+  spec.add_development_dependency 'rake-compiler'
 end

data/fbcrawl.proto

@@ -1,33 +1,51 @@
 syntax = "proto3";
 
+package fbcrawl_colly;
 option go_package = "./fbcrawl;fbcrawl";
 
 // The request message containing the user's name.
 message FacebookGroup {
-  string id = 1;
+  int64 id = 1;
   string name = 2;
+  int64 member_count = 3;
 }
 
 message FacebookUser {
-  string id = 1;
+  int64 id = 1;
   string name = 2;
 }
 
 message FacebookPost {
-  string id = 1;
+  int64 id = 1;
   FacebookGroup group = 2;
   FacebookUser user = 3;
   string content = 4;
+  string content_link = 6;
+  FacebookImage content_image = 8;
+  repeated FacebookImage content_images = 7;
   repeated FacebookComment comments = 5;
+  int64 created_at = 9;
+  int64 reaction_count = 10;
+  int64 comment_count = 11;
+}
+
+message FacebookImage {
+  int64 id = 1;
+  string url = 2;
 }
 
 message FacebookComment {
-  string id = 1;
+  int64 id = 1;
   FacebookPost post = 2;
   FacebookUser user = 3;
   string content = 4;
+  int64 created_at = 5;
 }
 
 message FacebookPostList {
   repeated FacebookPost posts = 1;
 }
+
+message FacebookImageList {
+  repeated FacebookImage images = 1;
+}

data/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/golang/protobuf v1.4.2
 	github.com/google/logger v1.1.0
 	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/olebedev/when v0.0.0-20190311101825-c3b538a97254
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
 	github.com/thoas/go-funk v0.7.0

data/go.sum

@@ -1,4 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/AlekSi/pointer v1.0.0 h1:KWCWzsvFxNLcmM5XmiqHsGTTsuwZMsLFwWF9Y+//bNE=
+github.com/AlekSi/pointer v1.0.0/go.mod h1:1kjywbfcPFCmncIxtk6fIEub6LKrfMz3gc5QKVOSOA8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
@@ -18,6 +20,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
@@ -50,6 +54,10 @@ github.com/google/logger v1.1.0 h1:saB74Etb4EAJNH3z74CVbCKk75hld/8T0CsXKetWCwM=
 github.com/google/logger v1.1.0/go.mod h1:w7O8nrRr0xufejBlQMI83MXqRusvREoJdaAxV+CoAB4=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
+github.com/olebedev/when v0.0.0-20190311101825-c3b538a97254 h1:JYoQR67E1vv1WGoeW8DkdFs7vrIEe/5wP+qJItd5tUE=
+github.com/olebedev/when v0.0.0-20190311101825-c3b538a97254/go.mod h1:DPucAeQGDPUzYUt+NaWw6qsF5SFapWWToxEiVDh2aV0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

data/lib/fbcrawl-colly.rb

@@ -1,16 +1,4 @@
-require 'ffi'
-require 'fbcrawl_pb'
-module FbcrawlColly
-  extend FFI::Library
 
-  ffi_lib File.expand_path("../ext/fbcrawl_colly/fbcolly.so", File.dirname(__FILE__))
-  attach_function :free, [ :pointer ], :void
+module FbcrawlColly
 
-  attach_function :Init, [], :pointer
-  attach_function :Login, [:pointer, :string, :string], :void
-  attach_function :FetchGroupFeed, [:pointer, :string], :string
-  attach_function :FetchPost, [:pointer, :string, :string], :string
-  # attach_function :FetchGroup, [:pointer, :string], :pointer
-  # attach_function :Login, [:pointer, :string, :string, :string], :void
-  # attach_function :FreePointer, [:pointer], :void
 end

data/lib/fbcrawl_colly/colly.rb

@@ -0,0 +1,57 @@
+require 'ffi'
+require_relative '../fbcrawl_pb'
+require_relative './ffi'
+
+module FbcrawlColly
+  class Colly
+    def initialize
+      super
+      @colly = ::FFI::AutoPointer.new(FbcrawlColly::FFI::Init(), FbcrawlColly::FFI.method(:FreeColly))
+    end
+
+    def login(email, password)
+      s, ptr = FbcrawlColly::FFI.Login(@colly, email, password)
+      FbcrawlColly::FFI.free(ptr)
+      s
+    end
+
+    def login_with_cookies(cookies)
+      FbcrawlColly::FFI.LoginWithCookies(@colly, cookies)
+    end
+
+    def fetch_group_info(group_id)
+      s, ptr = FbcrawlColly::FFI.FetchGroupInfo(@colly, group_id)
+      list = FbcrawlColly::FacebookGroup.decode(s)
+      FbcrawlColly::FFI.free(ptr)
+      list
+    end
+
+    def fetch_group_feed(group_id)
+      s, ptr = FbcrawlColly::FFI.FetchGroupFeed(@colly, group_id)
+      list = FbcrawlColly::FacebookPostList.decode(s)
+      FbcrawlColly::FFI.free(ptr)
+      list
+    end
+
+    def fetch_post(group_id, post_id)
+      s, ptr = FbcrawlColly::FFI.FetchPost(@colly, group_id, post_id)
+      post = FbcrawlColly::FacebookPost.decode(s)
+      FbcrawlColly::FFI.free(ptr)
+      post
+    end
+
+    def fetch_content_images(post_id)
+      s, ptr = FbcrawlColly::FFI.FetchContentImages(@colly, post_id)
+      imageList = FbcrawlColly::FacebookImageList.decode(s)
+      FbcrawlColly::FFI.free(ptr)
+      imageList
+    end
+
+    def fetch_image_url(image_id)
+      s, ptr = FbcrawlColly::FFI.FetchImageUrl(@colly, image_id)
+      image = FbcrawlColly::FacebookImage.decode(s)
+      FbcrawlColly::FFI.free(ptr)
+      image
+    end
+  end
+end

data/lib/fbcrawl_colly/ffi.rb

@@ -0,0 +1,18 @@
+require 'ffi'
+module FbcrawlColly::FFI
+  extend FFI::Library
+
+  ffi_lib File.expand_path("../../ext/fbcrawl_colly/fbcolly.so", File.dirname(__FILE__))
+  attach_function :free, [ :pointer ], :void
+
+  attach_function :Init, [], :pointer
+  attach_function :FreeColly, [:pointer], :pointer
+  attach_function :Login, [:pointer, :string, :string], :strptr
+  attach_function :LoginWithCookies, [:pointer, :string], :void
+  attach_function :FetchGroupInfo, [:pointer, :int64], :strptr
+  attach_function :FetchGroupFeed, [:pointer, :int64], :strptr
+  attach_function :FetchPost, [:pointer, :int64, :int64], :strptr
+  attach_function :FetchContentImages, [:pointer, :int64], :strptr
+  attach_function :FetchImageUrl, [:pointer, :int64], :strptr
+  # attach_function :FetchGroup, [:pointer, :string], :pointer
+end

data/lib/{fbcrawl-colly → fbcrawl_colly}/version.rb

@@ -1,3 +1,3 @@
 module FbcrawlColly
-  VERSION = "0.1.0"
+  VERSION = "0.2.4"
 end

data/main.go

@@ -27,36 +27,76 @@ var password = flag.String("password", "change_me", "facebook password")
 var otp = flag.String("otp", "123456", "facebook otp")
 var groupId = flag.String("groupId", "334294967318328", "facebook group id, default is 334294967318328")
 
-var tmp = fbcolly.New()
+var allInstances = map[uintptr]*fbcolly.Fbcolly{}
 
 //export Init
 func Init() uintptr {
-	return (uintptr)(unsafe.Pointer(tmp))
+	instance := fbcolly.New()
+	ptr := (uintptr)(unsafe.Pointer(instance))
+	allInstances[ptr] = instance
+	return ptr
+}
+
+//export FreeColly
+func FreeColly(pointer unsafe.Pointer) {
+	delete(allInstances, uintptr(pointer))
 }
 
 //export Login
-func Login(pointer unsafe.Pointer, email *C.char, password *C.char) {
+func Login(pointer unsafe.Pointer, email *C.char, password *C.char) *C.char {
 	p := (*fbcolly.Fbcolly)(pointer)
-	//print(p.E)
-	p.Login(C.GoString(email), C.GoString(password), "")
+	cookies, err := p.Login(C.GoString(email), C.GoString(password), "")
+	if err == nil {
+		return C.CString(cookies)
+	}
+	return nil
 }
 
+//export LoginWithCookies
+func LoginWithCookies(pointer unsafe.Pointer, cookies *C.char) {
+	p := (*fbcolly.Fbcolly)(pointer)
+	p.LoginWithCookies(C.GoString(cookies))
+}
+
+//export FetchGroupInfo
+func FetchGroupInfo(pointer unsafe.Pointer, groupId int64) unsafe.Pointer {
+	p := (*fbcolly.Fbcolly)(pointer)
+	_, groupInfo := p.FetchGroupInfo(groupId)
+	marshaled, _ := proto.Marshal(groupInfo)
+	return C.CBytes(append(marshaled, 0))
+}
 //export FetchGroupFeed
-func FetchGroupFeed(pointer unsafe.Pointer, groupId *C.char) unsafe.Pointer {
+func FetchGroupFeed(pointer unsafe.Pointer, groupId int64) unsafe.Pointer {
 	p := (*fbcolly.Fbcolly)(pointer)
-	_, postsList := p.FetchGroupFeed(C.GoString(groupId))
+	_, postsList := p.FetchGroupFeed(groupId)
 	marshaledPostsList, _ := proto.Marshal(postsList)
 	return C.CBytes(append(marshaledPostsList, 0))
 }
 
 //export FetchPost
-func FetchPost(pointer unsafe.Pointer, groupId *C.char, postId *C.char) unsafe.Pointer {
+func FetchPost(pointer unsafe.Pointer, groupId int64, postId int64) unsafe.Pointer {
 	p := (*fbcolly.Fbcolly)(pointer)
-	_, post := p.FetchPost(C.GoString(groupId), C.GoString(postId))
+	_, post := p.FetchPost(groupId, postId)
 	marshaledPost, _ := proto.Marshal(post)
 	return C.CBytes(append(marshaledPost, 0))
 }
 
+//export FetchContentImages
+func FetchContentImages(pointer unsafe.Pointer, postId int64) unsafe.Pointer {
+	p := (*fbcolly.Fbcolly)(pointer)
+	_, imageList := p.FetchContentImages(postId)
+	marshaled, _ := proto.Marshal(imageList)
+	return C.CBytes(append(marshaled, 0))
+}
+
+//export FetchImageUrl
+func FetchImageUrl(pointer unsafe.Pointer, imageId int64) unsafe.Pointer {
+	p := (*fbcolly.Fbcolly)(pointer)
+	_, image := p.FetchImageUrl(imageId)
+	marshaled, _ := proto.Marshal(image)
+	return C.CBytes(append(marshaled, 0))
+}
+
 func main() {
 	//r := regexp.MustCompile("/([\\d\\w.]+)").FindStringSubmatch()[1]
 	//print(r.FindStringSubmatch("/liem.phamthanh.161?refid=18&__tn__=R")[1])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fbcrawl-colly
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.4
 platform: ruby
 authors:
 - Duy Le
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-05 00:00:00.000000000 Z
+date: 2020-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Crawl mbasic.facebook.com using GO Colly
 email:
 - duyleekun@gmail.com
@@ -47,9 +61,9 @@ extensions:
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -64,7 +78,9 @@ files:
 - go.mod
 - go.sum
 - lib/fbcrawl-colly.rb
-- lib/fbcrawl-colly/version.rb
+- lib/fbcrawl_colly/colly.rb
+- lib/fbcrawl_colly/ffi.rb
+- lib/fbcrawl_colly/version.rb
 - main.go
 homepage: http://github.com/duyleekun/fbcrawl-colly
 licenses:

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.7.1
-before_install: gem install bundler -v 2.1.4