fbcrawl-colly 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e43a203c1c38131f0d9eb32b67a1859bd1530aac9249751ff0329a5901388f6
-  data.tar.gz: f3520762f00d2b1a2d475a9b97d2141fe8cd98faf718ccf3d8a797e61489f976
+  metadata.gz: 0fc0e07942d352bb9b49c93a106d138bc9946f8e1943bcd81f6181082e79c413
+  data.tar.gz: 1e31f7fe0bc3bf83c82b90d84080a257734cfe63eaa4d79fdb89aa52204e2eb4
 SHA512:
-  metadata.gz: 2560aa6a3239671a14f82a95034bbad03684dd44b0fc32cb2092b404678b2ec0e124a33db64a92f5c22f4cd3d9d94a42ccc40a6ab2e1e362088d0f02f05d22f3
-  data.tar.gz: 305f3d463d75bc3edcfcd1b5a985d9a9882e990e850786132eb1b6d4e93f4e3e31a2c0e9aeb23141cc0633cd1d5ad8e1d88ec4c0e3e4d2d201a6298a7c8699c6
+  metadata.gz: 55d00db7f51b078c1ca7c46a59b06eb4d62b1859fac6e11a2da3c117f7c2dd3958d442559f9b84573f4d4f5e2d02a77539287d5628ff60f421405a3e2910e2b7
+  data.tar.gz: 550b26405d7bbd13356f1ca7ef30edfa682c04364f2f8fc409269c74fe25e8cb0535464d395083109b04e17de129bd851da3f035dd2dec6af2095ed357a1ddd7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fbcrawl-colly (0.2.3)
+    fbcrawl-colly (0.2.4)
       ffi
       google-protobuf
 
@@ -9,7 +9,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     ffi (1.13.1)
-    google-protobuf (3.12.4)
+    google-protobuf (3.13.0)
     minitest (5.14.1)
     rake (12.3.3)
     rake-compiler (1.1.1)

data/fbcolly/fbcolly.go

@@ -112,20 +112,23 @@ func (f *Fbcolly) Login(email string, password string, otp string) (string, erro
 
 	logger.Info("Login using email", email)
 	loggedIn := false
-
+	firstLogin := true
 	collector.OnHTML("#login_form", func(element *colly.HTMLElement) {
-		logger.Info("OnHTML login_form")
-		loginURL, err, reqMap := getForm(element, err)
-		if err != nil {
-			logger.Error(err)
-			return
-		}
-		reqMap["email"] = email
-		reqMap["pass"] = password
-		logger.Info("req map:", reqMap)
-		err = collector.Post(loginURL, reqMap)
-		if err != nil {
-			logger.Error("post err:", err)
+		if firstLogin {
+			firstLogin = false
+			logger.Info("OnHTML login_form")
+			loginURL, err, reqMap := getForm(element, err)
+			if err != nil {
+				logger.Error(err)
+				return
+			}
+			reqMap["email"] = email
+			reqMap["pass"] = password
+			logger.Info("req map:", reqMap)
+			err = collector.Post(loginURL, reqMap)
+			if err != nil {
+				logger.Error("post err:", err)
+			}
 		}
 	})
 
@@ -256,6 +259,25 @@ func (f *Fbcolly) FetchGroupFeed(groupId int64) (error, *fbcrawl.FacebookPostLis
 	return err, &fbcrawl.FacebookPostList{Posts: result}
 }
 
+func (f *Fbcolly) FetchGroupInfo(groupId int64) (error, *fbcrawl.FacebookGroup) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	result :=  &fbcrawl.FacebookGroup{Id: groupId}
+
+	collector.OnHTML("a[href=\"#groupMenuBottom\"] h1", func(element *colly.HTMLElement) {
+		result.Name = element.Text
+	})
+	collector.OnHTML("a[href*=\"view=member\"]", func(element *colly.HTMLElement) {
+		result.MemberCount, _ = strconv.ParseInt(element.DOM.Closest("tr").Find("td:last-child").Text(),10,64)
+	})
+
+	err = collector.Visit(fmt.Sprintf("https://mbasic.facebook.com/groups/%d?view=info", groupId))
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, result
+}
+
 func (f *Fbcolly) FetchContentImages(postId int64) (error, *fbcrawl.FacebookImageList) {
 	collector := f.collector.Clone()
 	err := setupSharedCollector(collector)

data/fbcrawl.proto

@@ -7,6 +7,7 @@ option go_package = "./fbcrawl;fbcrawl";
 message FacebookGroup {
   int64 id = 1;
   string name = 2;
+  int64 member_count = 3;
 }
 
 message FacebookUser {

data/go.sum

@@ -20,6 +20,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=

data/lib/fbcrawl_colly/colly.rb

@@ -19,6 +19,13 @@ module FbcrawlColly
       FbcrawlColly::FFI.LoginWithCookies(@colly, cookies)
     end
 
+    def fetch_group_info(group_id)
+      s, ptr = FbcrawlColly::FFI.FetchGroupInfo(@colly, group_id)
+      list = FbcrawlColly::FacebookGroup.decode(s)
+      FbcrawlColly::FFI.free(ptr)
+      list
+    end
+
     def fetch_group_feed(group_id)
       s, ptr = FbcrawlColly::FFI.FetchGroupFeed(@colly, group_id)
       list = FbcrawlColly::FacebookPostList.decode(s)

data/lib/fbcrawl_colly/ffi.rb

@@ -9,6 +9,7 @@ module FbcrawlColly::FFI
   attach_function :FreeColly, [:pointer], :pointer
   attach_function :Login, [:pointer, :string, :string], :strptr
   attach_function :LoginWithCookies, [:pointer, :string], :void
+  attach_function :FetchGroupInfo, [:pointer, :int64], :strptr
   attach_function :FetchGroupFeed, [:pointer, :int64], :strptr
   attach_function :FetchPost, [:pointer, :int64, :int64], :strptr
   attach_function :FetchContentImages, [:pointer, :int64], :strptr

data/lib/fbcrawl_colly/version.rb

@@ -1,3 +1,3 @@
 module FbcrawlColly
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/main.go

@@ -58,6 +58,13 @@ func LoginWithCookies(pointer unsafe.Pointer, cookies *C.char) {
 	p.LoginWithCookies(C.GoString(cookies))
 }
 
+//export FetchGroupInfo
+func FetchGroupInfo(pointer unsafe.Pointer, groupId int64) unsafe.Pointer {
+	p := (*fbcolly.Fbcolly)(pointer)
+	_, groupInfo := p.FetchGroupInfo(groupId)
+	marshaled, _ := proto.Marshal(groupInfo)
+	return C.CBytes(append(marshaled, 0))
+}
 //export FetchGroupFeed
 func FetchGroupFeed(pointer unsafe.Pointer, groupId int64) unsafe.Pointer {
 	p := (*fbcolly.Fbcolly)(pointer)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fbcrawl-colly
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Duy Le
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-14 00:00:00.000000000 Z
+date: 2020-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi