fbcrawl-colly 0.2.1 → 1.0.0

data/lib/fbcrawl-colly.rb

@@ -1,4 +1 @@
-
-module FbcrawlColly
-
-end
+require_relative './fbcrawl_colly'

data/lib/fbcrawl_colly.rb

@@ -0,0 +1,5 @@
+require 'fbcrawl_services_pb'
+require 'fbcrawl_colly/client'
+module FbCrawlColly
+
+end

data/lib/fbcrawl_colly/client.rb

@@ -0,0 +1,46 @@
+module FbcrawlColly
+  class Client
+
+    def initialize(host_and_port)
+      @host_and_port = host_and_port
+      @client = new_grpc_client
+      @colly = @client.init(FbcrawlColly::Empty.new)
+
+      ObjectSpace.define_finalizer(self) do
+        new_grpc_client.free_colly(@colly)
+      end
+    end
+
+    def login(email, password, totp_secret = "")
+      s = @client.login(FbcrawlColly::LoginRequest.new(pointer: @colly, email: email, password: password, totp_secret: totp_secret)).cookies
+    end
+
+    def login_with_cookies(cookies)
+      s = @client.login_with_cookies(FbcrawlColly::LoginWithCookiesRequest.new(pointer: @colly, cookies: cookies))
+    end
+
+    def fetch_group_info(group_id_or_username)
+      s = @client.fetch_group_info(FbcrawlColly::FetchGroupInfoRequest.new(pointer: @colly, group_username: group_id_or_username))
+    end
+
+    def fetch_group_feed(group_id, next_cursor = nil)
+      s = @client.fetch_group_feed(FbcrawlColly::FetchGroupFeedRequest.new(pointer: @colly, group_id: group_id, next_cursor: next_cursor))
+    end
+
+    def fetch_post(group_id, post_id, comment_next_cursor = nil)
+      s = @client.fetch_post(FbcrawlColly::FetchPostRequest.new(pointer: @colly, group_id: group_id, post_id: post_id, comment_next_cursor: comment_next_cursor))
+    end
+
+    def fetch_content_images(post_id, next_cursor = nil)
+      s = @client.fetch_content_images(FbcrawlColly::FetchContentImagesRequest.new(pointer: @colly, post_id: post_id, next_cursor: next_cursor))
+    end
+
+    def fetch_image_url(image_id)
+      s = @client.fetch_image_url(FbcrawlColly::FetchImageUrlRequest.new(pointer: @colly, image_id: image_id))
+    end
+    private
+    def new_grpc_client
+      FbcrawlColly::Grpc::Stub.new(@host_and_port, :this_channel_is_insecure)
+    end
+  end
+end

data/lib/fbcrawl_colly/version.rb

@@ -1,3 +1,3 @@
 module FbcrawlColly
-  VERSION = "0.2.1"
+  VERSION = "1.0.0"
 end

data/lib/pb/fbcrawl_pb.rb

@@ -0,0 +1,117 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: fbcrawl.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("fbcrawl.proto", :syntax => :proto3) do
+    add_message "fbcrawl_colly.Empty" do
+    end
+    add_message "fbcrawl_colly.Pointer" do
+      optional :address, :int64, 1
+    end
+    add_message "fbcrawl_colly.LoginRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :email, :string, 2
+      optional :password, :string, 3
+      optional :totp_secret, :string, 4
+    end
+    add_message "fbcrawl_colly.LoginResponse" do
+      optional :cookies, :string, 1
+    end
+    add_message "fbcrawl_colly.LoginWithCookiesRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :cookies, :string, 2
+    end
+    add_message "fbcrawl_colly.FetchGroupInfoRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :group_username, :string, 2
+    end
+    add_message "fbcrawl_colly.FetchGroupFeedRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :group_id, :int64, 2
+      optional :next_cursor, :string, 3
+    end
+    add_message "fbcrawl_colly.FetchPostRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :group_id, :int64, 2
+      optional :post_id, :int64, 3
+      optional :comment_next_cursor, :string, 4
+    end
+    add_message "fbcrawl_colly.FetchContentImagesRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :post_id, :int64, 2
+      optional :next_cursor, :string, 3
+    end
+    add_message "fbcrawl_colly.FetchImageUrlRequest" do
+      optional :pointer, :message, 1, "fbcrawl_colly.Pointer"
+      optional :image_id, :int64, 2
+    end
+    add_message "fbcrawl_colly.FacebookGroup" do
+      optional :id, :int64, 1
+      optional :name, :string, 2
+      optional :member_count, :int64, 3
+    end
+    add_message "fbcrawl_colly.FacebookUser" do
+      optional :id, :int64, 1
+      optional :name, :string, 2
+    end
+    add_message "fbcrawl_colly.FacebookPost" do
+      optional :id, :int64, 1
+      optional :group, :message, 2, "fbcrawl_colly.FacebookGroup"
+      optional :user, :message, 3, "fbcrawl_colly.FacebookUser"
+      optional :content, :string, 4
+      optional :comments, :message, 5, "fbcrawl_colly.CommentList"
+      optional :content_link, :string, 6
+      repeated :content_images, :message, 7, "fbcrawl_colly.FacebookImage"
+      optional :content_image, :message, 8, "fbcrawl_colly.FacebookImage"
+      optional :created_at, :int64, 9
+      optional :reaction_count, :int64, 10
+      optional :comment_count, :int64, 11
+    end
+    add_message "fbcrawl_colly.CommentList" do
+      repeated :comments, :message, 5, "fbcrawl_colly.FacebookComment"
+      optional :next_cursor, :string, 12
+    end
+    add_message "fbcrawl_colly.FacebookImage" do
+      optional :id, :int64, 1
+      optional :url, :string, 2
+    end
+    add_message "fbcrawl_colly.FacebookComment" do
+      optional :id, :int64, 1
+      optional :post, :message, 2, "fbcrawl_colly.FacebookPost"
+      optional :user, :message, 3, "fbcrawl_colly.FacebookUser"
+      optional :content, :string, 4
+      optional :created_at, :int64, 5
+    end
+    add_message "fbcrawl_colly.FacebookPostList" do
+      repeated :posts, :message, 1, "fbcrawl_colly.FacebookPost"
+      optional :next_cursor, :string, 2
+    end
+    add_message "fbcrawl_colly.FacebookImageList" do
+      repeated :images, :message, 1, "fbcrawl_colly.FacebookImage"
+      optional :next_cursor, :string, 2
+    end
+  end
+end
+
+module FbcrawlColly
+  Empty = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.Empty").msgclass
+  Pointer = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.Pointer").msgclass
+  LoginRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.LoginRequest").msgclass
+  LoginResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.LoginResponse").msgclass
+  LoginWithCookiesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.LoginWithCookiesRequest").msgclass
+  FetchGroupInfoRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FetchGroupInfoRequest").msgclass
+  FetchGroupFeedRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FetchGroupFeedRequest").msgclass
+  FetchPostRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FetchPostRequest").msgclass
+  FetchContentImagesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FetchContentImagesRequest").msgclass
+  FetchImageUrlRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FetchImageUrlRequest").msgclass
+  FacebookGroup = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookGroup").msgclass
+  FacebookUser = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookUser").msgclass
+  FacebookPost = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookPost").msgclass
+  CommentList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.CommentList").msgclass
+  FacebookImage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookImage").msgclass
+  FacebookComment = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookComment").msgclass
+  FacebookPostList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookPostList").msgclass
+  FacebookImageList = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fbcrawl_colly.FacebookImageList").msgclass
+end

data/lib/pb/fbcrawl_services_pb.rb

@@ -0,0 +1,31 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: fbcrawl.proto for package 'fbcrawl_colly'
+
+require 'grpc'
+require 'fbcrawl_pb'
+
+module FbcrawlColly
+  module Grpc
+    class Service
+
+      include GRPC::GenericService
+
+      self.marshal_class_method = :encode
+      self.unmarshal_class_method = :decode
+      self.service_name = 'fbcrawl_colly.Grpc'
+
+      # Sends a greeting
+      rpc :Init, FbcrawlColly::Empty, FbcrawlColly::Pointer
+      rpc :FreeColly, FbcrawlColly::Pointer, FbcrawlColly::Empty
+      rpc :Login, FbcrawlColly::LoginRequest, FbcrawlColly::LoginResponse
+      rpc :LoginWithCookies, FbcrawlColly::LoginWithCookiesRequest, FbcrawlColly::Empty
+      rpc :FetchGroupInfo, FbcrawlColly::FetchGroupInfoRequest, FbcrawlColly::FacebookGroup
+      rpc :FetchGroupFeed, FbcrawlColly::FetchGroupFeedRequest, FbcrawlColly::FacebookPostList
+      rpc :FetchPost, FbcrawlColly::FetchPostRequest, FbcrawlColly::FacebookPost
+      rpc :FetchContentImages, FbcrawlColly::FetchContentImagesRequest, FbcrawlColly::FacebookImageList
+      rpc :FetchImageUrl, FbcrawlColly::FetchImageUrlRequest, FbcrawlColly::FacebookImage
+    end
+
+    Stub = Service.rpc_stub_class
+  end
+end

data/main.go

@@ -1,21 +1,17 @@
 package main
 
-/*
-#include <stdio.h>
-#include <stdlib.h>
-
-static void myprint(char* s) {
-  printf("%s\n", s);
-}
-
-
-*/
 import "C"
-
 import (
+	context "context"
 	"flag"
-	"github.com/golang/protobuf/proto"
-	"qnetwork.net/fbcrawl/fbcolly"
+	"github.com/google/logger"
+	lru "github.com/hashicorp/golang-lru"
+	"google.golang.org/grpc"
+	"log"
+	"net"
+	"os"
+	"qnetwork.net/fbcrawl/fbcrawl"
+	"qnetwork.net/fbcrawl/fbcrawl/pb"
 	"unsafe"
 )
 
@@ -27,84 +23,93 @@ var password = flag.String("password", "change_me", "facebook password")
 var otp = flag.String("otp", "123456", "facebook otp")
 var groupId = flag.String("groupId", "334294967318328", "facebook group id, default is 334294967318328")
 
-var allInstances = map[uintptr]*fbcolly.Fbcolly{}
+var allInstances, _ = lru.New(20)
+
+func getColly(pointer *pb.Pointer) *fbcolly.Fbcolly {
+	c, ok := allInstances.Get(pointer.Address)
+	if ok {
+		return c.(*fbcolly.Fbcolly)
+	}
+	return nil
+}
+
+// server is used to implement helloworld.GreeterServer.
+type server struct {
+	pb.GrpcServer
+}
 
-//export Init
-func Init() uintptr {
+func (s server) Init(ctx context.Context, empty *pb.Empty) (*pb.Pointer, error) {
 	instance := fbcolly.New()
 	ptr := (uintptr)(unsafe.Pointer(instance))
-	allInstances[ptr] = instance
-	return ptr
+	allInstances.Add(int64(ptr), instance)
+	return &pb.Pointer{Address: int64(ptr)}, nil
 }
 
-//export FreeColly
-func FreeColly(pointer unsafe.Pointer) {
-	delete(allInstances, uintptr(pointer))
+func (s server) FreeColly(ctx context.Context, pointer *pb.Pointer) (*pb.Empty, error) {
+	logger.Info("FreeColly")
+	allInstances.Remove(pointer.Address)
+	return &pb.Empty{}, nil
 }
 
-//export Login
-func Login(pointer unsafe.Pointer, email *C.char, password *C.char) *C.char {
-	p := (*fbcolly.Fbcolly)(pointer)
-	cookies, err := p.Login(C.GoString(email), C.GoString(password), "")
+func (s server) Login(ctx context.Context, request *pb.LoginRequest) (*pb.LoginResponse, error) {
+	p := getColly(request.Pointer)
+
+	cookies, err := p.Login(request.Email, request.Password, request.TotpSecret)
 	if err == nil {
-		return C.CString(cookies)
+		return &pb.LoginResponse{Cookies: cookies}, err
 	}
-	return nil
+	return nil, err
 }
 
-//export LoginWithCookies
-func LoginWithCookies(pointer unsafe.Pointer, cookies *C.char) {
-	p := (*fbcolly.Fbcolly)(pointer)
-	p.LoginWithCookies(C.GoString(cookies))
+func (s server) LoginWithCookies(ctx context.Context, request *pb.LoginWithCookiesRequest) (*pb.Empty, error) {
+	p := getColly(request.Pointer)
+	err := p.LoginWithCookies(request.Cookies)
+	return &pb.Empty{}, err
 }
 
-//export FetchGroupFeed
-func FetchGroupFeed(pointer unsafe.Pointer, groupId int64) unsafe.Pointer {
-	p := (*fbcolly.Fbcolly)(pointer)
-	_, postsList := p.FetchGroupFeed(groupId)
-	marshaledPostsList, _ := proto.Marshal(postsList)
-	return C.CBytes(append(marshaledPostsList, 0))
+func (s server) FetchGroupInfo(ctx context.Context, request *pb.FetchGroupInfoRequest) (*pb.FacebookGroup, error) {
+	p := getColly(request.Pointer)
+	err, groupInfo := p.FetchGroupInfo(request.GroupUsername)
+	return groupInfo, err
 }
 
-//export FetchPost
-func FetchPost(pointer unsafe.Pointer, groupId int64, postId int64) unsafe.Pointer {
-	p := (*fbcolly.Fbcolly)(pointer)
-	_, post := p.FetchPost(groupId, postId)
-	marshaledPost, _ := proto.Marshal(post)
-	return C.CBytes(append(marshaledPost, 0))
+func (s server) FetchGroupFeed(ctx context.Context, request *pb.FetchGroupFeedRequest) (*pb.FacebookPostList, error) {
+	p := getColly(request.Pointer)
+	err, postsList := p.FetchGroupFeed(request.GroupId, request.NextCursor)
+	return postsList, err
 }
 
-//export FetchContentImages
-func FetchContentImages(pointer unsafe.Pointer, postId int64) unsafe.Pointer {
-	p := (*fbcolly.Fbcolly)(pointer)
-	_, imageList := p.FetchContentImages(postId)
-	marshaled, _ := proto.Marshal(imageList)
-	return C.CBytes(append(marshaled, 0))
+func (s server) FetchPost(ctx context.Context, request *pb.FetchPostRequest) (*pb.FacebookPost, error) {
+	p := getColly(request.Pointer)
+	err, post := p.FetchPost(request.GroupId, request.PostId, request.CommentNextCursor)
+	return post, err
 }
 
-//export FetchImageUrl
-func FetchImageUrl(pointer unsafe.Pointer, imageId int64) unsafe.Pointer {
-	p := (*fbcolly.Fbcolly)(pointer)
-	_, image := p.FetchImageUrl(imageId)
-	marshaled, _ := proto.Marshal(image)
-	return C.CBytes(append(marshaled, 0))
+func (s server) FetchContentImages(ctx context.Context, request *pb.FetchContentImagesRequest) (*pb.FacebookImageList, error) {
+	p := getColly(request.Pointer)
+	err, imageList := p.FetchContentImages(request.PostId, request.NextCursor)
+	return imageList, err
+}
+
+func (s server) FetchImageUrl(ctx context.Context, request *pb.FetchImageUrlRequest) (*pb.FacebookImage, error) {
+	p := getColly(request.Pointer)
+	err, image := p.FetchImageUrl(request.ImageId)
+	return image, err
 }
 
 func main() {
-	//r := regexp.MustCompile("/([\\d\\w.]+)").FindStringSubmatch()[1]
-	//print(r.FindStringSubmatch("/liem.phamthanh.161?refid=18&__tn__=R")[1])
-	//flag.Parse()
-	//
-	////post := fbcrawl.FacebookPost{}
-	////bPost, err := proto.Marshal(&post)
-	//
-	//lf, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
-	//if err != nil {
-	//	logger.Fatalf("Failed to open log file: %v", err)
-	//}
-	//defer lf.Close()
-	//defer logger.Init("fb-colly", *verbose, false, lf).Close()
-	//f := fbcolly.New()
-	//err = f.Login(*email, *password, *otp)
-	//f.FetchGroupFeed(*groupId)
+	port := os.Getenv("PORT")
+	if len(port) == 0 {
+		port = "50051"
+	}
+	lis, err := net.Listen("tcp", ":"+port)
+	logger.Info("Port listened at", port)
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+	s := grpc.NewServer()
+	pb.RegisterGrpcServer(s, &server{})
+	if err := s.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
 }

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fbcrawl-colly
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Duy Le
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-08-17 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: ffi
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -39,13 +25,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake-compiler
+  name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -56,12 +42,12 @@ description: Crawl mbasic.facebook.com using GO Colly
 email:
 - duyleekun@gmail.com
 executables: []
-extensions:
-- ext/fbcrawl_colly/extconf.rb
+extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -69,18 +55,19 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- ext/fbcrawl_colly/.gitignore
-- ext/fbcrawl_colly/Makefile
-- ext/fbcrawl_colly/extconf.rb
-- fbcolly/fbcolly.go
 - fbcrawl-colly.gemspec
 - fbcrawl.proto
+- fbcrawl/fbcolly.go
+- fbcrawl/pb/fbcrawl.pb.go
+- fbcrawl/pb/fbcrawl_grpc.pb.go
 - go.mod
 - go.sum
 - lib/fbcrawl-colly.rb
-- lib/fbcrawl_colly/colly.rb
-- lib/fbcrawl_colly/ffi.rb
+- lib/fbcrawl_colly.rb
+- lib/fbcrawl_colly/client.rb
 - lib/fbcrawl_colly/version.rb
+- lib/pb/fbcrawl_pb.rb
+- lib/pb/fbcrawl_services_pb.rb
 - main.go
 homepage: http://github.com/duyleekun/fbcrawl-colly
 licenses:
@@ -93,6 +80,7 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
+- lib/pb
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="