fbcrawl-colly 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 96d233862f21d1dff4a447d57f0f4a4ee6311506d7b5cf1990d75bf92043defa
+  data.tar.gz: a238ee4a34eac5f33fa90bba0d5bdb22169bd0b54034682f8fbb25c1ca4de039
+SHA512:
+  metadata.gz: 36d58d4ef38bf94164f94fe0a841171cc5bdd12a4f2503e818858d77cae029e1fc79cddd3213b2bb9e130e70833703df97ad18700d92c2cc3e31401b8e1f1443
+  data.tar.gz: 5c56a623910f87369812c0d74908e618490b4aeead9d22b055f8076ca040ba9fe5fff0bb380c44e7e435e0d83a13823b259b0f6c57233d7fb3ec38ff83de0832

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+
+*.iml
+/parse.log
+last.html
+*.db
+/fbcrawl/fbcrawl.pb.go
+/lib/fbcrawl_pb.rb

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.1
+before_install: gem install bundler -v 2.1.4

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at duyleekun@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fbcrawl-colly.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "minitest", "~> 5.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Duy Le
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# FbcrawlColly
+
+This project is to crawl mbasic.facebook.com using GO Colly. 
+
+Ruby gem will available to use
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fbcrawl-colly'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install fbcrawl-colly
+
+<!---
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+-->
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fbcrawl-colly. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/fbcrawl-colly/blob/master/CODE_OF_CONDUCT.md).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the FbcrawlColly project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/fbcrawl-colly/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fbcrawl-colly"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ext/fbcrawl_colly/.gitignore

@@ -0,0 +1,2 @@
+*.so
+*.h

data/ext/fbcrawl_colly/Makefile

@@ -0,0 +1,6 @@
+all:
+	cd ../../ && protoc fbcrawl.proto --go_out ./ --ruby_out ./lib/ && go build -o ./ext/fbcrawl_colly/fbcolly.so -buildmode=c-shared #-v -ldflags=-v .
+clean:
+	rm -rf fbcolly.h fbcolly.so ../../lib/fbcrawl_pb.rb
+install:
+	true

data/ext/fbcrawl_colly/extconf.rb

@@ -0,0 +1,5 @@
+require 'mkmf'
+MakeMakefile::find_executable 'go'
+MakeMakefile::find_executable 'protoc'
+MakeMakefile::find_executable 'protoc-gen-go'
+$makefile_created = true

data/fbcolly/fbcolly.go

@@ -0,0 +1,234 @@
+package fbcolly
+
+import "C"
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	"github.com/gocolly/colly"
+	"github.com/gocolly/colly/extensions"
+	"github.com/gocolly/colly/storage"
+	"github.com/google/logger"
+	"net/url"
+	"qnetwork.net/fbcrawl/fbcrawl"
+	"regexp"
+	"strings"
+)
+
+type Fbcolly struct {
+	collector *colly.Collector
+}
+
+func sharedOnRequest(request *colly.Request) {
+	logger.Info("OnRequest")
+	//request.Headers.Set("Host", "facebook.com")
+	request.Headers.Set("Accept-Language", "en-US,en;q=0.9")
+	request.Headers.Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
+	request.Headers.Set("origin", "https://mbasic.facebook.com")
+
+	//logger.Info("Saved referrer is", request.Ctx.Get("_referer"))
+	request.Headers.Set("referer", "https://mbasic.facebook.com/checkpoint/?_rdr")
+	request.Headers.Set("cache-control", "max-age=0")
+	request.Headers.Set("upgrade-insecure-requests", "1")
+	//accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
+	//origin: https://mbasic.facebook.com
+	//referer: https://mbasic.facebook.com/checkpoint/?_rdr
+	request.Headers.Set("User-Agent", "Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Mobile Safari/537.36")
+	request.ResponseCharacterEncoding = "utf-8"
+}
+
+func setupSharedCollector(collector *colly.Collector) error {
+	var err error
+	extensions.Referer(collector)
+
+	collector.OnRequest(sharedOnRequest)
+	collector.OnResponse(sharedOnResponse)
+	collector.OnError(func(resp *colly.Response, errHttp error) {
+		err = errHttp
+		logger.Error("OnError", err)
+	})
+	return err
+}
+
+func sharedOnResponse(response *colly.Response) {
+	logger.Info("OnResponse ./last.html")
+	_ = response.Save("./last.html")
+	//logger.Info(string(resp.Body))
+}
+
+func getForm(element *colly.HTMLElement, err error) (string, error, map[string]string) {
+	submitUrl, exists := element.DOM.Attr("action")
+	if !exists {
+		err = errors.New("doesn't have action label")
+		return "", nil, nil
+	}
+	submitUrl = fmt.Sprintf("https://mbasic.facebook.com%s", submitUrl)
+	logger.Info("form url is:", submitUrl)
+	reqMap := make(map[string]string)
+	element.DOM.Find("input").Each(func(i int, s *goquery.Selection) {
+		name, _ := s.Attr("name")
+		value, _ := s.Attr("value")
+		if name != "" && name != "sign_up" && name != "submit[logout-button-with-confirm]" {
+			reqMap[name] = value
+		}
+	})
+	return submitUrl, err, reqMap
+}
+
+func New() *Fbcolly {
+	f := Fbcolly{}
+	f.collector = colly.NewCollector()
+	return &f
+}
+
+func (f *Fbcolly) Login(email string, password string, otp string) error {
+	collector := f.collector.Clone()
+	setupSharedCollector(collector)
+
+	logger.Info("Login using email", email)
+
+	var err error
+	collector.OnHTML("#login_form", func(element *colly.HTMLElement) {
+		logger.Info("OnHTML login_form")
+		loginURL, err, reqMap := getForm(element, err)
+		if err != nil {
+			logger.Error(err)
+			return
+		}
+		reqMap["email"] = email
+		reqMap["pass"] = password
+		logger.Info("req map:", reqMap)
+		err = collector.Post(loginURL, reqMap)
+		if err != nil {
+			logger.Error("post err:", err)
+		}
+	})
+
+	collector.OnHTML("a[href=\"/login/save-device/cancel/?flow=interstitial_nux&nux_source=regular_login\"]", func(element *colly.HTMLElement) {
+		collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+	})
+
+	collector.OnHTML("form[action=\"/login/checkpoint/\"]", func(element *colly.HTMLElement) {
+
+		checkpointUrl, err, reqMap := getForm(element, err)
+		if err != nil {
+			logger.Error(err)
+			return
+		}
+
+		if element.DOM.Find("input[name=\"name_action_selected\"]").Length() > 0 {
+			//Save Device
+			logger.Info("OnHTML Save Device checkpoint")
+			reqMap["name_action_selected"] = "dont_save"
+		} else if element.DOM.Find("input[name=\"approvals_code\"]").Length() > 0 {
+			logger.Info("OnHTML OTP checkpoint")
+			//logger.Info("Please input OTP")
+			//reader := bufio.NewReader(os.Stdin)
+			//code, _ := reader.ReadString('\n')
+			code := otp[0:6]
+			reqMap["approvals_code"] = code
+		} else {
+			logger.Info("OnHTML Only Continue checkpoint")
+		}
+		logger.Info("req map:", reqMap)
+		err = collector.Post(checkpointUrl, reqMap)
+		if err != nil {
+			logger.Error("post err:", err)
+		}
+	})
+
+	collector.OnHTML("form[action=\"/search/\"]", func(element *colly.HTMLElement) {
+		//We're in home
+		logger.Info("I'm IN HOME, navigate to page now")
+	})
+
+	err = collector.Visit("https://mbasic.facebook.com/")
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	logger.Info(storage.StringifyCookies(collector.Cookies("https://mbasic.facebook.com/")))
+	//return err, storage.StringifyCookies(collector.Cookies("https://mbasic.facebook.com/"))
+	//return err, collector.getS.Cookies("https://mbasic.facebook.com/")
+	return err
+}
+
+func (f *Fbcolly) FetchGroupFeed(groupId string) (error, *fbcrawl.FacebookPostList) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	currentPage := 1
+	var result []*fbcrawl.FacebookPost
+
+	collector.OnHTML("#m_group_stories_container > :last-child a", func(element *colly.HTMLElement) {
+		currentPage++
+		if currentPage < 3 {
+			logger.Info("Will fetch page", currentPage)
+			collector.Visit("http://mbasic.facebook.com" + element.Attr("href"))
+		}
+	})
+
+	//TODO: May not need this
+	collector.OnXML("//a[text()=\"Full Story\"]", func(element *colly.XMLElement) {
+		u, _ := url.Parse("http://mbasic.facebook.com" + element.Attr("href"))
+
+		result = append(result, &fbcrawl.FacebookPost{
+			Id:    u.Query().Get("id"),
+			Group: &fbcrawl.FacebookGroup{Id: groupId},
+		})
+		//f.detailCollector.Visit(url)
+	})
+
+	err = collector.Visit("https://mbasic.facebook.com/groups/" + groupId)
+	if err != nil {
+		logger.Error("crawl by colly err:", err)
+	}
+	return err, &fbcrawl.FacebookPostList{Posts: result}
+}
+
+func (f *Fbcolly) FetchPost(groupId string, postId string) (error, *fbcrawl.FacebookPost) {
+	collector := f.collector.Clone()
+	err := setupSharedCollector(collector)
+	post := &fbcrawl.FacebookPost{}
+	collector.OnHTML("#m_story_permalink_view", func(element *colly.HTMLElement) {
+		dataElement := element.DOM.Find("div[data-ft]")
+		if dataElement.Length() > 0 {
+			var result map[string]string
+			jsonData, isExist := dataElement.Attr("data-ft")
+			if isExist {
+				json.Unmarshal([]byte(jsonData), &result)
+				logger.Info("Post ", result)
+				post.Id = result["top_level_post_id"]
+				post.Group = &fbcrawl.FacebookGroup{Id: result["page_id"], Name: dataElement.Find("h3 strong:last-child a").Text()}
+				post.User = &fbcrawl.FacebookUser{Id: result["content_owner_id_new"], Name: dataElement.Find("h3 strong:first-child a").Text()}
+				//Content
+				post.Content = strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
+					return selection.Text()
+				}), "\n")
+
+				logger.Info("content", strings.Join(dataElement.Find("p").Map(func(i int, selection *goquery.Selection) string {
+					return selection.Text()
+				}), "\n"))
+			}
+			post.Comments = []*fbcrawl.FacebookComment{}
+			//Comment
+			element.DOM.Find("h3 + div + div + div").Parent().Parent().Each(func(i int, selection *goquery.Selection) {
+				//author
+				commentId := selection.AttrOr("id", "")
+				logger.Info("comment", commentId)
+				//idRegex, _ := regexp.Compile("")
+				//idRegex.FindString()
+				post.Comments = append(post.Comments, &fbcrawl.FacebookComment{
+					Id:   commentId,
+					Post: &fbcrawl.FacebookPost{Id: post.Id},
+					User: &fbcrawl.FacebookUser{
+						Id:   regexp.MustCompile("/([\\d\\w.]+)").FindStringSubmatch(selection.Find("h3 > a").AttrOr("href", ""))[1],
+						Name: selection.Find("h3 > a").Text(),
+					},
+					Content: selection.Find("h3 + div").Text(),
+				})
+			})
+		}
+	})
+	collector.Visit(fmt.Sprintf("http://mbasic.facebook.com/groups/%s?view=permalink&id=%s&_rdr", groupId, postId))
+	return err, post
+}

data/fbcrawl-colly.gemspec

@@ -0,0 +1,35 @@
+require_relative 'lib/fbcrawl-colly/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fbcrawl-colly"
+  spec.version       = FbcrawlColly::VERSION
+  spec.authors       = ["Duy Le"]
+  spec.email         = ["duyleekun@gmail.com"]
+
+  spec.summary       = %q{Crawl mbasic.facebook.com using GO Colly}
+  spec.description   = %q{Crawl mbasic.facebook.com using GO Colly}
+  spec.homepage      = "http://github.com/duyleekun/fbcrawl-colly"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "http://github.com/duyleekun/fbcrawl-colly"
+  spec.metadata["changelog_uri"] = "http://github.com/duyleekun/fbcrawl-colly"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  # spec.bindir        = "exe"
+  # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.extensions = [
+      'ext/fbcrawl_colly/extconf.rb'
+  ]
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'ffi'
+  spec.add_runtime_dependency 'google-protobuf'
+end

data/fbcrawl.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+
+option go_package = "./fbcrawl;fbcrawl";
+
+// The request message containing the user's name.
+message FacebookGroup {
+  string id = 1;
+  string name = 2;
+}
+
+message FacebookUser {
+  string id = 1;
+  string name = 2;
+}
+
+message FacebookPost {
+  string id = 1;
+  FacebookGroup group = 2;
+  FacebookUser user = 3;
+  string content = 4;
+  repeated FacebookComment comments = 5;
+}
+
+message FacebookComment {
+  string id = 1;
+  FacebookPost post = 2;
+  FacebookUser user = 3;
+  string content = 4;
+}
+
+message FacebookPostList {
+  repeated FacebookPost posts = 1;
+}

data/go.mod

@@ -0,0 +1,23 @@
+module qnetwork.net/fbcrawl
+
+go 1.14
+
+require (
+	github.com/PuerkitoBio/goquery v1.5.1
+	github.com/andybalholm/cascadia v1.2.0 // indirect
+	github.com/antchfx/htmlquery v1.2.3 // indirect
+	github.com/antchfx/xmlquery v1.2.4 // indirect
+	github.com/antchfx/xpath v1.1.9 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gocolly/colly v1.2.0
+	github.com/golang/protobuf v1.4.2
+	github.com/google/logger v1.1.0
+	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/temoto/robotstxt v1.1.1 // indirect
+	github.com/thoas/go-funk v0.7.0
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	golang.org/x/text v0.3.3 // indirect
+	google.golang.org/appengine v1.6.6 // indirect
+	google.golang.org/protobuf v1.25.0
+)

data/go.sum

@@ -0,0 +1,131 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
+github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
+github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
+github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
+github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
+github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
+github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.1.9 h1:s3QDSTRoZMK6EqLajCUt/BJMUATPHvzwlRqKqg12yjU=
+github.com/antchfx/xpath v1.1.9/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
+github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/logger v1.1.0 h1:saB74Etb4EAJNH3z74CVbCKk75hld/8T0CsXKetWCwM=
+github.com/google/logger v1.1.0/go.mod h1:w7O8nrRr0xufejBlQMI83MXqRusvREoJdaAxV+CoAB4=
+github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
+github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y=
+github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

data/lib/fbcrawl-colly.rb

@@ -0,0 +1,16 @@
+require 'ffi'
+require 'fbcrawl_pb'
+module FbcrawlColly
+  extend FFI::Library
+
+  ffi_lib File.expand_path("../ext/fbcrawl_colly/fbcolly.so", File.dirname(__FILE__))
+  attach_function :free, [ :pointer ], :void
+
+  attach_function :Init, [], :pointer
+  attach_function :Login, [:pointer, :string, :string], :void
+  attach_function :FetchGroupFeed, [:pointer, :string], :string
+  attach_function :FetchPost, [:pointer, :string, :string], :string
+  # attach_function :FetchGroup, [:pointer, :string], :pointer
+  # attach_function :Login, [:pointer, :string, :string, :string], :void
+  # attach_function :FreePointer, [:pointer], :void
+end

data/lib/fbcrawl-colly/version.rb

@@ -0,0 +1,3 @@
+module FbcrawlColly
+  VERSION = "0.1.0"
+end

data/main.go

@@ -0,0 +1,77 @@
+package main
+
+/*
+#include <stdio.h>
+#include <stdlib.h>
+
+static void myprint(char* s) {
+  printf("%s\n", s);
+}
+
+
+*/
+import "C"
+
+import (
+	"flag"
+	"github.com/golang/protobuf/proto"
+	"qnetwork.net/fbcrawl/fbcolly"
+	"unsafe"
+)
+
+const logPath = "parse.log"
+
+var verbose = flag.Bool("verbose", true, "print info level logs to stdout")
+var email = flag.String("email", "change_me@gmail.com", "facebook email")
+var password = flag.String("password", "change_me", "facebook password")
+var otp = flag.String("otp", "123456", "facebook otp")
+var groupId = flag.String("groupId", "334294967318328", "facebook group id, default is 334294967318328")
+
+var tmp = fbcolly.New()
+
+//export Init
+func Init() uintptr {
+	return (uintptr)(unsafe.Pointer(tmp))
+}
+
+//export Login
+func Login(pointer unsafe.Pointer, email *C.char, password *C.char) {
+	p := (*fbcolly.Fbcolly)(pointer)
+	//print(p.E)
+	p.Login(C.GoString(email), C.GoString(password), "")
+}
+
+//export FetchGroupFeed
+func FetchGroupFeed(pointer unsafe.Pointer, groupId *C.char) unsafe.Pointer {
+	p := (*fbcolly.Fbcolly)(pointer)
+	_, postsList := p.FetchGroupFeed(C.GoString(groupId))
+	marshaledPostsList, _ := proto.Marshal(postsList)
+	return C.CBytes(append(marshaledPostsList, 0))
+}
+
+//export FetchPost
+func FetchPost(pointer unsafe.Pointer, groupId *C.char, postId *C.char) unsafe.Pointer {
+	p := (*fbcolly.Fbcolly)(pointer)
+	_, post := p.FetchPost(C.GoString(groupId), C.GoString(postId))
+	marshaledPost, _ := proto.Marshal(post)
+	return C.CBytes(append(marshaledPost, 0))
+}
+
+func main() {
+	//r := regexp.MustCompile("/([\\d\\w.]+)").FindStringSubmatch()[1]
+	//print(r.FindStringSubmatch("/liem.phamthanh.161?refid=18&__tn__=R")[1])
+	//flag.Parse()
+	//
+	////post := fbcrawl.FacebookPost{}
+	////bPost, err := proto.Marshal(&post)
+	//
+	//lf, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
+	//if err != nil {
+	//	logger.Fatalf("Failed to open log file: %v", err)
+	//}
+	//defer lf.Close()
+	//defer logger.Init("fb-colly", *verbose, false, lf).Close()
+	//f := fbcolly.New()
+	//err = f.Login(*email, *password, *otp)
+	//f.FetchGroupFeed(*groupId)
+}

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: fbcrawl-colly
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Duy Le
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-08-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Crawl mbasic.facebook.com using GO Colly
+email:
+- duyleekun@gmail.com
+executables: []
+extensions:
+- ext/fbcrawl_colly/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ext/fbcrawl_colly/.gitignore
+- ext/fbcrawl_colly/Makefile
+- ext/fbcrawl_colly/extconf.rb
+- fbcolly/fbcolly.go
+- fbcrawl-colly.gemspec
+- fbcrawl.proto
+- go.mod
+- go.sum
+- lib/fbcrawl-colly.rb
+- lib/fbcrawl-colly/version.rb
+- main.go
+homepage: http://github.com/duyleekun/fbcrawl-colly
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://github.com/duyleekun/fbcrawl-colly
+  source_code_uri: http://github.com/duyleekun/fbcrawl-colly
+  changelog_uri: http://github.com/duyleekun/fbcrawl-colly
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Crawl mbasic.facebook.com using GO Colly
+test_files: []