facebook_dumper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 464f40adf07b41485a860d342997ea980fba1ebbb499644f64c9f7b3d2298895
+  data.tar.gz: '0914b8dcd756512c4724b58a05787bdb6b8a8d131c7283348f342cc8728051ef'
+SHA512:
+  metadata.gz: 065f8e630ee552e5cfce878667c75cc1c8649c9725d09a3d49cd97fbfa874de0d55994edbc22c86283051890f2dfadd9f42b6e67cc597e634e6cbff16e5df551
+  data.tar.gz: b5812af0302b4d8d5392d3a425386e0a7221bcd597de3d9a33311b0155aef6a2d2ced9cc55a24c8c2c1dec052282316acc53bd2ad4ebcc6067844016803a9c3b

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+Gemfile.lock
+*~
+facebook-friends.txt
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.1
+before_install: gem install bundler -v 2.1.4

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at koichiro.eto@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,12 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in facebook_dumper.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+#gem "pathname"
+#gem "open-uri"
+gem "nokogiri"
+#gem "pp"
+#gem "rexml"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Koichiro Eto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# FacebookDumper
+
+FacebookDumperは、Facebookの保存されたWebページを解析して、テキストとしてダンプするプログラムです。
+現在は、友達リストをダンプすることができます。
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'facebook_dumper'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install facebook_dumper
+
+## Usage
+
+- Facebookの友達リストのページを開きます。
+https://www.facebook.com/【あなたのID】/friends
+
+- たくさん友達がいる場合は、スクロールすると表示が増えます。スペースバーを押し続けると最後まで表示されるので、しばらくスペースバーを押し続けるといいでしょう。
+- 全て表示されたら、Webブラウザから、「ファイル」→「名前を付けてページを保存…」を選び、ファイルとして保存します。
+- 以下のように、そのHTMLファイルを指定して、facebook_dumperを起動します。引数に、ファイルを指定します。
+
+    $ facebook_dumper friends.html
+
+- カレントディレクトリーの`facebook-friends.txt`に、内容が保存されます。
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/eto/facebook_dumper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/eto/facebook_dumper/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the FacebookDumper project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/eto/facebook_dumper/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+task :run do
+  sh "ruby bin/facebook_dumper ~/dev/200711/*.html"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "facebook_dumper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/facebook_dumper

@@ -0,0 +1,21 @@
+#!/usr/bin/ruby -w
+# coding: utf-8
+
+#$LOAD_PATH.unshift("..") if !$LOAD_PATH.include?("..")
+$LOAD_PATH.unshift("lib") if !$LOAD_PATH.include?("lib")
+require "facebook_dumper"
+
+dumper = FacebookDumper::FacebookFriendsDumper.new
+dumper.run(ARGV)
+
+#.main(ARGV)
+#unless ARGV[0] == "--test"
+#else
+#  ARGV.shift
+#  require "test/unit"
+#  class TestFacebookFriendsDump < Test::Unit::TestCase
+#    def test_it
+#      assert_equal(2, 1+1)
+#    end
+#  end
+#end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/facebook_dumper.gemspec

@@ -0,0 +1,29 @@
+require_relative 'lib/facebook_dumper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "facebook_dumper"
+  spec.version       = FacebookDumper::VERSION
+  spec.authors       = ["Koichiro Eto"]
+  spec.email         = ["k-eto@aist.go.jp"]
+
+  spec.summary       = %q{Dump Facebook friends list from a web page.}
+  spec.description   = %q{You can create a list of facebook friends.}
+  spec.homepage      = "https://github.com/eto/facebook_dumper/"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org/"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/eto/facebook_dumper/"
+  spec.metadata["changelog_uri"] = "https://github.com/eto/facebook_dumper/"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/lib/facebook_dumper.rb

@@ -0,0 +1,263 @@
+#!/usr/bin/ruby -w
+# coding: utf-8
+
+require "facebook_dumper/version"
+
+require 'pathname'
+require 'open-uri'
+require 'nokogiri'
+require 'pp'
+require "rexml/document"
+
+module FacebookDumper
+  class Error < StandardError; end
+
+  class Person
+    def initialize
+      #@uid = 0
+      @url = ""
+      @name = ""
+      @imgsrc = ""
+      @friends_mutual = 0
+      #@friends_num = 0
+    end
+    attr_accessor :url
+    attr_accessor :name
+    attr_accessor :imgsrc
+    attr_accessor :friends_mutual
+    #attr_accessor :uid, :url, :name, :friends_num, :mutual_friends_num
+
+    def inspect
+      #"#{@url}	#{@name}	#{@friends_mutual}	#{@imgsrc}"
+      "#{@url}	#{@name}	#{@friends_mutual}"
+    end
+    def <=> other
+      #return @uid <=> other.uid
+      return @url <=> other.url
+    end
+  end
+
+  class FacebookFriendsDumper
+    def initialize
+      @friends = []
+    end
+
+    def run(argv)
+      file = argv[0]
+      p file
+      extract_from_file file
+      out = friends_list
+      open("facebook-friends.txt", "w") {|f| f.print out}
+    end
+
+    def take1
+      doc.search(:img).each {|node|
+        node.remove_attribute("alt")
+        node.remove_attribute("role")
+      }
+      doc.search(:div).each {|node|
+        node.remove_attribute("data-ft")
+        node.remove_attribute("data-testid")
+        node.remove_attribute("style")
+      }
+      doc.search(:noscript).map &:remove
+      doc.search(:a).each {|node|
+=begin
+        node.remove_attribute("data-gt")
+        node.remove_attribute("ajaxify")
+        node.remove_attribute("rel")
+        node.remove_attribute("role")
+        node.remove_attribute("data-hover")
+        node.remove_attribute("data-tooltip-uri")
+        node.remove_attribute("tabindex")
+        node.remove_attribute("aria-hidden")
+        node.remove_attribute("data-profileid")
+        node.remove_attribute("data-flloc")
+        node.remove_attribute("data-unref")
+        node.remove_attribute("data-floc")
+=end
+        node.remove_attribute("data-hovercard")
+        node.remove_attribute("data-hovercard-prefer-more-content-show")
+        node.remove_attribute("style")
+        node.remove_attribute("aria-haspopup")
+      }
+      doc.search(:li).each {|node|
+        node.remove_attribute("data-ft")
+        node.remove_attribute("data-gt")
+        node.remove_attribute("data-alert-id")
+      }
+      doc.search(:button).each {|node|
+        node.remove_attribute("class")
+        node.remove_attribute("data-flloc")
+        node.remove_attribute("data-profileid")
+        node.remove_attribute("type")
+        node.remove_attribute("data-cancelref")
+        node.remove_attribute("data-floc")
+      }
+      #open("t3.html", "w") {|f| f.print doc.to_s }
+      open("t4.html", "w") {|f| f.print doc.to_xml(:indent => 2) }
+
+      num = 0
+      @friends = []
+      doc.xpath('//li[@class="_698"]').each {|node|	#<li class="_698">
+        node.search(:button).map &:remove
+        node.search(:span).each {|n|
+          n.remove_attribute("class")
+          n.remove_attribute("aria-hidden")
+        }
+
+        person = Person.new
+        node.search(:div).each {|n|
+          if n.attribute('class').value == "uiProfileBlockContent"
+            n.search(:a).each {|a|
+              text = a.inner_text
+              if text =~ /共通の友達(.+)人/
+                person.mutual_friends_num = $1.gsub(",", "").to_i
+              elsif text =~ /友達(.+)人/
+                person.friends_num = $1.gsub(",", "").to_i
+              elsif a.attribute('ajaxify') && a.attribute('ajaxify').value =~ /\/ajax\/friends\/inactive\/dialog\?id=(.+)/
+                #<a href="https://www.facebook.com/etocom/friends#" rel="dialog" ajaxify="/ajax/friends/inactive/dialog?id=100008321654013" role="button">干場 隆志
+                #ajaxify="/ajax/browser/dialog/mutual_friends/?uid=1036721103"
+                uid = $1.to_i
+                person.uid = uid
+                person.url = "https://www.facebook.com/profile.php?id=#{uid}"
+                person.name = a.inner_text.chomp
+                person.friends_num = -1
+              elsif a.attribute('data-gt')
+                person.name = text.chomp
+                url = a.attribute('href').value
+                url.gsub!("?fref=profile_friend_list&hc_location=friends_tab", "")
+                url.gsub!("&fref=profile_friend_list&hc_location=friends_tab", "")
+                url.gsub!("?fref=pb&hc_location=friends_tab", "")
+                url.gsub!("&fref=pb&hc_location=friends_tab", "")
+                person.url = url
+              else
+                # ignore
+              end
+            }
+            #friends << [friend_url, friend_name, friend_num, friend_mutual]
+            @friends << person
+          end
+        }
+        num += 1
+      }
+      #p num	#4995
+      return @friends
+    end
+
+    def parse_a_person(gpa)
+      person = Person.new
+      gpa.search(:img).each {|img|
+        person.imgsrc = img.attribute('src').value
+        pa = img.parent
+        if pa.name != "a"
+          gpa.search(:span).each {|span|
+            text = span.inner_text
+            unless text =~ /^友達$/
+              person.name = text
+              person.url = "no_longer_on_Facebook"
+            end
+          }
+          return person	# return here, since the user is no longer on Facebook.
+        end
+      }
+      gpa.search(:a).each {|a|
+        if a.attribute('tabindex').value == "0"
+          url = a.attribute('href').value
+          text = a.inner_text
+          if url =~ /friends_mutual/
+            if text =~ /共通の友達(.+)人/
+              person.friends_mutual = $1.gsub(",", "").to_i
+            end
+          else
+            person.url = url
+            person.name = text
+          end
+          #f.puts [person.url, person.name]
+        end
+        #f.puts a.to_html
+      }
+      return person
+    end
+
+    def extract_from_file(file)
+      charset = nil
+      html = ""
+      open(file) {|f|
+        html = f.read
+      }
+      #open("t1.html", "w") {|f| f.print html }
+
+      doc = Nokogiri::HTML.parse(html, nil, charset)
+      #open("t2.html", "w") {|f| f.print doc.to_s }
+
+      doc.search(:meta).map &:remove
+      doc.search(:link).map &:remove
+      doc.search(:style).map &:remove
+      doc.search(:script).map &:remove
+      doc.search(:svg).map &:remove
+      doc.search(:div).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:span).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:i).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:a).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:label).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:img).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:input).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:ul).each {|node|
+        node.remove_attribute("class")
+      }
+      doc.search(:li).each {|node|
+        node.remove_attribute("class")
+      }
+      #open("t3.html", "w") {|f| f.print doc.to_s }
+
+      #$f = open("t4.html", "w")
+
+      num = 0
+      @friends = []
+      doc.xpath('//div[@aria-label="友達"]').each {|node|
+        gpa = node.parent.parent	# get grand parent
+        person = parse_a_person(gpa)
+        #$f.puts person.inspect
+        #$f.puts "----------------------------------------------------------------------"
+        @friends << person
+        num += 1
+      }
+      #$f.puts num	# 4993
+      #$f.close
+
+      return @friends
+    end
+
+    def friends_list_take1
+        ar << [p.url, p.name, p.num]
+        if num < 0
+          out << "#{p.url}	#{p.name}	-1\n"
+        else
+          out << "#{p.url}	#{p.name}\n"
+        end
+    end
+
+    def friends_list
+      out = ""
+      @friends.sort.each {|p|
+        out << p.inspect + "\n"
+      }
+      return out
+    end
+  end
+end

data/lib/facebook_dumper/version.rb

@@ -0,0 +1,3 @@
+module FacebookDumper
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: facebook_dumper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Koichiro Eto
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2020-08-21 00:00:00.000000000 Z
+dependencies: []
+description: You can create a list of facebook friends.
+email:
+- k-eto@aist.go.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/facebook_dumper
+- bin/setup
+- facebook_dumper.gemspec
+- lib/facebook_dumper.rb
+- lib/facebook_dumper/version.rb
+homepage: https://github.com/eto/facebook_dumper/
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org/
+  homepage_uri: https://github.com/eto/facebook_dumper/
+  source_code_uri: https://github.com/eto/facebook_dumper/
+  changelog_uri: https://github.com/eto/facebook_dumper/
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Dump Facebook friends list from a web page.
+test_files: []