network_profile 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: eaa6afbf057402529a5e99c22ff61b98b1af6bcb8ca03cd869ddf03ba0e6893f
+  data.tar.gz: 1cd9b2fe99baba7cbc8f6774d8cd981bffa7e6860349df584d7041e9bb23aeb9
+SHA512:
+  metadata.gz: 93002910a6cb60f7ca0434b2231a583e440e2d4bc9a4e23e3721d021ee6af70acf959f0b012932f65dc175e5e209edba490add5780137d9e6afb9af3ebe6fa7b
+  data.tar.gz: 649cea60a5a341322cb3927aac5cd35b59bf374632694d9443c6dc37cfacf00a4afefe42336e350dce742d06c3ee81a3512eaf5407a82e31f10af77c5e86d97d

data/.github/workflows/verify.yml

@@ -0,0 +1,23 @@
+name: Verify
+on: [push]
+
+jobs:
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # ruby: [ '2.5', '2.6', '2.7' ]
+        ruby: [ '2.6' ]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Install gems
+        run: |
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+      - name: Run tests
+        run: bin/rspec
+

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+spec/.failures.txt
+

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in network_profile.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "pry"
+gem "rspec", ">= 3.5"
+gem "vcr"
+gem "webmock"

data/Gemfile.lock

@@ -0,0 +1,92 @@
+PATH
+  remote: .
+  specs:
+    network_profile (0.1.0)
+      activesupport (>= 5.0.0)
+      nokogiri
+      rdf-microdata
+      typhoeus
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.7)
+    crack (0.4.4)
+    diff-lcs (1.4.4)
+    ethon (0.12.0)
+      ffi (>= 1.3.0)
+    ffi (1.13.1)
+    hamster (3.0.0)
+      concurrent-ruby (~> 1.0)
+    hashdiff (1.0.1)
+    htmlentities (4.3.4)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    link_header (0.0.8)
+    method_source (1.0.0)
+    mini_portile2 (2.4.0)
+    minitest (5.14.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.6)
+    rake (12.3.3)
+    rdf (3.1.6)
+      hamster (~> 3.0)
+      link_header (~> 0.0, >= 0.0.8)
+    rdf-microdata (3.1.1)
+      htmlentities (~> 4.3)
+      nokogiri (~> 1.10)
+      rdf (~> 3.1)
+      rdf-xsd (~> 3.1)
+    rdf-xsd (3.1.0)
+      rdf (~> 3.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    thread_safe (0.3.6)
+    typhoeus (1.4.0)
+      ethon (>= 0.9.0)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    vcr (6.0.0)
+    webmock (3.9.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    zeitwerk (2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  network_profile!
+  pry
+  rake (~> 12.0)
+  rspec (>= 3.5)
+  vcr
+  webmock
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Stefan Wienert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# NetworkProfile
+
+Extractor Gem to analyse random strings for profile links of user. E.g. User uploads a PDF, scan it for all references to a social network profile.
+
+This work is extracted from the German Applicant Tracking System EBMS (https://bms.empfehlungsbund.de).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'network_profile'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install network_profile
+
+## Usage
+
+### Parse and extract one link
+
+
+    extraction = NetworkProfile.parse('https://github.com/zealot128', include_fallback_custom: true)
+
+- ``include_fallback_custom: true`` uses the default extractor (og/meta-tags) if no other more specific extractor is found
+- ``include_fallback_custom: false`` only use the specific website extractors and return nil if none matches the link
+
+### Scan a whole long string for links
+
+    links = NetworkProfile::Extractor.call("Very long String with even broken links in it www . github . com/zealot128")
+
+### Config
+
+    NetworkProfile.headers = {
+      'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+      'Accept-Language' => 'de,en-US;q=0.7,en;q=0.3',
+      'Referer' => 'https://www.google.com',
+      'DNT' => '1',
+      'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:73.0) Gecko/20100101 Firefox/73.0',
+    }
+    NetworkProfile.github_api_key = nil
+
+
+## Extractor
+
+The following network profiles are supported:
+
+**GithubProfile/Company, GithubProject**:
+
+  - uses GH's GraphQL API (Thus a API KEY is required)
+
+**Instagram** **Facebook** **Linkedin**
+
+  - Because those websites are closed and defensive as hell, there is no extraction, just a simple matching (e.g. "Facebook profile")
+
+**Stackoverflow**
+
+  - Uses SO's API
+
+**Upwork** **XING**  **ResearchGate**
+
+- Custom Website Scraper/Extract JSON+LD
+
+**Default Fallback** (Custom)
+
+- OG-Meta-Tags / HTML-Meta-Tags
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "network_profile"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/network_profile.rb

@@ -0,0 +1,23 @@
+require "network_profile/version"
+require 'network_profile/extractor'
+require 'network_profile/extractors/default_profile'
+require 'active_support/core_ext/module/attribute_accessors'
+
+module NetworkProfile
+  class Error < StandardError; end
+
+  mattr_accessor :headers, :github_api_key
+
+  self.headers = {
+    'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language' => 'de,en-US;q=0.7,en;q=0.3',
+    'Referer' => 'https://www.google.com',
+    'DNT' => '1',
+    'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:73.0) Gecko/20100101 Firefox/73.0',
+  }
+  self.github_api_key = nil
+
+  def self.parse(link, include_fallback_custom: false)
+    NetworkProfile::DefaultProfile.parse(link, include_fallback_custom: include_fallback_custom)
+  end
+end

data/lib/network_profile/extractor.rb

@@ -0,0 +1,65 @@
+require 'active_support/core_ext/string/filters'
+class NetworkProfile::Extractor
+  # Logic from:
+  # https://github.com/tenderlove/rails_autolink/blob/master/lib/rails_autolink/helpers.rb
+  AUTO_LINK_RE = %r{
+    (?: ((?:ed2k|ftp|http|https|irc|mailto|news|gopher|nntp|telnet|webcal|xmpp|callto|feed|svn|urn|aim|rsync|tag|ssh|sftp|rtsp|afs|file):)// | www\. )
+    [^\s<\u00A0"]+
+  }ix.freeze
+  WORD_PATTERN = '\p{Word}'.freeze
+  BRACKETS = { ']' => '[', ')' => '(', '}' => '{' }.freeze
+
+  def self.call(string)
+    new(string).extracted_links!
+  end
+
+  def initialize(string)
+    @string = string
+  end
+
+  def extracted_links!
+    extracted = links.map do |l|
+      NetworkProfile.parse(l)
+    rescue StandardError => e
+      p e
+      nil
+    end
+    extracted.compact
+  end
+
+  def links
+    return @links if @links
+
+    @links ||= []
+    mapped_string.scan(AUTO_LINK_RE) { |_|
+      scheme = Regexp.last_match(1)
+      href = $&
+      punctuation = []
+      while href.sub!(%r{[^#{WORD_PATTERN}/-=&]$}, '')
+        punctuation.push($&)
+        if opening = BRACKETS[punctuation.last] and href.scan(opening).size > href.scan(punctuation.last).size
+          href << punctuation.pop
+          break
+        end
+      end
+      href = 'https://' + href unless scheme
+      @links << href
+    }
+    @links.uniq
+  end
+
+  TLD = /(?<tld>com|de|net|fr|at|ch|info)/.freeze
+  HOST_PART = %r{(?<host>[a-z\-\.0-9]+)}.freeze
+
+  def mapped_string
+    @string.
+      gsub(%r{ (#{HOST_PART}\.#{TLD}/)}) { |_|
+        host = Regexp.last_match['host']
+        "https://#{host}.#{Regexp.last_match['tld']}/"
+      }.
+      gsub(%r{ www *\. +#{HOST_PART} *\. *#{TLD}(?<path>[^<\u00A0"]+)}) { |_|
+        path = Regexp.last_match['path'].remove(' ')
+        "www.#{Regexp.last_match['host']}.#{Regexp.last_match['tld']}#{path}"
+      }
+  end
+end

data/lib/network_profile/extractors/custom.rb

@@ -0,0 +1,9 @@
+module NetworkProfile
+  class Custom < DefaultProfile
+    self.mdi_icon = 'open-in-new'
+
+    def self.handle?(link)
+      true
+    end
+  end
+end

data/lib/network_profile/extractors/default_profile.rb

@@ -0,0 +1,129 @@
+require 'rdf/microdata'
+require 'active_support/descendants_tracker'
+require 'active_support/core_ext/module/attribute_accessors'
+require 'active_support/core_ext/string/inflections'
+require 'nokogiri'
+require 'typhoeus'
+
+module NetworkProfile
+  class DefaultProfile
+    include ActiveSupport::DescendantsTracker
+
+    cattr_accessor :mdi_icon
+
+    class << self
+      attr_accessor :headers
+    end
+
+    def self.auto_extractor_link_types
+      [
+        NetworkProfile::GithubProfile,
+        NetworkProfile::GithubProject,
+        NetworkProfile::LinkedinProfile,
+        NetworkProfile::InstagramProfile,
+        NetworkProfile::XingProfile,
+        NetworkProfile::ResearchgateProfile,
+        NetworkProfile::UpworkProfile,
+        NetworkProfile::FacebookProfile,
+        NetworkProfile::StackoverflowProfile,
+      ].freeze
+    end
+
+    def self.all_types
+      auto_extractor_link_types + [NetworkProfile::Custom]
+    end
+
+    def self.parse(link, include_fallback_custom: false)
+      link_type = (include_fallback_custom ? all_types : auto_extractor_link_types).find { |i| i.handle?(link) }
+      if link_type
+        link_type.new(link.strip).data
+      end
+    end
+
+    def initialize(link)
+      @link = link
+    end
+
+    def image
+      img = doc.at('meta[property=og\:image]')&.[]('content')
+      if img && img[%r{^/\w+}]
+        img = URI.join(@link, img).to_s
+      end
+      img
+    end
+
+    def title
+      doc.at('title')&.text
+    end
+
+    def text
+      doc.at('meta[property=og\:description]')&.[]('content') || doc.at('meta[name=description]')&.[]('content')
+    end
+
+    def data
+      {
+        site_icon: mdi_icon,
+        link: @link,
+        title: title,
+        text: text,
+        image: image,
+        type: self.class.name.underscore.split('/').last
+      }.merge(extra_data)
+    end
+
+    def extra_data
+      {}
+    end
+
+    private
+
+    def response
+      @response ||= Typhoeus.get(@link, headers: NetworkProfile.headers, followlocation: true)
+    end
+
+    def doc
+      @doc ||= Nokogiri.parse(response.body)
+    end
+
+    def json_ld
+      @json_ld ||= JSON.parse(doc.search('script[type*=ld]').first.text)
+    end
+
+    def rdf
+      @rdf ||= map_rdf(
+        RDF::Microdata::Reader.new(response.body).to_h
+      )
+    end
+
+    def map_rdf(tree)
+      tree.
+        transform_keys { |v| map_rdf_value(v) }.
+        transform_values { |v| map_rdf_value(v) }
+    end
+
+    def map_rdf_value(value)
+      case value
+      when RDF::Vocabulary::Term then value.fragment
+      when RDF::URI then value.to_base
+      when RDF::Node then value.id
+      when RDF::Literal then value.value
+      when Hash then map_rdf(value)
+      when Array then value.map { |i| map_rdf_value(i) }
+      else
+        value
+      end
+    end
+  end
+end
+
+require_relative './custom'
+require_relative './github_profile'
+require_relative './github_project'
+require_relative './linkedin_profile'
+require_relative './instagram_profile'
+require_relative './xing_profile'
+require_relative './researchgate_profile'
+require_relative './upwork_profile'
+require_relative './facebook_profile'
+require_relative './stackoverflow_profile'
+

data/lib/network_profile/extractors/facebook_profile.rb

@@ -0,0 +1,15 @@
+require_relative './network_profile_without_extraction'
+
+module NetworkProfile
+  class FacebookProfile < NetworkProfileWithoutExtraction
+    self.mdi_icon = 'facebook'
+
+    def self.handle?(link)
+      (e = link[%r{facebook.com/([\w\.]+)}, 1]) && e.length > 3 && e != 'groups'
+    end
+
+    def profile_description
+      "Facebook Profil:"
+    end
+  end
+end

data/lib/network_profile/extractors/github_graphql.rb

@@ -0,0 +1,24 @@
+require 'active_support/core_ext/hash/indifferent_access'
+
+module NetworkProfile::GithubGraphql
+  def query!(query)
+    r = Typhoeus.post("https://api.github.com/graphql",
+                      body: { query: query }.to_json,
+                      headers: {
+                        "Authorization": "bearer #{NetworkProfile.github_api_key}"
+                      })
+    if r.success?
+      JSON.parse(r.body).with_indifferent_access
+    else
+      raise ArgumentError, "Fetching query failed: #{r.code}"
+    end
+  end
+
+  def json
+    @json ||= query!(query)
+  end
+
+  def doc
+    raise NotImplementedError
+  end
+end

data/lib/network_profile/extractors/github_profile.rb

@@ -0,0 +1,98 @@
+require_relative './github_graphql'
+
+module NetworkProfile
+  class GithubProfile < DefaultProfile
+    include GithubGraphql
+    self.mdi_icon = 'github'
+
+    def self.handle?(link)
+      link.to_s[%r{github.com/[^/]+/?$}] && NetworkProfile.github_api_key
+    end
+
+    def query
+      username = @link[%r{github.com/([^/]+)}, 1]
+      <<~DOC
+        query {
+          organization(login:"#{username}") {
+            avatarUrl
+            name
+            bio: description
+            location
+            websiteUrl
+            ...RepoFragment
+          }
+
+          user(login:"#{username}") {
+            avatarUrl
+            name
+            bio
+            company
+            location
+            websiteUrl
+            followers {
+              totalCount
+            }
+            ...RepoFragment
+          }
+        }
+        fragment RepoFragment on ProfileOwner {
+          pinnedItems(first: 9, types: [REPOSITORY]) {
+            edges {
+              node {
+                ... on Repository {
+                  nameWithOwner,
+                  url,
+                  createdAt,
+                  updatedAt
+                  stargazers { totalCount }
+                  watchers {
+                    totalCount
+                  },
+                  primaryLanguage {
+                    name
+                  }
+                }
+              }
+            }
+          }
+        }
+      DOC
+    end
+
+    def profile_data
+      json.dig('data', 'organization') || json.dig('data', 'user')
+    end
+
+    def title
+      profile_data['name']
+    end
+
+    def text
+      profile_data['bio']
+    end
+
+    def image
+      profile_data['avatarUrl']
+    end
+
+    def extra_data
+      {
+        company: profile_data['company'],
+        location: profile_data['location'],
+        profile_type: json.dig('data', 'organization') ? "organization" : "user",
+        followers: profile_data.dig('followers', 'totalCount'),
+        website: profile_data.dig('websiteUrl'),
+        pinned: profile_data.dig('pinnedItems', 'edges').map { |i|
+          n = i['node']
+          { name: n['nameWithOwner'],
+            url: n['url'],
+            created: Time.parse(n['createdAt']).to_date,
+            updated: Time.parse(n['updatedAt']).to_date,
+            language: n.dig('primaryLanguage', 'name'),
+            stars: n['stargazers']['totalCount'],
+            watchers: n['watchers']['totalCount'] }
+        }
+      }
+    end
+  end
+end

data/lib/network_profile/extractors/github_project.rb

@@ -0,0 +1,88 @@
+require_relative './github_graphql'
+
+module NetworkProfile
+  class GithubProject < DefaultProfile
+    include GithubGraphql
+    self.mdi_icon = 'github'
+
+    def self.handle?(link)
+      link.to_s[%r{github.com/[^/]+/.+}] && NetworkProfile.github_api_key
+    end
+
+    def query
+      _, author, repo = @link.match(%r{github.com/([^/]+)/([^/\?]+)(\.git)?}).to_a
+      <<~DOC
+        query {
+          repository(name:"#{repo}", owner: "#{author}") {
+            createdAt
+            description
+            nameWithOwner
+            updatedAt
+            languages(first:10) {
+              edges {
+                node {
+                  name
+                }
+                size
+              }
+              totalCount
+            }
+            licenseInfo { name }
+            forkCount
+            isFork
+            defaultBranchRef {
+              name
+              target {
+                ... on Commit {
+                  committedDate
+                  history(first: 0) {
+                    totalCount
+                  }
+                }
+              }
+            }
+            issues {
+              totalCount
+            }
+            stargazers {
+              totalCount
+            }
+            watchers {
+              totalCount
+            }
+          }
+        }
+      DOC
+    end
+
+    def title
+      json.dig('data', 'repository', 'nameWithOwner')
+    end
+
+    def text
+      json.dig('data', 'repository', 'description')
+    end
+
+    def last_commit
+      Time.parse(json.dig('data', 'repository', 'defaultBranchRef', 'target', 'committedDate')).to_date
+    end
+
+    def image
+      nil
+    end
+
+    def extra_data
+      {
+        watchers: json.dig('data', 'repository', 'watchers', 'totalCount'),
+        forks: json.dig('data', 'repository', 'forkCount'),
+        stars: json.dig('data', 'repository', 'stargazers', 'totalCount'),
+        issue_count: json.dig('data', 'repository', 'issues', 'totalCount'),
+        commits: json.dig('data', 'repository', 'defaultBranchRef', 'target', 'history', 'totalCount'),
+        license: json.dig('data', 'repository', 'licenseInfo', 'name'),
+        created: Time.parse(json.dig('data', 'repository', 'createdAt')).to_date,
+        language_bytes: json.dig('data', 'repository', 'languages', 'edges')&.map { |l| [l.dig('node', 'name'), l['size']] }&.sort_by { |_a, b| -b },
+        last_commit: last_commit
+      }
+    end
+  end
+end

data/lib/network_profile/extractors/instagram_profile.rb

@@ -0,0 +1,14 @@
+require_relative './network_profile_without_extraction'
+
+module NetworkProfile
+  class InstagramProfile < NetworkProfileWithoutExtraction
+    self.mdi_icon = 'instagram'
+    def self.handle?(link)
+      (e = link[%r{instagram.com/([\w\.]+)}, 1]) && e.length > 3 && e != 'groups'
+    end
+
+    def profile_description
+      "Instagram Profil:"
+    end
+  end
+end

data/lib/network_profile/extractors/linkedin_profile.rb

@@ -0,0 +1,14 @@
+require_relative './network_profile_without_extraction'
+
+module NetworkProfile
+  class LinkedinProfile < NetworkProfileWithoutExtraction
+    self.mdi_icon = 'linkedin'
+    def self.handle?(link)
+      link['linkedin.com/in/']
+    end
+
+    def profile_description
+      "LinkedIn Profil:"
+    end
+  end
+end

data/lib/network_profile/extractors/network_profile_without_extraction.rb

@@ -0,0 +1,25 @@
+require 'active_support/core_ext/string/inflections'
+require_relative './default_profile.rb'
+
+module NetworkProfile
+  class NetworkProfileWithoutExtraction < DefaultProfile
+    def profile_description
+      "Profil: "
+    end
+
+    def title
+      "#{profile_description} #{@link.split('/').last}"
+    end
+
+    def data
+      {
+        title: title,
+        text: "",
+        image: nil,
+        type: self.class.name.underscore.split('/').last,
+        link: @link,
+        site_icon: mdi_icon,
+      }
+    end
+  end
+end

data/lib/network_profile/extractors/researchgate_profile.rb

@@ -0,0 +1,44 @@
+module NetworkProfile
+  class ResearchgateProfile < DefaultProfile
+    self.mdi_icon = ''
+
+    def self.handle?(link)
+      link['researchgate.net/profile/']
+    end
+
+    def json_ld
+      original = super
+      if original['@graph']
+        original['@graph'].first
+      else
+        original
+      end
+    end
+
+    def title
+      json_ld['name']
+    end
+
+    def text
+      json_ld.dig('affiliation', 'name') || doc.at('.org')&.text
+    end
+
+    def last_item
+      item = rdf.find { |_, v| v['type'].to_s['ScholarlyArticle'] }.last
+      return unless item
+
+      title = item.dig('<http://schema.org/headline>', 0)
+      date = item.dig('<http://schema.org/datePublished>', 0)
+      "#{title} (#{date})"
+    end
+
+    def extra_data
+      items, reads, citations = doc.at(".profile-content-item .nova-c-card").
+        search(".nova-o-grid__column").
+        map { |col| col.search('.nova-e-text').map(&:text) }.
+        map(&:first).map { |i| i.gsub(',', '').to_i }
+
+      { items: items, reads: reads, citations: citations, last_item: last_item }
+    end
+  end
+end

data/lib/network_profile/extractors/stackoverflow_profile.rb

@@ -0,0 +1,81 @@
+module NetworkProfile
+  # Tags:
+  # https://api.stackexchange.com/2.2/users/220292/top-tags?pagesize=10&site=stackoverflow
+
+  class StackoverflowProfile < DefaultProfile
+    self.mdi_icon = 'stack-overflow'
+    SITES = [
+      ["https://stackoverflow.com", "stackoverflow", "Stack Overflow"],
+      ["https://serverfault.com", "serverfault", "Server Fault"],
+      ["https://superuser.com", "superuser", "Super User"],
+      ["https://webapps.stackexchange.com", "webapps", "Web Applications"],
+      ["https://gaming.stackexchange.com", "gaming", "Arqade"],
+      ["https://webmasters.stackexchange.com", "webmasters", "Webmasters"],
+      ["https://cooking.stackexchange.com", "cooking", "Seasoned Advice"],
+      ["https://gamedev.stackexchange.com", "gamedev", "Game Development"],
+      ["https://photo.stackexchange.com", "photo", "Photography"],
+      ["https://stats.stackexchange.com", "stats", "Cross Validated"],
+      ["https://math.stackexchange.com", "math", "Mathematics"],
+      ["https://diy.stackexchange.com", "diy", "Home Improvement"],
+      ["https://gis.stackexchange.com", "gis", "Geographic Information Systems"],
+      ["https://tex.stackexchange.com", "tex", "TeX - LaTeX"],
+      ["https://askubuntu.com", "askubuntu", "Ask Ubuntu"],
+    ].freeze
+
+    def self.handle?(link)
+      SITES.any? { |s, _, _| link.include?(s + "/users/") }
+    end
+
+    def site
+      @site ||= SITES.find { |s, _, _| @link.include?(s) }
+    end
+
+    def title
+      user_api['display_name']
+    end
+
+    def image
+      user_api['profile_image']
+    end
+
+    def text
+    end
+
+    def extra_data
+      {
+        reputation: user_api.dig('reputation'),
+        created: Time.at(user_api['creation_date']).to_date,
+        location: user_api.dig('location'),
+        site: site[2],
+        site_logo: "https://cdn.sstatic.net/Sites/#{site[1]}/img/apple-touch-icon.png",
+        tags: tags_api.map { |j| [j['tag_name'], j['answer_score'] + j['question_score']] }
+      }
+    end
+
+    def user_id
+      URI.parse(@link).path[%r{/users/(\d+)/?}, 1]
+    end
+
+    private
+
+    def user_api
+      @user_api ||=
+        begin
+          url = "https://api.stackexchange.com/2.2/users/#{user_id}?order=desc&sort=reputation&site=#{site[1]}"
+          api_call(url).dig('items', 0)
+        end
+    end
+
+    def tags_api
+      @tags_api ||=
+        begin
+          url = "https://api.stackexchange.com/2.2/users/#{user_id}/top-tags?pagesize=10&site=#{site[1]}"
+          api_call(url).dig('items')
+        end
+    end
+
+    def api_call(url)
+      JSON.parse(Typhoeus.get(url, accept_encoding: 'gzip').body)
+    end
+  end
+end

data/lib/network_profile/extractors/upwork_profile.rb

@@ -0,0 +1,42 @@
+require 'active_support/core_ext/string/filters'
+
+module NetworkProfile
+  class UpworkProfile < DefaultProfile
+    self.mdi_icon = 'upwork'
+    def self.handle?(link)
+      link[%r{upwork.com/o/profiles/users/.+}]
+    end
+
+    def title
+      php_vars.dig('profile', 'profile', 'name')
+    end
+
+    def text
+      doc.at('h2 strong').text
+    end
+
+    def php_vars
+      @php_vars ||=
+        begin
+          t = doc.search('script').find { |i| i && i.text['PROFILE_RESPONSE'] }.text
+          t.remove!(/window.PROFILE_RESPONSE=.*summary:/)
+          JSON.parse(t.remove(/\}$/))
+        end
+    end
+
+    def extra_data
+      profile = php_vars.dig('profile')
+      rate = profile.dig('stats', 'hourlyRate')
+      {
+        country: profile.dig('profile', 'location').yield_self { |v| "#{v['city']}, #{v['country']}" },
+        hours: profile.dig('stats', 'totalHours').floor,
+        jobs: profile.dig('stats', 'totalJobsWorked').floor,
+        rating: profile.dig('stats', 'rating').round(2),
+        hourly_rate: "#{rate['amount']} #{rate['currencyCode']}",
+        english_level: profile['stats']['englishLevel'],
+        hire_again: profile.dig('stats', 'hireAgainPercentage'),
+        skills: profile.dig('profile', 'skills').map { |i| i['prettyName'] }
+      }
+    end
+  end
+end

data/lib/network_profile/extractors/xing_profile.rb

@@ -0,0 +1,25 @@
+module NetworkProfile
+  class XingProfile < DefaultProfile
+    self.mdi_icon = 'xing'
+
+    def self.handle?(link)
+      link['xing.com/profile/']
+    end
+
+    def title
+      doc.at('h1').text.strip
+    end
+
+    def text
+      json_ld.dig('jobTitle')
+    end
+
+    def extra_data
+      {
+        employment_status: doc.at('[data-qa=xing-id-work_experience]')&.text&.split(', ')&.first,
+        tags: json_ld&.fetch('makesOffer', [])&.map { |i| i['name'] } || [],
+        languages: doc.at('[data-qa=language-skills-section]')&.search('li')&.map { |i| "#{i.at('h3').text} (#{i.at('div').text})" },
+      }
+    end
+  end
+end

data/lib/network_profile/version.rb

@@ -0,0 +1,3 @@
+module NetworkProfile
+  VERSION = "0.1.0"
+end

data/network_profile.gemspec

@@ -0,0 +1,31 @@
+require_relative 'lib/network_profile/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "network_profile"
+  spec.version       = NetworkProfile::VERSION
+  spec.authors       = ["Stefan Wienert"]
+  spec.email         = ["info@stefanwienert.de"]
+
+  spec.summary       = %q{Extract profile metadata from various social-media-profiles}
+  spec.description   = %q{Extract profile metadata from various social-media-profiles, such as Twitter, XING, Github, Stackoverflow or generic og-metatags.}
+  spec.homepage      = "https://github.com/pludoni/network_profile"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/pludoni/network_profile"
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_dependency "typhoeus"
+  spec.add_dependency "rdf-microdata"
+  spec.add_dependency "activesupport", ">= 5.0.0"
+  spec.add_dependency "nokogiri"
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: network_profile
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Stefan Wienert
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-09-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdf-microdata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Extract profile metadata from various social-media-profiles, such as
+  Twitter, XING, Github, Stackoverflow or generic og-metatags.
+email:
+- info@stefanwienert.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/verify.yml"
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/rake
+- bin/rspec
+- bin/setup
+- lib/network_profile.rb
+- lib/network_profile/extractor.rb
+- lib/network_profile/extractors/custom.rb
+- lib/network_profile/extractors/default_profile.rb
+- lib/network_profile/extractors/facebook_profile.rb
+- lib/network_profile/extractors/github_graphql.rb
+- lib/network_profile/extractors/github_profile.rb
+- lib/network_profile/extractors/github_project.rb
+- lib/network_profile/extractors/instagram_profile.rb
+- lib/network_profile/extractors/linkedin_profile.rb
+- lib/network_profile/extractors/network_profile_without_extraction.rb
+- lib/network_profile/extractors/researchgate_profile.rb
+- lib/network_profile/extractors/stackoverflow_profile.rb
+- lib/network_profile/extractors/upwork_profile.rb
+- lib/network_profile/extractors/xing_profile.rb
+- lib/network_profile/version.rb
+- network_profile.gemspec
+homepage: https://github.com/pludoni/network_profile
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/pludoni/network_profile
+  source_code_uri: https://github.com/pludoni/network_profile
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Extract profile metadata from various social-media-profiles
+test_files: []