turl 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: edc78b86ab00971907c4135b49e0aa815fbf50f8ca0c51331811a528c2ba0733
+  data.tar.gz: 84c27498e752a7bd98ec9e944d0b446bfad8c249b66ce29d01405f6f9403db3e
+SHA512:
+  metadata.gz: d43560abcea1393bf44d5f18cedd04ec7c433eb83553f256f5d86dc73193d25153a226c7b8221e0339387223f8cc1880f169a5cc2652a691e8c1e80e5f81db70
+  data.tar.gz: 111723347b196e5bbd810ae14948524149e34011611d2abd99bb0c90a6d8d6d828766a0f10b59d42e2a31be02b7521b826e286a90d90155aa19065d60df37aae

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/Gemfile.lock

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in turl.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/README.md

@@ -0,0 +1,36 @@
+# Turl
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/turl`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'turl'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install turl
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/pocke/turl.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "turl"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+Turl.prepare_database!
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/turl-collect

@@ -0,0 +1,5 @@
+#!ruby
+
+require 'turl'
+
+Turl::Collect.run(ARGV)

data/exe/turl-web

@@ -0,0 +1,7 @@
+#!ruby
+
+require 'turl'
+require 'turl/web'
+require 'erubi'
+
+Turl::Web.run(ARGV)

data/lib/turl.rb

@@ -0,0 +1,96 @@
+require 'twitter'
+require 'logger'
+require 'pathname'
+require 'active_record'
+require 'nokogiri'
+require 'open-uri'
+require 'net/http'
+
+require "turl/version"
+require 'turl/collect'
+require 'turl/application_record'
+require 'turl/link'
+require 'turl/tweet'
+require 'turl/twitter_user'
+require 'turl/tweet_link'
+require 'turl/url_normalization'
+require 'turl/normalizer'
+
+module Turl
+  CACHE_PATH = Pathname('~/.cache/turl').expand_path
+  DATABASE_PATH = CACHE_PATH / 'db.sqlite3'
+
+  def self.prepare_database!
+    CACHE_PATH.mkpath
+    ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: DATABASE_PATH.to_s)
+    unless DATABASE_PATH.exist?
+      <<~SQL.split(';').select(&:present?).each { |sql| ActiveRecord::Base.connection.execute(sql) }
+        create table links (
+          id integer primary key,
+
+          normalized_url text not null,
+          title text,
+
+          created_at datetime not null,
+          updated_at datetime not null
+        );
+
+        create unique index uniq_links_url_tweet on links(normalized_url);
+
+        create table tweets (
+          id integer primary key,
+          twitter_id text not null,
+
+          twitter_user_id integer not null,
+          content text not null,
+          tweeted_at datetime not null,
+
+          created_at datetime not null,
+          updated_at datetime not null
+        );
+
+        create unique index uniq_tweets_twitter_id on tweets(twitter_id);
+
+        create table tweet_links (
+          id integer primary key,
+
+          tweet_id integer not null,
+          link_id integer not null,
+
+          created_at datetime not null,
+          updated_at datetime not null
+        );
+
+        create unique index uniq_tweet_links_tweet_link on tweet_links(tweet_id, link_id);
+
+        create table twitter_users (
+          id integer primary key,
+          twitter_id text not null,
+
+          screen_name text not null,
+
+          created_at datetime not null,
+          updated_at datetime not null
+        );
+
+        create unique index uniq_twitter_users_twitter_id on twitter_users(twitter_id);
+
+        create table url_normalizations (
+          id integer primary key,
+
+          original_url text not null,
+          normalized_url text not null,
+
+          created_at datetime not null,
+          updated_at datetime not null
+        );
+
+        create unique index uniq_url_normalizations_original_url on url_normalizations(original_url);
+      SQL
+    end
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+end

data/lib/turl/application_record.rb

@@ -0,0 +1,5 @@
+module Turl
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/lib/turl/collect.rb

@@ -0,0 +1,46 @@
+module Turl
+  class Collect
+    def self.run(argv)
+      Turl.prepare_database!
+      self.new.run(argv)
+    end
+
+    def run(argv)
+      since_id = nil
+      loop do
+        new_since_id = fetch_and_save(since_id: since_id)
+        since_id = new_since_id if new_since_id
+        sleep 60
+      end
+    end
+
+    def fetch_and_save(since_id:)
+      Turl.logger.info "start Turl::Collect#fetch_and_save(since_id: #{since_id.inspect})"
+
+      tweets = client.home_timeline(count: 200)
+      tweets.each do |tweet_resp|
+        next if tweet_resp.urls.empty?
+
+        tweet = Tweet.from_response!(tweet_resp)
+
+        tweet_resp.urls.each do |url|
+          next if Link.ignored?(url)
+
+          Link.from_response!(url, tweet)
+        end
+      end
+      Turl.logger.info "done Turl::Collect#fetch_and_save(since_id: #{since_id.inspect})"
+      tweets.first&.id
+    end
+
+    private def client
+      @client ||= Twitter::REST::Client.new do |config|
+        config.consumer_key = ENV.fetch('TURL_CONSUMER_KEY')
+        config.consumer_secret = ENV.fetch('TURL_CONSUMER_SECRET')
+        config.access_token = ENV.fetch('TURL_ACCESS_TOKEN')
+        config.access_token_secret = ENV.fetch('TURL_ACCESS_TOKEN_SECRET')
+      end
+    end
+
+  end
+end

data/lib/turl/link.rb

@@ -0,0 +1,38 @@
+module Turl
+  class Link < ApplicationRecord
+    has_many :tweet_links
+    has_many :tweets, through: :tweet_links
+
+    # TODO: Make it configurable
+    def self.ignored?(url)
+      url.expanded_url.host == 'twitter.com'
+    end
+
+    def self.from_response!(resp, tweet)
+      url = resp.expanded_url.to_s
+      find_or_initialize_by(normalized_url: normalize(url)).tap do |u|
+        if u.new_record?
+          begin
+            title = URI.open(u.normalized_url) do |resp|
+              Nokogiri::HTML(resp.read).title
+            end
+          rescue => ex
+            Turl.logger.error "Error when detecting title for #{u.normalized_url}: #{ex}"
+          end
+
+          u.update!(
+            title: title,
+          )
+        else
+          u.save!
+        end
+
+        u.tweets << tweet unless u.tweets.include?(tweet)
+      end
+    end
+
+    def self.normalize(url)
+      Normalizer.normalize(url)
+    end
+  end
+end

data/lib/turl/normalizer.rb

@@ -0,0 +1,45 @@
+module Turl
+  module Normalizer
+    extend self
+
+    def normalize(url)
+      path = []
+      normalize_internal(url, path: path).tap do |result|
+        path.each do |before|
+          record = UrlNormalization.find_or_initialize_by(original_url: before)
+          record.update!(normalized_url: result)
+        end
+      end
+    end
+
+    private def normalize_internal(url, path:)
+      return url if path.include?(url) || path.size > 30
+
+      path << url
+
+      ret = UrlNormalization.find_by(original_url: url)
+      return ret.normalized_url if ret
+
+      parsed = URI.parse(url)
+      case
+      when parsed.host == 'htn.to'
+        resp = head(parsed)
+        normalize_internal(resp['x-redirect-to'] || url, path: path)
+      else
+        resp = head(parsed)
+        if resp.is_a?(Net::HTTPRedirection) && resp['location']
+          normalize_internal(resp['location'], path: path)
+        else
+          url
+        end
+      end
+    end
+
+    private def head(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == 'https'
+      path = [uri.path, uri.query].compact.join('?').presence || '/'
+      http.request_head(path)
+    end
+  end
+end

data/lib/turl/tweet.rb

@@ -0,0 +1,17 @@
+module Turl
+  class Tweet < ApplicationRecord
+    belongs_to :twitter_user
+    has_many :links, through: :tweet_links
+
+    def self.from_response!(tweet_resp)
+      user = TwitterUser.from_response!(tweet_resp.user)
+      find_or_initialize_by(twitter_id: tweet_resp.id).tap do |t|
+        t.update!(
+          content: tweet_resp.text,
+          twitter_user: user,
+          tweeted_at: tweet_resp.created_at,
+        )
+      end
+    end
+  end
+end

data/lib/turl/tweet_link.rb

@@ -0,0 +1,6 @@
+module Turl
+  class TweetLink < ApplicationRecord
+    belongs_to :tweet
+    belongs_to :link
+  end
+end

data/lib/turl/twitter_user.rb

@@ -0,0 +1,11 @@
+module Turl
+  class TwitterUser < ApplicationRecord
+    has_many :tweets
+
+    def self.from_response!(resp)
+      find_or_initialize_by(twitter_id: resp.id).tap do |u|
+        u.update!(screen_name: resp.screen_name)
+      end
+    end
+  end
+end

data/lib/turl/url_normalization.rb

@@ -0,0 +1,4 @@
+module Turl
+  class UrlNormalization < ApplicationRecord
+  end
+end

data/lib/turl/version.rb

@@ -0,0 +1,3 @@
+module Turl
+  VERSION = "0.1.0"
+end

data/lib/turl/views/root.html.erb

@@ -0,0 +1,10 @@
+<ul>
+  <% links.each do |link| %>
+    <li>
+      <a href="<%= link.normalized_url %>">
+        <%= link.title %>
+        <br />
+        <small><%= link.normalized_url %></small>
+    </li>
+  <% end %>
+</ul>

data/lib/turl/web.rb

@@ -0,0 +1,23 @@
+require 'sinatra/base'
+
+module Turl
+  class Web
+    def self.run(argv)
+      Turl.prepare_database!
+      self.new.run(argv)
+    end
+
+    def run(argv)
+      App.run!
+    end
+
+    class App < Sinatra::Base
+      set :erb, :escape_html => true
+
+      get '/' do
+        links = Link.where('updated_at > ?', 1.week.ago).order(updated_at: :desc)
+        erb :'root.html', locals: { links: links }
+      end
+    end
+  end
+end

data/turl.gemspec

@@ -0,0 +1,33 @@
+require_relative 'lib/turl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "turl"
+  spec.version       = Turl::VERSION
+  spec.authors       = ["Masataka Pocke Kuwabara"]
+  spec.email         = ["kuwabara@pocke.me"]
+
+  spec.summary       = %q{}
+  spec.description   = %q{}
+  spec.homepage      = "https://github.com/pocke/turl"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'twitter'
+  spec.add_runtime_dependency 'activerecord'
+  spec.add_runtime_dependency 'sqlite3'
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'sinatra'
+  spec.add_runtime_dependency 'erubi'
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: turl
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Masataka Pocke Kuwabara
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: erubi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- kuwabara@pocke.me
+executables:
+- turl-collect
+- turl-web
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/turl-collect
+- exe/turl-web
+- lib/turl.rb
+- lib/turl/application_record.rb
+- lib/turl/collect.rb
+- lib/turl/link.rb
+- lib/turl/normalizer.rb
+- lib/turl/tweet.rb
+- lib/turl/tweet_link.rb
+- lib/turl/twitter_user.rb
+- lib/turl/url_normalization.rb
+- lib/turl/version.rb
+- lib/turl/views/root.html.erb
+- lib/turl/web.rb
+- turl.gemspec
+homepage: https://github.com/pocke/turl
+licenses: []
+metadata:
+  homepage_uri: https://github.com/pocke/turl
+  source_code_uri: https://github.com/pocke/turl
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: ''
+test_files: []