slg 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 138f7e46a189d5f94aec8149144f460938dff2f74e3f86650ac323874be91c3e
+  data.tar.gz: 7b5b412986a25443036a2f8dabef508d2b59242884542f443837a9389c512bda
+SHA512:
+  metadata.gz: b9c3b92bad04d21f66128e0cfdd1cc6001f8a70eb3323c86b2e42dafa95756d7af2ff5e4a109c9ccc70e6876d6d50bd8ca9fd19ad176098bce6152520e35e071
+  data.tar.gz: 8e04ce6ff7b6086edcfc408587a8f451861c2a0055859da63c92f8c53d177946413f52dd2727e2a5ab2ccbdf7525096f4429f32c51544ebf44d0816a5b61fde2

data/bin/slg

@@ -0,0 +1,39 @@
+#! /usr/bin/env ruby
+# -*- coding: UTF-8 -*-
+
+# clean interrupt
+trap("INT") { abort }
+
+require "optimist"
+require "slg"
+
+opts = Optimist.options do
+  version "slg #{Slg.version}"
+  banner <<-EOS
+Slg is a command-line tool to scrape definitions from Slengo.it.
+
+Usage:
+    slg [options] <word(s)>
+where [options] are:
+EOS
+
+  opt :count, "Limit the number of definitions", :type => :int, :default => 1, :short => "-n"
+  opt :color, "Use colorized output", :default => true
+  opt :browser, "Open the results in a browser window", :short => "-b"
+end
+
+Optimist.die :count, "must be non-negative" if opts[:count] < 0
+
+if ARGV.empty?
+  puts "Error: No word provided. Use -h or --help to see the help."
+  exit 1
+end
+
+term = ARGV.join " "
+
+if opts[:browser]
+  Slg.open_url(term)
+else
+  q = Slg.query(ARGV.join(" "), opts)
+  puts Slg.format_results(q, opts[:color])
+end

data/lib/slg.rb

@@ -0,0 +1,107 @@
+# -*- coding: UTF-8 -*-
+
+require "uri"
+require "open-uri"
+require "nokogiri"
+
+require "ud/formatting"
+
+# Most of this code is copy/pasted from UD.
+# TODO generalize the code in UD and reuse it here.
+
+# This module provide some methods to scrape definitions from Slengo.it.
+module Slg
+  class << self
+    # @return [String] the current gem's version
+    def version
+      "0.0.1"
+    end
+
+    WWW_ROOT = "https://slengo.it"
+
+    # Get the search URL to query for a given term.
+    # @param term [String] the term to search for. It must be a string, spaces
+    #                      are allowed.
+    # @return [String]
+    def search_url(term)
+      param = URI.encode_www_form_component(term).gsub(/\+/, "%20")
+      "#{WWW_ROOT}/define/#{param}"
+    end
+
+    # Open the search URL in the user's browser
+    # @param term [String] the term to search for. It must be a string, spaces
+    #                      are allowed.
+    # @return [Nil]
+    def open_url(term)
+      system open_cmd, search_url(term)
+    end
+
+    # Query the website and return a list of definitions for the provided term.
+    # This list may be empty if there's no result.
+    # @param term [String] the term to search for
+    # @param opts [Hash] options. This is used by the command-line tool.
+    #                    +:count+ is the maximum number of results to return
+    # @return [Array<Hash>]
+    def query(term, opts = {})
+      url = search_url(term)
+      text = OpenURI.open_uri url
+
+      opts = { :count => 1 }.merge(opts || {})
+
+      return [] if opts[:count] <= 0
+
+      doc = Nokogiri::HTML.parse text
+
+      doc.css("article.definition-card").map do |elt|
+        examples = elt.css(".word-examples li").map(&:text).map(&:strip)
+        example = examples.join "\n\n"
+
+        votes = elt.css(".votes-container .v-progress-linear")
+        # Slengo.it uses percentages instead of number of votes
+        upvotes = votes.attr("aria-valuenow").to_s.to_i
+        downvotes = 100 - upvotes
+
+        header_p = elt.css("header p").first
+
+        definition = elt.css("div.word-definition").text
+        # Remove 'see also'
+        definition.gsub! /\bCfr\..+/m, ""
+        definition.strip!
+
+        {
+          # There's no :id nor :author
+          # For some reason the selector 'h1.word-title' work in JS in the
+          # browser but not with Nokogiri.
+          :word => header_p.text.strip,
+          :permalink => url,
+          :definition => definition,
+          :example => example,
+          :upvotes => upvotes,
+          :downvotes => downvotes,
+          # TODO add region as well
+        }
+
+      end.take opts[:count]
+    end
+
+    # Format results for output
+    # @param results [Array] this must be an array of results, as returned by
+    #                        +UD.query+.
+    # @param color [Boolean] colored output
+    # @return [String]
+    def format_results(results, color = true)
+      UD::Formatting.text(results, color)
+    end
+
+    def open_cmd
+      case RbConfig::CONFIG["host_os"]
+      when /darwin/
+        "open"
+      when /bsd|linux/
+        "xdg-open"
+      when /cygwin|mingw|mswin/
+        "start"
+      end
+    end
+  end
+end

data/tests/query_tests.rb

@@ -0,0 +1,44 @@
+#! /usr/bin/env ruby
+# -*- coding: UTF-8 -*-
+
+require File.dirname(__FILE__) + '/fake_responses'
+
+class Slg_Query_test < Test::Unit::TestCase
+  def setup
+    @foo, @bar = [
+      {
+        :definition=>"Foo foo.",
+        :downvotes=>0,
+        :example=> "foo ex 1\n\nfoo ex 2",
+        :permalink=>"https://slengo.it/define/culo",
+        :upvotes=>100,
+        :word=>"foo",
+      },
+      {
+        :definition=>"Bar bar.",
+        :downvotes=>10,
+        :example=>"bar ex",
+        :permalink=>"https://slengo.it/define/culo",
+        :upvotes=>90,
+        :word=>"foo",
+      }
+    ]
+  end
+
+  def test_search_url
+    assert_equal("https://slengo.it/define/foo", Slg.search_url('foo'))
+  end
+
+  def test_query_no_results
+    assert_equal([], Slg.query('foooo'))
+  end
+
+  def test_query_two_results
+    assert_equal([@foo, @bar], Slg.query('culo', :count => 2))
+  end
+
+  def test_query_count
+    assert_equal([@foo], Slg.query('culo', :count => 1))
+  end
+end
+

data/tests/tests.rb

@@ -0,0 +1,53 @@
+#! /usr/bin/env ruby
+# -*- coding: UTF-8 -*-
+
+ci = ENV["CI"] || ENV["CONTINUOUS_INTEGRATION"]
+
+if ci
+  require 'coveralls'
+  Coveralls.wear!
+end
+
+require 'test/unit'
+require 'simplecov'
+
+test_dir = File.expand_path( File.dirname(__FILE__) )
+
+SimpleCov.formatter = Coveralls::SimpleCov::Formatter if ci
+SimpleCov.start { add_filter '/tests/' }
+
+require 'slg'
+
+for t in Dir.glob( File.join( test_dir,  '*_tests.rb' ) )
+  require t
+end
+
+class SlgTests < Test::Unit::TestCase
+
+  def test_slg_version
+    assert(Slg.version =~ /^\d+\.\d+\.\d+/)
+  end
+
+  def test_slg_open_cmd
+    os = RbConfig::CONFIG["host_os"]
+
+    RbConfig::CONFIG["host_os"] = "darwin"
+    assert_equal "open", Slg.send(:open_cmd)
+
+    RbConfig::CONFIG["host_os"] = "linux"
+    assert_equal "xdg-open", Slg.send(:open_cmd)
+
+    RbConfig::CONFIG["host_os"] = "bsd"
+    assert_equal "xdg-open", Slg.send(:open_cmd)
+
+    RbConfig::CONFIG["host_os"] = "cygwin"
+    assert_equal "start", Slg.send(:open_cmd)
+
+  ensure
+    RbConfig::CONFIG["host_os"] = os
+  end
+
+end
+
+
+exit Test::Unit::AutoRunner.run

metadata

@@ -0,0 +1,200 @@
+--- !ruby/object:Gem::Specification
+name: slg
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Baptiste Fontaine
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIID/DCCAmSgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAkMSIwIAYDVQQDDBliL0RD
+  PXB0aXN0ZWZvbnRhaW5lL0RDPWZyMB4XDTIwMDMwNDE5MDMyMVoXDTIxMDMwNDE5
+  MDMyMVowJDEiMCAGA1UEAwwZYi9EQz1wdGlzdGVmb250YWluZS9EQz1mcjCCAaIw
+  DQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKdjTwN/gnJ3WHxKeusaRN1o902Q
+  BmIRgdtbhpsHtgUruKNujutEBpmhG1fka8+mujhqh885blckeC8hXgqpHGK1MsJU
+  tgsNB2F2Nvof/uGBzi+OZ76HPLmfvJFb/KUiwrh08jTbFftMBCmuekbxBajt2N2F
+  shF6DFEqO2hl/itVt5UeIZhpIZvO7qSZ9FtLRXs0q2O27h4QfAv6XobNDwH5TBs5
+  z+EYufgTtupf26xW0v6z2NDG6z36Eap7QAwoivuPCuXO+IPgxhubtGvFYSo7ygP4
+  RbYsaGucRneCCzezTD+i1LnhGt9TCbQjdvidgM+i616JC/HRXglM0apgBZSjayoI
+  XHoajIajnU+570cu01Wsa+SF9m4stKao0hLnCUMeYybmA/Ilk58nJ2xZJWMULu76
+  7Ttzead1Ah25ZdWefKG5FIGkpezEp4z/YihGEJt+hxS7Aa2xYiIlnYFwUno530/8
+  So5Ibqdd3pMKJDUQJvGIBhsQIXF4Iogk4txoPwIDAQABozkwNzAJBgNVHRMEAjAA
+  MAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUsV1eKnEFtVRwyz6/nTuharH3G+UwDQYJ
+  KoZIhvcNAQELBQADggGBAJz7tbRe/iyMthV7fdqnhFv0SZo1UVWZSaIjB39W4lUi
+  7X1Smag2Fe7IqARUPLn2e49ObyupAmTi0zHoSG/+HsBUqNPPoVW1Y67dtrMM49o7
+  qSN66EogB4nvGdvqJ9eOlioiAX7vczWmQjfSONuk0ihOfaTChD7LwoPptTUu/Q6+
+  aT8zz7TH+sxxx+D09Q9+Ywf1RVnUe+i1PTo8kadnHIQdeBPudKrw5GWcIoLz1V16
+  A+j/SNEUqY1ui8TBX6axkd8vh6C1YcVAFCcdrxTMFTeB1irvL8jns6SulRHM08UC
+  9NR+YthP16QuQzb5R2wIGX6WT11cnIpo0LzTnY/QSZGgwJkuLalBLzUssMiVnoSX
+  rVdmQBLrs38YcJouqHpf0wCGygf5430Bq4nCu0xUaH5umPcwJE9qaF1ZfgSCfZEz
+  Qe5fflfwGHtlmT6SWZoQD48xF/ILJ+cLTpF79ardGYnWT6N7VUAI5lKp7OjZYd+q
+  wwYlmlBW0Femxy5WoxzECQ==
+  -----END CERTIFICATE-----
+date: 2020-03-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: optimist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: colored
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: ud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: Get words' definitions from Slengo on the command-line.
+email: b@ptistefontaine.fr
+executables:
+- slg
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/slg
+- lib/slg.rb
+- tests/query_tests.rb
+- tests/tests.rb
+homepage: https://github.com/bfontaine/slg
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Slengo.it unofficial scrapper
+test_files:
+- tests/tests.rb
+- tests/query_tests.rb