uri_scanner 0.0.1

data/lib/uri_scanner/uri_scanner.rl

@@ -0,0 +1,45 @@
+#
+# @LANG: ruby
+# Scan text and extracts URI(s)
+# 
+
+=begin
+%%{
+  machine uri_scanner;
+  include actions "../machines/ruby_actions.rl";
+  include ip_addr "../machines/ip_addr.rl";
+  include uri     "../machines/uri.rl";
+  include sip_uri "../machines/sip_uri.rl";
+
+  main := |*
+    URI => {@collection << data[ts..te-1]};
+    any;
+  *|;
+}%%
+=end
+
+class MachineURIScanner
+  attr_accessor :collection
+
+  def initialize(data)
+    @collection = []
+    eof = data.length
+    %% write data;
+    %% write init;
+    %% write exec;
+    #%
+  end
+
+  class << self
+    def scan(data, to_objects=false)
+      if to_objects
+        self.new(data).collection.map do |url|
+          MachineURI.new(url)
+        end
+      else
+        self.new(data).collection
+      end
+    end
+  end
+end
+

data/lib/uri_scanner/version.rb

@@ -0,0 +1,3 @@
+module URIScanner
+  VERSION = "0.0.1"
+end

data/spec/ip_addr_spec.rb

@@ -0,0 +1,64 @@
+require 'uri_scanner/ip_address'
+
+RSpec.describe MachineIpAddr do
+  let(:ip){ MachineIpAddr.new }
+  context "parsing IPv4" do
+    specify { expect(ip.is_valid? "8.8.8.8").to be true }
+    specify { expect(ip.is_valid? "0.0.0.0").to be true }
+    specify { expect(ip.is_valid? "0.0.0.1").to be true }
+    specify { expect(ip.is_valid? "10.0.0.1").to be true }
+    specify { expect(ip.is_valid? "192.168.100.10").to be true }
+    specify { expect(ip.is_valid? "172.31.1.34").to be true }
+    specify { expect(ip.is_valid? "10.00.00.01").to be true }
+    specify { expect(ip.is_valid? "010.000.000.001").to be true }
+
+    specify { expect(ip.is_valid? "").to be false }
+    specify { expect(ip.is_valid? "12.12.a.b").to be false }
+    specify { expect(ip.is_valid? "1.2.3.4.5").to be false }
+    specify { expect(ip.is_valid? "1.2.3.4.").to be false }
+    specify { expect(ip.is_valid? "111.222.333.444").to be false }
+    specify { expect(ip.is_valid? "256.222.33.44").to be false }
+    specify { expect(ip.is_valid? "25.299.33.44").to be false }
+    specify { expect(ip.is_valid? "25.99.333.44").to be false }
+    specify { expect(ip.is_valid? "25.99.3.440").to be false }
+  end
+
+  # examples from RFC5952
+  context "parsing IPv6" do
+    specify { expect(ip.is_valid? "2001:db8:0:0:1:0:0:1").to be true }
+    specify { expect(ip.is_valid? "2001:0db8:0:0:1:0:0:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8::1:0:0:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8::0:1:0:0:1").to be true }
+    specify { expect(ip.is_valid? "2001:0db8::1:0:0:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0:0:1::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0000:0:1::1").to be true }
+    specify { expect(ip.is_valid? "2001:DB8:0:0:1::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd:eeee:0001").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd:eeee:001").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd:eeee:01").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd:eeee:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0:0:0::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0:0::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8::aaaa:0:0:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0:0:aaaa::1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd:eeee:AAAA").to be true }
+    specify { expect(ip.is_valid? "2001:db8:aaaa:bbbb:cccc:dddd:eeee:AaAa").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0:0:0:0:2:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8::2:1").to be true }
+    specify { expect(ip.is_valid? "2001:db8:0:1:1:1:1:1").to be true }
+    specify { expect(ip.is_valid? "::1").to be true }
+    specify { expect(ip.is_valid? "0000:0000:0000:0000:0000:0000:0000:0001").to be true }
+
+    specify { expect(ip.is_valid? "").to be false }
+    specify { expect(ip.is_valid? "2001:db8::1:1:1:::1").to be false }
+    specify { expect(ip.is_valid? "2016:::1:1").to be false }
+    specify { expect(ip.is_valid? "ab::1:2::a").to be false }
+    specify { expect(ip.is_valid? "qwe:1:2:3:4:5:6:7").to be false }
+    specify { expect(ip.is_valid? "aa:bb:cc:dd:ff:11:22:33:44").to be false }
+    specify { expect(ip.is_valid? "aa:bb:11:").to be false }
+    specify { expect(ip.is_valid? ":1234:aaaa:22:bb:ff").to be false }
+  end
+end

data/spec/scanner_spec.rb

@@ -0,0 +1,40 @@
+require 'uri_scanner/uri_parser'
+require 'uri_scanner/uri_scanner'
+
+RSpec.describe MachineURIScanner do
+  describe "#scan" do
+    it "returns single address" do
+      data = "text before http://example.com:8042/index.php?name=ferret#nose text after"
+      expect(MachineURIScanner.scan(data)).to eql(%w{http://example.com:8042/index.php?name=ferret#nose})
+    end
+    it "returns three URIs" do
+      data  = "Lorem ipsum dolor sit amet, consectetur "
+      data += "adipiscing elit, \"ftp://ftp.is.co.za/rfc/rfc1808.txt\" sed "
+      data += "do eiusmod tempor incididunt  <http://www.ietf.org/rfc/rfc2396.txt> "
+      data += "ut labore et ldap://[2001:db8::7]/c=GB?objectClass?one dolore magna aliqua."
+      expect(MachineURIScanner.scan(data).length).to be 3
+      expect(MachineURIScanner.scan(data)).to eql(%w{
+        ftp://ftp.is.co.za/rfc/rfc1808.txt
+        http://www.ietf.org/rfc/rfc2396.txt
+        ldap://[2001:db8::7]/c=GB?objectClass?one
+      })
+    end
+    it "scans text from file" do
+      f = File.open("spec/url.txt")
+      expect(MachineURIScanner.scan(f.read).length).to be 15
+    end
+
+    context "to array of MachineURI instances" do
+      let(:data) {%Q{text before http://example.com:8042/index.php?name=ferret#nose text after
+                  Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam 
+                  nec erat "foo://info.example.com?fred" Proin sollicitudin <sip:12345@sip-provider.info:5060> }}
+      let(:scanner) { MachineURIScanner.scan(data, true) }
+      specify{expect(scanner.first.scheme).to eq("http")}
+      specify{expect(scanner.first.port).to eq(8042)}
+      specify{expect(scanner.last.host).to eq("sip-provider.info")}
+      specify{expect(scanner[1].query).to eq("fred")}
+    end
+
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,96 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

data/spec/uri_scanner_spec.rb

@@ -0,0 +1,43 @@
+require 'uri_scanner'
+
+RSpec.describe URIScanner do
+  describe "#is_ip_valid?" do
+    it{expect(URIScanner.is_ip_valid?("172.31.1.222")).to be true}
+    it{expect(URIScanner.is_ip_valid?("172.311.1.222")).to be false}
+    it{expect(URIScanner.is_ip_valid?("::1")).to be true}
+    it{expect(URIScanner.is_ip_valid?("2016:::1:1")).to be false}
+    it{expect(URIScanner.is_ip_valid?("")).to be false}
+  end
+
+  describe "#parse_uri" do
+    context "parsing 'ldap://[2001:db8::7]/c=GB?objectClass?one'" do
+      let(:uri){ URIScanner.parse_uri "ldap://[2001:db8::7]/c=GB?objectClass?one"}
+      it {expect(uri.scheme).to eq("ldap")}
+      it {expect(uri.host).to eq("[2001:db8::7]")}
+      it {expect(uri.port).to be_nil}
+      it {expect(uri.path).to eq("/c=GB")}
+      it {expect(uri.query).to eq("objectClass?one")}
+    end
+  end
+
+  describe "#scan" do
+    context "text and returns array of strings" do
+      let(:data) {File.read "spec/url.txt" }
+      let(:uris) {URIScanner.scan data}
+      it{expect(uris.length).to be 15}
+      it{uris.each{|u| expect(u).to be_a String}}
+    end
+
+    context "text and returns array of uri objects" do
+      let(:data) {%Q{text before http://example.com:8042/index.php?name=ferret#nose text after
+                  Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam 
+                  nec erat "foo://info.example.com?fred=homme#world" Proin sollicitudin <sip:12345@sip-provider.info:5060> }}
+      let(:uri) {URIScanner.scan_and_parse(data)[1]}
+      it{expect(uri.scheme).to eq("foo")}
+      it{expect(uri.host).to eq("info.example.com")}
+      it{expect(uri.port).to be_nil}
+      it{expect(uri.query).to eq("fred=homme")}
+      it{expect(uri.fragment).to eq("world")}
+    end
+  end
+end

data/spec/uri_spec.rb

@@ -0,0 +1,185 @@
+require 'uri_scanner/uri_parser'
+
+RSpec.describe MachineURI do
+  context "validate URI" do
+    specify {
+      expect(MachineURI.new("http://www.ietf.org/rfc/rfc2396.txt")
+        .is_valid?).to be true}
+
+    specify {
+      expect{ MachineURI.new("1http://www.ietf.org/rfc/rfc2396.txt") }
+        .to raise_error(URIParserError)}
+
+    it "testing multiple samples with authority part" do
+      %W{
+        http://localhost/
+        ftp://ftp.is.co.za/rfc/rfc1808.txt
+        http://www.ietf.org/rfc/rfc2396.txt
+        ldap://[2001:db8::7]/c=GB?objectClass?one
+        telnet://192.0.2.16:80/
+        example://a/b/c/%7Bfoo%7D
+        eXAMPLE://a/./b/../b/%63/%7bfoo%7d
+        foo://example.com:8042/over/there?name=ferret#nose
+        foo://info.example.com?fred
+        redis://host:6503/dbindex/keyname
+      }.each do |uri|
+          expect( MachineURI.new(uri).is_valid? ).to be true
+        end
+    end
+    it "testing multiple samples without authority part" do
+      %W{
+        urn:oasis:names:specification:docbook:dtd:xml:4.1.2
+        urn:example:animal:ferret:nose
+        mailto:John.Doe@example.com
+        news:comp.infosystems.www.servers.unix
+        sip:12345@sip-provider.info:5060
+        sips:johndoe@sip.secure.com
+        tel:+1-816-555-1212
+      }.each do |uri|
+          expect( MachineURI.new(uri).is_valid? ).to be true
+        end
+    end
+  end
+
+  context "component parts parsing" do
+    describe "#scheme" do
+      it "should be 'foo'" do
+        uri = MachineURI.new "foo://example.com:8042/over/there?name=ferret#nose"
+        expect(uri.scheme).to eq("foo")
+      end
+      it "should be 'sip'" do
+        uri = MachineURI.new "sip:username@example.com"
+        expect(uri.scheme).to eq("sip")
+      end
+      it "raises error on invalid scheme" do
+        expect {
+          MachineURI.new "f~oo://google.com"
+        }.to raise_error(URIParserError)
+        expect {
+          MachineURI.new "1http://google.com"
+        }.to raise_error(URIParserError)
+      end
+    end
+
+    describe "#host" do
+      it "should be 'example.com'" do
+        uri = MachineURI.new "foo://example.com:8042/over/there?name=ferret#nose"
+        expect(uri.host).to eq("example.com")
+      end
+      it "should be 'sip-provider.info'" do
+        uri = MachineURI.new "sip:12345@sip-provider.info:5060"
+        expect(uri.host).to eq("sip-provider.info")
+      end
+      it "should be '[2001:db8::7]'" do
+        uri = MachineURI.new "ldap://[2001:db8::7]/c=GB?objectClass?one"
+        expect(uri.host).to eq("[2001:db8::7]")
+      end
+      it "should be '192.0.2.16'" do
+        uri = MachineURI.new "telnet://192.0.2.16:80/"
+        expect(uri.host).to eq("192.0.2.16")
+      end
+      it "should be nil" do
+        uri = MachineURI.new "urn:example:animal:ferret:nose"
+        expect(uri.host).to be_nil
+      end
+    end
+
+    describe "#userinfo" do
+      it "should be 'john'" do
+        uri = MachineURI.new "foo://john@example.com:8042/over/there?name=ferret#nose"
+        expect(uri.userinfo).to eq("john")
+      end
+      it "should be nil" do
+        uri = MachineURI.new "ldap://[2001:db8::7]/c=GB?objectClass?one"
+        expect(uri.userinfo).to be_nil
+      end
+      it "should be '12345'" do
+        uri = MachineURI.new "sip:12345@sip-provider.info:5060"
+        expect(uri.userinfo).to eq("12345")
+      end
+    end
+
+    describe "#port" do
+      it "should be 8042" do
+        uri = MachineURI.new "foo://john@example.com:8042/over/there?name=ferret#nose"
+        expect(uri.port).to eq(8042)
+      end
+      it "should be nil" do
+        uri = MachineURI.new "http://www.ietf.org/rfc/rfc2396.txt"
+        expect(uri.port).to be_nil
+      end
+      it "should be '5060'" do
+        uri = MachineURI.new "sip:12345@sip-provider.info:5060"
+        expect(uri.port).to eq(5060)
+      end
+    end
+
+    describe "#path" do
+      it "should be '/rfc/rfc2396.txt'" do
+        uri = MachineURI.new "http://www.ietf.org/rfc/rfc2396.txt#section%205.2"
+        expect(uri.path).to eq("/rfc/rfc2396.txt")
+      end
+      it "should be nil'" do
+        uri = MachineURI.new "telnet://192.0.2.16:80"
+        expect(uri.path).to be_empty
+      end
+      it "should be 'example:animal:ferret:nose'" do
+        uri = MachineURI.new "urn:example:animal:ferret:nose"
+        expect(uri.path).to eq("example:animal:ferret:nose")
+      end
+    end
+
+    describe "#query" do
+      it "should be 'name=ferret'" do
+        uri = MachineURI.new "foo://john@example.com:8042/over/there?name=ferret#nose"
+        expect(uri.query).to eq("name=ferret")
+      end
+      it "should be nil" do
+        uri = MachineURI.new "http://www.ietf.org/rfc/rfc2396.txt"
+        expect(uri.query).to be_nil
+      end
+    end
+
+    describe "#fragment" do
+      it "should be 'nose'" do
+        uri = MachineURI.new "foo://john@example.com:8042/over/there?name=ferret#nose"
+        expect(uri.fragment).to eq("nose")
+      end
+      it "should be nil" do
+        uri = MachineURI.new "http://www.ietf.org/rfc/rfc2396.txt?user=john"
+        expect(uri.fragment).to be_nil
+      end
+      it "should be 'section%205.2'" do
+        uri = MachineURI.new "http://www.ietf.org/rfc/rfc2396.txt#section%205.2"
+        expect(uri.fragment).to eq("section%205.2")
+      end
+    end
+
+    context "should be parsed to fields" do
+      let(:uri) {MachineURI.new "foo://john@example.com:8042/over/there?name=ferret#nose"}
+      it{expect(uri.scheme).to eq("foo")}
+      it{expect(uri.userinfo).to eq("john")}
+      it{expect(uri.host).to eq("example.com")}
+      it{expect(uri.port).to eq(8042)}
+      it{expect(uri.path).to eq("/over/there")}
+      it{expect(uri.query).to eq("name=ferret")}
+      it{expect(uri.fragment).to eq("nose")}
+    end
+
+    context "should parse SIP to fields" do
+      let(:uri){MachineURI.new "sips:alice:secretW0rd@gateway.com:5061;transport=udp;user=phone;method=REGISTER?subject=sales%20meeting&priority=urgent&to=sales%40city.com"}
+      it{expect(uri.scheme).to eq("sips")}
+      it{expect(uri.username).to eq("alice")}
+      it{expect(uri.password).to eq("secretW0rd")}
+      it{expect(uri.host).to eq("gateway.com")}
+      it{expect(uri.port).to eq(5061)}
+      it{expect(uri.param[:transport]).to eq("udp")}
+      it{expect(uri.param[:user]).to eq("phone")}
+      it{expect(uri.param[:method]).to eq("REGISTER")}
+      it{expect(uri.header[:subject]).to eq("sales meeting")}
+      it{expect(uri.header[:priority]).to eq("urgent")}
+      it{expect(uri.header[:to]).to eq("sales@city.com")}
+    end
+  end
+end
+