uri_scanner 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0c1cf86b6e9135a1fb326524b5069413b1f3bff9
+  data.tar.gz: 687da77c9318bbdb712e1bac6d1f164518526c47
+SHA512:
+  metadata.gz: 33be598298215e59cb044256b6ccd25a3d40394a5bf0a23c9ea5d173938a095f6fbd8ae7bdf9bfb63fe3ef34202cfda1ed0b692187ec31ab6ad523ce0d25c547
+  data.tar.gz: 344bed1a5df1cc61cace654f3d3d34a875383654f0af5076eaf866db00920cbe4f592b3c286bd71087b1a936350762c1f8d4c9fa62b205ce1631771e58914852

data/.gitignore

@@ -0,0 +1,14 @@
+*.sw?
+.DS_Store
+doc
+pkg
+bin
+Gemfile.lock
+.bundle
+*.rbc
+*.log
+*.dot
+*.svg
+bundle
+.bundle
+specs.out

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in uri_scanner.gemspec
+gemspec
+
+gem 'rspec', '~> 3.0'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Stas Kobzar
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,73 @@
+# UriScanner
+
+Simple library that parses URI or scans input text for URIs.
+RFC3986 compliant. SIP URIs parsing implemented following RFC3261.
+
+This library is based on [Ragel State Machine Compiler](http://www.colm.net/open-source/ragel/).
+Ragel is great software created by Dr. Adrian D. Thurston.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'uri_scanner'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install uri_scanner
+
+## Usage
+
+Start with:
+```ruby
+require 'uri_scanner'
+```
+There are only four core methods:
+
+```scan```: Scans text and return array of found URIs
+```ruby
+URIScanner.scan(text)
+```
+
+```parse_uri```: Parses uri and return object that allows access to URI segments.
+Raises ```URIParserError```
+```ruby
+uri = URIScanner.parse_uri(uri_string)
+uri.scheme
+uri.host
+uri.port
+uri.userinfo
+uri.username
+uri.password
+uri.path
+uri.query
+uri.fragment
+uri.param
+uri.header
+```
+
+```scan_and_parse```: Same as ```scan```, but retruns array of parsed URI objects (see parse_uri)
+```ruby
+URIScanner.scan_and_parse(text)
+```
+
+```is_ip_valid?```: Additional methos that validates IPv4/IPv6 (RFC3986 ABNF)
+```ruby
+URIScanner.is_ip_valid?(ip_string)
+```
+
+Check folder "example".
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/uri_scanner/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,35 @@
+require 'rake'
+require 'mkmf'
+require 'rspec/core/rake_task'
+require "bundler/gem_tasks"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default do
+  Rake::Task['ragel'].invoke
+  Rake::Task['spec'].invoke
+end
+
+desc "Build ragel machines into ruby classes."
+task :ragel do
+  ragel = find_executable('ragel')
+  raise ArgumentError, "Ragel executable not found" unless ragel
+  Dir["lib/uri_scanner/*.rl"].each do |file|
+    sh "#{ragel} -R #{file}"
+  end
+end
+
+desc "Create transition graphs."
+task :graph do
+  format = "svg"
+  ragel = find_executable('ragel')
+  raise ArgumentError, "Ragel executable not found" unless ragel
+  dot = find_executable('dot')
+  raise ArgumentError, "Graphviz executable not found" unless dot
+  Dir["lib/uri_scanner/*.rl"].each do |file|
+    sh "#{ragel} -R #{file}"
+    sh "#{ragel} -Vp #{file} -o #{file}.dot" 
+    sh "#{dot} #{file}.dot -T#{format} -o #{file}.#{format}"
+    sh "rm #{file}.dot"
+  end
+end

data/example/parse.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+# Scan text and get all uris
+# How to use
+# >$ gem install uri_scanner
+# >$ ruby example/parse.rb URL
+#
+# Try:
+# ruby example/parse.rb "foo://user:pass@example.com:8042/over/there?name=ferret#nose"
+# ruby example/parse.rb "sips:alice:secretW0rd@gateway.com:5061;transport=udp;user=phone;method=REGISTER?subject=sales%20meeting&priority=urgent&to=sales%40city.com"
+#
+
+require 'uri_scanner'
+
+begin
+  raise "Usage: #{__FILE__} URI" unless ARGV.first
+  uri = URIScanner.parse_uri ARGV.first
+  puts "URI #{ARGV.first} segments:"
+  puts "scheme:     #{uri.scheme}"
+  puts "host:       #{uri.host}"
+  puts "port:       #{uri.port}"
+  puts "userinfo:   #{uri.userinfo}"
+  puts "username:   #{uri.username}"
+  puts "password:   #{uri.password}"
+  puts "path:       #{uri.path}"
+  puts "query:      #{uri.query}"
+  puts "fragment:   #{uri.fragment}"
+  puts "param:      #{uri.param}"
+  puts "header:     #{uri.header}"
+rescue Exception => e
+  puts e.message
+end

data/example/scanner.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+# Scan text and get all uris
+# How to use
+# >$ gem install uri_scanner
+# >$ curl https://rubygems.org/ | ruby example/scanner.rb
+
+require 'uri_scanner'
+
+URIScanner.scan( $stdin.read ).each do |uri|
+  puts uri
+end
+# To parse to uri objects:
+# URIScanner.scan_and_parse( $stdin.read ).each ...

data/lib/machines/ip_addr.rl

@@ -0,0 +1,60 @@
+# Ragel machine : IP address
+# IPv4 and IPv6 addresses 
+%%{
+  machine ip_addr;
+
+  # Ragel machine: IPv4 addresses
+  # ===
+  # Implements RFC 3986 [Section 3.2.2]
+  #      IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
+  #
+  #      dec-octet   = DIGIT                 ; 0-9
+  #                  / %x31-39 DIGIT         ; 10-99
+  #                  / "1" 2DIGIT            ; 100-199
+  #                  / "2" %x30-34 DIGIT     ; 200-249
+  #                  / "25" %x30-35          ; 250-255
+  #
+  # Additionally also treating leading "0".
+  # For example 001.010.100.1 == 1.10.100.1
+  OCTET4 =  ("0" | "00")? digit |
+            "0"? [1-9] digit    |
+            "1" digit{2}        |
+            "2" [0-4] digit     |
+            "25" [0-5]          ;
+
+  IPv4_ADDR = (OCTET4 "."){3} OCTET4;
+
+  # IPv6 addresses
+  # Implements RFC 3986 [Section 3.2.2]
+  # IPv6address =                            6( h16 ":" ) ls32
+  #                   /                       "::" 5( h16 ":" ) ls32
+  #                   / [               h16 ] "::" 4( h16 ":" ) ls32
+  #                   / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
+  #                   / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
+  #                   / [ *3( h16 ":" ) h16 ] "::"    h16 ":"   ls32
+  #                   / [ *4( h16 ":" ) h16 ] "::"              ls32
+  #                   / [ *5( h16 ":" ) h16 ] "::"              h16
+  #                   / [ *6( h16 ":" ) h16 ] "::"
+  # 
+  #       ls32        = ( h16 ":" h16 ) / IPv4address
+  #                   ; least-significant 32 bits of address
+  # 
+  #       h16         = 1*4HEXDIG
+  #                   ; 16 bits of address represented in hexadecimal
+  H16       = xdigit{1,4} ;
+  LS32      = (H16 ":" H16) | IPv4_ADDR ;
+  IPv6_ADDR =                        (H16 ":"){6} LS32 |
+                                "::"              H16  |
+                                "::" (H16 ":"){5} LS32 |
+                            H16 "::" (H16 ":"){4} LS32 |
+              (H16 ":"){,1} H16 "::" (H16 ":"){3} LS32 |
+              (H16 ":"){,2} H16 "::" (H16 ":"){2} LS32 |
+              (H16 ":"){,3} H16 "::"  H16 ":"     LS32 |
+              (H16 ":"){,4} H16 "::"              LS32 |
+              (H16 ":"){,5} H16 "::"              H16  |
+              (H16 ":"){,6} H16 "::"                   ;
+
+  # IP address
+  IP_ADDR = IPv4_ADDR | IPv6_ADDR;
+
+}%%

data/lib/machines/ruby_actions.rl

@@ -0,0 +1,49 @@
+%%{
+  machine actions;
+
+  action mark_start{
+    mark = p
+  }
+
+  action fetch_scheme{
+    @scheme = data[0..p-1]
+  }
+
+  action fetch_host{
+    @host = data[mark..p-1]
+  }
+
+  action fetch_userinfo{
+    @userinfo = data[mark..p-1]
+    @username, @password = @userinfo.split(":")
+  }
+
+  action fetch_port{
+    @port = data[mark..p-1].to_i
+  }
+
+  action fetch_path{
+    @path = data[mark..p-1]
+  }
+
+  action fetch_query{
+    @query = data[mark..p-1]
+  }
+
+  action fetch_fragment{
+    @fragment = data[mark..p-1]
+  }
+
+# Actions for SIP URI
+  action fetch_uri_params{
+    data[mark..p-1].split(";").each do |param|
+      next if param.empty?
+      k,v = param.split("=")
+      @param[k.to_sym] = v
+    end
+  }
+  action fetch_sipuri_header{
+    k,v = data[mark..p-1].split("=")
+    @header[k.to_sym] = URI.unescape(v)
+  }
+}%%

data/lib/machines/sip_uri.rl

@@ -0,0 +1,52 @@
+# Ragel machine : SIP URI
+# RFC 3261
+# SIP URI scheme seems to be somehow not compatible with RFC986.
+# http://www.ietf.org/mail-archive/web/sip/current/msg26338.html
+# http://www.ietf.org/mail-archive/web/sip/current/msg26385.html
+# Here comes RFC3261 ABN form.
+
+%%{
+  machine sip_uri;
+
+  ESCAPED         = PCT_ENC;
+  SIP_UNRESERVED  = alnum | [\-_\.!~\*'()];
+  USER_UNRESERVED = [&=+$,;?/];
+  PARAM_UNRESERVED= [[\]/:&+$];
+  TOKEN           = (alnum | [\-\.!%*_+`'~]){1,};
+  PARAMCHAR       = PARAM_UNRESERVED | SIP_UNRESERVED | ESCAPED;
+  PNAME           = PARAMCHAR{1,};
+  PVALUE          = PARAMCHAR{1,};
+  HNV_UNRESERVED  = [[\]/?:+$];
+  HNAME           = (HNV_UNRESERVED | SIP_UNRESERVED | ESCAPED){1,};
+  HVALUE          = (HNV_UNRESERVED | SIP_UNRESERVED | ESCAPED)*;
+  HEADER          = HNAME >mark_start "=" HVALUE %fetch_sipuri_header;
+
+  TEL_SUBSCRIBER  = zlen; # will implement in future
+
+  USER            = ( SIP_UNRESERVED | ESCAPED | USER_UNRESERVED ){1,};
+  PASSWORD        = ( SIP_UNRESERVED | ESCAPED | [&=+$,] )*;
+  SIP_UINFO       = ( USER | TEL_SUBSCRIBER ) >mark_start ( ":" PASSWORD )? %fetch_userinfo "@";
+  DOMAINLABEL     = alnum | (alnum (alnum | "-")* alnum);
+  TOPLABEL        = alpha | (alpha (alnum | "-")* alnum);
+  HOSTNAME        = (DOMAINLABEL ".")* TOPLABEL "."?;
+  SIPHOST         = (HOSTNAME | IPv4_ADDR | IPv6_ADDR) >mark_start %fetch_host;
+
+  HOSTPORT        = SIPHOST (":" digit{1,} >mark_start %fetch_port)? ;
+
+  OTHER_PARAM     = PNAME ("=" PVALUE)?;
+  LR_PARAM        = "lr";
+  MADDR_PARAM     = "maddr=" SIPHOST;
+  TTL_PARAM       = "ttl=" digit{1,3};
+  METHOD_PARAM    = "method=" TOKEN{1,};
+  USER_PARAM      = "user=" ( "phone" | "ip" | TOKEN);
+  TRANSPORT_PARAM = "transport=" ( "udp" | "tcp" | "sctp" | "tls" | TOKEN );
+  URI_PARAM       = TRANSPORT_PARAM | USER_PARAM | METHOD_PARAM |
+                    TTL_PARAM | MADDR_PARAM | LR_PARAM | OTHER_PARAM;
+
+  URI_PARAMS      = ( ";" URI_PARAM )* >mark_start %fetch_uri_params;
+
+  HEADERS         = "?" HEADER ("&" HEADER)*;
+
+  SIP_URI         = ("sip"i [sS]?) %fetch_scheme ":" 
+                    SIP_UINFO? HOSTPORT URI_PARAMS HEADERS?;
+}%%

data/lib/machines/uri.rl

@@ -0,0 +1,92 @@
+# Ragel machine : URI
+# RFC 3986
+#         foo://example.com:8042/over/there?name=ferret#nose
+#         \_/   \______________/\_________/ \_________/ \__/
+#          |           |            |            |        |
+#       scheme     authority       path        query   fragment
+#          |   _____________________|__
+#         / \ /                        \
+#         urn:example:animal:ferret:nose
+
+%%{
+  machine uri;
+
+# Percent-Encoding. Example: "%20" (space)
+  PCT_ENC         = "%" xdigit xdigit;
+
+# These characters are called "reserved" because 
+# they may (or may not) be defined as delimiters by
+# the generic syntax.
+  GEN_DELIMS      = [:/?#[\]@];
+  SUB_DELIMS      = [!$&'()*+,;=];
+  RESERVED        = GEN_DELIMS | SUB_DELIMS;
+
+# Characters that are allowed in a URI but do not 
+# have a reserved purpose.
+  UNRESERVED      = alnum | [\-\._~];
+
+# URI  Scheme
+# scheme      = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+  SCHEME          = alpha (alnum | [+\-\.])* %fetch_scheme;
+
+# Hierarchical element for a naming authority
+# authority   = [ userinfo "@" ] host [ ":" port ]
+  USERINFO        = (UNRESERVED | PCT_ENC | SUB_DELIMS | ":")* ;
+  IPv_FUTURE      = "v" xdigit{1,} "." (UNRESERVED | SUB_DELIMS | ":"){1,};
+  IP_LITERAL      = "[" (IPv6_ADDR | IPv_FUTURE) "]";
+  REG_NAME        = (UNRESERVED | PCT_ENC | SUB_DELIMS)*;
+  HOST            = (IP_LITERAL | IPv4_ADDR | REG_NAME) >mark_start %fetch_host;
+  PORT            = digit*;
+  AUTHORITY       = (USERINFO >mark_start %fetch_userinfo "@")? HOST (":" PORT >mark_start %fetch_port)?;
+  
+# Path RFC 3986 Section 3.3
+  PCHAR           = UNRESERVED | PCT_ENC | SUB_DELIMS | ":" | "@";
+  SEG_NZ_NC       = (UNRESERVED | PCT_ENC | SUB_DELIMS | "@"){1,};
+  SEG_NZ          = PCHAR{1,};
+  SEGMENT         = PCHAR*;
+  PATH_EMPTY      = '\0';
+  PATH_ROOTLESS   = (SEG_NZ ("/" SEGMENT)*)  >mark_start %fetch_path;
+  PATH_NOSCHEME   = (SEG_NZ_NC ("/" SEGMENT)*) >mark_start %fetch_path;
+  PATH_ABSOLUTE   = ("/" (SEG_NZ ("/" SEGMENT)*)?) >mark_start %fetch_path;
+  PATH_ABEMPTY    = ("/" SEGMENT)* >mark_start %fetch_path;
+
+  PATH            = PATH_ABEMPTY | # begins with "/" or is empty
+                    PATH_ABSOLUTE | # begins with "/" but not "//"
+                    PATH_NOSCHEME | # begins with a non-colon segment
+                    PATH_ROOTLESS | # begins with a segment
+                    PATH_EMPTY ; # zero characters
+
+# The query component contains non-hierarchical
+# Section 3.4
+  QUERY           = (PCHAR | "/" | "?")* >mark_start %fetch_query;
+
+# Fragment
+# Section 3.5
+  FRAGMENT        = (PCHAR | "/" | "?")* >mark_start %fetch_fragment;
+
+# Relative Reference
+# Section 4.2
+  REL_PART        = ("//" AUTHORITY PATH_ABEMPTY) |
+                    PATH_ABSOLUTE |
+                    PATH_NOSCHEME |
+                    PATH_EMPTY ;
+
+  REL_REF         = REL_PART  ("?" QUERY)? ("#" FRAGMENT)?;
+
+# Absolute URI
+# Section 4.3
+  HIER_PART       = ("//" AUTHORITY PATH_ABEMPTY) |
+                    PATH_ABSOLUTE |
+                    PATH_ROOTLESS |
+                    PATH_EMPTY;
+
+  URI_ABS         = SCHEME ":" HIER_PART ("?" QUERY)?;
+
+# URI Reference
+# Section 4.1; Section 3
+  URI             = SCHEME ":" HIER_PART ("?" QUERY)? ("#" FRAGMENT)?;
+
+# generic
+  URI_REF         = URI | REL_REF;
+
+}%%