rdf-turtle 1.0.3 → 1.0.4

data/README.md

@@ -45,6 +45,36 @@ In some cases, the specification is unclear on certain issues:
   datatype are saved as non-datatyped triples in the graph. This will be updated in the future when the rest of the
   library suite is brought up to date with RDF 1.1.
 
+### Freebase-specific Reader
+There is a special reader useful for processing [Freebase Dumps][]. To invoke
+this, add the `:freebase => true` option to the {RDF::Turtle::Reader.new}, or
+use {RDF::Turtle::FreebaseReader} directly. As with {RDF::Turtle::Reader},
+prefix definitions may be passed in using the `:prefixes` option to
+{RDF::Turtle::FreebaseReader#initialize} using the standard mechanism defined
+for `RDF::Reader`.
+
+The [Freebase Dumps][] have a very normalized form, similar to N-Triples but
+with prefixes. They also have a large amount of garbage. This Reader is
+optimized for this format and will perform faster error recovery.
+
+An example of reading Freebase dumps:
+
+    require "rdf/turtle"
+    fb = "../freebase/freebase-rdf-2013-03-03-00-00.ttl"
+    fb_prefixes = {
+      :ns => "http://rdf.freebase.com/ns/",
+      :key => "http://rdf.freebase.com/key/",
+      :owl => "http://www.w3.org/2002/07/owl#>",
+      :rdfs => "http://www.w3.org/2000/01/rdf-schema#",
+      :rdf => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+      :xsd => "http://www.w3.org/2001/XMLSchema#"
+    }
+    RDF::Turtle::Reader.open(fb,
+      :freebase => true,
+      :prefixes => fb_prefixes) do |r|
+
+      r.each_statement {|stmt| puts stmt.to_ntriples}
+    end
 ## Implementation Notes
 The reader uses the [EBNF][] gem to generate first, follow and branch tables, and uses
 the `Parser` and `Lexer` modules to implement the Turtle parser.
@@ -102,4 +132,4 @@ see <http://unlicense.org/> or the accompanying {file:UNLICENSE} file.
 [Turtle]:       http://www.w3.org/TR/2012/WD-turtle-20120710/
 [Turtle doc]:   http://rubydoc.info/github/ruby-rdf/rdf-turtle/master/file/README.md
 [Turtle EBNF]:  http://dvcs.w3.org/hg/rdf/file/default/rdf-turtle/turtle.bnf
-[Swap]:         http://www.w3.org/2000/10/swap/
+[Freebase Dumps]: https://developers.google.com/freebase/data

data/VERSION

@@ -1 +1 @@
-1.0.3
+1.0.4

data/lib/rdf/turtle.rb

@@ -21,10 +21,11 @@ module RDF
   # @author [Gregg Kellogg](http://greggkellogg.net/)
   module Turtle
     require  'rdf/turtle/format'
-    autoload :Reader,     'rdf/turtle/reader'
-    autoload :Terminals,  'rdf/turtle/terminals'
-    autoload :VERSION,    'rdf/turtle/version'
-    autoload :Writer,     'rdf/turtle/writer'
+    autoload :Reader,         'rdf/turtle/reader'
+    autoload :FreebaseReader, 'rdf/turtle/freebase_reader'
+    autoload :Terminals,      'rdf/turtle/terminals'
+    autoload :VERSION,        'rdf/turtle/version'
+    autoload :Writer,         'rdf/turtle/writer'
 
     def self.debug?; @debug; end
     def self.debug=(value); @debug = value; end

data/lib/rdf/turtle/freebase_reader.rb

@@ -0,0 +1,96 @@
+require 'rdf'
+require 'rdf/ntriples'
+
+module RDF::Turtle
+  ##
+  # Parser specifically for Freebase, which has a very regular form.
+  #
+  # @see https://developers.google.com/freebase/data
+  class FreebaseReader < RDF::NTriples::Reader
+    include RDF::Turtle::Terminals
+
+    def self.format; RDF::Turtle::Format; end
+
+    ##
+    # Extension to N-Triples reader, includes reading
+    # pnames and prefixes
+    def read_triple
+      loop do
+        begin
+          readline.strip!
+          line = @line
+          unless blank? || read_prefix
+            subject   = read_pname(:intern => true) || fail_subject
+            predicate = read_pname(:intern => true) || fail_predicate
+            object    = read_pname || read_uriref || read_boolean || read_numeric || read_literal || fail_object
+            if validate? && !read_eos
+              raise RDF::ReaderError, "expected end of statement in line #{lineno}: #{current_line.inspect}"
+            end
+            return [subject, predicate, object]
+          end
+        rescue RDF::ReaderError => e
+          raise e if validate?
+          $stderr.puts e.message
+        end
+      end
+    end
+
+    ##
+    # Read a prefix of the form `@prefix pfx: <uri> .
+    #
+    # Add prefix definition to `prefixes`
+    # @return [RDF::URI]
+    def read_prefix
+      if prefix_str = match(/^@prefix\s+(\w+:\s+#{IRIREF})\s*.$/)
+        prefix, iri = prefix_str.split(/:\s+/)
+        return nil unless iri
+        prefix(prefix, iri[1..-2])
+      end
+    end
+
+    ##
+    # Read a PNAME of the form `prefix:suffix`.
+    # @return [RDF::URI]
+    def read_pname(options = {})
+      if pname_str = match(/^(\w+:\S+)/)
+        ns, suffix = pname_str.split(':', 2)
+        if suffix[-1,1] == "."
+          suffix.chop!  # Remove end of statement
+          @line.insert(0, ".")
+        end
+        pfx_iri = prefix(ns)
+        raise RDF::ReaderError, "prefix #{ns.inspect} is not defined" unless pfx_iri
+        uri = RDF::URI(pfx_iri + suffix)
+        uri.validate!     if validate?
+        uri
+      end
+    rescue ArgumentError => e
+      raise RDF::ReaderError, "invalid PName"
+    end
+
+    ##
+    # Read a numeric value
+    # @return [RDF::Literal::Integer, RDF::Literal::Float, RDF::Literal::Double]
+    def read_numeric
+      case
+      when double_str = match(/^(#{DOUBLE})/)
+        double_str = double_str.sub(/\.([eE])/, '.0\1')
+        RDF::Literal::Double.new(double_str, :canonicalize => canonicalize?)
+      when decimal_str = match(/^(#{DECIMAL})/)
+        decimal_str = "0#{decimal_str}" if decimal_str[0,1] == "."
+        RDF::Literal::Decimal.new(decimal_str, :canonicalize => canonicalize?)
+      when integer_str = match(/^(#{INTEGER})/)
+        RDF::Literal::Integer.new(integer_str, :canonicalize => canonicalize?)
+      end
+    end
+
+    ##
+    # Read a boolean value
+    # @return [RDF::Literal::Boolean]
+    def read_boolean
+      if bool_str = match(/^(true|false)/)
+        RDF::Literal::Boolean.new(bool_str, :canonicalize => canonicalize?)
+      end
+    end
+  end # class Reader
+end # module RDF::Turtle

data/lib/rdf/turtle/reader.rb

@@ -18,11 +18,7 @@ module RDF::Turtle
       input[:resource] = self.bnode(token.value[2..-1])
     end
     terminal(:IRIREF,               IRIREF, :unescape => true) do |prod, token, input|
-      begin
-        input[:resource] = process_iri(token.value[1..-2])
-      rescue ArgumentError => e
-        raise RDF::ReaderError, e.message
-      end
+      input[:resource] = process_iri(token.value[1..-2])
     end
     terminal(:DOUBLE,               DOUBLE) do |prod, token, input|
       # Note that a Turtle Double may begin with a '.[eE]', so tack on a leading
@@ -204,6 +200,21 @@ module RDF::Turtle
       input[:resource] = literal(current[:string_value], opts)
     end
 
+    ##
+    # Redirect for Freebase Reader
+    #
+    # @private
+    def self.new(input = nil, options = {}, &block)
+      klass = if options[:freebase]
+        FreebaseReader
+      else
+        self
+      end
+      reader = klass.allocate
+      reader.send(:initialize, input, options, &block)
+      reader
+    end
+
     ##
     # Initializes a new reader instance.
     #
@@ -228,18 +239,29 @@ module RDF::Turtle
     #   the parser will attempt to recover from errors.
     # @option options [Boolean] :progress
     #   Show progress of parser productions
-    # @option options [Boolean] :debug
-    #   Detailed debug output
+    # @option options [Boolean, Integer, Array] :debug
+    #   Detailed debug output. If set to an Integer, output is restricted
+    #   to messages of that priority: `0` for errors, `1` for warnings,
+    #   `2` for processor tracing, and anything else for various levels
+    #   of debug. If set to an Array, information is collected in the array
+    #   instead of being output to `$stderr`.
+    # @option options [Boolean] :freebase (false)
+    #   Use optimized Freebase reader
     # @return [RDF::Turtle::Reader]
     def initialize(input = nil, options = {}, &block)
       super do
         @options = {
           :anon_base => "b0",
           :validate => false,
-          :debug => RDF::Turtle.debug?,
         }.merge(options)
         @options = {:prefixes => {nil => ""}}.merge(@options) unless @options[:validate]
+        @options[:debug] ||= case
+        when RDF::Turtle.debug? then true
+        when @options[:progress] then 2
+        when @options[:validate] then 1
+        end
 
+        @options[:base_uri] = RDF::URI(base_uri || "")
         debug("base IRI") {base_uri.inspect}
         
         debug("validate") {validate?.inspect}
@@ -273,13 +295,27 @@ module RDF::Turtle
                                                  :follow => FOLLOW,
                                                  :reset_on_start => true)
       ) do |context, *data|
-        loc = data.shift
         case context
         when :statement
-          add_statement(loc, RDF::Statement.from(data))
+          loc = data.shift
+          s = RDF::Statement.from(data, :lineno => lineno)
+          add_statement(loc, s) unless !s.valid? && validate?
+        when :trace
+          level, lineno, depth, *args = data
+          message = "#{args.join(': ')}"
+          d_str = depth > 100 ? ' ' * 100 + '+' : ' ' * depth
+          str = "[#{lineno}](#{level})#{d_str}#{message}"
+          case @options[:debug]
+          when Array
+            @options[:debug] << str
+          when TrueClass
+            $stderr.puts str
+          when Integer
+            $stderr.puts(str) if level <= @options[:debug]
+          end
         end
       end
-    rescue ArgumentError, EBNF::LL1::Parser::Error => e
+    rescue EBNF::LL1::Parser::Error => e
       progress("Parsing completed with errors:\n\t#{e.message}")
       raise RDF::ReaderError, e.message if validate?
     end
@@ -305,25 +341,23 @@ module RDF::Turtle
     # @return [RDF::Statement] Added statement
     # @raise [RDF::ReaderError] Checks parameter types and raises if they are incorrect if parsing mode is _validate_.
     def add_statement(node, statement)
-      error(node, "Statement is invalid: #{statement.inspect.inspect}") unless statement.valid?
+      error(node, "Statement is invalid: #{statement.inspect.inspect}") if validate? && statement.invalid?
       progress(node) {"generate statement: #{statement.to_ntriples}"}
-      @callback.call(statement)
+      @callback.call(statement) if statement.subject &&
+                                   statement.predicate &&
+                                   statement.object &&
+                                   (validate? ? statement.valid? : true)
     end
 
+    # Process a URI against base
     def process_iri(iri)
-      iri(base_uri, iri)
-    end
-    
-    # Create IRIs
-    def iri(value, append = nil)
-      value = RDF::URI.new(value)
-      value = value.join(append) if append
-      value.validate! if validate? && value.respond_to?(:validate)
+      value = base_uri.join(iri)
+      value.validate! if validate?
       value.canonicalize! if canonicalize?
       value = RDF::URI.intern(value) if intern?
       value
     end
-
+    
     # Create a literal
     def literal(value, options = {})
       options = options.dup

metadata

@@ -1,141 +1,166 @@
 --- !ruby/object:Gem::Specification
 name: rdf-turtle
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
+  prerelease:
 platform: ruby
 authors:
 - Gregg Kellogg
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdf
-  requirement: !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    none: false
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: ebnf
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
-  type: :runtime
+        version: 0.3.0
+    none: false
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: open-uri-cached
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
-- !ruby/object:Gem::Dependency
-  name: open-uri-cached
+        version: 0.0.5
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.5
-  type: :development
+    none: false
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.5
-- !ruby/object:Gem::Dependency
-  name: rspec
+        version: 2.12.0
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.12.0
-  type: :development
+    none: false
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rdf-isomorphic
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.12.0
-- !ruby/object:Gem::Dependency
-  name: rdf-isomorphic
+        version: !binary |-
+          MA==
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: !binary |-
+          MA==
+    none: false
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: json-ld
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: json-ld
+        version: !binary |-
+          MA==
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: !binary |-
+          MA==
+    none: false
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: yard
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: yard
+        version: 0.8.3
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.3
-  type: :development
+    none: false
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rdf-spec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.3
-- !ruby/object:Gem::Dependency
-  name: rdf-spec
+        version: '1.0'
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+    none: false
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: rake
+        version: !binary |-
+          MA==
+    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: !binary |-
+          MA==
+    none: false
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  type: :development
 description: RDF::Turtle is an Turtle reader/writer for the RDF.rb library suite.
 email: public-rdf-ruby@w3.org
 executables: []
@@ -147,36 +172,42 @@ files:
 - History
 - UNLICENSE
 - VERSION
+- lib/rdf/turtle.rb
 - lib/rdf/turtle/format.rb
+- lib/rdf/turtle/freebase_reader.rb
 - lib/rdf/turtle/meta.rb
 - lib/rdf/turtle/reader.rb
 - lib/rdf/turtle/terminals.rb
 - lib/rdf/turtle/version.rb
 - lib/rdf/turtle/writer.rb
-- lib/rdf/turtle.rb
 homepage: http://github.com/ruby-rdf/rdf-turtle
 licenses:
 - Public Domain
-metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.8.1
+  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      segments:
+      - 0
+      version: !binary |-
+        MA==
+      hash: 2
+  none: false
 requirements: []
 rubyforge_project: rdf-turtle
-rubygems_version: 2.0.3
-signing_key: 
-specification_version: 4
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
 summary: Turtle reader/writer for Ruby.
 test_files: []
 has_rdoc: false

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NmFjNDZhYjI5YjkzMzUwMzNjMGVkZGQ2NDg1ZjIzZDRmOTZhZTFkOA==
-  data.tar.gz: !binary |-
-    ZjRmYTU5N2I2MzU5OGQzMmNjMzFkNDhkYzE5YWI5NTYxODZmYzg4Yw==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ODBiMWZhMDUzMDNjODhmOTc5MDZmODQ2YTYwMzYyN2I1MWYzMDRmZDZjMmU3
-    MDFkODc0M2RlNDU4MjdhMGFlMzgxMWJmMWMzOTQxNDkwNjVkOGM5YWM4NTg3
-    ZGJhNzJkMWVjM2E1Y2UzOWI2MDI0ZmY1YmM2NTFkZWY2ZWEwYTY=
-  data.tar.gz: !binary |-
-    NmFlMGUzMjZhMjQwNGVjMDViOGU4ODUwNTE1MmYyYmE1ZWU0YjA2YmEzYmI3
-    YTY0ZDhiNzc0ZTUzYWFmNzY5NDRlNTJhNjkyMDdmMDc3ZWNjZmFlYzI5ZWEy
-    NTIzMWEyYTE1MzlkYzQ0YWY1NjQzNjg2NjJkNjRmZDg1MDNjOTk=