rdf-tabular 0.1.1

data/spec/reader_spec.rb

@@ -0,0 +1,221 @@
+# coding: utf-8
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'rdf/spec/reader'
+
+describe RDF::Tabular::Reader do
+  let!(:doap) {File.expand_path("../../etc/doap.ttl", __FILE__)}
+  let!(:doap_count) {File.open(doap).each_line.to_a.length}
+
+  before(:each) do
+    @reader = RDF::Tabular::Reader.new(StringIO.new(""), base_uri: "file:#{File.expand_path("..", __FILE__)}")
+
+    WebMock.stub_request(:any, %r(.*example.org.*)).
+      to_return(lambda {|request|
+        file = request.uri.to_s.split('/').last
+        content_type = case file
+        when /\.json/ then 'application/json'
+        when /\.csv/  then 'text/csv'
+        else 'text/plain'
+        end
+
+        case file
+        when "metadata.json", "country-codes-and-names.csv-metadata.json"
+          {status: 401}
+        else
+          {
+            body: File.read(File.expand_path("../data/#{file}", __FILE__)),
+            status: 200,
+            headers: {'Content-Type' => content_type}
+          }
+        end
+      })
+
+    @debug = []
+  end
+  
+  # @see lib/rdf/spec/reader.rb in rdf-spec
+  #include RDF_Reader
+
+  it "should be discoverable" do
+    readers = [
+      RDF::Reader.for(:tabular),
+      RDF::Reader.for("etc/doap.csv"),
+      RDF::Reader.for(file_name:      "etc/doap.csv"),
+      RDF::Reader.for(file_extension: "csv"),
+      RDF::Reader.for(content_type:   "text/csv"),
+    ]
+    readers.each { |reader| expect(reader).to eq RDF::Tabular::Reader }
+  end
+
+  context "Test Files" do
+    test_files = {
+      "tree-ops.csv" => "tree-ops-standard.ttl",
+      "tree-ops.csv-metadata.json" => "tree-ops-standard.ttl",
+      "tree-ops-ext.json" => "tree-ops-ext-standard.ttl",
+      "tree-ops-virtual.json" => "tree-ops-virtual-standard.ttl",
+      "country-codes-and-names.csv" => "country-codes-and-names-standard.ttl",
+      "countries.json" => "countries-standard.ttl",
+      "countries.csv" => "countries.csv-standard.ttl",
+      "roles.json" => "roles-standard.ttl",
+    }
+    context "#each_statement" do
+      test_files.each do |csv, ttl|
+        context csv do
+          let(:about) {RDF::URI("http://example.org").join(csv)}
+          let(:input) {File.expand_path("../data/#{csv}", __FILE__)}
+
+          it "standard mode" do
+            expected = File.expand_path("../data/#{ttl}", __FILE__)
+            RDF::Reader.open(input, format: :tabular, base_uri: about, noProv: true, validate: true, debug: @debug) do |reader|
+              graph = RDF::Graph.new << reader
+              graph2 = RDF::Graph.load(expected, base_uri: about)
+              expect(graph).to be_equivalent_graph(graph2,
+                                                   debug: @debug,
+                                                   id: about,
+                                                   action: about,
+                                                   result: expected,
+                                                   metadata: reader.metadata)
+            end
+          end
+
+          it "minimal mode" do
+            ttl = ttl.sub("standard", "minimal")
+            expected = File.expand_path("../data/#{ttl}", __FILE__)
+            RDF::Reader.open(input, format: :tabular, base_uri: about, minimal: true, debug: @debug) do |reader|
+              graph = RDF::Graph.new << reader
+              graph2 = RDF::Graph.load(expected, base_uri: about)
+              expect(graph).to be_equivalent_graph(graph2,
+                                                   debug: @debug,
+                                                   id: about,
+                                                   action: about,
+                                                   result: expected,
+                                                   metadata: reader.metadata)
+            end
+          end
+        end
+      end
+    end
+
+    describe "#to_json" do
+      test_files.each do |csv, ttl|
+        context csv do
+          let(:about) {RDF::URI("http://example.org").join(csv)}
+          let(:input) {File.expand_path("../data/#{csv}", __FILE__)}
+          it "standard mode" do
+            json = ttl.sub("-standard.ttl", "-standard.json")
+            expected = File.expand_path("../data/#{json}", __FILE__)
+
+            RDF::Reader.open(input, format: :tabular, base_uri: about, noProv: true, debug: @debug) do |reader|
+              expect(JSON.parse(reader.to_json)).to produce(
+                JSON.parse(File.read(expected)),
+                debug: @debug,
+                id: about,
+                action: about,
+                result: expected,
+                noProv: true,
+                metadata: reader.metadata
+              )
+            end
+          end
+
+          it "minimal mode" do
+            json = ttl.sub("-standard.ttl", "-minimal.json")
+            expected = File.expand_path("../data/#{json}", __FILE__)
+
+            RDF::Reader.open(input, format: :tabular, base_uri: about, minimal: true, debug: @debug) do |reader|
+              expect(JSON.parse(reader.to_json)).to produce(
+                JSON.parse(File.read(expected)),
+                debug: @debug,
+                id: about,
+                action: about,
+                result: expected,
+                minimal: true,
+                metadata: reader.metadata
+              )
+            end
+          end
+
+          it "ADT mode", skip: true do
+            json = ttl.sub("-standard.ttl", "-atd.json")
+            expected = File.expand_path("../data/#{json}", __FILE__)
+
+            RDF::Reader.open(input, format: :tabular, base_uri: about, noProv: true, debug: @debug) do |reader|
+              expect(JSON.parse(reader.to_json(atd: true))).to produce(
+                JSON.parse(File.read(expected)),
+                debug: @debug,
+                id: about,
+                action: about,
+                result: expected,
+                noProv: true,
+                metadata: reader.metadata
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+
+  context "Provenance" do
+    {
+      "country-codes-and-names.csv" => %(
+        PREFIX csvw: <http://www.w3.org/ns/csvw#>
+        PREFIX prov: <http://www.w3.org/ns/prov#>
+        PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+        ASK WHERE {
+          [ prov:wasGeneratedBy [
+              a prov:Activity;
+              prov:wasAssociatedWith <http://rubygems.org/gems/rdf-tabular>;
+              prov:startedAtTime ?start;
+              prov:endedAtTime ?end;
+              prov:qualifiedUsage [
+                a prov:Usage ;
+                prov:entity <http://example.org/country-codes-and-names.csv> ;
+                prov:hadRole csvw:csvEncodedTabularData
+              ];
+            ]
+          ]
+          FILTER (
+            DATATYPE(?start) = xsd:dateTime &&
+            DATATYPE(?end) = xsd:dateTime
+          )
+        }
+      ),
+      "countries.json" => %(
+        PREFIX csvw: <http://www.w3.org/ns/csvw#>
+        PREFIX prov: <http://www.w3.org/ns/prov#>
+        PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+        ASK WHERE {
+          [ prov:wasGeneratedBy [
+              a prov:Activity;
+              prov:wasAssociatedWith <http://rubygems.org/gems/rdf-tabular>;
+              prov:startedAtTime ?start;
+              prov:endedAtTime ?end;
+              prov:qualifiedUsage [
+                a prov:Usage ;
+                prov:entity <http://example.org/countries.csv>, <http://example.org/country_slice.csv>;
+                prov:hadRole csvw:csvEncodedTabularData
+              ], [
+                a prov:Usage ;
+                prov:entity <http://example.org/countries.json> ;
+                prov:hadRole csvw:tabularMetadata
+              ];
+            ]
+          ]
+          FILTER (
+            DATATYPE(?start) = xsd:dateTime &&
+            DATATYPE(?end) = xsd:dateTime
+          )
+        }
+      )
+    }.each do |file, query|
+      it file do
+        about = RDF::URI("http://example.org").join(file)
+        input = File.expand_path("../data/#{file}", __FILE__)
+        graph = RDF::Graph.load(input, format: :tabular, base_uri: about, debug: @debug)
+
+        expect(graph).to pass_query(query, debug: @debug, id: about, action: about)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,47 @@
+$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$:.unshift File.dirname(__FILE__)
+
+require "bundler/setup"
+require 'rspec'
+require 'rspec/its'
+require 'rdf/isomorphic'
+require 'rdf/tabular'
+require 'rdf/turtle'
+require 'rdf/spec/matchers'
+require 'json'
+require 'webmock/rspec'
+require 'matchers'
+require 'suite_helper'
+require 'simplecov'
+SimpleCov.start
+
+JSON_STATE = JSON::State.new(
+  :indent       => "  ",
+  :space        => " ",
+  :space_before => "",
+  :object_nl    => "\n",
+  :array_nl     => "\n"
+)
+
+::RSpec.configure do |c|
+  c.filter_run focus:  true
+  c.run_all_when_everything_filtered = true
+  c.include(RDF::Spec::Matchers)
+end
+
+# Heuristically detect the input stream
+def detect_format(stream)
+  # Got to look into the file to see
+  if stream.is_a?(IO) || stream.is_a?(StringIO)
+    stream.rewind
+    string = stream.read(1000)
+    stream.rewind
+  else
+    string = stream.to_s
+  end
+  case string
+  when /<html/i   then RDF::Microdatea::Reader
+  when /@prefix/i then RDF::Turtle::Reader
+  else                 RDF::Turtle::Reader
+  end
+end

data/spec/suite_helper.rb

@@ -0,0 +1,161 @@
+$:.unshift "."
+require 'spec_helper'
+require 'rdf/turtle'
+require 'json/ld'
+require 'open-uri'
+
+# For now, override RDF::Utils::File.open_file to look for the file locally before attempting to retrieve it
+module RDF::Util
+  module File
+    REMOTE_PATH = "http://w3c.github.io/csvw/"
+    LOCAL_PATH = ::File.expand_path("../w3c-csvw", __FILE__) + '/'
+
+    class << self
+      alias_method :original_open_file, :open_file
+    end
+
+    ##
+    # Override to use Patron for http and https, Kernel.open otherwise.
+    #
+    # @param [String] filename_or_url to open
+    # @param  [Hash{Symbol => Object}] options
+    # @option options [Array, String] :headers
+    #   HTTP Request headers.
+    # @return [IO] File stream
+    # @yield [IO] File stream
+    def self.open_file(filename_or_url, options = {}, &block)
+      case filename_or_url.to_s
+      when /^file:/
+        path = filename_or_url.to_s[5..-1]
+        Kernel.open(path.to_s, &block)
+      when -> (k) {k =~ %r{^#{REMOTE_PATH}} && ::File.exist?(filename_or_url.to_s.sub(REMOTE_PATH, LOCAL_PATH))}
+        begin
+          #puts "attempt to open #{filename_or_url} locally"
+          localpath = filename_or_url.to_s.sub(REMOTE_PATH, LOCAL_PATH)
+          response = begin
+            ::File.open(localpath)
+          rescue Errno::ENOENT
+            Kernel.open(filename_or_url.to_s, "r:utf-8", 'Accept' => "application/ld+json, application/json, text/csv")
+          end
+          document_options = {
+            base_uri:     RDF::URI(filename_or_url),
+            charset:      Encoding::UTF_8,
+            code:         200,
+            headers:      {}
+          }
+          #puts "use #{filename_or_url} locally"
+          document_options[:headers][:content_type] = case filename_or_url.to_s
+          when /\.csv$/   then 'text/csv'
+          when /\.json$/   then 'application/json'
+          when /\.jsonld$/ then 'application/ld+json'
+          else                  'unknown'
+          end
+
+          document_options[:headers][:content_type] = response.content_type if response.respond_to?(:content_type)
+          # For overriding content type from test data
+          document_options[:headers][:content_type] = options[:contentType] if options[:contentType]
+
+          # For overriding Link header from test data
+          document_options[:headers][:link] = options[:httpLink] if options[:httpLink]
+
+          remote_document = RDF::Util::File::RemoteDocument.new(response.read, document_options)
+          if block_given?
+            yield remote_document
+          else
+            remote_document
+          end
+        end
+      else
+        original_open_file(filename_or_url, options, &block)
+      end
+    end
+  end
+end
+
+module Fixtures
+  module SuiteTest
+    BASE = "http://w3c.github.io/csvw/tests/"
+    class Manifest < JSON::LD::Resource
+      def self.open(file, base)
+        #puts "open: #{file}"
+        RDF::Util::File.open_file(file) do |file|
+          json = ::JSON.load(file.read)
+          yield Manifest.new(json, context: json['@context'].merge('@base' => base))
+        end
+      end
+
+      def entries
+        # Map entries to resources
+        attributes['entries'].map {|e| Entry.new(e, context: context)}
+      end
+    end
+ 
+    class Entry < JSON::LD::Resource
+      attr_accessor :debug
+      attr_accessor :metadata
+
+      def id
+        attributes['id']
+      end
+
+      def base
+        action
+      end
+
+      # Apply base to action and result
+      def action
+        RDF::URI(context['@base']).join(attributes["action"]).to_s
+      end
+
+      def result
+        RDF::URI(context['@base']).join(attributes["result"]).to_s
+      end
+
+      def input
+        @input ||= RDF::Util::File.open_file(action) {|f| f.read}
+      end
+
+      def expected
+        @expected ||= RDF::Util::File.open_file(result) {|f| f.read}
+      end
+      
+      def evaluate?
+        type.include?("To")
+      end
+      
+      def sparql?
+        type.include?("Sparql")
+      end
+
+      def rdf?
+        result.to_s.end_with?(".ttl")
+      end
+
+      def json?
+        result.to_s.end_with?(".json")
+      end
+
+      def validation?
+        type.include?("Validation")
+      end
+
+      def positive_test?
+        !negative_test?
+      end
+      
+      def negative_test?
+        type.include?("Negative")
+      end
+
+      def reader_options
+        res = {}
+        res[:noProv] = option['noProv'] if option
+        res[:metadata] = RDF::URI(context['@base']).join(option['metadata']).to_s if option && option.has_key?('metadata')
+        res[:httpLink] = httpLink if attributes['httpLink']
+        res[:minimal] = option['minimal'] if option
+        res[:contentType] = contentType if attributes['contentType']
+        res
+      end
+    end
+  end
+end

data/spec/suite_spec.rb

@@ -0,0 +1,76 @@
+$:.unshift "."
+require 'spec_helper'
+require 'fileutils'
+
+WebMock.allow_net_connect!(net_http_connect_on_start: true)
+describe RDF::Tabular::Reader do
+  require 'suite_helper'
+
+  before(:all) {WebMock.allow_net_connect!(net_http_connect_on_start: true)}
+  after(:all) {WebMock.allow_net_connect!(net_http_connect_on_start: false)}
+
+  %w(rdf json validation).each do |variant|
+    describe "w3c csvw #{variant.upcase} tests" do
+      manifest = Fixtures::SuiteTest::BASE + "manifest-#{variant}.jsonld"
+
+      Fixtures::SuiteTest::Manifest.open(manifest, manifest[0..-8]) do |m|
+        describe m.comment do
+          m.entries.each do |t|
+            specify "#{t.id.split("/").last}: #{t.name} - #{t.comment}" do
+              t.debug = []
+              RDF::Tabular::Reader.open(t.action,
+                t.reader_options.merge(
+                  base_uri: t.base,
+                  validate: t.validation?,
+                  debug:    t.debug
+                )
+              ) do |reader|
+                expect(reader).to be_a RDF::Reader
+
+                t.metadata = reader.metadata # for debug output
+                t.metadata = t.metadata.parent if t.metadata && t.metadata.parent
+
+                graph = RDF::Repository.new
+
+                if t.positive_test?
+                  if t.json?
+                    result = reader.to_json
+                    if t.evaluate?
+                      RDF::Util::File.open_file(t.result) do |res|
+                        expect(::JSON.parse(result)).to produce(::JSON.parse(res.read), t)
+                      end
+                    else
+                      expect(::JSON.parse(result)).to be_a(Hash)
+                    end
+                  else # RDF or Validation
+                    begin
+                      graph << reader
+                    rescue Exception => e
+                      expect(e.message).to produce("Not exception #{e.inspect}\n#{e.backtrace.join("\n")}", t.debug)
+                    end
+
+                    if t.sparql?
+                      RDF::Util::File.open_file(t.result) do |query|
+                        expect(graph).to pass_query(query, t)
+                      end
+                    elsif t.evaluate?
+                      output_graph = RDF::Repository.load(t.result, format: :ttl, base_uri:  t.base)
+                      expect(graph).to be_equivalent_graph(output_graph, t)
+                    else
+                      expect(graph).to be_a(RDF::Enumerable)
+                    end
+                  end
+                else
+                  expect {
+                    graph << reader
+                    expect(graph.dump(:ntriples)).to produce("not this", t.debug)
+                  }.to raise_error(RDF::Tabular::Error)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end unless ENV['CI']  # Skip for continuous integration