spread2rdf 0.0.1pre.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzBlOGJjZDMyMjBjMjNmNDA5MTk3MmY1NjMzMmE2OWNiZGE3MTljYw==
+  data.tar.gz: !binary |-
+    Y2IxMjY2MDBjZDk1OGYxNjE2YjlmM2UyZGZkYTQ2NmZjNDk5MWYyYQ==
+SHA512:
+  metadata.gz: !binary |-
+    ODlkODhkZDQ1NzZmNGVhY2E0MGIwNDgyOWVmZDFkODkyYjJlYmNiZjYyMzlh
+    ZmEzNjk4YjJiNjIyZjM1OTM2YTQ1ZGZlZDJjZTFhOTA1Mzc0ZWJlZTY4NjY1
+    YmEyNGZmNGM1ZjBkYTMxMTNmZWQ2YTZkNzVkOTViMDA2Y2I0ZTk=
+  data.tar.gz: !binary |-
+    MDExZWZiZjY4YjBlNDdkYzhkN2ExZjNmNTVhNWVkNGEwMDBmYzg0YTY0ZGM2
+    Njk3ZmQ4NmU3MmY3MjAxYzkwZjkzZThjM2VhZWMzYjYyZmJjYjBjMTc0MTdi
+    YzUwNzlhNTRhMjY0NDJiNzIxZTdiYTkzNTZlM2U0ZTBjZTNmODY=

data/.gitignore

@@ -0,0 +1,23 @@
+.DS_Store
+.idea
+.idea45
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+testdata
+tmp
+
+lib/**/-*

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in spread2rdf.gemspec
+gemspec
+
+group :development, :test do
+  gem 'rake'
+  gem 'pry',     '~> 0.9.12.2'
+  gem 'pry-nav', '~> 0.2.3'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Marcel Otto
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,44 @@
+# Spread2RDF
+
+Spread2RDF is a converter for complex spreadsheets to RDF and a DSL for
+specifying the mapping rules for this conversion.
+
+## Features
+
+* Supports Excel/Excelx, Google spreadsheets, OpenOffice, LibreOffice and CSV
+  spreadsheets as input, thanks to [Roo](https://github.com/Empact/roo).
+  (Currently, it's tested for Excel only.
+  If you have a problem with another spreadsheet type,
+  [raise an issue](https://github.com/marcelotto/spread2rdf/issues).)
+* Supports many RDF serialization formats for the output, thanks to
+  [RDF.rb](https://github.com/ruby-rdf/rdf).
+
+## Installation
+
+Install [Ruby](http://www.ruby-lang.org/) and execute the following command
+in a terminal:
+
+    $ gem install spread2rdf
+
+## Command-line interface
+
+For a description of all available parameters, type the following in a terminal:
+
+    $ spread2rdf --help
+
+## Mapping DSL
+
+Description is following soon.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## Authors
+
+* Marcel Otto

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/VERSION

@@ -0,0 +1 @@
+0.0.1pre.1

data/bin/spread2rdf

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'spread2rdf'
+Spread2RDF::Cli.new.run

data/lib/spread2rdf/attributes.rb

@@ -0,0 +1,45 @@
+module Spread2RDF
+  module Attributes
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def attributes
+        if superclass.respond_to?(:attributes) and
+            (super_attributes = superclass.attributes).is_a? Hash
+          @attributes.reverse_merge(super_attributes)
+        else
+          @attributes
+        end
+      end
+
+      def attributes=(defaults)
+        defaults.each { |attribute, default_value| attr_accessor attribute }
+        @attributes = @attributes.try(:merge, defaults) || defaults
+      end
+    end
+
+    def init_attributes(initial_values)
+      self.class.attributes.each do |attribute, default_value|
+        instance_variable_set("@#{attribute}".to_sym,
+          initial_values.delete(attribute) || default_value)
+      end
+      initial_values
+    end
+
+    def update_attributes(update_values)
+      update_values.each do |attribute, value|
+        next unless self.class.attributes.include? attribute
+        instance_variable_set("@#{attribute}".to_sym, value)
+      end
+      update_values
+    end
+
+    def inspect
+      "#{self}: " +
+        self.class.attributes.map do |attribute, default_value|
+          "#{attribute}=#{self.send(attribute)}"
+        end.join(', ')
+    end
+
+  end
+end

data/lib/spread2rdf/cli.rb

@@ -0,0 +1,91 @@
+# coding: utf-8
+
+module Spread2RDF
+  class Cli
+    def initialize
+      parse_command_line!
+    end
+
+    def run(schema_spec_file = nil)
+      schema_spec_file ||= @options[:schema_spec_file]
+      abort "No schema specification file given" if schema_spec_file.nil?
+      abort "Couldn't find schema specification file #{schema_spec_file}" unless
+          File.exist?(schema_spec_file)
+      load schema_spec_file
+      abort "No schema specification found" if Spreadsheet.definitions.empty?
+      puts "Reading #{@input_file} ..."
+      @table = Spreadsheet.definitions.first
+      @table.read(@input_file)
+      write_output
+      self
+    end
+
+  private
+
+    # Parse command line options
+    def parse_command_line!(options={})
+      @options = options
+      optparse = OptionParser.new do |opts|
+        opts.banner = 'Usage: spread2rdf [options] -s SPEC_FILE SPREAD_SHEET_FILE'
+
+        opts.on( '-h', '--help', 'Display this information' ) do
+          puts opts
+          exit
+        end
+
+        @options[:output_dir] = '.'
+        opts.on( '-o', '--output DIR', 'Output directory (default: current directory)' ) do |dir|
+          abort "Output directory #{dir} doesn't exist" unless Dir.exist?(dir)
+          @options[:output_dir] = dir
+        end
+
+        @options[:output_format] = 'ttl'
+        opts.on( '-f', '--output-format FORMAT', 'Serialization format for the RDF data',
+          "FORMAT being one of: nt, n3, ttl, rdf, xml, html, json (default: ttl)") do |format|
+          #format = 'turtle' if format == 'ttl'
+          @options[:output_format] = format.strip.downcase
+        end
+
+        @options[:schema_spec_file] = nil
+        opts.on( '-s', '--schema SPEC_FILE', 'Schema specification file (required)' ) do |file|
+          @options[:schema_spec_file] = file
+        end
+
+      end
+
+      optparse.parse!
+      raise OptionParser::ParseError, 'required file arguments missing' if ARGV.empty?
+      raise OptionParser::ParseError, 'required schema specification file missing' if @options[:schema_spec_file].nil?
+
+      @input_file = ARGV.first
+    rescue OptionParser::ParseError => e
+      puts e.message
+      puts optparse.help
+      exit
+    end
+
+    def output_filename
+      output_dir = @options[:output_dir]
+      name = File.basename(@input_file, File.extname(@input_file))
+      "#{output_dir}/#{name}.#{@options[:output_format]}"
+    end
+
+    def write_output
+      filename = output_filename
+      abort 'No RDF data to write!' if @table.try(:to_rdf).blank?
+      graph = @table.to_rdf
+      puts "Writing #{graph.count} RDF statements to #{filename} ... "
+      # TODO: base_uri: ... for writer constructor
+      RDF::Writer.open(filename) do |writer|
+        RDF::Vocabulary.each do |vocabulary|
+          writer.prefix vocabulary.__prefix__, vocabulary.to_s
+        end
+        Namespace.namespace.each do |name, namespace|
+          writer.prefix name.to_s.downcase, namespace.to_s
+        end
+        graph.each_statement { |statement| writer << statement }
+      end
+    end
+    self
+  end
+end

data/lib/spread2rdf/extensions/roo_xlsm_fix.rb

@@ -0,0 +1,28 @@
+require 'zip'
+#require 'zip/zipfilesystem'
+# source (adapted to newer version of Roo and ruby-zip): https://gist.github.com/roblingle/1333908
+
+# Easy access to xlsm files through the roo gem, version 1.10.0.
+# The error that led me to write this is in the file below for google fodder. Not exactly sure what was causing
+# the problem, so I'm not sure that this change won't break everything on your computer or summon zombies.
+#
+# Be sure to tell roo that you don't care about the extension mismatch:
+# xl = Roo::Excelx.new("C:/path/to/spreadsheet_with_macro.xlsm", :zip, :warning)
+#
+class Roo::Excelx
+
+  alias :old_initialize :initialize
+  def initialize(filename, options = {}) # , packed=nil, file_warning = :error)
+    @original_file = filename
+    old_initialize(filename, options)
+  end
+
+  # extract files from the zip file, rewrites a method of the same name in lib/roo/excelx.rb
+  def extract_content(tmpdir, zipfilename_unused)
+    #Zip::ZipFile.open(@original_file) do |zip|
+    Zip::File.open(@original_file) do |zip|
+      process_zipfile(tmpdir, @original_file, zip)
+    end
+  end
+
+end

data/lib/spread2rdf/helper.rb

@@ -0,0 +1,14 @@
+module Spread2RDF
+  module Helper
+
+  module_function
+
+    # TODO: include this in the MappingContext(s)
+    def resource_name(string)
+      string
+        .gsub(', ', '-')
+        .gsub(' ', '-')
+    end
+
+  end
+end

data/lib/spread2rdf/namespace.rb

@@ -0,0 +1,50 @@
+module Spread2RDF
+  module Namespace
+    class << self
+      def [](name)
+        name = name.to_sym
+        self.namespace[name] ||
+            ( RDF.const_defined?(name) && RDF.const_get(name)) ||
+            nil
+      end
+
+      def []=(name, namespace)
+        name = name.to_sym
+        self.namespace[name] = case namespace
+          when RDF::Vocabulary          then namespace
+          when String, RDF::URI         then RDF::Vocabulary.new(namespace)
+          else raise ArgumentError, "expecting a namespace but got #{namespace}:#{namespace.class}"
+        end
+      end
+
+      def namespace
+        @namespace ||= {}
+      end
+
+      def namespaces
+        namespace.values
+      end
+
+      def resolve_to_namespace(namespace_descriptor)
+        case namespace_descriptor
+          when Symbol
+            Namespace[namespace_descriptor]
+          when RDF::Vocabulary, RDF::URI, String
+            namespace_descriptor.to_s
+          else
+            raise "invalid namespace: #{namespace_descriptor.inspect}"
+        end
+      end
+
+      def const_missing(name)
+        self[name] or super
+      end
+    end
+  end
+  NS = Namespace
+
+  def self.const_missing(name)
+    Namespace[name] or super
+  end
+
+end

data/lib/spread2rdf/spreadsheet/column.rb

@@ -0,0 +1,48 @@
+module Spread2RDF
+  class Spreadsheet
+    class Sheet
+      class Column < Element
+
+        attr_reader :coord # this is set by Worksheet#index_columns!
+
+        self.attributes = {
+            predicate:  nil,
+            object:     nil,
+            statement:  nil
+        }
+
+        def initialize(sheet, options = {}, &block)
+          super
+        end
+
+        alias sheet parent
+
+        def worksheet
+          parent = self.parent
+          parent = parent.parent until parent.is_a? Worksheet or parent.nil?
+          parent
+        end
+
+        def map(range, context)
+          #puts "mapping #{self} in #{range} ..."
+          case range
+            when Integer
+              coord = Coord[row: range, column: self.coord]
+              worksheet.cell_mapping[coord.to_sym] ||= mapping =
+                  create_context(context, row: range,
+                                 subject: context.subject, predicate: predicate)
+              mapping.object
+            when Range
+              range.map { |row| self.map(row, context) }
+            else raise ArgumentError
+          end
+        end
+
+        def to_s
+          "#{super} of #{sheet}"
+        end
+
+      end
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/column_mapping_context.rb

@@ -0,0 +1,156 @@
+module Spread2RDF
+  class Spreadsheet
+    class Sheet
+      class Column
+        class MappingContext < Spreadsheet::MappingContext
+
+          self.attributes = {
+              subject:    nil,
+              predicate:  nil,
+              row:        nil
+          }
+
+          alias column element
+          alias property predicate
+
+          attr_reader :value
+
+          def initialize(sheet, parent_context = nil, attr = {})
+            super
+            @value = cell_value(row: row, column: column.coord)
+            return if @value.blank?
+            statements_to_object
+            worksheet.graph << self.graph
+          end
+
+          def cell_coord
+            Coord[row: row, column: column.coord]
+          end
+
+          def subject
+            @subject or parent_context.try(:subject)
+          end
+
+          def object
+            @object ||= @value && map_to_object(value)
+          end
+
+          def value_of_column(name)
+            other_column = sheet.column[name]
+            raise "couldn't find column #{name} when mapping #{column}" if
+                other_column.nil?
+            cell_value(row: row, column: other_column.coord)
+          end
+
+          def object_of_column(name)
+            other_column = sheet.column[name]
+            raise "couldn't find column #{name} when mapping #{column}" if
+                other_column.nil?
+            cell(row: row, column: other_column.coord).object
+          end
+
+          ######################################################################
+          # Value-to-object mapping
+
+          private
+
+          def map_to_object(value)
+            case object_mapping_mode
+              when :to_string
+                value
+              when :resource_ref
+                resolve_resource_ref
+              when :new_resource
+                create_resource_object
+              when :custom
+                # TODO execute a mapping block in the context of Column::MappingContext
+              else
+                raise 'internal error: unknown column mapping type'
+            end
+          end
+
+
+          def object_mapping_mode
+            case
+              when column.object.nil?         then :to_string
+              when column.object.is_a?(Proc)  then :custom
+              when !column.object[:uri].nil?  then :new_resource
+              when !column.object[:from].nil? then :resource_ref
+              else
+                raise "mapping specification error: don't know how to map #{column}"
+            end
+          end
+
+          def resolve_resource_ref
+            source = column.object[:from]
+            source = { worksheet: source } if source.is_a? Symbol
+            raise ArgumentError, "expecting a Hash as source, but got #{source}" unless source.is_a? Hash
+            source_worksheet = source[:worksheet]
+            source_worksheet = spreadsheet.worksheet[source_worksheet]
+            raise "#{column}: couldn't find source worksheet #{source[:worksheet]}" if source_worksheet.nil?
+            source_predicate = source[:predicate] || RDF::RDFS.label
+            result = source_worksheet.graph.query([nil, source_predicate, value])
+            raise "#{column}: couldn't find a resource for #{value} in #{source_worksheet}" if result.empty?
+            raise "#{column}: found multiple resources for #{value} in #{source_worksheet}: #{result.map(&:subject)}" if result.count > 1
+            result.first.subject
+          end
+
+          # TODO: Should we reuse/share mapping logic with Sheet::MappingContext (#subject etc.)?
+          def create_resource_object
+            case
+              when (column.object.try(:fetch, :uri, nil) || object) == :bnode
+                RDF::Node.new
+              else
+                raise NotImplementedError
+            end
+          end
+
+
+          ######################################################################
+          # Statement mapping
+
+          def statement_mapping_mode
+            case
+              when column.statement == :none then :ignore
+              when column.statement == :none then :ignore
+              when column.predicate.nil?     then :ignore
+              when restriction_mode          then :restriction
+              else                                :default
+            end
+          end
+
+          def restriction_mode
+            restriction_mode = column.statement
+            case restriction_mode
+              when :restriction then RDF::OWL.hasValue
+              when Hash         then restriction_mode[:restriction]
+              else nil
+            end
+          end
+
+          def statements_to_object
+            case statement_mapping_mode
+              when :default
+                statement(subject, predicate, object)
+              when :restriction
+                restriction_class = RDF::Node.new
+                statements(
+                    [ subject, RDF::RDFS.subClassOf, restriction_class ],
+                    [ restriction_class, RDF.type, RDF::OWL.Restriction ],
+                    [ restriction_class, RDF::OWL.onProperty, predicate ],
+                    [ restriction_class, restriction_mode, object ]
+                )
+            end
+            exec(value, &column.block) if column.block
+          end
+
+          def exec(value, &block)
+            #puts "executing block of #{@___column___} in row #{row}"
+            self.instance_exec(value, &block)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/coord.rb

@@ -0,0 +1,51 @@
+module Spread2RDF
+  class Spreadsheet
+    class Coord < Struct.new(:column, :row)
+      def initialize(*args)
+        case args.length
+          when 2 then super
+          when 1
+            case args = args.first
+              when Hash
+                super(args[:column], args[:row])
+              when Symbol, String
+                coord = args.to_s
+                raise "Invalid cell coordinates #{coord}" unless coord =~ /(\w+)(\d+)/
+                super(Regexp.last_match[1], Regexp.last_match[2].to_i)
+              else raise ArgumentError, "can't handle argument #{args}"
+            end
+          else raise ArgumentError, "too many arguments: #{args}"
+        end
+      end
+
+      def column_as_number
+        Roo::Base.letter_to_number(column)
+      end
+
+      def column_as_index
+        column_as_number - 1
+      end
+
+      def increment_column(count = 1)
+        self.class.increment_column(self.column, count)
+      end
+
+      def to_s
+        "#{column}#{row}"
+      end
+
+      def to_sym
+        to_s.to_sym
+      end
+
+      class << self
+        alias [] new
+
+        def increment_column(column, count=1)
+          Roo::Base.number_to_letter(Roo::Base.letter_to_number(column) + count)
+        end
+      end
+
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/element.rb

@@ -0,0 +1,53 @@
+module Spread2RDF
+  class Spreadsheet
+    class Element
+      include Attributes
+
+      self.attributes = {
+          name:         nil,
+          source_name:  nil
+      }
+
+      attr_reader :parent
+      attr_reader :block
+
+
+      def initialize(parent, attr={}, &block)
+        @parent   = parent
+        @block    = block
+        init_attributes(attr)
+      end
+
+      def init
+
+      end
+
+      def name
+        (@name or @source_name).try(:to_sym)
+      end
+
+      def source_name
+        (@source_name or @name).try(:to_s)
+      end
+
+      def spreadsheet
+        parent.spreadsheet
+      end
+
+      def to_s
+        name = (self.name.to_s == self.source_name.to_s ?
+            self.name : "#{self.name} (#{self.source_name})" )
+        "#{self.class.name.split('::').last} #{name}"
+      end
+
+    private
+
+      def create_context(parent_context, attr)
+        context_class = self.class.const_get(:MappingContext)
+        context_class.new(self, parent_context, attr)
+      end
+
+    end
+  end
+end
+