spread2rdf 0.0.1pre.1

data/lib/spread2rdf/spreadsheet/mapping_context.rb

@@ -0,0 +1,67 @@
+module Spread2RDF
+  class Spreadsheet
+    class MappingContext
+      include Attributes
+
+      self.attributes = {
+      }
+
+      attr_reader :element
+      attr_reader :parent_context
+
+      attr_reader :graph
+
+      def initialize(element, parent_context, attr = {})
+        @element = element
+        @parent_context = parent_context
+        @graph = RDF::Repository.new
+        init_attributes(attr)
+      end
+
+      def sheet
+        @element.sheet
+      end
+
+      def worksheet
+        @element.worksheet
+      end
+
+      def spreadsheet
+        @element.spreadsheet
+      end
+
+      def cell_value(coord)
+        worksheet.cell(coord)
+      end
+
+      def cell(coord)
+        coord = Coord[coord] unless coord.is_a? Coord
+        worksheet.cell_mapping[coord.to_sym]
+      end
+
+      def to_s
+        "#{self.class.name.split('::')[-2..-1].join('::')} of #{element}"
+      end
+
+      ##########################################################################
+      # statement generators
+
+      private
+
+      def add_statement(*args)
+        args = args.first if args.count == 1 and args.first.is_a? Array
+        #puts "adding statement: #{args.inspect}"
+        raise "internal error: trying to add a bad triple with nil value: #{args}" if args.count != 3 or args.one? { |arg| arg.nil? }
+        @graph << RDF::Statement.new(*args)
+      end
+      alias statement add_statement
+
+      def add_statements(*args)
+        args = args.first if args.count == 1 and args.first.is_a? Array
+        args.each { |arg| statement(arg) }
+      end
+      alias statements add_statements
+
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/mapping_dsl.rb

@@ -0,0 +1,23 @@
+module Spread2RDF
+  class Spreadsheet
+    class MappingDSL
+
+      def initialize(schema)
+        @schema = schema
+      end
+
+      def namespaces(namespaces)
+        namespaces.each { |name, namespace| Namespace[name] = namespace }
+      end
+
+      def worksheet(name, options={}, &block)
+        source_name = options[:source_name] = name
+        name = ( options.delete(:name) || source_name ).to_sym
+        worksheet = @schema.worksheet[name] ||= Worksheet.new(@schema)
+        worksheet.update_attributes options.merge(name: name, source_name: source_name)
+        Sheet::DSL.new(worksheet, &block)
+      end
+
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/sheet.rb

@@ -0,0 +1,128 @@
+module Spread2RDF
+  class Spreadsheet
+    class Sheet < Element
+
+      self.attributes = {
+          subject:          nil,
+          start:            :A2,
+          row_count_per_resource: nil
+      }
+
+      def initialize(parent, options={}, &block)
+        super(parent, options, &block)
+        @column = {}
+      end
+
+      def worksheet
+        return self if self.is_a? Worksheet
+        parent = self.parent
+        parent = parent.parent until parent.is_a? Worksheet or parent.nil?
+        parent
+      end
+
+      def column(name = nil)
+        return @column if name.nil?
+        name = name.to_sym
+        @column[name] or ( parent.is_a?(Sheet) and parent.column(name) ) or nil
+      end
+
+      def columns
+        @column.values
+      end
+
+      def cell(coord)
+        coord = Coord[coord]
+        spreadsheet.roo.cell(coord.column, coord.row, worksheet.source_name)
+      end
+
+      def start_coord
+        Coord[start]
+      end
+
+      def row_range
+        raise NotImplementedError, 'subclasses of Sheet must implement this method'
+      end
+
+      def column_range
+        first = columns.first.coord
+        first = first.begin if first.is_a? Range
+        last = columns.last.coord
+        last = last.end if last.is_a? Range
+        first .. last
+      end
+
+      def each_column(&block)
+        columns.each do |column|
+          if column.is_a? SubSheet
+            column.each_column(&block)
+          else
+            yield column
+          end
+        end
+      end
+
+      def subject_column
+        #return nil unless subject_mapping_type == :from_column
+        column_name = self.subject.try(:fetch, :column, nil) || :uri
+        @column[column_name]
+      end
+
+      def fix_row_count_per_resource
+        row_count_per_resource or ( !subject_column && 1 ) or nil
+      end
+
+      def add_triple(*args)
+        raise "internal error: trying to add a bad triple with nil value in #{self}: #{args}" if args.count != 3 or args.one? { |arg| arg.nil? }
+        worksheet.graph << RDF::Statement.new(*args)
+      end
+
+      def map(row_range = self.row_range, context = nil)
+        #puts "processing #{self} ..."
+        return [] if row_range.nil?
+        subjects = rows_per_resource(row_range).map do |resource_range|
+          mapping = create_context(context, row_range: resource_range)
+          spreadsheet.worksheet_mapping[worksheet.name] = mapping unless self.is_a? SubSheet
+          mapping.subject
+        end
+        subjects
+      end
+
+    private
+
+      def rows_per_resource(row_range)
+        return [] if row_range.nil?
+        rows = if fix_row_count = fix_row_count_per_resource
+            row_range.find_all do |row|
+              (row - row_range.begin) % fix_row_count == 0
+            end
+          else
+            subject_column_coord = self.subject_column.try(:coord)
+            raise "no subject column for #{self}" if subject_column_coord.blank?
+            row_range.find_all do |row|
+              not cell(row: row, column: subject_column_coord).blank?
+            end
+          end
+        rows_per_resource = []
+        rows.each_with_index do |first_row, i|
+          last_row = (i+1 == rows.count ? row_range.end : rows[i+1]-1)
+          rows_per_resource << Range.new(first_row, last_row)
+        end
+        rows_per_resource
+      end
+
+      def roo_select
+        spreadsheet.roo.default_sheet = worksheet.source_name
+      end
+
+      def roo
+        last_default_sheet = spreadsheet.roo.default_sheet
+        roo_select
+        return nil unless block_given?
+        result = yield
+        spreadsheet.roo.default_sheet = last_default_sheet
+        result
+      end
+
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/sheet_dsl.rb

@@ -0,0 +1,34 @@
+module Spread2RDF
+  class Spreadsheet
+    class Sheet
+      class DSL
+        def initialize(sheet, &block)
+          @sheet = sheet
+          instance_exec(&block) if block_given?
+        end
+
+        def column(name, options={}, &block)
+          name = name.to_sym
+          column = @sheet.column[name] ||= Column.new(@sheet, &block)
+          column.update_attributes options.merge(name: name)
+          column # TODO: chaining logic ...?
+        end
+
+        def sub_sheet(name, options={}, &block)
+          name = name.to_sym
+          sub_sheet = @sheet.column[name] ||= SubSheet.new(@sheet)
+          sub_sheet.update_attributes options.merge(name: name)
+          Sheet::DSL.new(sub_sheet, &block)
+        end
+        alias column_block sub_sheet
+
+        def cell(coord, options = {}, &block)
+          content = @sheet.cell(coord)
+          content = block.call(content) if block_given?
+          content
+        end
+
+      end
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/sheet_mapping_context.rb

@@ -0,0 +1,90 @@
+module Spread2RDF
+  class Spreadsheet
+    class Sheet
+      class MappingContext < Spreadsheet::MappingContext
+
+        self.attributes = {
+            row_range:  nil
+        }
+
+        alias sheet element
+
+        def initialize(sheet, parent_context = nil, attr = {})
+          super
+          @objects = ( sheet.columns - [ sheet.subject_column ] ).map do |column|
+            column.map(row_range, self).compact.presence
+          end.compact
+          return if @objects.empty?
+          subject_description
+          statements_to_object
+          worksheet.graph << self.graph
+        end
+
+        ########################################################################
+        # subject mapping
+
+        # TODO: every new context instance (for the same cell) returns a different bnode, it must be stored ...
+        def subject
+          @subject ||= case subject_mapping_mode
+             when :bnode         then RDF::Node.new
+             when :from_column   then subject_resource_from_column
+             else raise 'unknown subject mapping type'
+           end
+        end
+        alias subject_resource subject
+
+        def subject_resource_type
+          sheet.subject.try(:fetch, :type, nil) or
+              (sheet.subject.try(:fetch, :sub_class_of, nil) && RDF::RDFS.Class) or
+              nil
+        end
+
+        def subject_namespace
+          subject_namespace_name =
+              sheet.subject.try(:fetch, :uri, nil).try(:fetch, :namespace, nil)
+          Namespace.resolve_to_namespace(subject_namespace_name)
+        end
+
+      private
+
+        def subject_mapping_mode
+          case
+            when ( sheet.subject.try(:fetch, :uri, nil) || sheet.subject ) == :bnode
+              :bnode
+            else
+              :from_column
+          end
+        end
+
+        def subject_name_suffix
+          cells = row_range.map do |row|
+            cell_value(row: row, column: sheet.subject_column.coord).presence
+          end.compact
+          raise "no subject found for #{sheet} in #{row_range}" if cells.empty?
+          raise "multiple subjects found for #{sheet} in #{row_range}: #{cells.inspect}" if cells.count > 1
+          cells.first
+        end
+
+        def subject_resource_from_column
+          namespace = subject_namespace
+          subject_suffix = Helper.resource_name(subject_name_suffix)
+          #puts "subject resource for #{sheet} in #{range}: " + RDF::URI.new("#{namespace}#{subject_suffix}" )
+          RDF::URI.new("#{namespace}#{subject_suffix}")
+        end
+
+        def subject_description
+          type = subject_resource_type
+          statement(subject, RDF.type, type) unless type.nil?
+          if type == RDF::RDFS.Class &&
+              super_class = sheet.subject.try(:fetch, :sub_class_of, nil)
+            statement(subject, RDF::RDFS.subClassOf, super_class)
+          end
+        end
+
+        def statements_to_object
+        end
+
+      end
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/sub_sheet.rb

@@ -0,0 +1,14 @@
+module Spread2RDF
+  class Spreadsheet
+    class SubSheet < Sheet
+
+      self.attributes = {
+          predicate:  nil,
+          statement:  nil
+      }
+
+      alias coord column_range
+
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/sub_sheet_mapping_context.rb

@@ -0,0 +1,55 @@
+module Spread2RDF
+  class Spreadsheet
+    class SubSheet
+      class MappingContext < Sheet::MappingContext
+
+        self.attributes = {
+        }
+
+        alias sub_sheet element
+        alias column_block element
+
+        private
+
+        ##########################################################################
+        # Statement mapping
+        # TODO: Duplication Column::MappingContext ! Share it ?
+
+        def statement_mapping_mode
+          case
+            when column_block.statement == :none then :ignore
+            when column_block.statement == :none then :ignore
+            when column_block.predicate.nil?     then :ignore
+            when restriction_mode                then :restriction
+            else                                      :default
+          end
+        end
+
+        def restriction_mode
+          restriction_mode = column_block.statement
+          case restriction_mode
+            when :restriction then RDF::OWL.hasValue
+            when Hash         then restriction_mode[:restriction]
+            else nil
+          end
+        end
+
+        def statements_to_object
+          case statement_mapping_mode
+            when :default
+              statement(parent_context.subject, column_block.predicate, subject)
+            when :restriction
+              restriction_class = RDF::Node.new
+              statements(
+                  [ parent_context.subject, RDF::RDFS.subClassOf, restriction_class ],
+                  [ restriction_class, RDF.type, RDF::OWL.Restriction ],
+                  [ restriction_class, RDF::OWL.onProperty, column_block.predicate ],
+                  [ restriction_class, restriction_mode, subject ]
+              )
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet/worksheet.rb

@@ -0,0 +1,49 @@
+module Spread2RDF
+  class Spreadsheet
+    class Worksheet < Sheet
+
+      self.attributes = {
+      }
+
+      attr_reader :cell_mapping
+      attr_reader :graph
+
+      def initialize(parent, options={}, &block)
+        super
+        @cell_mapping = {}
+        @graph = RDF::Repository.new
+      end
+
+      def init
+        index_columns!
+      end
+
+=begin
+      def cell_mapping_by_name(name)
+
+      end
+
+      def cell_mapping_by_coord(coord)
+
+      end
+=end
+
+      def index_columns!
+        column_index = start_coord.column_as_number
+        each_column do |column|
+          column.instance_variable_set :@coord,
+                                       Roo::Base.number_to_letter(column_index)
+          column_index += 1
+        end
+      end
+
+      def row_range
+        range = roo { (Coord[start].row .. spreadsheet.roo.last_row) }
+        range.begin <= range.end ? range : nil
+      end
+
+    private
+
+    end
+  end
+end

data/lib/spread2rdf/spreadsheet.rb

@@ -0,0 +1,92 @@
+require 'spread2rdf/spreadsheet/coord'
+require 'spread2rdf/spreadsheet/element'
+require 'spread2rdf/spreadsheet/mapping_context'
+require 'spread2rdf/spreadsheet/sheet'
+require 'spread2rdf/spreadsheet/sub_sheet'
+require 'spread2rdf/spreadsheet/worksheet'
+require 'spread2rdf/spreadsheet/sheet_mapping_context'
+require 'spread2rdf/spreadsheet/sub_sheet_mapping_context'
+require 'spread2rdf/spreadsheet/column'
+require 'spread2rdf/spreadsheet/column_mapping_context'
+
+require 'spread2rdf/spreadsheet/sheet_dsl'
+require 'spread2rdf/spreadsheet/mapping_dsl'
+
+module Spread2RDF
+  class Spreadsheet
+
+    attr_reader :name
+    attr_reader :worksheet
+    attr_reader :worksheet_mapping
+    attr_reader :input_file
+    attr_reader :roo
+
+    def initialize(name, &block)
+      @name = name
+      @worksheet = {}
+      @worksheet_mapping = {}
+      @schema_spec = block
+    end
+
+    def spreadsheet
+      self
+    end
+
+    def worksheets
+      @worksheet.values
+    end
+
+    def templates
+      @template.values
+    end
+
+    def read(filename)
+      @input_file = filename
+      load_roo
+      load_schema
+      load_resources
+      self
+    end
+
+    def graph
+      graph = RDF::Repository.new
+      worksheets.each { |worksheet| graph << worksheet.graph }
+      graph
+    end
+    alias to_rdf graph
+
+  private
+
+    # TODO: make this work with other spreadsheets than Excel
+    def load_roo
+      options = {}
+      options[:packed], options[:file_warning] = :zip, :ignore if
+          File.extname(@input_file).downcase == '.xlsm'
+      @roo = Roo::Excelx.new(@input_file, options)
+    end
+
+    def load_schema
+      Spreadsheet::MappingDSL.new(self).instance_exec(&@schema_spec)
+      worksheets.each { |worksheet| worksheet.init }
+    end
+
+    def load_resources
+      worksheets.each do |worksheet|
+        next if worksheet.column.empty?
+        worksheet.map
+      end
+    end
+
+    class << self
+      def definition(*args, &block)
+        definitions << new(*args, &block)
+      end
+      private :new
+
+      def definitions
+        @@definitions ||= []
+      end
+    end
+
+  end
+end

data/lib/spread2rdf/version.rb

@@ -0,0 +1,19 @@
+module Spread2RDF
+  module VERSION
+    FILE = File.expand_path('../../../VERSION', __FILE__)
+    MAJOR, MINOR, TINY, EXTRA = File.read(FILE).chomp.split('.')
+    STRING = [MAJOR, MINOR, TINY, EXTRA].compact.join('.').freeze
+
+    ##
+    # @return [String]
+    def self.to_s() STRING end
+
+    ##
+    # @return [String]
+    def self.to_str() STRING end
+
+    ##
+    # @return [Array(Integer, Integer, Integer)]
+    def self.to_a() [MAJOR, MINOR, TINY] end
+  end
+end

data/lib/spread2rdf.rb

@@ -0,0 +1,23 @@
+# coding: utf-8
+require 'rubygems/package'
+
+require 'optparse'
+
+require 'active_support/core_ext'
+require 'awesome_print'
+
+require 'roo'
+require 'spread2rdf/extensions/roo_xlsm_fix'
+
+require 'linkeddata'
+
+require 'spread2rdf/attributes'
+require 'spread2rdf/version'
+require 'spread2rdf/helper'
+require 'spread2rdf/namespace'
+require 'spread2rdf/spreadsheet'
+require 'spread2rdf/cli'
+
+module Spread2RDF
+end
+

data/spread2rdf.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/spread2rdf/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = 'spread2rdf'
+  gem.authors       = ['Marcel Otto']
+  gem.email         = %w[marcelotto.de@gmail.com]
+  gem.summary       = %q{a DSL-based converter for spreadsheets to RDF}
+  gem.description   = %q{Spread2RDF is a converter for complex spreadsheets to RDF and a DSL for specifying the mapping rules for this conversion.}
+  gem.homepage      = 'http://github.com/marcelotto/spread2rdf'
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+  gem.version       = Spread2RDF::VERSION.to_s
+  gem.bindir        = 'bin'
+  gem.executables   = ['spread2rdf']
+
+  gem.required_ruby_version = '>= 1.9.3'
+
+  gem.add_dependency('activesupport', '~> 3.2.3')
+  gem.add_dependency('awesome_print')
+
+  gem.add_dependency('roo', '~> 1.12.2')
+  gem.add_dependency('rubyzip', '~> 1.0.0') # for the roo-xlsm-fix
+
+  gem.add_dependency('linkeddata')
+
+  gem.add_development_dependency('rake')
+  gem.add_development_dependency('pry', '~> 0.9.12.2')
+  gem.add_development_dependency('pry-nav', '~> 0.2.3')
+
+end