spread2rdf 0.0.1pre.1 → 0.0.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzBlOGJjZDMyMjBjMjNmNDA5MTk3MmY1NjMzMmE2OWNiZGE3MTljYw==
+    NTJjZDkxNmJhNmM2ZjFmYjM4ZGJlZjE2MGQyZmZkNWMzMWI1NGQ4Yw==
   data.tar.gz: !binary |-
-    Y2IxMjY2MDBjZDk1OGYxNjE2YjlmM2UyZGZkYTQ2NmZjNDk5MWYyYQ==
+    YjFmZjA5ZmZmOTk5ZWJjODAwOTgwZDNkZjkyYWY3NTgwMjllYzBkOA==
 SHA512:
   metadata.gz: !binary |-
-    ODlkODhkZDQ1NzZmNGVhY2E0MGIwNDgyOWVmZDFkODkyYjJlYmNiZjYyMzlh
-    ZmEzNjk4YjJiNjIyZjM1OTM2YTQ1ZGZlZDJjZTFhOTA1Mzc0ZWJlZTY4NjY1
-    YmEyNGZmNGM1ZjBkYTMxMTNmZWQ2YTZkNzVkOTViMDA2Y2I0ZTk=
+    OTEzM2JjNjkyNTkyMGJiNDUwZTE1OTE0NjE4ZjcyZmNkZTk5MWFjMDYyYzU5
+    Nzk0NWU5MDVhMmIwYmJkNjFiMjA5ZmI5NjU1Yjg4NzMxNWM5Yjk3MzFjMTgy
+    MGRjMWJiN2VlMzU5OWNkYmNhMDkxZjNlN2ExOWUwN2ZhYmQzOGI=
   data.tar.gz: !binary |-
-    MDExZWZiZjY4YjBlNDdkYzhkN2ExZjNmNTVhNWVkNGEwMDBmYzg0YTY0ZGM2
-    Njk3ZmQ4NmU3MmY3MjAxYzkwZjkzZThjM2VhZWMzYjYyZmJjYjBjMTc0MTdi
-    YzUwNzlhNTRhMjY0NDJiNzIxZTdiYTkzNTZlM2U0ZTBjZTNmODY=
+    N2MxZGQ4YjMxYmI2Y2IyN2NiMjFiN2U4NmRkZjA4NGJlMWNkNzgxYjg5MWU2
+    YTkyZmY1ZGM5YWIyZTkwMjFmODI5ZDUwZTc4ZWQ2ZmFlMzVmMzc3NGE1Nzcz
+    ZGIzZDBmMTA2MWRhNjJmZDJmMGY3NzM3NDEyOGNiODFlOTQwNGU=

data/Gemfile.ocra

@@ -0,0 +1,6 @@
+# we need this dedicated Gemfile currently, due to this issue:
+# https://github.com/larsch/ocra/issues/59
+
+source 'https://rubygems.org'
+
+gem 'spread2rdf', '~> 0.0.1'

data/VERSION

@@ -1 +1 @@
-0.0.1pre.1
+0.0.1

data/bin/spread2rdf

@@ -1,3 +1,3 @@
 #!/usr/bin/env ruby
 require 'spread2rdf'
-Spread2RDF::Cli.new.run
+Spread2RDF::CLI.run

data/lib/spread2rdf/attributes.rb

@@ -6,6 +6,7 @@ module Spread2RDF
       def attributes
         if superclass.respond_to?(:attributes) and
             (super_attributes = superclass.attributes).is_a? Hash
+          @attributes ||= {}
           @attributes.reverse_merge(super_attributes)
         else
           @attributes

data/lib/spread2rdf/cli.rb

@@ -1,63 +1,119 @@
 # coding: utf-8
+require 'singleton'
 
 module Spread2RDF
   class Cli
-    def initialize
+    include Singleton
+
+    attr_accessor :mapping_schema
+    
+    def run(options = {})
+      @running = true
+      init(options)
       parse_command_line!
+      case
+        when compile? then compile(@mapping_schema)
+        else convert
+      end
+      self
+    rescue => e
+      if Spread2RDF.debug_mode
+        raise e
+      else
+        abort e.to_s
+      end
     end
 
-    def run(schema_spec_file = nil)
-      schema_spec_file ||= @options[:schema_spec_file]
-      abort "No schema specification file given" if schema_spec_file.nil?
-      abort "Couldn't find schema specification file #{schema_spec_file}" unless
-          File.exist?(schema_spec_file)
-      load schema_spec_file
-      abort "No schema specification found" if Spreadsheet.definitions.empty?
-      puts "Reading #{@input_file} ..."
-      @table = Spreadsheet.definitions.first
-      @table.read(@input_file)
-      write_output
-      self
+    def init(options)
+      @options = options
+      @input_file = @options.delete(:input_file) if @options[:input_file]
+      self.mapping_schema = @options.delete(:schema) if @options[:schema]
+    end
+    private :init
+
+    def running?
+      @running
+    end
+
+    def compile?
+      !!@options[:compile]
+    end
+
+    def mapping_schema=(schema)
+      @mapping_schema =
+        case schema
+          when nil then nil
+          when Schema::Spreadsheet then schema
+          when String
+            abort "no schema specification file given" if schema.nil?
+            abort "couldn't find schema specification file #{schema}" unless
+                File.exist?(schema)
+            load schema
+            abort "no schema specification found" if Schema.definitions.empty?
+            Schema.definitions.first
+          else raise ArgumentError
+        end
     end
 
   private
 
-    # Parse command line options
-    def parse_command_line!(options={})
-      @options = options
+    def parse_command_line!
       optparse = OptionParser.new do |opts|
-        opts.banner = 'Usage: spread2rdf [options] -s SPEC_FILE SPREAD_SHEET_FILE'
+        if mapping_schema
+          opts.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options] SPREAD_SHEET_FILE"
+        else
+          opts.banner = 'Usage: spread2rdf [options] -s SPEC_FILE SPREAD_SHEET_FILE'
+        end
 
         opts.on( '-h', '--help', 'Display this information' ) do
           puts opts
           exit
         end
 
-        @options[:output_dir] = '.'
-        opts.on( '-o', '--output DIR', 'Output directory (default: current directory)' ) do |dir|
-          abort "Output directory #{dir} doesn't exist" unless Dir.exist?(dir)
+        opts.on( '-v', '--version', 'Print version information' ) do
+          puts "Spread2RDF #{VERSION}"
+          exit
+        end
+
+        if not mapping_schema
+          opts.on( '-c', '--compile', 'Compile the schema specification to an executable' ) do
+            @options[:compile] = true
+          end
+
+          opts.on( '-s', '--schema SPEC_FILE', 'Schema specification file (required)' ) do |file|
+            if @options[:compile]
+              @mapping_schema = file
+            else
+              self.mapping_schema = file
+            end
+          end
+        end
+
+        @options[:output_dir] ||= '.'
+        opts.on( '-o', '--output DIR', "Output directory (default: #{@options[:output_dir]})" ) do |dir|
+          abort "Output directory #{dir} doesn't exist" unless compile? or Dir.exist?(dir)
           @options[:output_dir] = dir
         end
 
-        @options[:output_format] = 'ttl'
+        @options[:output_format] ||= 'ttl'
         opts.on( '-f', '--output-format FORMAT', 'Serialization format for the RDF data',
-          "FORMAT being one of: nt, n3, ttl, rdf, xml, html, json (default: ttl)") do |format|
-          #format = 'turtle' if format == 'ttl'
+          "FORMAT being one of: nt, n3, ttl, rdf, xml, html, json (default: #{@options[:output_format]})") do |format|
           @options[:output_format] = format.strip.downcase
         end
 
-        @options[:schema_spec_file] = nil
-        opts.on( '-s', '--schema SPEC_FILE', 'Schema specification file (required)' ) do |file|
-          @options[:schema_spec_file] = file
+        opts.on( '-d', '--debug', 'Run in debug mode' ) do
+          Spread2RDF.debug_mode = true
         end
 
       end
-
       optparse.parse!
-      raise OptionParser::ParseError, 'required file arguments missing' if ARGV.empty?
-      raise OptionParser::ParseError, 'required schema specification file missing' if @options[:schema_spec_file].nil?
-
-      @input_file = ARGV.first
+      if compile?
+        @mapping_schema ||= ARGV.first or raise OptionParser::ParseError, 'required schema specification file missing'
+      else
+        raise OptionParser::ParseError, 'required schema specification file missing' if @mapping_schema.nil?
+        @input_file = ARGV.first or @input_file or
+            raise OptionParser::ParseError, 'required file arguments missing'
+      end
     rescue OptionParser::ParseError => e
       puts e.message
       puts optparse.help
@@ -72,8 +128,8 @@ module Spread2RDF
 
     def write_output
       filename = output_filename
-      abort 'No RDF data to write!' if @table.try(:to_rdf).blank?
-      graph = @table.to_rdf
+      abort 'No RDF data to write!' if @mapping.try(:graph).blank?
+      graph = @mapping.graph
       puts "Writing #{graph.count} RDF statements to #{filename} ... "
       # TODO: base_uri: ... for writer constructor
       RDF::Writer.open(filename) do |writer|
@@ -85,7 +141,45 @@ module Spread2RDF
         end
         graph.each_statement { |statement| writer << statement }
       end
+
+    end
+
+    def convert
+      puts "Reading #{@input_file} ..."
+      @mapping = mapping_schema.map(@input_file)
+      write_output
+    end
+
+    def compile(mapping_file)
+      output = if File.directory?(@options[:output_dir])
+        output_dir = @options[:output_dir]
+        output_file = File.basename(mapping_file, File.extname(mapping_file)) + '.exe'
+        File.join(output_dir, output_file)
+      else
+        @options[:output_dir] += '.exe' unless File.extname(@options[:output_dir]) == '.exe'
+        @options[:output_dir]
+      end
+      ocra_options = [
+        '--gem-full',
+        '--add-all-core',
+        '--no-autoload',
+        '--no-dep-run',
+        '--no-enc',
+        '--console'
+      ]
+      #ocra_options << '--quiet' unless Spread2RDF.debug_mode
+      ocra_options << '--debug' if Spread2RDF.debug_mode
+      ocra_gemfile = File.join(Spread2RDF::ROOT, 'Gemfile.ocra')
+      ocra_options << "--gemfile #{ocra_gemfile}"
+      ocra_options << "--output #{output}"
+      ocra_cmd = "ocra #{ocra_options.join(' ')} #{mapping_file}"
+      puts "compiling #{mapping_file} to #{output}"
+      #puts ocra_cmd
+      Kernel.system(ocra_cmd)
     end
+
     self
   end
-end
+
+  CLI = Cli.instance
+end

data/lib/spread2rdf/coord.rb

@@ -0,0 +1,49 @@
+module Spread2RDF
+  class Coord < Struct.new(:column, :row)
+    def initialize(*args)
+      case args.length
+        when 2 then super
+        when 1
+          case args = args.first
+            when Hash
+              super(args[:column], args[:row])
+            when Symbol, String
+              coord = args.to_s
+              raise "Invalid cell coordinates #{coord}" unless coord =~ /(\w+)(\d+)/
+              super(Regexp.last_match[1], Regexp.last_match[2].to_i)
+            else raise ArgumentError, "can't handle argument #{args}"
+          end
+        else raise ArgumentError, "too many arguments: #{args}"
+      end
+    end
+
+    def column_as_number
+      Roo::Base.letter_to_number(column)
+    end
+
+    def column_as_index
+      column_as_number - 1
+    end
+
+    def increment_column(count = 1)
+      self.class.increment_column(self.column, count)
+    end
+
+    def to_s
+      "#{column}#{row}"
+    end
+
+    def to_sym
+      to_s.to_sym
+    end
+
+    class << self
+      alias [] new
+
+      def increment_column(column, count=1)
+        Roo::Base.number_to_letter(Roo::Base.letter_to_number(column) + count)
+      end
+    end
+
+  end
+end

data/lib/spread2rdf/mapping/cell.rb

@@ -0,0 +1,105 @@
+module Spread2RDF
+  module Mapping
+    class Cell < Element
+
+      attr_reader :row
+
+      def_delegators :parent, :subject, :worksheet, :resource
+      def_delegators :schema, :predicate
+
+      def initialize(sheet, parent, row)
+        super(sheet, parent)
+        @row = row
+        map
+      end
+
+      def map
+        exec(&schema.block) unless empty?
+      end
+
+      def coord
+        Coord[column: schema.coord, row: row]
+      end
+
+      def value
+        @value ||= worksheet.cell_value(coord)
+      end
+
+      def object
+        @object ||= value && map_to_object(value)
+      end
+
+      def empty?
+        value.blank?
+      end
+
+      def map_to_object(value)
+        case schema.object_mapping_mode
+          when :to_string     then value
+          when :resource_ref  then resolve_resource_ref
+          when :new_resource  then create_resource_object
+          when :custom        then  exec(&schema.cell_mapping)
+          else raise 'internal error: unknown column mapping type'
+        end
+      end
+      private :map_to_object
+
+      def resolve_resource_ref
+        source = schema.object[:from]
+        source = { worksheet: source } if source.is_a? Symbol
+        raise ArgumentError, "expecting a Hash as source, but got #{source}" unless source.is_a? Hash
+        source_worksheet = source[:worksheet]
+        source_worksheet = spreadsheet.worksheet(source_worksheet)
+        raise "#{self}: couldn't find source worksheet #{source[:worksheet]}" if source_worksheet.nil?
+        source_predicate = source[:predicate] || RDF::RDFS.label
+        result = source_worksheet.graph.query([nil, source_predicate, value])
+        raise "#{self}: couldn't find a resource for #{value} in #{source_worksheet}" if result.empty?
+        raise "#{self}: found multiple resources for #{value} in #{source_worksheet}: #{result.map(&:subject)}" if result.count > 1
+        result.first.subject
+      end
+      private :resolve_resource_ref
+
+      def create_resource_object
+        case
+          when (schema.object.try(:fetch, :uri, nil) || object) == :bnode
+            RDF::Node.new
+          else
+            raise NotImplementedError
+        end
+      end
+      private :create_resource_object
+
+      ##########################################################################
+      # for the DSL for column statement blocks
+
+      def value_of_column(name)
+        sheet_schema = parent.parent.schema
+        other_column = sheet_schema.column(name)
+        raise "couldn't find column #{name} when mapping #{self}" if
+            other_column.nil?
+        worksheet.cell_value(column: other_column.coord, row: row)
+      end
+
+      def object_of_column(name)
+        other_column = resource.column!(name)
+        raise "couldn't find column #{name} when mapping #{self}" if
+            other_column.nil?
+        other_column.cell!(row).object
+      end
+
+      def exec(&block)
+        #puts "executing block of #{@___column___} in row #{row}"
+        self.instance_exec(value, &block) if block_given?
+      end
+      private :exec
+
+      ##########################################################################
+      # Element#_children_
+
+      def _children_
+        nil
+      end
+
+    end
+  end
+end

data/lib/spread2rdf/mapping/column.rb

@@ -0,0 +1,63 @@
+module Spread2RDF
+  module Mapping
+    class Column < Element
+      include Statement
+
+      def_delegators :parent, :subject, :worksheet, :row_range
+      def_delegators :schema, :predicate
+
+      alias resource parent
+
+      def initialize(sheet, parent)
+        super
+        @cells = {}
+        map(row_range)
+      end
+
+      def map(range)
+        #puts "mapping #{self} in #{range} ..."
+        case range
+          when Integer
+            cell = cell!(range)
+            statements_to_object(cell.object) unless cell.empty?
+          when Range
+            range.each { |row| self.map(row) }
+          else raise ArgumentError
+        end
+      end
+
+      def objects
+        cells.map(&:object)
+      end
+
+      def cell_coord(row)
+        case row
+          when Integer then Coord[column: schema.coord, row: row]
+          when Coord   then Coord
+          when Hash    then Coord[row]
+          else raise ArgumentError
+        end
+      end
+
+      ##########################################################################
+      # Mapping::Element structure
+
+      def cells
+        @cells.values
+      end
+      alias _children_ cells
+
+      def cell(coord)
+        coord = cell_coord(coord)
+        @cells[coord.to_sym] # TODO: search @sub_sheet_mappings also
+      end
+
+      def cell!(coord)
+        coord = cell_coord(coord)
+        @cells[coord.to_sym] ||= Cell.new(schema, self, coord.row)
+      end
+
+    end
+  end
+end
+

data/lib/spread2rdf/mapping/column_block.rb

@@ -0,0 +1,23 @@
+module Spread2RDF
+  module Mapping
+    class ColumnBlock < Sheet
+      include Statement
+
+      def_delegators :parent, :subject, :row_range
+      def_delegators :schema, :predicate
+
+      def map
+        super
+        @resources.each do |resource|
+          statements_to_object(resource.subject) unless resource.empty?
+        end
+        self
+      end
+
+      def objects
+        @resources.map(&:subject)
+      end
+
+    end
+  end
+end

data/lib/spread2rdf/mapping/default_cell_mappings.rb

@@ -0,0 +1,26 @@
+module Spread2RDF
+  module Mapping
+    class Cell
+      module Default
+        def self.unit_mapping(value)
+          @qudt ||= RDF::Graph.load File.join(ONTOLOGY_DIR, 'unit-v1.1.ttl')
+          query = RDF::Query.new(
+              unit: { RDF::URI.new('http://qudt.org/schema/qudt#symbol') =>
+                        RDF::Literal.new(value, datatype: RDF::XSD.string) })
+          solutions = query.execute(@qudt)
+          raise "unit #{value} is not unique; possible candidates:
+                    #{solutions.map(&:unit).ai}" if solutions.count > 1
+          raise "couldn't find a QUDT unit for unit '#{value}''" if solutions.empty?
+          solutions.first.unit
+        end
+
+        def self.uri_normalization(string)
+          string
+            .gsub(', ', '-')
+            .gsub(' ', '-')
+        end
+
+      end
+    end
+  end
+end

data/lib/spread2rdf/mapping/element.rb

@@ -0,0 +1,64 @@
+require 'forwardable'
+
+module Spread2RDF
+  module Mapping
+    class Element
+      extend Forwardable
+
+      attr_reader :schema
+      attr_reader :parent
+
+      def_delegators :parent, :spreadsheet
+
+      def initialize(schema, parent)
+        @graph = RDF::Repository.new
+        @schema = schema
+        @parent = parent
+      end
+
+      def to_s
+        "#{self.class.name.split('::').last}-mapping of #{schema}"
+      end
+
+      ##########################################################################
+      # children
+
+      def _children_
+        raise NotImplementedError, 'subclasses must implement this method'
+      end
+
+      def empty?
+        _children_.empty? or _children_.all?(&:empty?)
+      end
+
+      ##########################################################################
+      # RDF graph
+
+      def graph
+        if _children_
+          _children_.inject(@graph.clone) { |graph, child| graph << child.graph }
+        else
+          @graph
+        end
+      end
+      alias to_rdf graph
+
+    private
+
+      def add_statement(*args)
+        args = args.first if args.count == 1 and args.first.is_a? Array
+        #puts "adding statement: #{args.inspect}"
+        raise "internal error: trying to add a bad triple with nil value: #{args}" if args.count != 3 or args.one? { |arg| arg.nil? }
+        @graph << RDF::Statement.new(*args)
+      end
+      alias statement add_statement
+
+      def add_statements(*args)
+        args = args.first if args.count == 1 and args.first.is_a? Array
+        args.each { |arg| statement(arg) }
+      end
+      alias statements add_statements
+
+    end
+  end
+end

data/lib/spread2rdf/mapping/resource.rb

@@ -0,0 +1,95 @@
+module Spread2RDF
+  module Mapping
+    class Resource < Element
+
+      attr_reader :row_range
+
+      def_delegators :parent, :worksheet
+
+      def initialize(sheet, parent, row_range)
+        super(sheet, parent)
+        @columns = {}
+        @row_range = row_range
+        map
+      end
+
+      def map
+        #puts "processing #{self} in #{row_range}"
+        object_columns = parent.schema.columns - [parent.schema.subject_column]
+        object_columns.each { |column| column!(column) }
+        subject_description unless empty?
+        self
+      end
+
+      ##########################################################################
+      # subject mapping
+
+      def subject
+        @subject ||=
+            case schema.subject_mapping_mode
+              when :bnode         then RDF::Node.new
+              when :from_column   then subject_resource_from_column
+              else raise 'unknown subject mapping type'
+            end
+      end
+      alias subject_resource subject
+
+      def subject_name_suffix
+        cells = row_range.map do |row|
+          parent.cell_value(row: row, column: schema.subject_column.coord).presence
+        end.compact
+        raise "no subject found for Resource in #{row_range} of #{sheet}" if cells.empty?
+        raise "multiple subjects found for Resource in #{row_range} of #{sheet}: #{cells.inspect}" if cells.count > 1
+        cells.first
+      end
+      private :subject_name_suffix
+
+      def subject_resource_from_column
+        namespace = schema.subject_namespace
+        subject_suffix = Mapping::Cell::Default.uri_normalization(subject_name_suffix)
+        #puts "subject resource for #{sheet} in #{range}: " + RDF::URI.new("#{namespace}#{subject_suffix}" )
+        RDF::URI.new("#{namespace}#{subject_suffix}")
+      end
+      private :subject_resource_from_column
+
+      def subject_description
+        type = schema.subject_resource_type
+        statement(subject, RDF.type, type) unless type.nil?
+        if type == RDF::RDFS.Class &&
+            super_class = schema.subject.try(:fetch, :sub_class_of, nil)
+          statement(subject, RDF::RDFS.subClassOf, super_class)
+        end
+      end
+      private :subject_description
+
+      ##########################################################################
+      # Mapping::Element structure
+
+      def columns
+        @columns.values
+      end
+      alias _children_ columns
+
+      def column(name)
+        @columns[column_schema(name).name]
+      end
+
+      def column!(name)
+        column_schema = column_schema(name)
+        @columns[column_schema.name] ||= case column_schema
+          when Schema::Column      then Column.new(column_schema, self)
+          when Schema::ColumnBlock then ColumnBlock.new(column_schema, self).map
+        end
+      end
+
+      def column_schema(name)
+        case name
+          when Schema::Column, Schema::ColumnBlock then name
+          when String, Symbol                      then schema.column(name)
+          else raise ArgumentError
+        end
+      end
+
+    end
+  end
+end