yarrrml_template_builder 0.1.54

data/lib/yarrrml_template_builder/yarrrml_transform.rb

@@ -0,0 +1,181 @@
+require 'tempfile'
+require 'rest-client'
+
+# note that SDMrdfizer needs to be running on port 4000 with the ./data folder mounted as /data
+# docker run --name rdfizer --rm -d -p 4000:4000 -v $PWD/data:/data fairdatasystems/sdmrdfizer:0.1.0
+
+class YARRRML_Transform
+  
+  attr_accessor :data_path_server
+  attr_accessor :data_path_client
+  attr_accessor :config_path_server
+  attr_accessor :config_path_client
+  attr_accessor :rdfizer_base_url
+  attr_accessor :yarrrml_transform_base_url
+  attr_accessor :datatype_tag
+  attr_accessor :datafile
+  attr_accessor :baseURI
+  
+  attr_accessor :yarrrmltemplate
+  attr_accessor :yarrrmlfilename_client
+  attr_accessor :yarrrmlfilename_server
+  attr_accessor :outputrmlfile
+  attr_accessor :formulation
+  attr_accessor :outputrdffolder
+  attr_accessor :inifile_client
+  attr_accessor :inifile_server  
+  attr_accessor :failure
+ 
+  
+
+# Creates the YARRRML Transformer object
+#
+#
+#  the datatype_tag is used to construct other input/output filenames.  e.g. a tag of "height" will
+# cause the output RML file to be "./data/height_rml.ttl", and the expected YARRRML template being named 
+# "./config/height_yarrrml_template.yaml"
+#
+# Note that this assumes a specific configuration, with a csv folder mounted as /data/, containing a subdirectory /data/triples
+# and a folder containing the yarrrml templates mounted as /config inside the docker container
+#
+# all params are passed as a hash, and retrieved by params.fetch(paramName).  the ini file used by yarrrml transform is auto-generated
+#
+# @param params [Hash]  params   a Hash of options
+# @option params   :datatype_tag [String] (required)  a one-word indicator of the data type.  This is considered the "base" for all other filenames (see below)
+# @option params   :data_path_server [String]  (/data) the path to data from the server perspective (default for Docker image)
+# @option params   :data_path_client  [String]  (/data) the path to data from the client perspective (default for Docker image)
+# @option params   :config_path_server [String]  (/config) the path to config from the server perspective (default for Docker image)
+# @option params   :config_path_client [String]  (/config) the path to config from the server perspective (default for Docker image)
+# @option params   :formulation [String]  (csv)
+# @option params   :rdfizer_base_url [String] (http://rdfizer:4000) the base URL of the yarrrml_parser (default for a docker)
+# @option params   :yarrrml_transform_base_url [String] ("http://yarrrml_transform:3000")  base URL of the yarrrml transform to rml script (default for a docker)
+
+# @option params   :outputrdffolder [String]  (/data/triples/) - defaults to /data/triples - this folder must exist, even if left to default.  NOTE - this path is not relative to the host, it is relative to the docker rdfizer, so it begins with /data not ./data)
+# @return [YARRRML_Transform]
+  def initialize(params = {}) # get a name from the "new" call, or set a default
+    
+    @datatype_tag = params.fetch( :datatype_tag, nil)
+    unless @datatype_tag
+          warn "must have a datatype_tag parameter.  Aborting"
+          self.failure = true
+          return nil
+    end
+    
+    @data_path_client = params.fetch(:data_path_client, "/data")   # from client perspective (default for docker)
+    @data_path_server = params.fetch(:data_path_server, self.data_path_client)     # potentially from Docker image perspective (default for docker)
+    @config_path_client = params.fetch(:config_path_client, "/config")  # from client perspective (default for docker)
+    @config_path_server = params.fetch(:config_path_server, self.config_path_client)  # potentially from Docker image perspective (default for docker)
+
+    @data_path_client.gsub!('/$', "")   # remove trailing slashes -I'll add them later if I need them
+    @data_path_server.gsub!('/$', "")
+    @config_path_client.gsub!('/$', "")
+    @config_path_server.gsub!('/$', "")
+
+
+    @rdfizer_base_url = params.fetch(:rdfizer_base_url, "http://rdfizer:4000")
+    @yarrrml_transform_base_url = params.fetch(:yarrrml_transform_base_url, "http://yarrrml_transform:3000")
+    @rdfizer_base_url.gsub!('/$', "")
+    @yarrrml_transform_base_url.gsub!('/$', "")
+    
+    
+    @formulation = params.fetch(:formulation, "csv")
+    @datafile = params.fetch(:datafile, "#{self.data_path_server}/#{self.datatype_tag}.csv")
+    @baseURI = params.fetch(:baseURI, ENV['baseURI'])
+    @baseURI = "http://example.org/data/" unless @baseURI
+
+    @outputrdffolder = params.fetch(:outputrdffolder, "#{self.data_path_server}/triples")
+    @outputrmlfile = params.fetch(:outputrmlfile, "#{self.data_path_server}/#{self.datatype_tag}_rml.ttl")
+    @yarrrmlfilename_client = params.fetch(:yarrrmlfilename, "#{self.data_path_client}/#{self.datatype_tag}_yarrrml.yaml")
+    @yarrrmlfilename_server = params.fetch(:yarrrmlfilename, "#{self.data_path_server}/#{self.datatype_tag}_yarrrml.yaml")
+    @yarrrmltemplate = params.fetch(:yarrrmltemplate, "#{self.config_path_client}/#{self.datatype_tag}_yarrrml_template.yaml")
+    @inifile_client = params.fetch(:inifile_client, "#{self.data_path_client}/#{self.datatype_tag}.ini")
+    @inifile_server = params.fetch(:inifile_server, "#{self.data_path_server}/#{self.datatype_tag}.ini")   # this will be a docker image in almost all cases
+    
+    
+    write_ini()
+    
+    transform_template()
+    
+    return self
+
+  end
+  
+
+
+  def transform_template()
+# transform appropriate template with this data
+
+    File.open(self.yarrrmltemplate, "r") {|f| @template = f.read}
+    @template.gsub!("|||DATA|||", self.datafile)
+    @template.gsub!("|||FORMULATION|||", self.formulation)
+    @template.gsub!("|||BASE|||", self.baseURI)
+    File.open(self.yarrrmlfilename_client, "w") {|f| f.puts @template}
+    warn "Ready to yarrrml transform #{self.datatype_tag} from #{self.yarrrmlfilename_client} "
+
+  end
+  
+  
+
+
+  def write_ini(inifile = self.inifile_client,
+                path = self.data_path_server,
+                rdffolder = self.outputrdffolder,
+                rmlfile = self.outputrmlfile,
+                datatype = self.datatype_tag)
+
+    configfilecontent = <<CONFIG
+[default]
+main_directory: #{path}
+
+[datasets]
+number_of_datasets: 1
+output_folder: #{rdffolder}
+all_in_one_file: yes
+remove_duplicate: yes
+enrichment: yes
+name: #{datatype}
+ordered: yes
+large_file: true
+
+[dataset1]
+name: #{datatype}
+mapping: #{rmlfile}
+
+CONFIG
+
+    File.open(inifile, "w"){|f| f.puts configfilecontent}
+
+  end
+  
+  
+  
+# Executes the yarrrml to rml transformation 
+#
+# no parameters
+#
+#
+  def yarrrml_transform
+    warn "running docker yarrrml-parser:ejp-latest"
+    #parser_start_string = "docker run -e PARSERIN=#{self.yarrrmlfilename} -e PARSEROUT=#{self.outputrmlfile} --rm --name yarrrml-parser -v $PWD/data:/data markw/yarrrml-parser-ejp:latest"
+    warn "yarrrml to rml starting with: #{self.yarrrml_transform_base_url} PARSERIN=#{self.yarrrmlfilename_server} -e PARSEROUT=#{self.outputrmlfile}"
+    #resp = RestClient.get("#{self.yarrrml_transform_base_url}/?parserin=#{self.yarrrmlfilename_server}&parserout=#{self.outputrmlfile}")
+    resp = RestClient::Request.execute(method: :get, url: "#{self.yarrrml_transform_base_url}/?parserin=#{self.yarrrmlfilename_server}&parserout=#{self.outputrmlfile}", timeout: 9000000)
+    warn "#{resp}: rml file has been created in #{self.outputrmlfile} - ready to make FAIR data"
+  end
+  
+  
+  
+# Executes the CSV to RDF based on the RML
+#
+# no parameters
+#
+#executes the sdmrdfizer transformation using the .ini file created by the 'initialize' routine
+  def make_fair_data
+    warn "making FAIR data with #{self.rdfizer_base_url}/graph_creation/#{self.inifile_server}"  # this is sdmrdfizer
+    response = RestClient::Request.execute(method: :get, url: self.rdfizer_base_url + "/graph_creation" + self.inifile_server, timeout: 900000000)
+    warn response.code
+    warn "FAIR data is avaialable in .#{self.outputrdffolder}/#{self.datatype_tag}.nt"
+  end
+  
+  
+end

data/lib/yarrrmltemplatebuilder.rb

@@ -0,0 +1,5 @@
+
+require File.dirname(__FILE__) + '/yarrrml_template_builder/version'
+require File.dirname(__FILE__) + '/yarrrml_template_builder/yarrrml_template_builder'
+require File.dirname(__FILE__) + '/yarrrml_template_builder/yarrrml_transform'
+

data/run_me_to_test.rb

@@ -0,0 +1,26 @@
+#!ruby
+
+require "yarrrml-template-builder"
+
+$stderr.puts "this script will do an rdf transformation of the height.csv file, in the ./data folder using the yarrrml file in the ./config folder"
+$stderr.puts "you must already have the docker-compose up before running this script.  If you see failures, that is likely why :-)"
+datatype_tag = "height"  # the "tag" of your data
+data_path_client = "./data"
+data_path_server = "/data"  # assumes you are running my docker image
+config_path_client = "./config"
+config_path_server = "/config"  # assumes you are running my docker image
+rdfizer_base_url = "http://localhost:4000"  # again, my docker
+yarrrml_transform_base_url = "http://localhost:3000"  # again, my docker
+
+y = YARRRML_Transform.new(
+	datatype_tag: datatype_tag, 
+	data_path_client: data_path_client,
+	data_path_server: data_path_server,
+	config_path_client: config_path_client,
+	config_path_server: config_path_server,
+	rdfizer_base_url: rdfizer_base_url,
+	yarrrml_transform_base_url: yarrrml_transform_base_url,
+	)
+y.yarrrml_transform
+y.make_fair_data
+

data/yarrrml_template_builder.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "lib/yarrrml_template_builder/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "yarrrml_template_builder"
+  spec.version       = YARRRMLTemplateBuilder::VERSION
+  spec.authors       = ["Mark Wilkinson"]
+  spec.email         = ["markw@illuminae.com"]
+
+  spec.summary       = "Creates YARRRML Template templates"
+  spec.description   = "Creates a templated version of YARRRML, missing the source and iteration type so they can be reused"
+  spec.homepage      = "https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder"
+  spec.metadata["changelog_uri"] = "https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder/Changelog.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+
+  spec.add_dependency "yaml", "~> 0.1.0"
+  spec.add_dependency "rest-client", "~> 2.1.0"
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: yarrrml_template_builder
+version: !ruby/object:Gem::Version
+  version: 0.1.54
+platform: ruby
+authors:
+- Mark Wilkinson
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2022-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+description: Creates a templated version of YARRRML, missing the source and iteration
+  type so they can be reused
+email:
+- markw@illuminae.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Changelog.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- bin/console
+- bin/setup
+- config/height_yarrrml_template.yaml
+- data/height.csv
+- doc/YARRRMLTemplateBuilder.html
+- doc/YARRRML_Template_Builder.html
+- doc/YARRRML_Transform.html
+- doc/_index.html
+- doc/class_list.html
+- doc/css/common.css
+- doc/css/full_list.css
+- doc/css/style.css
+- doc/file.README.html
+- doc/file_list.html
+- doc/frames.html
+- doc/index.html
+- doc/js/app.js
+- doc/js/full_list.js
+- doc/js/jquery.js
+- doc/method_list.html
+- doc/top-level-namespace.html
+- examples/example_output_timerange.rb
+- examples/example_timed_process.rb
+- examples/example_timeinstant_process.rb
+- examples/example_two_linked_processes.rb
+- exemplar_docker_compose.yml
+- lib/yarrrml-template-builder.rb
+- lib/yarrrml_template_builder/version.rb
+- lib/yarrrml_template_builder/yarrrml_template_builder.rb
+- lib/yarrrml_template_builder/yarrrml_transform.rb
+- lib/yarrrmltemplatebuilder.rb
+- run_me_to_test.rb
+- yarrrml_template_builder.gemspec
+homepage: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder
+  source_code_uri: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder
+  changelog_uri: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder/Changelog.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.28
+signing_key: 
+specification_version: 4
+summary: Creates YARRRML Template templates
+test_files: []