yarrrml_template_builder 0.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/Changelog.md +1 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +39 -0
- data/LICENSE.txt +21 -0
- data/README.md +139 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/config/height_yarrrml_template.yaml +190 -0
- data/data/height.csv +2 -0
- data/doc/YARRRMLTemplateBuilder.html +121 -0
- data/doc/YARRRML_Template_Builder.html +6392 -0
- data/doc/YARRRML_Transform.html +1940 -0
- data/doc/_index.html +117 -0
- data/doc/class_list.html +51 -0
- data/doc/css/common.css +1 -0
- data/doc/css/full_list.css +58 -0
- data/doc/css/style.css +497 -0
- data/doc/file.README.html +214 -0
- data/doc/file_list.html +56 -0
- data/doc/frames.html +17 -0
- data/doc/index.html +214 -0
- data/doc/js/app.js +314 -0
- data/doc/js/full_list.js +216 -0
- data/doc/js/jquery.js +4 -0
- data/doc/method_list.html +459 -0
- data/doc/top-level-namespace.html +112 -0
- data/examples/example_output_timerange.rb +56 -0
- data/examples/example_timed_process.rb +56 -0
- data/examples/example_timeinstant_process.rb +56 -0
- data/examples/example_two_linked_processes.rb +88 -0
- data/exemplar_docker_compose.yml +36 -0
- data/lib/yarrrml-template-builder.rb +2 -0
- data/lib/yarrrml_template_builder/version.rb +5 -0
- data/lib/yarrrml_template_builder/yarrrml_template_builder.rb +1485 -0
- data/lib/yarrrml_template_builder/yarrrml_transform.rb +181 -0
- data/lib/yarrrmltemplatebuilder.rb +5 -0
- data/run_me_to_test.rb +26 -0
- data/yarrrml_template_builder.gemspec +39 -0
- metadata +115 -0
@@ -0,0 +1,181 @@
|
|
1
|
+
require 'tempfile'
|
2
|
+
require 'rest-client'
|
3
|
+
|
4
|
+
# note that SDMrdfizer needs to be running on port 4000 with the ./data folder mounted as /data
|
5
|
+
# docker run --name rdfizer --rm -d -p 4000:4000 -v $PWD/data:/data fairdatasystems/sdmrdfizer:0.1.0
|
6
|
+
|
7
|
+
class YARRRML_Transform
|
8
|
+
|
9
|
+
attr_accessor :data_path_server
|
10
|
+
attr_accessor :data_path_client
|
11
|
+
attr_accessor :config_path_server
|
12
|
+
attr_accessor :config_path_client
|
13
|
+
attr_accessor :rdfizer_base_url
|
14
|
+
attr_accessor :yarrrml_transform_base_url
|
15
|
+
attr_accessor :datatype_tag
|
16
|
+
attr_accessor :datafile
|
17
|
+
attr_accessor :baseURI
|
18
|
+
|
19
|
+
attr_accessor :yarrrmltemplate
|
20
|
+
attr_accessor :yarrrmlfilename_client
|
21
|
+
attr_accessor :yarrrmlfilename_server
|
22
|
+
attr_accessor :outputrmlfile
|
23
|
+
attr_accessor :formulation
|
24
|
+
attr_accessor :outputrdffolder
|
25
|
+
attr_accessor :inifile_client
|
26
|
+
attr_accessor :inifile_server
|
27
|
+
attr_accessor :failure
|
28
|
+
|
29
|
+
|
30
|
+
|
31
|
+
# Creates the YARRRML Transformer object
|
32
|
+
#
|
33
|
+
#
|
34
|
+
# the datatype_tag is used to construct other input/output filenames. e.g. a tag of "height" will
|
35
|
+
# cause the output RML file to be "./data/height_rml.ttl", and the expected YARRRML template being named
|
36
|
+
# "./config/height_yarrrml_template.yaml"
|
37
|
+
#
|
38
|
+
# Note that this assumes a specific configuration, with a csv folder mounted as /data/, containing a subdirectory /data/triples
|
39
|
+
# and a folder containing the yarrrml templates mounted as /config inside the docker container
|
40
|
+
#
|
41
|
+
# all params are passed as a hash, and retrieved by params.fetch(paramName). the ini file used by yarrrml transform is auto-generated
|
42
|
+
#
|
43
|
+
# @param params [Hash] params a Hash of options
|
44
|
+
# @option params :datatype_tag [String] (required) a one-word indicator of the data type. This is considered the "base" for all other filenames (see below)
|
45
|
+
# @option params :data_path_server [String] (/data) the path to data from the server perspective (default for Docker image)
|
46
|
+
# @option params :data_path_client [String] (/data) the path to data from the client perspective (default for Docker image)
|
47
|
+
# @option params :config_path_server [String] (/config) the path to config from the server perspective (default for Docker image)
|
48
|
+
# @option params :config_path_client [String] (/config) the path to config from the server perspective (default for Docker image)
|
49
|
+
# @option params :formulation [String] (csv)
|
50
|
+
# @option params :rdfizer_base_url [String] (http://rdfizer:4000) the base URL of the yarrrml_parser (default for a docker)
|
51
|
+
# @option params :yarrrml_transform_base_url [String] ("http://yarrrml_transform:3000") base URL of the yarrrml transform to rml script (default for a docker)
|
52
|
+
|
53
|
+
# @option params :outputrdffolder [String] (/data/triples/) - defaults to /data/triples - this folder must exist, even if left to default. NOTE - this path is not relative to the host, it is relative to the docker rdfizer, so it begins with /data not ./data)
|
54
|
+
# @return [YARRRML_Transform]
|
55
|
+
def initialize(params = {}) # get a name from the "new" call, or set a default
|
56
|
+
|
57
|
+
@datatype_tag = params.fetch( :datatype_tag, nil)
|
58
|
+
unless @datatype_tag
|
59
|
+
warn "must have a datatype_tag parameter. Aborting"
|
60
|
+
self.failure = true
|
61
|
+
return nil
|
62
|
+
end
|
63
|
+
|
64
|
+
@data_path_client = params.fetch(:data_path_client, "/data") # from client perspective (default for docker)
|
65
|
+
@data_path_server = params.fetch(:data_path_server, self.data_path_client) # potentially from Docker image perspective (default for docker)
|
66
|
+
@config_path_client = params.fetch(:config_path_client, "/config") # from client perspective (default for docker)
|
67
|
+
@config_path_server = params.fetch(:config_path_server, self.config_path_client) # potentially from Docker image perspective (default for docker)
|
68
|
+
|
69
|
+
@data_path_client.gsub!('/$', "") # remove trailing slashes -I'll add them later if I need them
|
70
|
+
@data_path_server.gsub!('/$', "")
|
71
|
+
@config_path_client.gsub!('/$', "")
|
72
|
+
@config_path_server.gsub!('/$', "")
|
73
|
+
|
74
|
+
|
75
|
+
@rdfizer_base_url = params.fetch(:rdfizer_base_url, "http://rdfizer:4000")
|
76
|
+
@yarrrml_transform_base_url = params.fetch(:yarrrml_transform_base_url, "http://yarrrml_transform:3000")
|
77
|
+
@rdfizer_base_url.gsub!('/$', "")
|
78
|
+
@yarrrml_transform_base_url.gsub!('/$', "")
|
79
|
+
|
80
|
+
|
81
|
+
@formulation = params.fetch(:formulation, "csv")
|
82
|
+
@datafile = params.fetch(:datafile, "#{self.data_path_server}/#{self.datatype_tag}.csv")
|
83
|
+
@baseURI = params.fetch(:baseURI, ENV['baseURI'])
|
84
|
+
@baseURI = "http://example.org/data/" unless @baseURI
|
85
|
+
|
86
|
+
@outputrdffolder = params.fetch(:outputrdffolder, "#{self.data_path_server}/triples")
|
87
|
+
@outputrmlfile = params.fetch(:outputrmlfile, "#{self.data_path_server}/#{self.datatype_tag}_rml.ttl")
|
88
|
+
@yarrrmlfilename_client = params.fetch(:yarrrmlfilename, "#{self.data_path_client}/#{self.datatype_tag}_yarrrml.yaml")
|
89
|
+
@yarrrmlfilename_server = params.fetch(:yarrrmlfilename, "#{self.data_path_server}/#{self.datatype_tag}_yarrrml.yaml")
|
90
|
+
@yarrrmltemplate = params.fetch(:yarrrmltemplate, "#{self.config_path_client}/#{self.datatype_tag}_yarrrml_template.yaml")
|
91
|
+
@inifile_client = params.fetch(:inifile_client, "#{self.data_path_client}/#{self.datatype_tag}.ini")
|
92
|
+
@inifile_server = params.fetch(:inifile_server, "#{self.data_path_server}/#{self.datatype_tag}.ini") # this will be a docker image in almost all cases
|
93
|
+
|
94
|
+
|
95
|
+
write_ini()
|
96
|
+
|
97
|
+
transform_template()
|
98
|
+
|
99
|
+
return self
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
|
105
|
+
def transform_template()
|
106
|
+
# transform appropriate template with this data
|
107
|
+
|
108
|
+
File.open(self.yarrrmltemplate, "r") {|f| @template = f.read}
|
109
|
+
@template.gsub!("|||DATA|||", self.datafile)
|
110
|
+
@template.gsub!("|||FORMULATION|||", self.formulation)
|
111
|
+
@template.gsub!("|||BASE|||", self.baseURI)
|
112
|
+
File.open(self.yarrrmlfilename_client, "w") {|f| f.puts @template}
|
113
|
+
warn "Ready to yarrrml transform #{self.datatype_tag} from #{self.yarrrmlfilename_client} "
|
114
|
+
|
115
|
+
end
|
116
|
+
|
117
|
+
|
118
|
+
|
119
|
+
|
120
|
+
def write_ini(inifile = self.inifile_client,
|
121
|
+
path = self.data_path_server,
|
122
|
+
rdffolder = self.outputrdffolder,
|
123
|
+
rmlfile = self.outputrmlfile,
|
124
|
+
datatype = self.datatype_tag)
|
125
|
+
|
126
|
+
configfilecontent = <<CONFIG
|
127
|
+
[default]
|
128
|
+
main_directory: #{path}
|
129
|
+
|
130
|
+
[datasets]
|
131
|
+
number_of_datasets: 1
|
132
|
+
output_folder: #{rdffolder}
|
133
|
+
all_in_one_file: yes
|
134
|
+
remove_duplicate: yes
|
135
|
+
enrichment: yes
|
136
|
+
name: #{datatype}
|
137
|
+
ordered: yes
|
138
|
+
large_file: true
|
139
|
+
|
140
|
+
[dataset1]
|
141
|
+
name: #{datatype}
|
142
|
+
mapping: #{rmlfile}
|
143
|
+
|
144
|
+
CONFIG
|
145
|
+
|
146
|
+
File.open(inifile, "w"){|f| f.puts configfilecontent}
|
147
|
+
|
148
|
+
end
|
149
|
+
|
150
|
+
|
151
|
+
|
152
|
+
# Executes the yarrrml to rml transformation
|
153
|
+
#
|
154
|
+
# no parameters
|
155
|
+
#
|
156
|
+
#
|
157
|
+
def yarrrml_transform
|
158
|
+
warn "running docker yarrrml-parser:ejp-latest"
|
159
|
+
#parser_start_string = "docker run -e PARSERIN=#{self.yarrrmlfilename} -e PARSEROUT=#{self.outputrmlfile} --rm --name yarrrml-parser -v $PWD/data:/data markw/yarrrml-parser-ejp:latest"
|
160
|
+
warn "yarrrml to rml starting with: #{self.yarrrml_transform_base_url} PARSERIN=#{self.yarrrmlfilename_server} -e PARSEROUT=#{self.outputrmlfile}"
|
161
|
+
#resp = RestClient.get("#{self.yarrrml_transform_base_url}/?parserin=#{self.yarrrmlfilename_server}&parserout=#{self.outputrmlfile}")
|
162
|
+
resp = RestClient::Request.execute(method: :get, url: "#{self.yarrrml_transform_base_url}/?parserin=#{self.yarrrmlfilename_server}&parserout=#{self.outputrmlfile}", timeout: 9000000)
|
163
|
+
warn "#{resp}: rml file has been created in #{self.outputrmlfile} - ready to make FAIR data"
|
164
|
+
end
|
165
|
+
|
166
|
+
|
167
|
+
|
168
|
+
# Executes the CSV to RDF based on the RML
|
169
|
+
#
|
170
|
+
# no parameters
|
171
|
+
#
|
172
|
+
#executes the sdmrdfizer transformation using the .ini file created by the 'initialize' routine
|
173
|
+
def make_fair_data
|
174
|
+
warn "making FAIR data with #{self.rdfizer_base_url}/graph_creation/#{self.inifile_server}" # this is sdmrdfizer
|
175
|
+
response = RestClient::Request.execute(method: :get, url: self.rdfizer_base_url + "/graph_creation" + self.inifile_server, timeout: 900000000)
|
176
|
+
warn response.code
|
177
|
+
warn "FAIR data is avaialable in .#{self.outputrdffolder}/#{self.datatype_tag}.nt"
|
178
|
+
end
|
179
|
+
|
180
|
+
|
181
|
+
end
|
data/run_me_to_test.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#!ruby
|
2
|
+
|
3
|
+
require "yarrrml-template-builder"
|
4
|
+
|
5
|
+
$stderr.puts "this script will do an rdf transformation of the height.csv file, in the ./data folder using the yarrrml file in the ./config folder"
|
6
|
+
$stderr.puts "you must already have the docker-compose up before running this script. If you see failures, that is likely why :-)"
|
7
|
+
datatype_tag = "height" # the "tag" of your data
|
8
|
+
data_path_client = "./data"
|
9
|
+
data_path_server = "/data" # assumes you are running my docker image
|
10
|
+
config_path_client = "./config"
|
11
|
+
config_path_server = "/config" # assumes you are running my docker image
|
12
|
+
rdfizer_base_url = "http://localhost:4000" # again, my docker
|
13
|
+
yarrrml_transform_base_url = "http://localhost:3000" # again, my docker
|
14
|
+
|
15
|
+
y = YARRRML_Transform.new(
|
16
|
+
datatype_tag: datatype_tag,
|
17
|
+
data_path_client: data_path_client,
|
18
|
+
data_path_server: data_path_server,
|
19
|
+
config_path_client: config_path_client,
|
20
|
+
config_path_server: config_path_server,
|
21
|
+
rdfizer_base_url: rdfizer_base_url,
|
22
|
+
yarrrml_transform_base_url: yarrrml_transform_base_url,
|
23
|
+
)
|
24
|
+
y.yarrrml_transform
|
25
|
+
y.make_fair_data
|
26
|
+
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/yarrrml_template_builder/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "yarrrml_template_builder"
|
7
|
+
spec.version = YARRRMLTemplateBuilder::VERSION
|
8
|
+
spec.authors = ["Mark Wilkinson"]
|
9
|
+
spec.email = ["markw@illuminae.com"]
|
10
|
+
|
11
|
+
spec.summary = "Creates YARRRML Template templates"
|
12
|
+
spec.description = "Creates a templated version of YARRRML, missing the source and iteration type so they can be reused"
|
13
|
+
spec.homepage = "https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder"
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.required_ruby_version = ">= 2.7.0"
|
16
|
+
|
17
|
+
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
18
|
+
|
19
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
20
|
+
spec.metadata["source_code_uri"] = "https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder"
|
21
|
+
spec.metadata["changelog_uri"] = "https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder/Changelog.md"
|
22
|
+
|
23
|
+
# Specify which files should be added to the gem when it is released.
|
24
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
25
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
26
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
27
|
+
end
|
28
|
+
spec.bindir = "exe"
|
29
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
30
|
+
spec.require_paths = ["lib"]
|
31
|
+
|
32
|
+
# Uncomment to register a new dependency of your gem
|
33
|
+
# spec.add_dependency "example-gem", "~> 1.0"
|
34
|
+
|
35
|
+
spec.add_dependency "yaml", "~> 0.1.0"
|
36
|
+
spec.add_dependency "rest-client", "~> 2.1.0"
|
37
|
+
# For more information and examples about making a new gem, checkout our
|
38
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
39
|
+
end
|
metadata
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: yarrrml_template_builder
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.54
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Mark Wilkinson
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2022-10-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: yaml
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.1.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.1.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rest-client
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 2.1.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 2.1.0
|
41
|
+
description: Creates a templated version of YARRRML, missing the source and iteration
|
42
|
+
type so they can be reused
|
43
|
+
email:
|
44
|
+
- markw@illuminae.com
|
45
|
+
executables: []
|
46
|
+
extensions: []
|
47
|
+
extra_rdoc_files: []
|
48
|
+
files:
|
49
|
+
- ".gitignore"
|
50
|
+
- Changelog.md
|
51
|
+
- Gemfile
|
52
|
+
- Gemfile.lock
|
53
|
+
- LICENSE.txt
|
54
|
+
- README.md
|
55
|
+
- bin/console
|
56
|
+
- bin/setup
|
57
|
+
- config/height_yarrrml_template.yaml
|
58
|
+
- data/height.csv
|
59
|
+
- doc/YARRRMLTemplateBuilder.html
|
60
|
+
- doc/YARRRML_Template_Builder.html
|
61
|
+
- doc/YARRRML_Transform.html
|
62
|
+
- doc/_index.html
|
63
|
+
- doc/class_list.html
|
64
|
+
- doc/css/common.css
|
65
|
+
- doc/css/full_list.css
|
66
|
+
- doc/css/style.css
|
67
|
+
- doc/file.README.html
|
68
|
+
- doc/file_list.html
|
69
|
+
- doc/frames.html
|
70
|
+
- doc/index.html
|
71
|
+
- doc/js/app.js
|
72
|
+
- doc/js/full_list.js
|
73
|
+
- doc/js/jquery.js
|
74
|
+
- doc/method_list.html
|
75
|
+
- doc/top-level-namespace.html
|
76
|
+
- examples/example_output_timerange.rb
|
77
|
+
- examples/example_timed_process.rb
|
78
|
+
- examples/example_timeinstant_process.rb
|
79
|
+
- examples/example_two_linked_processes.rb
|
80
|
+
- exemplar_docker_compose.yml
|
81
|
+
- lib/yarrrml-template-builder.rb
|
82
|
+
- lib/yarrrml_template_builder/version.rb
|
83
|
+
- lib/yarrrml_template_builder/yarrrml_template_builder.rb
|
84
|
+
- lib/yarrrml_template_builder/yarrrml_transform.rb
|
85
|
+
- lib/yarrrmltemplatebuilder.rb
|
86
|
+
- run_me_to_test.rb
|
87
|
+
- yarrrml_template_builder.gemspec
|
88
|
+
homepage: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder
|
89
|
+
licenses:
|
90
|
+
- MIT
|
91
|
+
metadata:
|
92
|
+
allowed_push_host: https://rubygems.org
|
93
|
+
homepage_uri: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder
|
94
|
+
source_code_uri: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder
|
95
|
+
changelog_uri: https://github.com/ejp-rd-vp/CDE-semantic-model-implementations/tree/master/YARRRML_Tools/yarrrml_template_builder/Changelog.md
|
96
|
+
post_install_message:
|
97
|
+
rdoc_options: []
|
98
|
+
require_paths:
|
99
|
+
- lib
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: 2.7.0
|
105
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
requirements: []
|
111
|
+
rubygems_version: 3.2.28
|
112
|
+
signing_key:
|
113
|
+
specification_version: 4
|
114
|
+
summary: Creates YARRRML Template templates
|
115
|
+
test_files: []
|