@jbr-experiment/largerdfbench 0.0.0-stage → 6.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +21 -0
- package/README.md +254 -2
- package/components/ExperimentHandlerLargeRdfBench.jsonld +44 -0
- package/components/ExperimentLargeRdfBench.jsonld +490 -0
- package/components/components.jsonld +12 -0
- package/components/context.jsonld +91 -0
- package/index.d.ts +2 -0
- package/index.js +19 -0
- package/index.ts +2 -0
- package/lib/DatasetSampler.d.ts +73 -0
- package/lib/DatasetSampler.js +164 -0
- package/lib/ExperimentHandlerLargeRdfBench.d.ts +12 -0
- package/lib/ExperimentHandlerLargeRdfBench.js +46 -0
- package/lib/ExperimentLargeRdfBench.d.ts +159 -0
- package/lib/ExperimentLargeRdfBench.js +483 -0
- package/lib/LineageComputer.d.ts +183 -0
- package/lib/LineageComputer.js +368 -0
- package/lib/QueryPatterns.d.ts +115 -0
- package/lib/QueryPatterns.js +264 -0
- package/lib/SampleValidator.d.ts +85 -0
- package/lib/SampleValidator.js +168 -0
- package/lib/ZenodoArchive.d.ts +92 -0
- package/lib/ZenodoArchive.js +199 -0
- package/lib/templates/equivalent-queries/C2.sparql +23 -0
- package/lib/templates/equivalent-queries/C5.sparql +27 -0
- package/package.json +89 -4
package/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright © 2021 - now Ruben Taelman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,255 @@
|
|
|
1
|
-
#
|
|
1
|
+
# JBR Experiment - LargeRDFBench
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://github.com/rubensworks/jbr.js/actions?query=workflow%3ACI)
|
|
4
|
+
[](https://coveralls.io/github/rubensworks/jbr.js?branch=master)
|
|
5
|
+
[](https://www.npmjs.com/package/@jbr-experiment/largerdfbench)
|
|
6
|
+
|
|
7
|
+
A [jbr](https://github.com/rubensworks/jbr.js/tree/master/packages/jbr) experiment type for the federated query benchmark [LargeRDFBench](https://github.com/dice-group/LargeRDFBench).
|
|
8
|
+
|
|
9
|
+
The datasets, queries, and expected results are taken from the [LargeRDFBench Interoperable Edition](https://zenodo.org/records/22140177) on Zenodo.
|
|
10
|
+
Each dataset is served by its own SPARQL endpoint, and the queries are sent to the SPARQL endpoint of a federation engine over these sources.
|
|
11
|
+
|
|
12
|
+
By default, this experiment uses the full datasets of the FedBench part of LargeRDFBench (9 datasets with 165 million triples, as HDT files, and the 23 queries S1-S14 and C1-C9).
|
|
13
|
+
Optionally, it can use a sample of about 4.1 million of these triples instead, which makes it small enough for CI.
|
|
14
|
+
The sample is built so that every query produces exactly the same results over it as over the full data,
|
|
15
|
+
and every triple pattern matches the same sources.
|
|
16
|
+
|
|
17
|
+
## Requirements
|
|
18
|
+
|
|
19
|
+
* [Node.js](https://nodejs.org/en/) _(18 or higher)_
|
|
20
|
+
* [jbr](https://github.com/rubensworks/jbr.js/tree/master/packages/jbr) _(required for initializing, preparing, and running experiments on the command line)_
|
|
21
|
+
* Only for generating a sample: the optional dependencies [`hdt`](https://www.npmjs.com/package/hdt) (a native module that requires a C++ compiler) and [`@comunica/query-sparql-hdt`](https://www.npmjs.com/package/@comunica/query-sparql-hdt).
|
|
22
|
+
|
|
23
|
+
## Quick start
|
|
24
|
+
|
|
25
|
+
### 1. Install jbr
|
|
26
|
+
|
|
27
|
+
[jbr](https://github.com/rubensworks/jbr.js/tree/master/packages/jbr) is a command line tool that enables experiments to be initialized, prepared, and started.
|
|
28
|
+
It can be installed from the npm registry:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
$ npm install -g jbr
|
|
32
|
+
```
|
|
33
|
+
or
|
|
34
|
+
```bash
|
|
35
|
+
$ yarn global add jbr
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### 2. Initialize a new experiment
|
|
39
|
+
|
|
40
|
+
Using the `jbr` CLI tool, initialize a new experiment:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
$ jbr init largerdfbench my-experiment
|
|
44
|
+
$ cd my-experiment
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
This will create a new `my-experiment` directory with default configs for this experiment type.
|
|
48
|
+
|
|
49
|
+
### 3. Configure the required hooks
|
|
50
|
+
|
|
51
|
+
This experiment type requires two hooks:
|
|
52
|
+
|
|
53
|
+
* `hookSparqlEndpoint`: The SPARQL endpoint of the federation engine that is benchmarked, which must be exposed at `endpointUrl`.
|
|
54
|
+
* `hookSparqlEndpointSources`: A list of hooks that start the SPARQL endpoints of the datasets.
|
|
55
|
+
They are started one by one, in the given order.
|
|
56
|
+
If `sourceUrlsExternal` contains a URL at the index of a hook, the next hook is only started once that URL is available.
|
|
57
|
+
|
|
58
|
+
This experiment does not dictate how the datasets are served,
|
|
59
|
+
but they are available in `generated/datasets/` after the prepare step (see below).
|
|
60
|
+
For example, with the [`cli`](https://github.com/rubensworks/jbr.js/tree/master/packages/hook-cli) hook,
|
|
61
|
+
each HDT file can be served by [Comunica](https://github.com/comunica/comunica),
|
|
62
|
+
with Comunica as federation engine over them, by setting the following in `jbr-experiment.json`:
|
|
63
|
+
|
|
64
|
+
```text
|
|
65
|
+
...
|
|
66
|
+
"sourceUrlsExternal": [
|
|
67
|
+
"http://localhost:4001/sparql",
|
|
68
|
+
"http://localhost:4002/sparql",
|
|
69
|
+
...
|
|
70
|
+
],
|
|
71
|
+
"hookSparqlEndpoint": {
|
|
72
|
+
"@id": "urn:jrb:my-experiment:hookSparqlEndpoint",
|
|
73
|
+
"@type": "HookCli",
|
|
74
|
+
"entrypoint": [
|
|
75
|
+
"npx", "comunica-sparql-http",
|
|
76
|
+
"sparql@http://localhost:4001/sparql",
|
|
77
|
+
"sparql@http://localhost:4002/sparql",
|
|
78
|
+
...
|
|
79
|
+
"-p", "3001"
|
|
80
|
+
],
|
|
81
|
+
"statsFilePath": "output/stats-engine.csv",
|
|
82
|
+
"logName": "engine"
|
|
83
|
+
},
|
|
84
|
+
"hookSparqlEndpointSources": [
|
|
85
|
+
{
|
|
86
|
+
"@id": "urn:jrb:my-experiment:hookSparqlEndpointSources_ChEBI",
|
|
87
|
+
"@type": "HookCli",
|
|
88
|
+
"entrypoint": [ "npx", "comunica-sparql-hdt-http", "hdt@generated/datasets/ChEBI.hdt", "-p", "4001" ],
|
|
89
|
+
"statsFilePath": "output/stats-ChEBI.csv",
|
|
90
|
+
"logName": "ChEBI"
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"@id": "urn:jrb:my-experiment:hookSparqlEndpointSources_DBPedia-Subset",
|
|
94
|
+
"@type": "HookCli",
|
|
95
|
+
"entrypoint": [ "npx", "comunica-sparql-hdt-http", "hdt@generated/datasets/DBPedia-Subset.hdt", "-p", "4002" ],
|
|
96
|
+
"statsFilePath": "output/stats-DBPedia-Subset.csv",
|
|
97
|
+
"logName": "DBPedia-Subset"
|
|
98
|
+
},
|
|
99
|
+
...
|
|
100
|
+
]
|
|
101
|
+
...
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
For a sample, the datasets are N-Triples files instead,
|
|
105
|
+
which can be served with `comunica-sparql-file-http generated/datasets/ChEBI.nt`.
|
|
106
|
+
|
|
107
|
+
Alternatively, the hooks can be set via `jbr set-hook`.
|
|
108
|
+
As `jbr init` creates a single entry for `hookSparqlEndpointSources`,
|
|
109
|
+
first turn it into a list with one entry per dataset in `jbr-experiment.json`,
|
|
110
|
+
after which each source can be set by its index:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
$ jbr set-hook hookSparqlEndpoint sparql-endpoint-comunica
|
|
114
|
+
$ jbr set-hook hookSparqlEndpointSources/0 cli
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### 4. Prepare the experiment
|
|
118
|
+
|
|
119
|
+
In order to run all preprocessing steps, such as downloading the data and creating the sample if enabled, invoke the prepare step:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
$ jbr prepare
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
All prepared files will be contained in the `generated/` directory:
|
|
126
|
+
```text
|
|
127
|
+
generated/
|
|
128
|
+
zenodo/ The files downloaded from Zenodo
|
|
129
|
+
hdt/ The full datasets as HDT files (only for a sample)
|
|
130
|
+
queries/
|
|
131
|
+
results/ The published expected results
|
|
132
|
+
lineage/ The triples that contribute to the solutions of each query, per dataset (only for a sample)
|
|
133
|
+
datasets/ The datasets to serve: <dataset>.nt for a sample, or <dataset>.hdt for the full data
|
|
134
|
+
queries/ The queries: <query>.sparql
|
|
135
|
+
results/ The expected results: <query>.srj
|
|
136
|
+
manifest.json The parameters, and statistics of the sample and its validation
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The prepare step is skipped if `manifest.json`, the datasets, and the queries already exist in `generated/`,
|
|
140
|
+
so that a sample that was built before can be reused by extracting it into `generated/`.
|
|
141
|
+
Use `jbr prepare -f` to force them to be generated again.
|
|
142
|
+
|
|
143
|
+
### 5. Run the experiment
|
|
144
|
+
|
|
145
|
+
Once the experiment has been fully configured and prepared, you can run it:
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
$ jbr run
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Once the run step completes, results will be present in the `output/` directory.
|
|
152
|
+
|
|
153
|
+
## Sampling
|
|
154
|
+
|
|
155
|
+
By default, the full HDT files (about 2.6 GB for the FedBench part) are used, with the published expected results.
|
|
156
|
+
With `sample` enabled, a sample is created instead:
|
|
157
|
+
|
|
158
|
+
1. The HDT files, queries, and published results are downloaded from Zenodo.
|
|
159
|
+
2. For each query, the triples that contribute to its solutions over the full data are computed.
|
|
160
|
+
3. The subjects of each dataset are sampled at their rate in `sampleSubjectRates`, keeping their triples that match a query pattern, and their other triples at `sampleOtherTriplesRate`.
|
|
161
|
+
4. The contributing triples are added, and one matching triple for each query pattern that matches the full data but not the sample, so that source selection is unchanged.
|
|
162
|
+
5. With `sampleValidate` enabled, prepare fails unless each query produces the same results over the sample as over the full data, which become its expected results.
|
|
163
|
+
|
|
164
|
+
The same parameters produce a byte-identical sample, which takes about 20 minutes and 7.6 GB of memory (`--max-old-space-size=8192`) with the default sampling parameters.
|
|
165
|
+
|
|
166
|
+
## Output
|
|
167
|
+
|
|
168
|
+
The following output is generated after an experiment has run.
|
|
169
|
+
|
|
170
|
+
`output/query-times.csv`: The aggregated results per query of [`sparql-benchmark-runner`](https://github.com/comunica/sparql-benchmark-runner.js):
|
|
171
|
+
```csv
|
|
172
|
+
name;id;error;errorDescription;failures;hash;httpRequests;httpRequestsMax;httpRequestsMin;httpRequestsStd;replication;results;resultsMax;resultsMin;time;timeMax;timeMin;times;timestamps;timestampsMax;timestampsMin;timestampsStd;timeStd;timestampsAll
|
|
173
|
+
C3;0;false;;0;6bb056bf1533b491893a1c6ceeb90869;438;438;438;0;3;9;9;9;5933.33;6873;5444;6873 5483 5444;...
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
`output/query-results-raw.json`: The results of each execution of each query.
|
|
177
|
+
|
|
178
|
+
`output/logs/load-time.csv`: The time in milliseconds to start the sources and the federation engine.
|
|
179
|
+
|
|
180
|
+
## Configuration
|
|
181
|
+
|
|
182
|
+
The default generated configuration file (`jbr-experiment.json`) for this experiment looks as follows:
|
|
183
|
+
|
|
184
|
+
```json
|
|
185
|
+
{
|
|
186
|
+
"@context": [
|
|
187
|
+
"https://linkedsoftwaredependencies.org/bundles/npm/jbr/^6.0.0/components/context.jsonld",
|
|
188
|
+
"https://linkedsoftwaredependencies.org/bundles/npm/@jbr-experiment/largerdfbench/^6.0.0/components/context.jsonld"
|
|
189
|
+
],
|
|
190
|
+
"@id": "urn:jrb:my-experiment",
|
|
191
|
+
"@type": "ExperimentLargeRdfBench",
|
|
192
|
+
"zenodoRecord": "22140177",
|
|
193
|
+
"datasets": [ "ChEBI", "DBPedia-Subset", "DrugBank", "GeoNames", "Jamendo", "KEGG", "LMDB", "NYT", "SWDFood" ],
|
|
194
|
+
"queries": [
|
|
195
|
+
"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13", "S14",
|
|
196
|
+
"C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9"
|
|
197
|
+
],
|
|
198
|
+
"sample": false,
|
|
199
|
+
"sampleSubjectRates": {
|
|
200
|
+
"DBPedia-Subset": 0.02,
|
|
201
|
+
"GeoNames": 0.02,
|
|
202
|
+
"LMDB": 0.25
|
|
203
|
+
},
|
|
204
|
+
"sampleOtherTriplesRate": 0.2,
|
|
205
|
+
"sampleSeed": "largerdfbench-sample-v1",
|
|
206
|
+
"sampleValidate": true,
|
|
207
|
+
"generationQueryTimeout": 1800,
|
|
208
|
+
"endpointUrl": "http://localhost:3001/sparql",
|
|
209
|
+
"endpointUrlExternal": "http://localhost:3001/sparql",
|
|
210
|
+
"sourceUrlsExternal": [],
|
|
211
|
+
"queryRunnerReplication": 3,
|
|
212
|
+
"queryRunnerWarmupRounds": 1,
|
|
213
|
+
"queryRunnerRequestDelay": 0,
|
|
214
|
+
"queryRunnerEndpointAvailabilityCheckTimeout": 1000,
|
|
215
|
+
"queryRunnerUrlParams": {},
|
|
216
|
+
"hookSparqlEndpoint": {
|
|
217
|
+
"@id": "urn:jrb:my-experiment:hookSparqlEndpoint",
|
|
218
|
+
"@type": "HookNonConfigured"
|
|
219
|
+
},
|
|
220
|
+
"hookSparqlEndpointSources": {
|
|
221
|
+
"@id": "urn:jrb:my-experiment:hookSparqlEndpointSources",
|
|
222
|
+
"@type": "HookNonConfigured"
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
Any config changes require re-running the prepare step.
|
|
228
|
+
|
|
229
|
+
### Configuration fields
|
|
230
|
+
|
|
231
|
+
* `zenodoRecord`: The Zenodo record of the LargeRDFBench Interoperable Edition to download from.
|
|
232
|
+
* `datasets`: The datasets to include, by their name in the Zenodo record. Defaults to the FedBench datasets.
|
|
233
|
+
* `queries`: The queries to include, by their name in the Zenodo record. Defaults to the FedBench queries.
|
|
234
|
+
* `sample`: If the datasets must be sampled. Otherwise, the full datasets are used, as HDT files. Defaults to `false`.
|
|
235
|
+
* `sampleSubjectRates`: The fraction of subjects to sample per dataset. Datasets that are not listed are not sampled by subject.
|
|
236
|
+
* `sampleOtherTriplesRate`: The fraction of the triples of sampled subjects that match no query pattern to keep.
|
|
237
|
+
* `sampleSeed`: The seed for sampling.
|
|
238
|
+
* `sampleValidate`: If the queries must be validated to produce the same results over the sample as over the full data.
|
|
239
|
+
* `generationQueryTimeout`: The timeout in seconds of each query over the full data while generating the sample.
|
|
240
|
+
* `endpointUrl`: URL through which the SPARQL endpoint of the `hookSparqlEndpoint` hook will be exposed.
|
|
241
|
+
* `endpointUrlExternal`: URL to check if the SPARQL endpoint of the `hookSparqlEndpoint` hook is available. Defaults to `endpointUrl`.
|
|
242
|
+
* `sourceUrlsExternal`: URLs to check if each source of `hookSparqlEndpointSources` is available, by index, before starting the next one.
|
|
243
|
+
* `queryRunnerReplication`: Number of replication runs for [`sparql-benchmark-runner`](https://github.com/comunica/sparql-benchmark-runner.js).
|
|
244
|
+
* `queryRunnerWarmupRounds`: Number of warmup runs for [`sparql-benchmark-runner`](https://github.com/comunica/sparql-benchmark-runner.js).
|
|
245
|
+
* `queryRunnerRequestDelay`: Delay between requests in milliseconds.
|
|
246
|
+
* `queryRunnerEndpointAvailabilityCheckTimeout`: Timeout in milliseconds of each check if the SPARQL endpoint is available.
|
|
247
|
+
* `queryRunnerUrlParams`: A JSON record of string mappings containing URL parameters that will be passed to the SPARQL endpoint.
|
|
248
|
+
* `queryTimeoutFallback`: An optional timeout value for a single query in milliseconds, to be used as fallback in case the SPARQL endpoint hook's timeout fails. This should always be higher than the timeout value configured in the SPARQL endpoint hook.
|
|
249
|
+
|
|
250
|
+
## License
|
|
251
|
+
|
|
252
|
+
jbr.js is written by [Ruben Taelman](http://www.rubensworks.net/).
|
|
253
|
+
|
|
254
|
+
This code is copyrighted by [Ghent University – imec](http://idlab.ugent.be/)
|
|
255
|
+
and released under the [MIT license](http://opensource.org/licenses/MIT).
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"@context": [
|
|
3
|
+
"https://linkedsoftwaredependencies.org/bundles/npm/@jbr-experiment/largerdfbench/^6.0.0/components/context.jsonld",
|
|
4
|
+
"https://linkedsoftwaredependencies.org/bundles/npm/jbr/^6.0.0/components/context.jsonld"
|
|
5
|
+
],
|
|
6
|
+
"@id": "npmd:@jbr-experiment/largerdfbench",
|
|
7
|
+
"components": [
|
|
8
|
+
{
|
|
9
|
+
"@id": "jel:components/ExperimentHandlerLargeRdfBench.jsonld#ExperimentHandlerLargeRdfBench",
|
|
10
|
+
"@type": "Class",
|
|
11
|
+
"requireElement": "ExperimentHandlerLargeRdfBench",
|
|
12
|
+
"extends": [
|
|
13
|
+
{
|
|
14
|
+
"@type": "GenericComponentExtension",
|
|
15
|
+
"component": "j:components/experiment/ExperimentHandler.jsonld#ExperimentHandler",
|
|
16
|
+
"genericTypeInstances": [
|
|
17
|
+
"jel:components/ExperimentLargeRdfBench.jsonld#ExperimentLargeRdfBench"
|
|
18
|
+
]
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"comment": "An experiment handler for LargeRDFBench.",
|
|
22
|
+
"parameters": [],
|
|
23
|
+
"memberFields": [
|
|
24
|
+
{
|
|
25
|
+
"@id": "jel:components/ExperimentHandlerLargeRdfBench.jsonld#ExperimentHandlerLargeRdfBench__member_constructor",
|
|
26
|
+
"memberFieldName": "constructor"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"@id": "jel:components/ExperimentHandlerLargeRdfBench.jsonld#ExperimentHandlerLargeRdfBench__member_getDefaultParams",
|
|
30
|
+
"memberFieldName": "getDefaultParams"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"@id": "jel:components/ExperimentHandlerLargeRdfBench.jsonld#ExperimentHandlerLargeRdfBench__member_getHookNames",
|
|
34
|
+
"memberFieldName": "getHookNames"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"@id": "jel:components/ExperimentHandlerLargeRdfBench.jsonld#ExperimentHandlerLargeRdfBench__member_init",
|
|
38
|
+
"memberFieldName": "init"
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
"constructorArguments": []
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|