@jbr-experiment/largerdfbench 0.0.0-stage → 6.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +21 -0
- package/README.md +254 -2
- package/components/ExperimentHandlerLargeRdfBench.jsonld +44 -0
- package/components/ExperimentLargeRdfBench.jsonld +490 -0
- package/components/components.jsonld +12 -0
- package/components/context.jsonld +91 -0
- package/index.d.ts +2 -0
- package/index.js +19 -0
- package/index.ts +2 -0
- package/lib/DatasetSampler.d.ts +73 -0
- package/lib/DatasetSampler.js +164 -0
- package/lib/ExperimentHandlerLargeRdfBench.d.ts +12 -0
- package/lib/ExperimentHandlerLargeRdfBench.js +46 -0
- package/lib/ExperimentLargeRdfBench.d.ts +159 -0
- package/lib/ExperimentLargeRdfBench.js +483 -0
- package/lib/LineageComputer.d.ts +183 -0
- package/lib/LineageComputer.js +368 -0
- package/lib/QueryPatterns.d.ts +115 -0
- package/lib/QueryPatterns.js +264 -0
- package/lib/SampleValidator.d.ts +85 -0
- package/lib/SampleValidator.js +168 -0
- package/lib/ZenodoArchive.d.ts +92 -0
- package/lib/ZenodoArchive.js +199 -0
- package/lib/templates/equivalent-queries/C2.sparql +23 -0
- package/lib/templates/equivalent-queries/C5.sparql +27 -0
- package/package.json +101 -4
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.ZenodoArchive = exports.HttpRandomAccessReader = exports.fetchRange = exports.TAIL_BYTES = exports.RETRY_DELAYS = exports.HTTP_HEADERS = void 0;
|
|
27
|
+
const Path = __importStar(require("node:path"));
|
|
28
|
+
const node_stream_1 = require("node:stream");
|
|
29
|
+
const promises_1 = require("node:stream/promises");
|
|
30
|
+
const fs = __importStar(require("fs-extra"));
|
|
31
|
+
const yauzl = __importStar(require("yauzl"));
|
|
32
|
+
/**
|
|
33
|
+
* Zenodo refuses requests with the default user agent of Node.js.
|
|
34
|
+
*/
|
|
35
|
+
exports.HTTP_HEADERS = { 'User-Agent': 'jbr (https://github.com/rubensworks/jbr.js)' };
|
|
36
|
+
/**
|
|
37
|
+
* The delays in milliseconds before retrying a request,
|
|
38
|
+
* as Zenodo sometimes answers range requests with a server error that succeeds when retried.
|
|
39
|
+
*/
|
|
40
|
+
exports.RETRY_DELAYS = [1000, 5000, 15000, 30000];
|
|
41
|
+
/**
|
|
42
|
+
* The number of bytes at the end of an archive that are downloaded in a single request.
|
|
43
|
+
* This covers small archives entirely, and the central directory of large ones,
|
|
44
|
+
* which would otherwise be read with two requests per entry.
|
|
45
|
+
*/
|
|
46
|
+
exports.TAIL_BYTES = 32 * 1024 * 1024;
|
|
47
|
+
/**
|
|
48
|
+
* Request a byte range, and retry on network errors, server errors, and rate limiting.
|
|
49
|
+
* @param url The URL.
|
|
50
|
+
* @param range The value of the Range header.
|
|
51
|
+
* @param retryDelays The delays in milliseconds before each retry.
|
|
52
|
+
*/
|
|
53
|
+
async function fetchRange(url, range, retryDelays) {
|
|
54
|
+
var _a;
|
|
55
|
+
for (const delay of retryDelays) {
|
|
56
|
+
try {
|
|
57
|
+
const response = await fetch(url, { headers: Object.assign(Object.assign({}, exports.HTTP_HEADERS), { Range: range }) });
|
|
58
|
+
if (response.status < 500 && response.status !== 429) {
|
|
59
|
+
return response;
|
|
60
|
+
}
|
|
61
|
+
await ((_a = response.body) === null || _a === void 0 ? void 0 : _a.cancel());
|
|
62
|
+
}
|
|
63
|
+
catch (_b) {
|
|
64
|
+
// Network errors are retried as well, until the last attempt.
|
|
65
|
+
}
|
|
66
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
67
|
+
}
|
|
68
|
+
return fetch(url, { headers: Object.assign(Object.assign({}, exports.HTTP_HEADERS), { Range: range }) });
|
|
69
|
+
}
|
|
70
|
+
exports.fetchRange = fetchRange;
|
|
71
|
+
/**
|
|
72
|
+
* Reads byte ranges of a remote file via HTTP range requests.
|
|
73
|
+
*/
|
|
74
|
+
class HttpRandomAccessReader extends yauzl.RandomAccessReader {
|
|
75
|
+
/**
|
|
76
|
+
* @param url The URL of the file.
|
|
77
|
+
* @param retryDelays The delays in milliseconds before each retry of a request.
|
|
78
|
+
* @param tail The bytes at the end of the file, if they were downloaded already.
|
|
79
|
+
*/
|
|
80
|
+
constructor(url, retryDelays = exports.RETRY_DELAYS, tail) {
|
|
81
|
+
super();
|
|
82
|
+
this.url = url;
|
|
83
|
+
this.retryDelays = retryDelays;
|
|
84
|
+
this.tail = tail;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Request a byte range, and fail if the response is not partial.
|
|
88
|
+
* @param start The first byte.
|
|
89
|
+
* @param end The byte after the last one.
|
|
90
|
+
*/
|
|
91
|
+
async fetchRange(start, end) {
|
|
92
|
+
var _a;
|
|
93
|
+
const response = await fetchRange(this.url, `bytes=${start}-${end - 1}`, this.retryDelays);
|
|
94
|
+
if (response.status !== 206) {
|
|
95
|
+
await ((_a = response.body) === null || _a === void 0 ? void 0 : _a.cancel());
|
|
96
|
+
throw new Error(`Expected a partial response for ${this.url}, but got status ${response.status}`);
|
|
97
|
+
}
|
|
98
|
+
return response;
|
|
99
|
+
}
|
|
100
|
+
_readStreamForRange(start, end) {
|
|
101
|
+
if (this.tail && start >= this.tail.start) {
|
|
102
|
+
return node_stream_1.Readable.from([this.tail.data.subarray(start - this.tail.start, end - this.tail.start)], {
|
|
103
|
+
objectMode: false,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
const stream = new node_stream_1.PassThrough();
|
|
107
|
+
this.fetchRange(start, end)
|
|
108
|
+
.then(response => node_stream_1.Readable.fromWeb(response.body).pipe(stream))
|
|
109
|
+
.catch((error) => stream.destroy(error));
|
|
110
|
+
return stream;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
exports.HttpRandomAccessReader = HttpRandomAccessReader;
|
|
114
|
+
/**
|
|
115
|
+
* A zip archive on Zenodo, from which entries can be extracted without downloading the whole archive.
|
|
116
|
+
*/
|
|
117
|
+
class ZenodoArchive {
|
|
118
|
+
constructor(record, name, retryDelays = exports.RETRY_DELAYS, tailBytes = exports.TAIL_BYTES) {
|
|
119
|
+
this.record = record;
|
|
120
|
+
this.name = name;
|
|
121
|
+
this.retryDelays = retryDelays;
|
|
122
|
+
this.tailBytes = tailBytes;
|
|
123
|
+
}
|
|
124
|
+
get url() {
|
|
125
|
+
return `https://zenodo.org/api/records/${this.record}/files/${this.name}/content`;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Open the remote zip file.
|
|
129
|
+
*/
|
|
130
|
+
async open() {
|
|
131
|
+
var _a, _b;
|
|
132
|
+
const response = await fetchRange(this.url, 'bytes=0-0', this.retryDelays);
|
|
133
|
+
await ((_a = response.body) === null || _a === void 0 ? void 0 : _a.cancel());
|
|
134
|
+
const size = Number((_b = response.headers.get('content-range')) === null || _b === void 0 ? void 0 : _b.split('/')[1]);
|
|
135
|
+
if (!size) {
|
|
136
|
+
throw new Error(`Could not determine the size of ${this.url}, got status ${response.status}`);
|
|
137
|
+
}
|
|
138
|
+
// Ranges are requested on the final URL, after redirects.
|
|
139
|
+
const tailStart = Math.max(0, size - this.tailBytes);
|
|
140
|
+
const tailResponse = await new HttpRandomAccessReader(response.url, this.retryDelays).fetchRange(tailStart, size);
|
|
141
|
+
const tail = { start: tailStart, data: Buffer.from(await tailResponse.arrayBuffer()) };
|
|
142
|
+
const reader = new HttpRandomAccessReader(response.url, this.retryDelays, tail);
|
|
143
|
+
return await new Promise((resolve, reject) => yauzl.fromRandomAccessReader(reader, size, { lazyEntries: true, autoClose: false }, (error, zip) => error ? reject(error) : resolve(zip)));
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Extract the entries whose file name (without directories) is accepted by the filter into the target directory.
|
|
147
|
+
* Files that already exist with the right size are not downloaded again.
|
|
148
|
+
* @param filter A function that decides if an entry with the given file name must be extracted.
|
|
149
|
+
* @param target The target directory.
|
|
150
|
+
* @param log A logger.
|
|
151
|
+
* @return The extracted file names.
|
|
152
|
+
*/
|
|
153
|
+
async extract(filter, target, log) {
|
|
154
|
+
const zip = await this.open();
|
|
155
|
+
try {
|
|
156
|
+
await fs.ensureDir(target);
|
|
157
|
+
const extracted = [];
|
|
158
|
+
for (const entry of await ZenodoArchive.listEntries(zip)) {
|
|
159
|
+
const fileName = Path.basename(entry.fileName);
|
|
160
|
+
if (entry.fileName.endsWith('/') || !filter(fileName)) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
extracted.push(fileName);
|
|
164
|
+
const file = Path.join(target, fileName);
|
|
165
|
+
if (await fs.pathExists(file) && (await fs.stat(file)).size === entry.uncompressedSize) {
|
|
166
|
+
log(` ${fileName} already exists`);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
log(` Downloading ${fileName} (${Math.round(entry.compressedSize / 1024 / 1024)} MiB)`);
|
|
170
|
+
const stream = await new Promise((resolve, reject) => zip.openReadStream(entry, (error, entryStream) => error ? reject(error) : resolve(entryStream)));
|
|
171
|
+
// Yauzl verifies the CRC-32 of each entry while it is read.
|
|
172
|
+
await (0, promises_1.pipeline)(stream, fs.createWriteStream(`${file}.part`));
|
|
173
|
+
await fs.rename(`${file}.part`, file);
|
|
174
|
+
}
|
|
175
|
+
return extracted;
|
|
176
|
+
}
|
|
177
|
+
finally {
|
|
178
|
+
zip.close();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* List all entries of a zip file opened with lazyEntries.
|
|
183
|
+
* @param zip A zip file.
|
|
184
|
+
*/
|
|
185
|
+
static listEntries(zip) {
|
|
186
|
+
return new Promise((resolve, reject) => {
|
|
187
|
+
const entries = [];
|
|
188
|
+
zip.on('entry', (entry) => {
|
|
189
|
+
entries.push(entry);
|
|
190
|
+
zip.readEntry();
|
|
191
|
+
});
|
|
192
|
+
zip.on('end', () => resolve(entries));
|
|
193
|
+
zip.on('error', reject);
|
|
194
|
+
zip.readEntry();
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
exports.ZenodoArchive = ZenodoArchive;
|
|
199
|
+
//# sourceMappingURL=ZenodoArchive.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
|
2
|
+
PREFIX owl: <http://www.w3.org/2002/07/owl#>
|
|
3
|
+
PREFIX drugbank: <http://www4.wiwiss.fu-berlin.de/drugbank/resource/drugbank/>
|
|
4
|
+
PREFIX kegg: <http://bio2rdf.org/ns/kegg#>
|
|
5
|
+
PREFIX chebi: <http://bio2rdf.org/ns/chebi#>
|
|
6
|
+
PREFIX purl: <http://purl.org/dc/elements/1.1/>
|
|
7
|
+
PREFIX bio2RDF: <http://bio2rdf.org/ns/bio2rdf#>
|
|
8
|
+
# Equivalent to C2 of LargeRDFBench, with FILTER(?drugbankInchi = ?chebiInchi) in the OPTIONAL replaced by the shared ?inchi.
|
|
9
|
+
# The InChI values are simple literals, for which = is term equality, so both forms have the same solutions.
|
|
10
|
+
SELECT ?drug ?keggmass ?chebiIupacName
|
|
11
|
+
WHERE
|
|
12
|
+
{
|
|
13
|
+
?drug rdf:type drugbank:drugs .
|
|
14
|
+
?drug drugbank:keggCompoundId ?keggDrug .
|
|
15
|
+
?keggDrug bio2RDF:mass ?keggmass .
|
|
16
|
+
?drug drugbank:genericName ?drugBankName .
|
|
17
|
+
?chebiDrug purl:title ?drugBankName .
|
|
18
|
+
?chebiDrug chebi:iupacName ?chebiIupacName .
|
|
19
|
+
OPTIONAL {
|
|
20
|
+
?drug drugbank:inchiIdentifier ?inchi .
|
|
21
|
+
?chebiDrug bio2RDF:inchi ?inchi .
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
PREFIX linkedmdb: <http://data.linkedmdb.org/resource/movie/>
|
|
2
|
+
PREFIX dcterms: <http://purl.org/dc/terms/>
|
|
3
|
+
PREFIX dbpedia: <http://dbpedia.org/ontology/>
|
|
4
|
+
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
5
|
+
# Equivalent to C5 of LargeRDFBench, with FILTER(STR(?actor_name_en) = STR(?actor_name)) replaced by a join on the shared ?name.
|
|
6
|
+
# As STR returns simple literals, = on them is term equality, so both forms have the same solutions.
|
|
7
|
+
SELECT ?actor ?movie ?movieTitle ?movieDate ?birthDate ?spouseName
|
|
8
|
+
{
|
|
9
|
+
{
|
|
10
|
+
SELECT * WHERE {
|
|
11
|
+
?actor rdfs:label ?actor_name_en ;
|
|
12
|
+
dbpedia:birthDate ?birthDate ;
|
|
13
|
+
dbpedia:spouse ?spouseURI .
|
|
14
|
+
?spouseURI rdfs:label ?spouseName .
|
|
15
|
+
BIND(STR(?actor_name_en) AS ?name)
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
{
|
|
19
|
+
SELECT * WHERE {
|
|
20
|
+
?imdbactor linkedmdb:actor_name ?actor_name .
|
|
21
|
+
?movie linkedmdb:actor ?imdbactor ;
|
|
22
|
+
dcterms:title ?movieTitle ;
|
|
23
|
+
dcterms:date ?movieDate .
|
|
24
|
+
BIND(STR(?actor_name) AS ?name)
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
} LIMIT 500
|
package/package.json
CHANGED
|
@@ -1,6 +1,103 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jbr-experiment/largerdfbench",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "6.7.1",
|
|
4
|
+
"description": "LargeRDFBench experiment handler for JBR",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"typings": "index",
|
|
7
|
+
"lsd:module": true,
|
|
8
|
+
"author": "Ruben Taelman <rubensworks@gmail.com>",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "https://github.com/rubensworks/jbr.js.git",
|
|
12
|
+
"directory": "packages/experiment-largerdfbench"
|
|
13
|
+
},
|
|
14
|
+
"publishConfig": {
|
|
15
|
+
"access": "public"
|
|
16
|
+
},
|
|
17
|
+
"bugs": {
|
|
18
|
+
"url": "https://github.com/rubensworks/jbr.js/issues"
|
|
19
|
+
},
|
|
20
|
+
"homepage": "https://github.com/rubensworks/jbr.js#readme",
|
|
21
|
+
"license": "MIT",
|
|
22
|
+
"keywords": [
|
|
23
|
+
"experiment",
|
|
24
|
+
"largerdfbench",
|
|
25
|
+
"fedbench",
|
|
26
|
+
"federation",
|
|
27
|
+
"jbr",
|
|
28
|
+
"benchmarking"
|
|
29
|
+
],
|
|
30
|
+
"files": [
|
|
31
|
+
"lib/**/*.d.ts",
|
|
32
|
+
"lib/**/*.js",
|
|
33
|
+
"lib/templates",
|
|
34
|
+
"index.d.ts",
|
|
35
|
+
"index.ts",
|
|
36
|
+
"components"
|
|
37
|
+
],
|
|
38
|
+
"jest": {
|
|
39
|
+
"globals": {
|
|
40
|
+
"ts-jest": {
|
|
41
|
+
"tsconfig": "../../tsconfig.json"
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"transform": {
|
|
45
|
+
"^.+\\.ts$": "ts-jest"
|
|
46
|
+
},
|
|
47
|
+
"testRegex": "(/test/.*|(\\.|/)(test|spec))\\.ts$",
|
|
48
|
+
"moduleFileExtensions": [
|
|
49
|
+
"ts",
|
|
50
|
+
"js"
|
|
51
|
+
],
|
|
52
|
+
"collectCoverage": true
|
|
53
|
+
},
|
|
54
|
+
"scripts": {
|
|
55
|
+
"test": "node \"../../node_modules/jest/bin/jest.js\" ${1}",
|
|
56
|
+
"test-watch": "node \"../../node_modules/jest/bin/jest.js\" ${1} --watch",
|
|
57
|
+
"build": "node \"../../node_modules/typescript/bin/tsc\"",
|
|
58
|
+
"validate": "npm ls"
|
|
59
|
+
},
|
|
60
|
+
"devDependencies": {
|
|
61
|
+
"@comunica/query-sparql-hdt": "^5.0.1",
|
|
62
|
+
"@comunica/query-sparql-rdfjs": "^5.4.1",
|
|
63
|
+
"@types/yazl": "^3.3.1",
|
|
64
|
+
"hdt": "^3.3.2",
|
|
65
|
+
"jbr": "^6.2.0",
|
|
66
|
+
"yazl": "^3.3.1"
|
|
67
|
+
},
|
|
68
|
+
"peerDependencies": {
|
|
69
|
+
"@comunica/query-sparql-hdt": "^5.0.1",
|
|
70
|
+
"@comunica/query-sparql-rdfjs": "^5.4.1",
|
|
71
|
+
"hdt": "^3.3.2",
|
|
72
|
+
"jbr": ">=2.0.0"
|
|
73
|
+
},
|
|
74
|
+
"peerDependenciesMeta": {
|
|
75
|
+
"@comunica/query-sparql-hdt": {
|
|
76
|
+
"optional": true
|
|
77
|
+
},
|
|
78
|
+
"@comunica/query-sparql-rdfjs": {
|
|
79
|
+
"optional": true
|
|
80
|
+
},
|
|
81
|
+
"hdt": {
|
|
82
|
+
"optional": true
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
"dependencies": {
|
|
86
|
+
"@rdfjs/types": "*",
|
|
87
|
+
"@traqula/algebra-sparql-1-1": "^1.4.0",
|
|
88
|
+
"@traqula/algebra-transformations-1-1": "^1.4.0",
|
|
89
|
+
"@traqula/generator-sparql-1-1": "^1.4.0",
|
|
90
|
+
"@traqula/parser-sparql-1-1": "^1.4.0",
|
|
91
|
+
"@traqula/rules-sparql-1-1": "^1.4.0",
|
|
92
|
+
"@types/fs-extra": "^11.0.0",
|
|
93
|
+
"@types/n3": "^1.26.1",
|
|
94
|
+
"@types/yauzl": "^3.4.0",
|
|
95
|
+
"fs-extra": "^11.0.0",
|
|
96
|
+
"n3": "^1.26.0",
|
|
97
|
+
"rdf-stores": "^2.10.0",
|
|
98
|
+
"rdf-string-ttl": "^2.0.1",
|
|
99
|
+
"sparql-benchmark-runner": "^5.3.0",
|
|
100
|
+
"yauzl": "^3.4.0"
|
|
101
|
+
},
|
|
102
|
+
"gitHead": "0382711111fe1e92a307963f3806f2fc45a461c0"
|
|
103
|
+
}
|