@jbr-experiment/largerdfbench 0.0.0-stage → 6.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,264 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.PatternMatcher = exports.projectedVariables = exports.isSelectAll = exports.mandatoryVariables = exports.collectPatterns = exports.valuesPattern = exports.termToAst = exports.generateQuery = exports.parseQuery = exports.compareStrings = exports.keep = exports.fnv1a = exports.tripleToLine = exports.AST_FACTORY = void 0;
4
+ const algebra_sparql_1_1_1 = require("@traqula/algebra-sparql-1-1");
5
+ const algebra_transformations_1_1_1 = require("@traqula/algebra-transformations-1-1");
6
+ const generator_sparql_1_1_1 = require("@traqula/generator-sparql-1-1");
7
+ const parser_sparql_1_1_1 = require("@traqula/parser-sparql-1-1");
8
+ const rules_sparql_1_1_1 = require("@traqula/rules-sparql-1-1");
9
+ const n3_1 = require("n3");
10
+ const rdf_string_ttl_1 = require("rdf-string-ttl");
11
+ const XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string';
12
+ const writer = new n3_1.Writer({ format: 'N-Triples' });
13
+ const parser = new parser_sparql_1_1_1.Parser();
14
+ const generator = new generator_sparql_1_1_1.Generator();
15
+ exports.AST_FACTORY = new rules_sparql_1_1_1.AstFactory();
16
+ /**
17
+ * The N-Triples line of a triple, including the trailing newline, which is used as its identity.
18
+ * @param subject The subject.
19
+ * @param predicate The predicate.
20
+ * @param object The object.
21
+ */
22
+ function tripleToLine(subject, predicate, object) {
23
+ return writer.quadToString(subject, predicate, object);
24
+ }
25
+ exports.tripleToLine = tripleToLine;
26
+ /**
27
+ * A 32-bit FNV-1a hash over UTF-16 code units, which is stable across platforms.
28
+ * @param value A string.
29
+ */
30
+ function fnv1a(value) {
31
+ let hash = 0x811C9DC5;
32
+ for (let i = 0; i < value.length; i++) {
33
+ // eslint-disable-next-line unicorn/prefer-code-point -- code units keep the hashes of existing samples
34
+ hash ^= value.charCodeAt(i);
35
+ hash = Math.imul(hash, 0x01000193);
36
+ }
37
+ return hash >>> 0;
38
+ }
39
+ exports.fnv1a = fnv1a;
40
+ /**
41
+ * Deterministically decide if something identified by the given key parts is kept at the given rate.
42
+ * @param seed A seed.
43
+ * @param rate A rate between 0 and 1.
44
+ * @param keyParts Strings that identify what is sampled.
45
+ */
46
+ function keep(seed, rate, ...keyParts) {
47
+ return rate >= 1 || fnv1a([seed, ...keyParts].join('\u0000')) < rate * 4294967296;
48
+ }
49
+ exports.keep = keep;
50
+ /**
51
+ * Compare two strings by their UTF-16 code units, which is independent of the locale.
52
+ * @param a A string.
53
+ * @param b A string.
54
+ */
55
+ function compareStrings(a, b) {
56
+ if (a < b) {
57
+ return -1;
58
+ }
59
+ return a > b ? 1 : 0;
60
+ }
61
+ exports.compareStrings = compareStrings;
62
+ /**
63
+ * Parse a SPARQL SELECT query.
64
+ * @param query A SPARQL query string.
65
+ */
66
+ function parseQuery(query) {
67
+ const ast = parser.parse(query);
68
+ if (ast.type !== 'query' || ast.subType !== 'select') {
69
+ throw new Error('Only SELECT queries are supported');
70
+ }
71
+ return { ast, algebra: (0, algebra_sparql_1_1_1.toAlgebra)(ast) };
72
+ }
73
+ exports.parseQuery = parseQuery;
74
+ /**
75
+ * Serialize a syntax tree to a SPARQL query string.
76
+ * @param ast A syntax tree of a SELECT query.
77
+ */
78
+ function generateQuery(ast) {
79
+ return generator.generate(ast);
80
+ }
81
+ exports.generateQuery = generateQuery;
82
+ /**
83
+ * Convert a term to a term in a syntax tree, for use in VALUES.
84
+ * @param term An IRI or literal.
85
+ */
86
+ function termToAst(term) {
87
+ const F = exports.AST_FACTORY;
88
+ switch (term.termType) {
89
+ case 'NamedNode':
90
+ return F.termNamed(F.gen(), term.value);
91
+ case 'Literal':
92
+ if (term.language) {
93
+ return F.termLiteral(F.gen(), term.value, term.language);
94
+ }
95
+ return term.datatype.value === XSD_STRING ?
96
+ F.termLiteral(F.gen(), term.value) :
97
+ F.termLiteral(F.gen(), term.value, F.termNamed(F.gen(), term.datatype.value));
98
+ default:
99
+ throw new Error(`Unsupported term type ${term.termType} in VALUES`);
100
+ }
101
+ }
102
+ exports.termToAst = termToAst;
103
+ /**
104
+ * A VALUES pattern over the variables of the given rows, in which missing variables are UNDEF.
105
+ * @param rows Rows of variable bindings.
106
+ */
107
+ function valuesPattern(rows) {
108
+ const F = exports.AST_FACTORY;
109
+ const variables = [...new Set(rows.flatMap(row => Object.keys(row)))].sort(compareStrings);
110
+ return F.patternValues(variables.map(variable => F.termVariable(variable, F.gen())), rows.map(row => Object.fromEntries(Object.entries(row).map(([variable, term]) => [variable, termToAst(term)]))), F.gen());
111
+ }
112
+ exports.valuesPattern = valuesPattern;
113
+ /**
114
+ * The child operations of an operation, without those in expressions such as EXISTS.
115
+ * @param operation An algebra operation.
116
+ */
117
+ function childOperations(operation) {
118
+ const input = operation.input;
119
+ if (input === undefined) {
120
+ return [];
121
+ }
122
+ return Array.isArray(input) ? input : [input];
123
+ }
124
+ /**
125
+ * All triple patterns of an operation, in query order, including those in OPTIONAL, UNION, and subqueries.
126
+ * @param operation An algebra operation.
127
+ * @param acc The array to add the triple patterns to.
128
+ */
129
+ function collectPatterns(operation, acc = []) {
130
+ if (operation.type === algebra_transformations_1_1_1.Types.BGP) {
131
+ acc.push(...operation.patterns);
132
+ }
133
+ for (const child of childOperations(operation)) {
134
+ collectPatterns(child, acc);
135
+ }
136
+ return acc;
137
+ }
138
+ exports.collectPatterns = collectPatterns;
139
+ /**
140
+ * Variables that are bound in every solution of an operation:
141
+ * those of its triple patterns outside OPTIONAL, MINUS, and UNION, that are projected by the subqueries they are in.
142
+ * @param operation An algebra operation.
143
+ */
144
+ function mandatoryVariables(operation) {
145
+ switch (operation.type) {
146
+ case algebra_transformations_1_1_1.Types.BGP:
147
+ return new Set(operation.patterns.flatMap(pattern => [pattern.subject, pattern.predicate, pattern.object])
148
+ .filter(term => term.termType === 'Variable')
149
+ .map(term => term.value));
150
+ case algebra_transformations_1_1_1.Types.JOIN:
151
+ return new Set(operation.input.flatMap(input => [...mandatoryVariables(input)]));
152
+ case algebra_transformations_1_1_1.Types.LEFT_JOIN:
153
+ case algebra_transformations_1_1_1.Types.MINUS:
154
+ return mandatoryVariables(operation.input[0]);
155
+ case algebra_transformations_1_1_1.Types.PROJECT: {
156
+ const mandatory = mandatoryVariables(operation.input);
157
+ return new Set(operation.variables.map(variable => variable.value).filter(variable => mandatory.has(variable)));
158
+ }
159
+ case algebra_transformations_1_1_1.Types.FILTER:
160
+ case algebra_transformations_1_1_1.Types.EXTEND:
161
+ case algebra_transformations_1_1_1.Types.DISTINCT:
162
+ case algebra_transformations_1_1_1.Types.REDUCED:
163
+ case algebra_transformations_1_1_1.Types.ORDER_BY:
164
+ case algebra_transformations_1_1_1.Types.SLICE:
165
+ return mandatoryVariables(operation.input);
166
+ default:
167
+ return new Set();
168
+ }
169
+ }
170
+ exports.mandatoryVariables = mandatoryVariables;
171
+ /**
172
+ * Check if a query projects all its variables.
173
+ * @param ast The syntax tree of a SELECT query.
174
+ */
175
+ function isSelectAll(ast) {
176
+ return ast.variables[0].type === 'wildcard';
177
+ }
178
+ exports.isSelectAll = isSelectAll;
179
+ /**
180
+ * The names of the projected variables of a query, which are taken from the solutions for SELECT *.
181
+ * @param ast The syntax tree of a SELECT query.
182
+ * @param solutions The solutions of the query.
183
+ */
184
+ function projectedVariables(ast, solutions) {
185
+ if (isSelectAll(ast)) {
186
+ const names = new Set();
187
+ for (const solution of solutions) {
188
+ for (const variable of solution.keys()) {
189
+ names.add(variable.value);
190
+ }
191
+ }
192
+ return [...names].sort();
193
+ }
194
+ return ast.variables
195
+ .map(variable => variable.type === 'term' ? variable.value : variable.variable.value);
196
+ }
197
+ exports.projectedVariables = projectedVariables;
198
+ /**
199
+ * Matches triples against the triple patterns of queries.
200
+ * Fully open patterns (?s ?p ?o) are excluded, as they match every triple.
201
+ */
202
+ class PatternMatcher {
203
+ constructor(queries) {
204
+ this.patterns = [];
205
+ this.byPredicate = new Map();
206
+ this.variablePredicate = [];
207
+ const seen = new Set();
208
+ for (const query of queries) {
209
+ for (const triple of collectPatterns(parseQuery(query).algebra)) {
210
+ const terms = [triple.subject, triple.predicate, triple.object]
211
+ .map(term => term.termType === 'Variable' ? undefined : term);
212
+ if (terms.every(term => term === undefined)) {
213
+ continue;
214
+ }
215
+ const [subject, predicate, object] = terms.map(term => term ? (0, rdf_string_ttl_1.termToString)(term) : undefined);
216
+ const key = `${subject !== null && subject !== void 0 ? subject : '?'} ${predicate !== null && predicate !== void 0 ? predicate : '?'} ${object !== null && object !== void 0 ? object : '?'}`;
217
+ if (seen.has(key)) {
218
+ continue;
219
+ }
220
+ seen.add(key);
221
+ const pattern = { index: this.patterns.length, key, subject, predicate, object, terms };
222
+ this.patterns.push(pattern);
223
+ if (predicate) {
224
+ if (!this.byPredicate.has(predicate)) {
225
+ this.byPredicate.set(predicate, []);
226
+ }
227
+ this.byPredicate.get(predicate).push(pattern);
228
+ }
229
+ else {
230
+ this.variablePredicate.push(pattern);
231
+ }
232
+ }
233
+ }
234
+ }
235
+ /**
236
+ * The indexes of the patterns that a triple matches.
237
+ * The key of the object is only computed when needed, as this is called for every triple of the full data.
238
+ * @param subject The key of the subject.
239
+ * @param predicate The key of the predicate.
240
+ * @param object The object.
241
+ */
242
+ match(subject, predicate, object) {
243
+ var _a;
244
+ const matches = [];
245
+ let objectKey;
246
+ for (const list of [(_a = this.byPredicate.get(predicate)) !== null && _a !== void 0 ? _a : [], this.variablePredicate]) {
247
+ for (const pattern of list) {
248
+ if (pattern.subject !== undefined && pattern.subject !== subject) {
249
+ continue;
250
+ }
251
+ if (pattern.object !== undefined) {
252
+ objectKey !== null && objectKey !== void 0 ? objectKey : (objectKey = (0, rdf_string_ttl_1.termToString)(object));
253
+ if (pattern.object !== objectKey) {
254
+ continue;
255
+ }
256
+ }
257
+ matches.push(pattern.index);
258
+ }
259
+ }
260
+ return matches;
261
+ }
262
+ }
263
+ exports.PatternMatcher = PatternMatcher;
264
+ //# sourceMappingURL=QueryPatterns.js.map
@@ -0,0 +1,85 @@
1
+ import type { IQueryEngineLike, ISparqlJsonResults } from './LineageComputer';
2
+ /**
3
+ * Rows that occur more often in one result than in another, with up to 3 examples.
4
+ */
5
+ export interface IRowDifference {
6
+ count: number;
7
+ examples: (string | null)[][];
8
+ }
9
+ /**
10
+ * The validation of a single query.
11
+ */
12
+ export interface IQueryValidation {
13
+ ok: boolean;
14
+ results?: number;
15
+ fullResults?: number;
16
+ limit?: number;
17
+ onlyInSample?: IRowDifference;
18
+ onlyInFull?: IRowDifference;
19
+ published?: {
20
+ results: number;
21
+ onlyInSample: IRowDifference;
22
+ onlyInPublished: IRowDifference;
23
+ };
24
+ error?: string;
25
+ }
26
+ /**
27
+ * Count how often each key occurs.
28
+ * @param keys Keys.
29
+ */
30
+ export declare function countKeys(keys: string[]): Map<string, number>;
31
+ /**
32
+ * The rows (with multiplicity) that occur more often in a than in b.
33
+ * @param a Counted row keys.
34
+ * @param b Counted row keys.
35
+ */
36
+ export declare function bagDifference(a: Map<string, number>, b: Map<string, number>): IRowDifference;
37
+ /**
38
+ * Validates a sample by evaluating each query over the sample and over the full data, with each dataset as a
39
+ * separate source, as in a federation. Queries without LIMIT must have exactly the same results over both, and
40
+ * queries with a LIMIT must have as many results as their expected results.
41
+ * The results over the sample are written as the expected results.
42
+ * They are also compared with the published expected results, whose differences are reported.
43
+ */
44
+ export declare class SampleValidator {
45
+ readonly options: {
46
+ sampleEngine: IQueryEngineLike;
47
+ sampleContext: any;
48
+ fullEngine: IQueryEngineLike;
49
+ fullContext: any;
50
+ queries: Record<string, string>;
51
+ evaluationQueries: Record<string, string>;
52
+ publishedResults: Record<string, ISparqlJsonResults | undefined>;
53
+ resultsDirectory: string;
54
+ sampleTimeoutSeconds: number;
55
+ fullTimeoutSeconds: number;
56
+ log: (message: string) => void;
57
+ };
58
+ constructor(options: {
59
+ sampleEngine: IQueryEngineLike;
60
+ sampleContext: any;
61
+ fullEngine: IQueryEngineLike;
62
+ fullContext: any;
63
+ queries: Record<string, string>;
64
+ evaluationQueries: Record<string, string>;
65
+ publishedResults: Record<string, ISparqlJsonResults | undefined>;
66
+ resultsDirectory: string;
67
+ sampleTimeoutSeconds: number;
68
+ fullTimeoutSeconds: number;
69
+ log: (message: string) => void;
70
+ });
71
+ /**
72
+ * Validate all queries.
73
+ */
74
+ validateAll(): Promise<Record<string, IQueryValidation>>;
75
+ /**
76
+ * Validate a single query.
77
+ * @param id A query id.
78
+ */
79
+ validate(id: string): Promise<IQueryValidation>;
80
+ /**
81
+ * A summary of the validation of a query.
82
+ * @param validation The validation of a query.
83
+ */
84
+ static summarize(validation: IQueryValidation): string;
85
+ }
@@ -0,0 +1,168 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.SampleValidator = exports.bagDifference = exports.countKeys = void 0;
27
+ const Path = __importStar(require("node:path"));
28
+ const fs = __importStar(require("fs-extra"));
29
+ const rdf_string_ttl_1 = require("rdf-string-ttl");
30
+ const LineageComputer_1 = require("./LineageComputer");
31
+ const QueryPatterns_1 = require("./QueryPatterns");
32
+ /**
33
+ * Count how often each key occurs.
34
+ * @param keys Keys.
35
+ */
36
+ function countKeys(keys) {
37
+ var _a;
38
+ const counts = new Map();
39
+ for (const key of keys) {
40
+ counts.set(key, ((_a = counts.get(key)) !== null && _a !== void 0 ? _a : 0) + 1);
41
+ }
42
+ return counts;
43
+ }
44
+ exports.countKeys = countKeys;
45
+ /**
46
+ * The rows (with multiplicity) that occur more often in a than in b.
47
+ * @param a Counted row keys.
48
+ * @param b Counted row keys.
49
+ */
50
+ function bagDifference(a, b) {
51
+ var _a;
52
+ let count = 0;
53
+ const examples = [];
54
+ for (const [key, n] of a) {
55
+ const extra = n - ((_a = b.get(key)) !== null && _a !== void 0 ? _a : 0);
56
+ if (extra > 0) {
57
+ count += extra;
58
+ if (examples.length < 3) {
59
+ examples.push(JSON.parse(key));
60
+ }
61
+ }
62
+ }
63
+ return { count, examples };
64
+ }
65
+ exports.bagDifference = bagDifference;
66
+ /**
67
+ * Validates a sample by evaluating each query over the sample and over the full data, with each dataset as a
68
+ * separate source, as in a federation. Queries without LIMIT must have exactly the same results over both, and
69
+ * queries with a LIMIT must have as many results as their expected results.
70
+ * The results over the sample are written as the expected results.
71
+ * They are also compared with the published expected results, whose differences are reported.
72
+ */
73
+ class SampleValidator {
74
+ constructor(options) {
75
+ this.options = options;
76
+ }
77
+ /**
78
+ * Validate all queries.
79
+ */
80
+ async validateAll() {
81
+ await fs.ensureDir(this.options.resultsDirectory);
82
+ const report = {};
83
+ for (const id of Object.keys(this.options.queries)) {
84
+ try {
85
+ report[id] = await this.validate(id);
86
+ }
87
+ catch (error) {
88
+ report[id] = { ok: false, error: error.message };
89
+ }
90
+ this.options.log(` ${id}: ${SampleValidator.summarize(report[id])}`);
91
+ }
92
+ return report;
93
+ }
94
+ /**
95
+ * Validate a single query.
96
+ * @param id A query id.
97
+ */
98
+ async validate(id) {
99
+ var _a, _b;
100
+ const parsed = (0, QueryPatterns_1.parseQuery)(this.options.queries[id]).ast;
101
+ const query = (_a = this.options.evaluationQueries[id]) !== null && _a !== void 0 ? _a : this.options.queries[id];
102
+ const sample = await (0, LineageComputer_1.evaluate)(this.options.sampleEngine, query, this.options.sampleContext, this.options.sampleTimeoutSeconds);
103
+ const variables = (0, QueryPatterns_1.projectedVariables)(parsed, sample);
104
+ const limit = (_b = parsed.solutionModifiers.limitOffset) === null || _b === void 0 ? void 0 : _b.limit;
105
+ const solutionKey = (solution) => JSON.stringify(variables
106
+ .map(variable => solution.has(variable) ? (0, rdf_string_ttl_1.termToString)(solution.get(variable)) : null));
107
+ const sampleKeys = countKeys(sample.map(solutionKey));
108
+ const resultsFile = Path.join(this.options.resultsDirectory, `${id}.srj`);
109
+ const validation = { ok: false, results: sample.length };
110
+ if (limit === undefined) {
111
+ const full = await (0, LineageComputer_1.evaluate)(this.options.fullEngine, query, this.options.fullContext, this.options.fullTimeoutSeconds);
112
+ const fullKeys = countKeys(full.map(solutionKey));
113
+ validation.fullResults = full.length;
114
+ validation.onlyInSample = bagDifference(sampleKeys, fullKeys);
115
+ validation.onlyInFull = bagDifference(fullKeys, sampleKeys);
116
+ validation.ok = validation.onlyInSample.count === 0 && validation.onlyInFull.count === 0;
117
+ // The results over the sample, in a stable order, become the expected results.
118
+ const sorted = sample
119
+ .map(solution => ({ solution, key: solutionKey(solution) }))
120
+ .sort((a, b) => (0, QueryPatterns_1.compareStrings)(a.key, b.key));
121
+ const results = {
122
+ head: { vars: variables },
123
+ results: {
124
+ bindings: sorted.map(({ solution }) => Object.fromEntries(variables
125
+ .filter(variable => solution.has(variable))
126
+ .map(variable => [variable, (0, LineageComputer_1.rdfTermToJson)(solution.get(variable))]))),
127
+ },
128
+ };
129
+ await fs.writeFile(resultsFile, JSON.stringify(results), 'utf8');
130
+ }
131
+ else {
132
+ // The expected result was computed over the full data, and any result of its size is correct.
133
+ const expected = JSON.parse(await fs.readFile(resultsFile, 'utf8'));
134
+ validation.limit = limit;
135
+ validation.ok = sample.length === expected.results.bindings.length;
136
+ }
137
+ // Differences with the published expected results, which can have defects, such as altered literals.
138
+ const published = this.options.publishedResults[id];
139
+ if (published) {
140
+ const publishedKeys = countKeys(published.results.bindings.map(row => JSON.stringify(variables
141
+ .map(variable => row[variable] ? (0, rdf_string_ttl_1.termToString)((0, LineageComputer_1.jsonTermToRdf)(row[variable])) : null))));
142
+ validation.published = {
143
+ results: published.results.bindings.length,
144
+ onlyInSample: bagDifference(sampleKeys, publishedKeys),
145
+ onlyInPublished: bagDifference(publishedKeys, sampleKeys),
146
+ };
147
+ }
148
+ return validation;
149
+ }
150
+ /**
151
+ * A summary of the validation of a query.
152
+ * @param validation The validation of a query.
153
+ */
154
+ static summarize(validation) {
155
+ if (validation.error) {
156
+ return `failed: ${validation.error}`;
157
+ }
158
+ const comparison = validation.limit === undefined ?
159
+ `${validation.fullResults} over the full data` :
160
+ `LIMIT ${validation.limit}`;
161
+ const published = validation.published ?
162
+ `, published ${validation.published.results} (${validation.published.onlyInSample.count} rows only in sample, ${validation.published.onlyInPublished.count} only in published)` :
163
+ '';
164
+ return `${validation.ok ? 'OK' : 'FAILED'}, ${validation.results} results over the sample, ${comparison}${published}`;
165
+ }
166
+ }
167
+ exports.SampleValidator = SampleValidator;
168
+ //# sourceMappingURL=SampleValidator.js.map
@@ -0,0 +1,92 @@
1
+ /// <reference types="node" />
2
+ /// <reference types="node" />
3
+ /// <reference types="node" />
4
+ import { Readable } from 'node:stream';
5
+ import * as yauzl from 'yauzl';
6
+ /**
7
+ * Zenodo refuses requests with the default user agent of Node.js.
8
+ */
9
+ export declare const HTTP_HEADERS: {
10
+ 'User-Agent': string;
11
+ };
12
+ /**
13
+ * The delays in milliseconds before retrying a request,
14
+ * as Zenodo sometimes answers range requests with a server error that succeeds when retried.
15
+ */
16
+ export declare const RETRY_DELAYS: number[];
17
+ /**
18
+ * The number of bytes at the end of an archive that are downloaded in a single request.
19
+ * This covers small archives entirely, and the central directory of large ones,
20
+ * which would otherwise be read with two requests per entry.
21
+ */
22
+ export declare const TAIL_BYTES: number;
23
+ /**
24
+ * Request a byte range, and retry on network errors, server errors, and rate limiting.
25
+ * @param url The URL.
26
+ * @param range The value of the Range header.
27
+ * @param retryDelays The delays in milliseconds before each retry.
28
+ */
29
+ export declare function fetchRange(url: string, range: string, retryDelays: number[]): Promise<Response>;
30
+ /**
31
+ * The bytes at the end of a file that were downloaded already.
32
+ */
33
+ export interface IFileTail {
34
+ /**
35
+ * The offset of the first byte in the file.
36
+ */
37
+ start: number;
38
+ /**
39
+ * The bytes from the offset until the end of the file.
40
+ */
41
+ data: Buffer;
42
+ }
43
+ /**
44
+ * Reads byte ranges of a remote file via HTTP range requests.
45
+ */
46
+ export declare class HttpRandomAccessReader extends yauzl.RandomAccessReader {
47
+ readonly url: string;
48
+ readonly retryDelays: number[];
49
+ readonly tail?: IFileTail | undefined;
50
+ /**
51
+ * @param url The URL of the file.
52
+ * @param retryDelays The delays in milliseconds before each retry of a request.
53
+ * @param tail The bytes at the end of the file, if they were downloaded already.
54
+ */
55
+ constructor(url: string, retryDelays?: number[], tail?: IFileTail | undefined);
56
+ /**
57
+ * Request a byte range, and fail if the response is not partial.
58
+ * @param start The first byte.
59
+ * @param end The byte after the last one.
60
+ */
61
+ fetchRange(start: number, end: number): Promise<Response>;
62
+ _readStreamForRange(start: number, end: number): Readable;
63
+ }
64
+ /**
65
+ * A zip archive on Zenodo, from which entries can be extracted without downloading the whole archive.
66
+ */
67
+ export declare class ZenodoArchive {
68
+ readonly record: string;
69
+ readonly name: string;
70
+ readonly retryDelays: number[];
71
+ readonly tailBytes: number;
72
+ constructor(record: string, name: string, retryDelays?: number[], tailBytes?: number);
73
+ get url(): string;
74
+ /**
75
+ * Open the remote zip file.
76
+ */
77
+ open(): Promise<yauzl.ZipFile>;
78
+ /**
79
+ * Extract the entries whose file name (without directories) is accepted by the filter into the target directory.
80
+ * Files that already exist with the right size are not downloaded again.
81
+ * @param filter A function that decides if an entry with the given file name must be extracted.
82
+ * @param target The target directory.
83
+ * @param log A logger.
84
+ * @return The extracted file names.
85
+ */
86
+ extract(filter: (fileName: string) => boolean, target: string, log: (message: string) => void): Promise<string[]>;
87
+ /**
88
+ * List all entries of a zip file opened with lazyEntries.
89
+ * @param zip A zip file.
90
+ */
91
+ static listEntries(zip: yauzl.ZipFile): Promise<yauzl.Entry[]>;
92
+ }