@lde/pipeline 0.22.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -33,6 +33,11 @@ export interface SparqlConstructExecutorOptions {
|
|
|
33
33
|
* @default 300000 (5 minutes)
|
|
34
34
|
*/
|
|
35
35
|
timeout?: number;
|
|
36
|
+
/**
|
|
37
|
+
* Number of retries for transient errors (network failures and HTTP 502/503/504).
|
|
38
|
+
* @default 3
|
|
39
|
+
*/
|
|
40
|
+
retries?: number;
|
|
36
41
|
/**
|
|
37
42
|
* Optional custom SparqlEndpointFetcher instance.
|
|
38
43
|
*/
|
|
@@ -69,6 +74,7 @@ export declare class SparqlConstructExecutor implements Executor {
|
|
|
69
74
|
private readonly rawQuery;
|
|
70
75
|
private readonly preParsed?;
|
|
71
76
|
private readonly fetcher;
|
|
77
|
+
private readonly retries;
|
|
72
78
|
private readonly generator;
|
|
73
79
|
constructor(options: SparqlConstructExecutorOptions);
|
|
74
80
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAWpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAmBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAuC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
|
package/dist/sparql/executor.js
CHANGED
|
@@ -3,6 +3,8 @@ import { readFile } from 'node:fs/promises';
|
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { Parser } from '@traqula/parser-sparql-1-1';
|
|
5
5
|
import { Generator } from '@traqula/generator-sparql-1-1';
|
|
6
|
+
import isNetworkError from 'is-network-error';
|
|
7
|
+
import pRetry from 'p-retry';
|
|
6
8
|
import { withDefaultGraph } from './graph.js';
|
|
7
9
|
import { injectValues } from './values.js';
|
|
8
10
|
/**
|
|
@@ -45,9 +47,11 @@ export class SparqlConstructExecutor {
|
|
|
45
47
|
rawQuery;
|
|
46
48
|
preParsed;
|
|
47
49
|
fetcher;
|
|
50
|
+
retries;
|
|
48
51
|
generator = new Generator();
|
|
49
52
|
constructor(options) {
|
|
50
53
|
this.rawQuery = options.query;
|
|
54
|
+
this.retries = options.retries ?? 3;
|
|
51
55
|
if (!options.query.includes('#subjectFilter#')) {
|
|
52
56
|
const parsed = new Parser().parse(options.query);
|
|
53
57
|
if (parsed.type !== 'query' || parsed.subType !== 'construct') {
|
|
@@ -92,7 +96,10 @@ export class SparqlConstructExecutor {
|
|
|
92
96
|
}
|
|
93
97
|
let query = this.generator.generate(ast);
|
|
94
98
|
query = query.replaceAll('?dataset', `<${dataset.iri}>`);
|
|
95
|
-
return await this.fetcher.fetchTriples(endpoint.toString(), query)
|
|
99
|
+
return await pRetry(() => this.fetcher.fetchTriples(endpoint.toString(), query), {
|
|
100
|
+
retries: this.retries,
|
|
101
|
+
shouldRetry: ({ error }) => isTransientError(error),
|
|
102
|
+
});
|
|
96
103
|
}
|
|
97
104
|
/**
|
|
98
105
|
* Create an executor from a query file.
|
|
@@ -111,3 +118,15 @@ export class SparqlConstructExecutor {
|
|
|
111
118
|
export async function readQueryFile(filename) {
|
|
112
119
|
return (await readFile(resolve(filename))).toString();
|
|
113
120
|
}
|
|
121
|
+
const transientStatusPattern = /HTTP status (\d+)/;
|
|
122
|
+
function isTransientError(error) {
|
|
123
|
+
if (isNetworkError(error))
|
|
124
|
+
return true;
|
|
125
|
+
if (!(error instanceof Error))
|
|
126
|
+
return false;
|
|
127
|
+
const match = error.message.match(transientStatusPattern);
|
|
128
|
+
if (!match)
|
|
129
|
+
return false;
|
|
130
|
+
const status = Number(match[1]);
|
|
131
|
+
return status === 502 || status === 503 || status === 504;
|
|
132
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -33,7 +33,9 @@
|
|
|
33
33
|
"@traqula/rules-sparql-1-1": "^1.0.3",
|
|
34
34
|
"fetch-sparql-endpoint": "^7.1.0",
|
|
35
35
|
"filenamify-url": "^4.0.0",
|
|
36
|
+
"is-network-error": "^1.1.0",
|
|
36
37
|
"n3": "^2.0.3",
|
|
38
|
+
"p-retry": "^7.1.1",
|
|
37
39
|
"tslib": "^2.3.0"
|
|
38
40
|
}
|
|
39
41
|
}
|