@lde/pipeline 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -34,7 +34,7 @@ export interface SparqlConstructExecutorOptions {
|
|
|
34
34
|
*/
|
|
35
35
|
timeout?: number;
|
|
36
36
|
/**
|
|
37
|
-
* Number of retries for transient
|
|
37
|
+
* Number of retries for transient errors (network failures and HTTP 502/503/504).
|
|
38
38
|
* @default 3
|
|
39
39
|
*/
|
|
40
40
|
retries?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAWpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAmBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAuC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
|
package/dist/sparql/executor.js
CHANGED
|
@@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { Parser } from '@traqula/parser-sparql-1-1';
|
|
5
5
|
import { Generator } from '@traqula/generator-sparql-1-1';
|
|
6
|
+
import isNetworkError from 'is-network-error';
|
|
6
7
|
import pRetry from 'p-retry';
|
|
7
8
|
import { withDefaultGraph } from './graph.js';
|
|
8
9
|
import { injectValues } from './values.js';
|
|
@@ -79,7 +80,7 @@ export class SparqlConstructExecutor {
|
|
|
79
80
|
ast = structuredClone(this.preParsed);
|
|
80
81
|
}
|
|
81
82
|
else {
|
|
82
|
-
const substituted = this.rawQuery.
|
|
83
|
+
const substituted = this.rawQuery.replaceAll('#subjectFilter#', distribution.subjectFilter ?? '');
|
|
83
84
|
const parsed = new Parser().parse(substituted);
|
|
84
85
|
if (parsed.type !== 'query' || parsed.subType !== 'construct') {
|
|
85
86
|
throw new Error('Query must be a CONSTRUCT query');
|
|
@@ -97,7 +98,7 @@ export class SparqlConstructExecutor {
|
|
|
97
98
|
query = query.replaceAll('?dataset', `<${dataset.iri}>`);
|
|
98
99
|
return await pRetry(() => this.fetcher.fetchTriples(endpoint.toString(), query), {
|
|
99
100
|
retries: this.retries,
|
|
100
|
-
shouldRetry: ({ error }) =>
|
|
101
|
+
shouldRetry: ({ error }) => isTransientError(error),
|
|
101
102
|
});
|
|
102
103
|
}
|
|
103
104
|
/**
|
|
@@ -118,7 +119,9 @@ export async function readQueryFile(filename) {
|
|
|
118
119
|
return (await readFile(resolve(filename))).toString();
|
|
119
120
|
}
|
|
120
121
|
const transientStatusPattern = /HTTP status (\d+)/;
|
|
121
|
-
function
|
|
122
|
+
function isTransientError(error) {
|
|
123
|
+
if (isNetworkError(error))
|
|
124
|
+
return true;
|
|
122
125
|
if (!(error instanceof Error))
|
|
123
126
|
return false;
|
|
124
127
|
const match = error.message.match(transientStatusPattern);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.1",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
"@traqula/rules-sparql-1-1": "^1.0.3",
|
|
34
34
|
"fetch-sparql-endpoint": "^7.1.0",
|
|
35
35
|
"filenamify-url": "^4.0.0",
|
|
36
|
+
"is-network-error": "^1.1.0",
|
|
36
37
|
"n3": "^2.0.3",
|
|
37
38
|
"p-retry": "^7.1.1",
|
|
38
39
|
"tslib": "^2.3.0"
|