@lde/pipeline-void 0.29.4 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/stage.d.ts +8 -8
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +4 -4
- package/dist/uriSpaceTransform.d.ts +4 -4
- package/dist/uriSpaceTransform.d.ts.map +1 -1
- package/dist/uriSpaceTransform.js +2 -2
- package/dist/vocabularyTransform.d.ts +5 -5
- package/dist/vocabularyTransform.d.ts.map +1 -1
- package/dist/vocabularyTransform.js +3 -3
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -12,8 +12,8 @@ Accepts an optional `VoidStagesOptions` object:
|
|
|
12
12
|
|
|
13
13
|
| Option | Default | Description |
|
|
14
14
|
| ---------------- | ------- | --------------------------------------------------------------------------------------------------------------- |
|
|
15
|
-
| `batchSize` | 10 | Maximum class bindings per
|
|
16
|
-
| `maxConcurrency` | 10 | Maximum concurrent in-flight
|
|
15
|
+
| `batchSize` | 10 | Maximum class bindings per reader call (per-class stages only) |
|
|
16
|
+
| `maxConcurrency` | 10 | Maximum concurrent in-flight reader batches (per-class stages only) |
|
|
17
17
|
| `perClass` | — | Override per-class iteration for all five per-class stages |
|
|
18
18
|
| `uriSpaces` | — | When provided, includes the object URI space stage |
|
|
19
19
|
| `vocabularies` | — | Additional vocabulary namespace URIs to detect beyond the built-in defaults |
|
|
@@ -74,7 +74,7 @@ Global and domain-specific factories accept `VoidStageOptions` (`transform`) and
|
|
|
74
74
|
|
|
75
75
|
## Stage transforms
|
|
76
76
|
|
|
77
|
-
A VoID stage decorates its
|
|
77
|
+
A VoID stage decorates its reader’s output with a `QuadTransform<ReaderContext>` attached as data (see [@lde/pipeline](../pipeline)’s extension model and [ADR 2](../../docs/decisions/0002-unify-pipeline-extension-on-quad-transforms.md)). It runs once per reader call and may fire its own SPARQL queries against the `distribution` in scope — so write it to accept being called more than once: a global stage calls it once over the complete output, a per-class stage with batching enabled once per batch (one class at `batchSize: 1`).
|
|
78
78
|
|
|
79
79
|
Two transform factories are built in:
|
|
80
80
|
|
|
@@ -87,9 +87,9 @@ Pass a `transform` to an individual factory, or route transforms through `voidSt
|
|
|
87
87
|
|
|
88
88
|
```typescript
|
|
89
89
|
import { voidStages, VOID_STAGE_NAMES } from '@lde/pipeline-void';
|
|
90
|
-
import type {
|
|
90
|
+
import type { ReaderContext, QuadTransform } from '@lde/pipeline-void';
|
|
91
91
|
|
|
92
|
-
const sampleSubjects: QuadTransform<
|
|
92
|
+
const sampleSubjects: QuadTransform<ReaderContext> = async function* (
|
|
93
93
|
quads,
|
|
94
94
|
{ dataset, distribution },
|
|
95
95
|
) {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { Stage, NotSupported } from '@lde/pipeline';
|
|
2
|
-
export type {
|
|
2
|
+
export type { AttachedReader, ReaderContext, QuadTransform, } from '@lde/pipeline';
|
|
3
3
|
export * from './stage.js';
|
|
4
4
|
export * from './vocabularyTransform.js';
|
|
5
5
|
export * from './uriSpaceTransform.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACpD,YAAY,EACV,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACpD,YAAY,EACV,cAAc,EACd,aAAa,EACb,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC"}
|
package/dist/stage.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Stage, type
|
|
1
|
+
import { Stage, type ReaderContext, type QuadTransform } from '@lde/pipeline';
|
|
2
2
|
import type { Quad } from '@rdfjs/types';
|
|
3
3
|
/**
|
|
4
4
|
* Stable names for every VoID stage, equal to the underlying query filename.
|
|
@@ -27,8 +27,8 @@ export declare const VOID_STAGE_NAMES: {
|
|
|
27
27
|
};
|
|
28
28
|
/** The name of a VoID stage. @see VOID_STAGE_NAMES */
|
|
29
29
|
export type VoidStageName = (typeof VOID_STAGE_NAMES)[keyof typeof VOID_STAGE_NAMES];
|
|
30
|
-
/** A transform, or transforms, decorating a VoID stage's
|
|
31
|
-
export type VoidStageTransform = QuadTransform<
|
|
30
|
+
/** A transform, or transforms, decorating a VoID stage's reader output. */
|
|
31
|
+
export type VoidStageTransform = QuadTransform<ReaderContext> | QuadTransform<ReaderContext>[];
|
|
32
32
|
/**
|
|
33
33
|
* Options for configuring VoID stage execution.
|
|
34
34
|
*
|
|
@@ -39,8 +39,8 @@ export type VoidStageTransform = QuadTransform<ExecutorContext> | QuadTransform<
|
|
|
39
39
|
*/
|
|
40
40
|
export interface VoidStageOptions {
|
|
41
41
|
/**
|
|
42
|
-
* Transform(s) decorating this stage's
|
|
43
|
-
* merges
|
|
42
|
+
* Transform(s) decorating this stage's reader output before the stage
|
|
43
|
+
* merges readers. For a global stage the transform sees the reader's
|
|
44
44
|
* complete output; for a per-class stage it sees one batch – one class at
|
|
45
45
|
* `batchSize: 1`. Built-in transforms (e.g. {@link uriSpaces}) compose
|
|
46
46
|
* with these, built-in first.
|
|
@@ -54,9 +54,9 @@ export interface VoidStageOptions {
|
|
|
54
54
|
* and processed concurrently — they have no effect on global (non-per-class) stages.
|
|
55
55
|
*/
|
|
56
56
|
export interface PerClassVoidStageOptions extends VoidStageOptions {
|
|
57
|
-
/** Maximum number of class bindings per
|
|
57
|
+
/** Maximum number of class bindings per reader call. @default 10 */
|
|
58
58
|
batchSize?: number;
|
|
59
|
-
/** Maximum concurrent in-flight
|
|
59
|
+
/** Maximum concurrent in-flight reader batches. @default 10 */
|
|
60
60
|
maxConcurrency?: number;
|
|
61
61
|
/** When true, iterate queries per class using a class selector. @default true */
|
|
62
62
|
perClass?: boolean;
|
|
@@ -76,7 +76,7 @@ export interface VoidStagesOptions extends Omit<PerClassVoidStageOptions, 'trans
|
|
|
76
76
|
/**
|
|
77
77
|
* Transforms to attach to bundled stages, keyed by {@link VOID_STAGE_NAMES}.
|
|
78
78
|
*
|
|
79
|
-
* Each transform decorates the
|
|
79
|
+
* Each transform decorates the reader of the named stage – so a consumer
|
|
80
80
|
* can wrap a stage it never constructs. Where a stage already carries a
|
|
81
81
|
* built-in transform ({@link uriSpaces}, {@link detectVocabularies}), the
|
|
82
82
|
* consumer transform composes after it. An invalid key is a compile error.
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EAKL,KAAK,
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EAKL,KAAK,aAAa,EAElB,KAAK,aAAa,EACnB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAezC;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;CAiBnB,CAAC;AAEX,sDAAsD;AACtD,MAAM,MAAM,aAAa,GACvB,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,OAAO,gBAAgB,CAAC,CAAC;AAE3D,2EAA2E;AAC3E,MAAM,MAAM,kBAAkB,GAC1B,aAAa,CAAC,aAAa,CAAC,GAC5B,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IAChE,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,iBAAkB,SAAQ,IAAI,CAC7C,wBAAwB,EACxB,WAAW,CACZ;IACC,yEAAyE;IACzE,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;IACjD,mFAAmF;IACnF,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC,CAAC;CACjE;AAgED,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAE3E;AAED,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAE1E;AAOD,wBAAgB,mBAAmB,CACjC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAK1E;AAED,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAK1E;AAED,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKxE;AAED,wBAAgB,YAAY,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKvE;AAED,wBAAgB,qBAAqB,CACnC,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,oBAAoB,CAClC,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKzE;AAED,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAEzE;AAID,wBAAgB,qBAAqB,CACnC,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAID,wBAAgB,SAAS,CACvB,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC,EACjD,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,KAAK,CAAC,CAOhB;AAED,MAAM,WAAW,yBAA0B,SAAQ,gBAAgB;IACjE,mFAAmF;IACnF,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,wBAAgB,kBAAkB,CAChC,OAAO,CAAC,EAAE,yBAAyB,GAClC,OAAO,CAAC,KAAK,CAAC,CAQhB;AAED;;;;;;;GAOG;AACH,wBAAsB,UAAU,CAC9B,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,KAAK,EAAE,CAAC,CAgDlB"}
|
package/dist/stage.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Stage,
|
|
1
|
+
import { Stage, SparqlConstructReader, SparqlItemSelector, readQueryFile, } from '@lde/pipeline';
|
|
2
2
|
import { assertSafeIri } from '@lde/dataset';
|
|
3
3
|
import { resolve, dirname } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
@@ -32,13 +32,13 @@ export const VOID_STAGE_NAMES = {
|
|
|
32
32
|
};
|
|
33
33
|
async function createVoidStage(filename, options) {
|
|
34
34
|
const query = await readQueryFile(resolve(queriesDir, filename));
|
|
35
|
-
const
|
|
36
|
-
|
|
35
|
+
const reader = {
|
|
36
|
+
reader: new SparqlConstructReader({ query }),
|
|
37
37
|
transform: options?.transform,
|
|
38
38
|
};
|
|
39
39
|
return new Stage({
|
|
40
40
|
name: filename,
|
|
41
|
-
|
|
41
|
+
readers: reader,
|
|
42
42
|
itemSelector: options?.perClass ? classSelector() : undefined,
|
|
43
43
|
batchSize: options?.batchSize,
|
|
44
44
|
maxConcurrency: options?.maxConcurrency,
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ReaderContext, QuadTransform } from '@lde/pipeline';
|
|
2
2
|
import type { Quad } from '@rdfjs/types';
|
|
3
3
|
/**
|
|
4
4
|
* Creates a {@link QuadTransform} that consumes `void:Linkset` quads from a
|
|
5
|
-
* stage's
|
|
5
|
+
* stage's reader output, matches each `void:objectsTarget` against the
|
|
6
6
|
* configured URI space prefixes using `startsWith`, and aggregates triple
|
|
7
7
|
* counts per matched space.
|
|
8
8
|
*
|
|
@@ -10,8 +10,8 @@ import type { Quad } from '@rdfjs/types';
|
|
|
10
10
|
* from the metadata quad subjects), not the raw URI space prefix. Unmatched
|
|
11
11
|
* linksets are discarded.
|
|
12
12
|
*
|
|
13
|
-
* Attach it to the `object-uri-space.rq` stage's
|
|
13
|
+
* Attach it to the `object-uri-space.rq` stage's reader – directly via
|
|
14
14
|
* {@link uriSpaces} or through the `transforms` map of {@link voidStages}.
|
|
15
15
|
*/
|
|
16
|
-
export declare function withUriSpaces(uriSpaces: ReadonlyMap<string, readonly Quad[]>): QuadTransform<
|
|
16
|
+
export declare function withUriSpaces(uriSpaces: ReadonlyMap<string, readonly Quad[]>): QuadTransform<ReaderContext>;
|
|
17
17
|
//# sourceMappingURL=uriSpaceTransform.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uriSpaceTransform.d.ts","sourceRoot":"","sources":["../src/uriSpaceTransform.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"uriSpaceTransform.d.ts","sourceRoot":"","sources":["../src/uriSpaceTransform.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC,GAC9C,aAAa,CAAC,aAAa,CAAC,CAG9B"}
|
|
@@ -4,7 +4,7 @@ import { rdf, _void, xsd } from '@tpluscode/rdf-ns-builders';
|
|
|
4
4
|
const { namedNode, quad, literal } = DataFactory;
|
|
5
5
|
/**
|
|
6
6
|
* Creates a {@link QuadTransform} that consumes `void:Linkset` quads from a
|
|
7
|
-
* stage's
|
|
7
|
+
* stage's reader output, matches each `void:objectsTarget` against the
|
|
8
8
|
* configured URI space prefixes using `startsWith`, and aggregates triple
|
|
9
9
|
* counts per matched space.
|
|
10
10
|
*
|
|
@@ -12,7 +12,7 @@ const { namedNode, quad, literal } = DataFactory;
|
|
|
12
12
|
* from the metadata quad subjects), not the raw URI space prefix. Unmatched
|
|
13
13
|
* linksets are discarded.
|
|
14
14
|
*
|
|
15
|
-
* Attach it to the `object-uri-space.rq` stage's
|
|
15
|
+
* Attach it to the `object-uri-space.rq` stage's reader – directly via
|
|
16
16
|
* {@link uriSpaces} or through the `transforms` map of {@link voidStages}.
|
|
17
17
|
*/
|
|
18
18
|
export function withUriSpaces(uriSpaces) {
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ReaderContext, QuadTransform } from '@lde/pipeline';
|
|
2
2
|
export declare const defaultVocabularies: readonly string[];
|
|
3
3
|
/**
|
|
4
4
|
* Creates a {@link QuadTransform} that passes through all quads from a stage's
|
|
5
|
-
*
|
|
5
|
+
* reader output and appends `void:vocabulary` triples for detected
|
|
6
6
|
* vocabulary prefixes.
|
|
7
7
|
*
|
|
8
8
|
* Inspects quads with predicate `void:property` to detect known vocabulary
|
|
9
9
|
* namespace prefixes, then yields the corresponding `void:vocabulary` quads
|
|
10
|
-
* after the
|
|
10
|
+
* after the reader output has been consumed.
|
|
11
11
|
*
|
|
12
|
-
* Attach it to the `entity-properties.rq` stage's
|
|
12
|
+
* Attach it to the `entity-properties.rq` stage's reader – directly via
|
|
13
13
|
* {@link detectVocabularies} or through the `transforms` map of
|
|
14
14
|
* {@link voidStages}.
|
|
15
15
|
*/
|
|
16
|
-
export declare function withVocabularies(vocabularies?: readonly string[]): QuadTransform<
|
|
16
|
+
export declare function withVocabularies(vocabularies?: readonly string[]): QuadTransform<ReaderContext>;
|
|
17
17
|
//# sourceMappingURL=vocabularyTransform.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vocabularyTransform.d.ts","sourceRoot":"","sources":["../src/vocabularyTransform.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"vocabularyTransform.d.ts","sourceRoot":"","sources":["../src/vocabularyTransform.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAQlE,eAAO,MAAM,mBAAmB,EAAE,SAAS,MAAM,EAEhD,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,YAAY,GAAE,SAAS,MAAM,EAAwB,GACpD,aAAa,CAAC,aAAa,CAAC,CAG9B"}
|
|
@@ -7,14 +7,14 @@ export const defaultVocabularies = [
|
|
|
7
7
|
];
|
|
8
8
|
/**
|
|
9
9
|
* Creates a {@link QuadTransform} that passes through all quads from a stage's
|
|
10
|
-
*
|
|
10
|
+
* reader output and appends `void:vocabulary` triples for detected
|
|
11
11
|
* vocabulary prefixes.
|
|
12
12
|
*
|
|
13
13
|
* Inspects quads with predicate `void:property` to detect known vocabulary
|
|
14
14
|
* namespace prefixes, then yields the corresponding `void:vocabulary` quads
|
|
15
|
-
* after the
|
|
15
|
+
* after the reader output has been consumed.
|
|
16
16
|
*
|
|
17
|
-
* Attach it to the `entity-properties.rq` stage's
|
|
17
|
+
* Attach it to the `entity-properties.rq` stage's reader – directly via
|
|
18
18
|
* {@link detectVocabularies} or through the `transforms` map of
|
|
19
19
|
* {@link voidStages}.
|
|
20
20
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline-void",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.30.0",
|
|
4
4
|
"description": "VOiD (Vocabulary of Interlinked Datasets) statistical analysis for RDF datasets",
|
|
5
5
|
"repository": {
|
|
6
6
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"tslib": "^2.3.0"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
|
-
"@lde/dataset": "0.7.
|
|
37
|
-
"@lde/pipeline": "0.
|
|
36
|
+
"@lde/dataset": "^0.7.8",
|
|
37
|
+
"@lde/pipeline": "^0.32.0"
|
|
38
38
|
}
|
|
39
39
|
}
|