@lde/pipeline-void 0.33.4 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -104
- package/dist/stage.d.ts +18 -4
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +51 -10
- package/dist/vocabularyTransform.d.ts +6 -0
- package/dist/vocabularyTransform.d.ts.map +1 -1
- package/dist/vocabularyTransform.js +20 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,25 +1,14 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @lde/pipeline-void
|
|
2
2
|
|
|
3
3
|
Extensions to [@lde/pipeline](../pipeline) for VoID (Vocabulary of Interlinked Datasets) statistical analysis of RDF datasets.
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## Installation
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
Accepts an optional `VoidStagesOptions` object:
|
|
12
|
-
|
|
13
|
-
| Option | Default | Description |
|
|
14
|
-
| ---------------- | ------- | --------------------------------------------------------------------------------------------------------------- |
|
|
15
|
-
| `batchSize` | 10 | Maximum class bindings per reader call (per-class stages only) |
|
|
16
|
-
| `maxConcurrency` | 10 | Maximum concurrent in-flight reader batches (per-class stages only) |
|
|
17
|
-
| `perClass` | — | Override per-class iteration for all five per-class stages |
|
|
18
|
-
| `uriSpaces` | — | When provided, includes the object URI space stage |
|
|
19
|
-
| `vocabularies` | — | Additional vocabulary namespace URIs to detect beyond the built-in defaults |
|
|
20
|
-
| `transforms` | — | Transforms to attach to bundled stages, keyed by `VOID_STAGE_NAMES` (see [Stage transforms](#stage-transforms)) |
|
|
7
|
+
```sh
|
|
8
|
+
npm install @lde/pipeline-void
|
|
9
|
+
```
|
|
21
10
|
|
|
22
|
-
|
|
11
|
+
## Usage
|
|
23
12
|
|
|
24
13
|
```typescript
|
|
25
14
|
import { voidStages } from '@lde/pipeline-void';
|
|
@@ -37,91 +26,6 @@ await new Pipeline({
|
|
|
37
26
|
}).run();
|
|
38
27
|
```
|
|
39
28
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
Global and domain-specific factories accept `VoidStageOptions` (`transform`) and return `Promise<Stage>`. Per-class factories accept `PerClassVoidStageOptions` (`transform`, `batchSize`, `maxConcurrency`, `perClass`) — they default `perClass` to `true`; set it to `false` to run them as monolithic queries instead.
|
|
43
|
-
|
|
44
|
-
#### Global stages (one CONSTRUCT query per dataset):
|
|
45
|
-
|
|
46
|
-
| Factory | Query |
|
|
47
|
-
| ----------------------- | ------------------------------------------------------------------------------- |
|
|
48
|
-
| `classPartitions()` | [`class-partition.rq`](queries/class-partition.rq) — Classes with entity counts |
|
|
49
|
-
| `countDatatypes()` | [`datatypes.rq`](queries/datatypes.rq) — Dataset-level datatypes |
|
|
50
|
-
| `countObjectLiterals()` | [`object-literals.rq`](queries/object-literals.rq) — Literal object counts |
|
|
51
|
-
| `countObjectUris()` | [`object-uris.rq`](queries/object-uris.rq) — URI object counts |
|
|
52
|
-
| `countProperties()` | [`properties.rq`](queries/properties.rq) — Distinct properties |
|
|
53
|
-
| `countSubjects()` | [`subjects.rq`](queries/subjects.rq) — Distinct subjects |
|
|
54
|
-
| `countTriples()` | [`triples.rq`](queries/triples.rq) — Total triple count |
|
|
55
|
-
| `detectLicenses()` | [`licenses.rq`](queries/licenses.rq) — License detection |
|
|
56
|
-
| `subjectUriSpaces()` | [`subject-uri-space.rq`](queries/subject-uri-space.rq) — Subject URI namespaces |
|
|
57
|
-
|
|
58
|
-
#### Per-class stages (iterated with a class selector):
|
|
59
|
-
|
|
60
|
-
| Factory | Query |
|
|
61
|
-
| ------------------------- | ------------------------------------------------------------------------------------------------------------------ |
|
|
62
|
-
| `classPropertySubjects()` | [`class-properties-subjects.rq`](queries/class-properties-subjects.rq) — Properties per class (subject counts) |
|
|
63
|
-
| `classPropertyObjects()` | [`class-properties-objects.rq`](queries/class-properties-objects.rq) — Properties per class (object counts) |
|
|
64
|
-
| `perClassDatatypes()` | [`class-property-datatypes.rq`](queries/class-property-datatypes.rq) — Per-class datatype partitions |
|
|
65
|
-
| `perClassLanguages()` | [`class-property-languages.rq`](queries/class-property-languages.rq) — Per-class language tags |
|
|
66
|
-
| `perClassObjectClasses()` | [`class-property-object-classes.rq`](queries/class-property-object-classes.rq) — Per-class object class partitions |
|
|
67
|
-
|
|
68
|
-
#### Domain-specific stages:
|
|
69
|
-
|
|
70
|
-
| Factory | Description |
|
|
71
|
-
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
72
|
-
| `detectVocabularies()` | [`entity-properties.rq`](queries/entity-properties.rq) — Entity properties with automatic `void:vocabulary` detection. Accepts `DetectVocabulariesOptions` with an optional `vocabularies` array to extend the built-in defaults. |
|
|
73
|
-
| `uriSpaces(uriSpaceMap)` | [`object-uri-space.rq`](queries/object-uri-space.rq) — Object URI namespace linksets, aggregated against a provided URI space map |
|
|
74
|
-
|
|
75
|
-
## Namespace normalization
|
|
76
|
-
|
|
77
|
-
Some vocabularies publish under both `http://` and `https://` variants of the same namespace (notably schema.org), and datasets mix them. Without normalization each variant gets its own `void:classPartition`/`void:propertyPartition`, so consumers see two partitions for one class — which crashed the Dataset Register browser (netwerk-digitaal-erfgoed/dataset-knowledge-graph#334).
|
|
78
|
-
|
|
79
|
-
`schemaOrgPartitionMergePlugin` normalizes `http://schema.org/` to `https://schema.org/` **and** merges the duplicate partition nodes the two variants produced. It is a [`beforeDatasetWrite`](../pipeline) plugin — it runs once over a whole dataset’s output at the pipeline edge, so the analysis queries stay unaware of namespace aliases (see [ADR 7](../../docs/decisions/0007-namespace-merge-as-a-dataset-plugin.md)):
|
|
80
|
-
|
|
81
|
-
```typescript
|
|
82
|
-
import { voidStages, schemaOrgPartitionMergePlugin } from '@lde/pipeline-void';
|
|
83
|
-
import { Pipeline, provenancePlugin } from '@lde/pipeline';
|
|
84
|
-
|
|
85
|
-
await new Pipeline({
|
|
86
|
-
stages: await voidStages(), // plain, no namespace options
|
|
87
|
-
plugins: [schemaOrgPartitionMergePlugin(), provenancePlugin()],
|
|
88
|
-
// …
|
|
89
|
-
}).run();
|
|
90
|
-
```
|
|
91
|
-
|
|
92
|
-
Use `namespacePartitionMergePlugin(aliases)` for namespaces other than schema.org. The transform streams — it buffers only partition quads (bounded by the summary), passing everything else straight through. Datasets typically use a single schema.org namespace, so within one dataset there is one variant per class and every count stays exact; a dataset that mixes both namespaces on one property has its `void:distinctObjects` summed (an over-count for shared objects) rather than deduped.
|
|
93
|
-
|
|
94
|
-
> This plugin does more than rename IRIs: rewriting the `void:class` objects alone would still leave two `void:classPartition` nodes for one class. If you only need a blanket namespace rewrite over a dataset’s own quads (not a VoID partition merge) — for example when mapping instance data to an application profile — use the generic [`schemaOrgNormalizationPlugin` / `namespaceNormalizationPlugin`](../pipeline) from `@lde/pipeline` instead.
|
|
95
|
-
|
|
96
|
-
## Stage transforms
|
|
97
|
-
|
|
98
|
-
A VoID stage decorates its reader’s output with a `QuadTransform<ReaderContext>` attached as data (see [@lde/pipeline](../pipeline)’s extension model and [ADR 2](../../docs/decisions/0002-unify-pipeline-extension-on-quad-transforms.md)). It runs once per reader call and may fire its own SPARQL queries against the `distribution` in scope — so write it to accept being called more than once: a global stage calls it once over the complete output, a per-class stage with batching enabled once per batch (one class at `batchSize: 1`).
|
|
29
|
+
## Documentation
|
|
99
30
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
- `withVocabularies(vocabularies?)` — passes through all quads and appends `void:vocabulary` triples for detected vocabulary namespace prefixes in `void:property` quads. The built-in defaults are exported as `defaultVocabularies` (sourced from `@zazuko/prefixes`); `detectVocabularies()` attaches it to the `entity-properties.rq` stage.
|
|
103
|
-
- `withUriSpaces(uriSpaceMap)` — consumes `void:Linkset` quads, matches each `void:objectsTarget` against the configured URI space prefixes using `startsWith`, and aggregates triple counts per matched space. Emits `void:objectsTarget` pointing to the target dataset IRI (taken from the metadata quad subjects), not the raw prefix; unmatched linksets are discarded. `uriSpaces(uriSpaceMap)` attaches it to the `object-uri-space.rq` stage.
|
|
104
|
-
|
|
105
|
-
### Attaching your own transform
|
|
106
|
-
|
|
107
|
-
Pass a `transform` to an individual factory, or route transforms through `voidStages` with the `transforms` map keyed by `VOID_STAGE_NAMES` — so you can decorate a stage you never construct. Where a stage already carries a built-in transform, your transform composes after it. An invalid stage name is a compile error.
|
|
108
|
-
|
|
109
|
-
```typescript
|
|
110
|
-
import { voidStages, VOID_STAGE_NAMES } from '@lde/pipeline-void';
|
|
111
|
-
import type { ReaderContext, QuadTransform } from '@lde/pipeline-void';
|
|
112
|
-
|
|
113
|
-
const sampleSubjects: QuadTransform<ReaderContext> = async function* (
|
|
114
|
-
quads,
|
|
115
|
-
{ dataset, distribution },
|
|
116
|
-
) {
|
|
117
|
-
yield* quads; // pass the stage’s subsets through unchanged …
|
|
118
|
-
// … then fire a sample SELECT against `distribution` and append measurements.
|
|
119
|
-
};
|
|
120
|
-
|
|
121
|
-
const stages = await voidStages({
|
|
122
|
-
batchSize: 1,
|
|
123
|
-
transforms: {
|
|
124
|
-
[VOID_STAGE_NAMES.subjectUriSpace]: sampleSubjects,
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
|
-
```
|
|
31
|
+
See the [full documentation](https://ldelements.org/reference/pipeline-void).
|
package/dist/stage.d.ts
CHANGED
|
@@ -50,11 +50,13 @@ export interface VoidStageOptions {
|
|
|
50
50
|
/**
|
|
51
51
|
* Options for per-class VoID stages that iterate over classes.
|
|
52
52
|
*
|
|
53
|
-
* `batchSize` and `maxConcurrency` control how
|
|
54
|
-
*
|
|
53
|
+
* `batchSize` and `maxConcurrency` control how item bindings are batched and
|
|
54
|
+
* processed concurrently. They apply to every iterating stage – the five
|
|
55
|
+
* per-class stages and (via {@link VoidStagesOptions}) the per-property
|
|
56
|
+
* vocabularies stage – and have no effect on global one-query stages.
|
|
55
57
|
*/
|
|
56
58
|
export interface PerClassVoidStageOptions extends VoidStageOptions {
|
|
57
|
-
/** Maximum number of
|
|
59
|
+
/** Maximum number of item bindings per reader call. @default 10 */
|
|
58
60
|
batchSize?: number;
|
|
59
61
|
/** Maximum concurrent in-flight reader batches. @default 10 */
|
|
60
62
|
maxConcurrency?: number;
|
|
@@ -73,6 +75,12 @@ export interface VoidStagesOptions extends Omit<PerClassVoidStageOptions, 'trans
|
|
|
73
75
|
uriSpaces?: ReadonlyMap<string, readonly Quad[]>;
|
|
74
76
|
/** Additional vocabulary namespace URIs to detect beyond the built-in defaults. */
|
|
75
77
|
vocabularies?: readonly string[];
|
|
78
|
+
/**
|
|
79
|
+
* When true, iterate the vocabularies (property-partition) stage per
|
|
80
|
+
* property (see {@link DetectVocabulariesOptions.perProperty}).
|
|
81
|
+
* @default true
|
|
82
|
+
*/
|
|
83
|
+
perProperty?: boolean;
|
|
76
84
|
/**
|
|
77
85
|
* Transforms to attach to bundled stages, keyed by {@link VOID_STAGE_NAMES}.
|
|
78
86
|
*
|
|
@@ -98,9 +106,15 @@ export declare function perClassObjectClasses(options?: PerClassVoidStageOptions
|
|
|
98
106
|
export declare function perClassDatatypes(options?: PerClassVoidStageOptions): Promise<Stage>;
|
|
99
107
|
export declare function perClassLanguages(options?: PerClassVoidStageOptions): Promise<Stage>;
|
|
100
108
|
export declare function uriSpaces(uriSpaceMap: ReadonlyMap<string, readonly Quad[]>, options?: VoidStageOptions): Promise<Stage>;
|
|
101
|
-
export interface DetectVocabulariesOptions extends
|
|
109
|
+
export interface DetectVocabulariesOptions extends Omit<PerClassVoidStageOptions, 'perClass'> {
|
|
102
110
|
/** Additional vocabulary namespace URIs to detect beyond the built-in defaults. */
|
|
103
111
|
vocabularies?: readonly string[];
|
|
112
|
+
/**
|
|
113
|
+
* When true, iterate the property-partition query per property using a
|
|
114
|
+
* property selector, so its memory use is bounded by the batch instead of
|
|
115
|
+
* the whole dataset (see {@link propertySelector}). @default true
|
|
116
|
+
*/
|
|
117
|
+
perProperty?: boolean;
|
|
104
118
|
}
|
|
105
119
|
export declare function detectVocabularies(options?: DetectVocabulariesOptions): Promise<Stage>;
|
|
106
120
|
/**
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EAKL,KAAK,aAAa,EAElB,KAAK,aAAa,EACnB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAgBzC;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;CAiBnB,CAAC;AAEX,sDAAsD;AACtD,MAAM,MAAM,aAAa,GACvB,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,OAAO,gBAAgB,CAAC,CAAC;AAE3D,2EAA2E;AAC3E,MAAM,MAAM,kBAAkB,
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EAKL,KAAK,aAAa,EAElB,KAAK,aAAa,EACnB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAgBzC;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;CAiBnB,CAAC;AAEX,sDAAsD;AACtD,MAAM,MAAM,aAAa,GACvB,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,OAAO,gBAAgB,CAAC,CAAC;AAE3D,2EAA2E;AAC3E,MAAM,MAAM,kBAAkB,GAC5B,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;AAEhE;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IAChE,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,iBAAkB,SAAQ,IAAI,CAC7C,wBAAwB,EACxB,WAAW,CACZ;IACC,yEAAyE;IACzE,SAAS,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;IACjD,mFAAmF;IACnF,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC,CAAC;CACjE;AAmHD,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAE3E;AAED,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAE1E;AAOD,wBAAgB,mBAAmB,CACjC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAK1E;AAED,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAK1E;AAED,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKxE;AAED,wBAAgB,YAAY,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKvE;AAED,wBAAgB,qBAAqB,CACnC,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,oBAAoB,CAClC,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKzE;AAED,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAEzE;AAID,wBAAgB,qBAAqB,CACnC,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,KAAK,CAAC,CAKhB;AAID,wBAAgB,SAAS,CACvB,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC,EACjD,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,KAAK,CAAC,CAOhB;AAED,MAAM,WAAW,yBAA0B,SAAQ,IAAI,CACrD,wBAAwB,EACxB,UAAU,CACX;IACC,mFAAmF;IACnF,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,wBAAgB,kBAAkB,CAChC,OAAO,CAAC,EAAE,yBAAyB,GAClC,OAAO,CAAC,KAAK,CAAC,CAYhB;AAED;;;;;;;GAOG;AACH,wBAAsB,UAAU,CAC9B,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,KAAK,EAAE,CAAC,CAqDlB"}
|
package/dist/stage.js
CHANGED
|
@@ -40,7 +40,11 @@ async function createVoidStage(filename, options) {
|
|
|
40
40
|
return new Stage({
|
|
41
41
|
name: filename,
|
|
42
42
|
readers: reader,
|
|
43
|
-
itemSelector: options?.perClass
|
|
43
|
+
itemSelector: options?.perClass
|
|
44
|
+
? classSelector()
|
|
45
|
+
: options?.perProperty
|
|
46
|
+
? propertySelector()
|
|
47
|
+
: undefined,
|
|
44
48
|
batchSize: options?.batchSize,
|
|
45
49
|
maxConcurrency: options?.maxConcurrency,
|
|
46
50
|
expectsOutput: options?.expectsOutput,
|
|
@@ -52,11 +56,20 @@ function asTransforms(transform) {
|
|
|
52
56
|
return [];
|
|
53
57
|
return Array.isArray(transform) ? [...transform] : [transform];
|
|
54
58
|
}
|
|
55
|
-
|
|
59
|
+
/**
|
|
60
|
+
* Build an {@link ItemSelector} that pages through the distinct bindings of
|
|
61
|
+
* one variable.
|
|
62
|
+
*
|
|
63
|
+
* The `ORDER BY` is load-bearing: the inner {@link SparqlItemSelector} pages
|
|
64
|
+
* with `LIMIT`/`OFFSET` (the query’s `LIMIT` is the page size), and SPARQL
|
|
65
|
+
* guarantees no result order without `ORDER BY` – unordered pages can skip or
|
|
66
|
+
* repeat items on endpoints with unstable ordering.
|
|
67
|
+
*/
|
|
68
|
+
function distinctItemSelector(variable, pattern) {
|
|
56
69
|
return {
|
|
57
70
|
// Forward `options` so the Pipeline’s per-dataset TimeoutPolicy
|
|
58
71
|
// reaches the inner SparqlItemSelector – without this the adaptive
|
|
59
|
-
// budget is silently bypassed for
|
|
72
|
+
// budget is silently bypassed for item selection.
|
|
60
73
|
select: (distribution, batchSize, options) => {
|
|
61
74
|
const subjectFilter = distribution.subjectFilter ?? '';
|
|
62
75
|
let fromClause = '';
|
|
@@ -64,13 +77,11 @@ function classSelector() {
|
|
|
64
77
|
assertSafeIri(distribution.namedGraph);
|
|
65
78
|
fromClause = `FROM <${distribution.namedGraph}>`;
|
|
66
79
|
}
|
|
67
|
-
// Exclude blank-node classes at the endpoint: the selector would drop
|
|
68
|
-
// them client-side anyway (no stable identity), but only after fetching
|
|
69
|
-
// them.
|
|
70
80
|
const selectorQuery = [
|
|
71
|
-
|
|
81
|
+
`SELECT DISTINCT ?${variable}`,
|
|
72
82
|
fromClause,
|
|
73
|
-
`WHERE { ${subjectFilter}
|
|
83
|
+
`WHERE { ${pattern(subjectFilter)} }`,
|
|
84
|
+
`ORDER BY ?${variable}`,
|
|
74
85
|
'LIMIT 1000',
|
|
75
86
|
].join('\n');
|
|
76
87
|
return new SparqlItemSelector({
|
|
@@ -79,6 +90,28 @@ function classSelector() {
|
|
|
79
90
|
},
|
|
80
91
|
};
|
|
81
92
|
}
|
|
93
|
+
function classSelector() {
|
|
94
|
+
// Exclude blank-node classes at the endpoint: the selector would drop
|
|
95
|
+
// them client-side anyway (no stable identity), but only after fetching
|
|
96
|
+
// them.
|
|
97
|
+
return distinctItemSelector('class', (subjectFilter) => `${subjectFilter} ?s a ?class . FILTER(!isBlank(?class))`);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Select the distinct properties of a dataset, so the property-partition
|
|
101
|
+
* query can iterate in bounded batches instead of aggregating every property
|
|
102
|
+
* at once.
|
|
103
|
+
*
|
|
104
|
+
* The chunking exists for memory, not style: the unchunked
|
|
105
|
+
* `entity-properties.rq` computes two `COUNT(DISTINCT …)` aggregates grouped
|
|
106
|
+
* over every property in one query, which materializes the dataset’s full
|
|
107
|
+
* scan – measured to exhaust a 16 GB query budget on a 608M-triple dataset,
|
|
108
|
+
* so large datasets got no property partitions (and, downstream, no
|
|
109
|
+
* `void:vocabulary`) at all. With a property batch injected as `VALUES`, the
|
|
110
|
+
* working set is bounded by one batch’s triples regardless of dataset size.
|
|
111
|
+
*/
|
|
112
|
+
function propertySelector() {
|
|
113
|
+
return distinctItemSelector('p', (subjectFilter) => `${subjectFilter} ?s ?p ?o .`);
|
|
114
|
+
}
|
|
82
115
|
// Global stages
|
|
83
116
|
export function subjectUriSpaces(options) {
|
|
84
117
|
return createVoidStage(VOID_STAGE_NAMES.subjectUriSpace, options);
|
|
@@ -170,12 +203,15 @@ export function uriSpaces(uriSpaceMap, options) {
|
|
|
170
203
|
});
|
|
171
204
|
}
|
|
172
205
|
export function detectVocabularies(options) {
|
|
173
|
-
const { vocabularies, transform } = options ?? {};
|
|
206
|
+
const { vocabularies, transform, batchSize, maxConcurrency, perProperty } = options ?? {};
|
|
174
207
|
const allVocabularies = vocabularies
|
|
175
208
|
? [...defaultVocabularies, ...vocabularies]
|
|
176
209
|
: undefined;
|
|
177
210
|
return createVoidStage(VOID_STAGE_NAMES.vocabularies, {
|
|
178
211
|
transform: [withVocabularies(allVocabularies), ...asTransforms(transform)],
|
|
212
|
+
perProperty: perProperty ?? true,
|
|
213
|
+
batchSize,
|
|
214
|
+
maxConcurrency,
|
|
179
215
|
});
|
|
180
216
|
}
|
|
181
217
|
/**
|
|
@@ -187,7 +223,11 @@ export function detectVocabularies(options) {
|
|
|
187
223
|
* when the cache is cold.
|
|
188
224
|
*/
|
|
189
225
|
export async function voidStages(options) {
|
|
190
|
-
const { uriSpaces: uriSpaceMap, vocabularies, transforms,
|
|
226
|
+
const { uriSpaces: uriSpaceMap, vocabularies, transforms,
|
|
227
|
+
// Destructured out of stageOptions so it cannot leak into the global
|
|
228
|
+
// stages’ createVoidStage options – it applies to the vocabularies stage
|
|
229
|
+
// only.
|
|
230
|
+
perProperty, ...stageOptions } = options ?? {};
|
|
191
231
|
// Merge the shared per-stage options with the transform routed to a stage.
|
|
192
232
|
const withTransform = (name) => ({
|
|
193
233
|
...stageOptions,
|
|
@@ -214,6 +254,7 @@ export async function voidStages(options) {
|
|
|
214
254
|
detectVocabularies({
|
|
215
255
|
...withTransform(VOID_STAGE_NAMES.vocabularies),
|
|
216
256
|
vocabularies,
|
|
257
|
+
perProperty,
|
|
217
258
|
}),
|
|
218
259
|
subjectUriSpaces(withTransform(VOID_STAGE_NAMES.subjectUriSpace)),
|
|
219
260
|
...(uriSpaceMap
|
|
@@ -12,6 +12,12 @@ export declare const defaultVocabularies: readonly string[];
|
|
|
12
12
|
* Attach it to the `entity-properties.rq` stage's reader – directly via
|
|
13
13
|
* {@link detectVocabularies} or through the `transforms` map of
|
|
14
14
|
* {@link voidStages}.
|
|
15
|
+
*
|
|
16
|
+
* With per-property iteration the transform sees one batch at a time, so it
|
|
17
|
+
* remembers which vocabularies it already emitted per {@link Distribution}
|
|
18
|
+
* and yields each `void:vocabulary` quad only once per stage run. The memory
|
|
19
|
+
* is keyed weakly on the distribution instance – one per run – so a fallback
|
|
20
|
+
* re-run of the same dataset (a fresh distribution) starts clean.
|
|
15
21
|
*/
|
|
16
22
|
export declare function withVocabularies(vocabularies?: readonly string[]): QuadTransform<ReaderContext>;
|
|
17
23
|
//# sourceMappingURL=vocabularyTransform.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vocabularyTransform.d.ts","sourceRoot":"","sources":["../src/vocabularyTransform.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"vocabularyTransform.d.ts","sourceRoot":"","sources":["../src/vocabularyTransform.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AASlE,eAAO,MAAM,mBAAmB,EAAE,SAAS,MAAM,EAEhD,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,gBAAgB,CAC9B,YAAY,GAAE,SAAS,MAAM,EAAwB,GACpD,aAAa,CAAC,aAAa,CAAC,CAe9B"}
|
|
@@ -17,11 +17,25 @@ export const defaultVocabularies = [
|
|
|
17
17
|
* Attach it to the `entity-properties.rq` stage's reader – directly via
|
|
18
18
|
* {@link detectVocabularies} or through the `transforms` map of
|
|
19
19
|
* {@link voidStages}.
|
|
20
|
+
*
|
|
21
|
+
* With per-property iteration the transform sees one batch at a time, so it
|
|
22
|
+
* remembers which vocabularies it already emitted per {@link Distribution}
|
|
23
|
+
* and yields each `void:vocabulary` quad only once per stage run. The memory
|
|
24
|
+
* is keyed weakly on the distribution instance – one per run – so a fallback
|
|
25
|
+
* re-run of the same dataset (a fresh distribution) starts clean.
|
|
20
26
|
*/
|
|
21
27
|
export function withVocabularies(vocabularies = defaultVocabularies) {
|
|
22
|
-
|
|
28
|
+
const emittedPerRun = new WeakMap();
|
|
29
|
+
return (quads, { dataset, distribution }) => {
|
|
30
|
+
let emitted = emittedPerRun.get(distribution);
|
|
31
|
+
if (!emitted) {
|
|
32
|
+
emitted = new Set();
|
|
33
|
+
emittedPerRun.set(distribution, emitted);
|
|
34
|
+
}
|
|
35
|
+
return appendVocabularies(quads, dataset.iri.toString(), vocabularies, emitted);
|
|
36
|
+
};
|
|
23
37
|
}
|
|
24
|
-
async function* appendVocabularies(quads, datasetIri, vocabularies) {
|
|
38
|
+
async function* appendVocabularies(quads, datasetIri, vocabularies, emitted) {
|
|
25
39
|
const detectedVocabularies = new Set();
|
|
26
40
|
for await (const q of quads) {
|
|
27
41
|
yield q;
|
|
@@ -37,6 +51,10 @@ async function* appendVocabularies(quads, datasetIri, vocabularies) {
|
|
|
37
51
|
}
|
|
38
52
|
const datasetNode = namedNode(datasetIri);
|
|
39
53
|
for (const vocabUri of detectedVocabularies) {
|
|
54
|
+
if (emitted.has(vocabUri)) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
emitted.add(vocabUri);
|
|
40
58
|
yield quad(datasetNode, _void.vocabulary, namedNode(vocabUri));
|
|
41
59
|
}
|
|
42
60
|
}
|
package/package.json
CHANGED