@lde/pipeline 0.15.0 → 0.15.1

package/README.md

@@ -1,9 +1,9 @@
 # Pipeline
 
-A framework for transforming large RDF datasets using pure [SPARQL](https://www.w3.org/TR/sparql11-query/) queries.
+A framework for transforming large RDF datasets, primarily using [SPARQL](https://www.w3.org/TR/sparql11-query/) queries with TypeScript for the parts that are hard to express in SPARQL alone.
 
 - **SPARQL-native.** Data transformations are plain SPARQL query files — portable, transparent, testable and version-controlled.
-- **Composable.** Decorators wrap executors and resolvers to add behaviour (provenance, vocabulary detection, data import) without subclassing.
+- **Composable.** Executors are an interface: wrap a SPARQL executor with custom TypeScript to handle edge cases like date parsing or string normalisation (see [Executor](#executor)).
 - **Extensible.** A plugin system lets packages like [@lde/pipeline-void](../pipeline-void) (or your own plugins) hook into the pipeline lifecycle.
 
 ## Components
@@ -61,7 +61,7 @@ const itemSelector: ItemSelector = {
 
 ### Executor
 
-Generates RDF triples. `SparqlConstructExecutor` runs a SPARQL CONSTRUCT query with template substitution and variable bindings:
+Generates RDF triples. The built-in `SparqlConstructExecutor` runs a SPARQL CONSTRUCT query with template substitution and variable bindings:
 
 ```typescript
 const executor = new SparqlConstructExecutor({
@@ -69,6 +69,65 @@ const executor = new SparqlConstructExecutor({
 });
 ```
 
+`Executor` is an interface, so you can implement your own for logic that's hard to express in pure SPARQL — for example, cleaning up messy date notations or converting locale-specific dates to ISO 8601. The decorator pattern lets you wrap a SPARQL executor and post-process its quad stream in TypeScript:
+
+```typescript
+import { DataFactory } from 'n3';
+import type { Quad, Literal } from '@rdfjs/types';
+import type { Dataset, Distribution } from '@lde/dataset';
+import {
+  type Executor,
+  type ExecuteOptions,
+  NotSupported,
+} from '@lde/pipeline';
+
+class TransformExecutor implements Executor {
+  constructor(
+    private readonly inner: Executor,
+    private readonly transform: (
+      quads: AsyncIterable<Quad>,
+      dataset: Dataset,
+    ) => AsyncIterable<Quad>,
+  ) {}
+
+  async execute(
+    dataset: Dataset,
+    distribution: Distribution,
+    options?: ExecuteOptions,
+  ): Promise<AsyncIterable<Quad> | NotSupported> {
+    const result = await this.inner.execute(dataset, distribution, options);
+    if (result instanceof NotSupported) return result;
+    return this.transform(result, dataset);
+  }
+}
+```
+
+Then use it to wrap any SPARQL executor:
+
+```typescript
+new Stage({
+  name: 'dates',
+  executors: new TransformExecutor(
+    await SparqlConstructExecutor.fromFile('dates.rq'),
+    async function* (quads) {
+      for await (const quad of quads) {
+        if (quad.object.termType === 'Literal' && isMessyDate(quad.object)) {
+          const cleaned = DataFactory.literal(
+            parseDutchDate(quad.object.value),
+            DataFactory.namedNode('http://www.w3.org/2001/XMLSchema#date'),
+          );
+          yield DataFactory.quad(quad.subject, quad.predicate, cleaned);
+        } else {
+          yield quad;
+        }
+      }
+    },
+  ),
+});
+```
+
+This keeps SPARQL doing the heavy lifting while TypeScript handles the edge cases. See [@lde/pipeline-void](../pipeline-void)'s `VocabularyExecutor` for a real-world example of this pattern.
+
 ### Writer
 
 Writes generated quads to a destination:

package/dist/selector.d.ts

@@ -1,5 +1,5 @@
 import { Dataset } from '@lde/dataset';
-import { Client, Paginator } from '@lde/dataset-registry-client';
+import { Client, Paginator, type SearchCriteria } from '@lde/dataset-registry-client';
 /**
  * Select {@link Dataset}s for processing in a pipeline.
  */
@@ -24,7 +24,7 @@ export declare class ManualDatasetSelection implements DatasetSelector {
  * @param {object} options
  * @param Client options.registry The Dataset Registry Client to query for datasets.
  * @param string options.query Optional custom SPARQL query to select datasets.
- * @param object options.criteria Optional search criteria to select datasets.
+ * @param SearchCriteria options.criteria Optional search criteria to select datasets.
  */
 export declare class RegistrySelector implements DatasetSelector {
     private readonly registry;
@@ -33,7 +33,7 @@ export declare class RegistrySelector implements DatasetSelector {
     constructor({ registry, query, criteria, }: {
         registry: Client;
         query?: string;
-        criteria?: object;
+        criteria?: SearchCriteria;
     });
     select(): Promise<Paginator<Dataset>>;
 }

package/dist/selector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../src/selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;CACvC;AAED,qBAAa,sBAAuB,YAAW,eAAe;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,OAAO,EAAE;IAE1C,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAG5C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;gBAEvB,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB;IAMK,MAAM;CAOb"}
+{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../src/selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EACL,MAAM,EACN,SAAS,EACT,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;CACvC;AAED,qBAAa,sBAAuB,YAAW,eAAe;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,OAAO,EAAE;IAE1C,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAG5C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAiB;gBAE/B,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,cAAc,CAAC;KAC3B;IAMK,MAAM;CAOb"}

package/dist/selector.js

@@ -1,4 +1,4 @@
-import { Paginator } from '@lde/dataset-registry-client';
+import { Paginator, } from '@lde/dataset-registry-client';
 export class ManualDatasetSelection {
     datasets;
     constructor(datasets) {
@@ -21,7 +21,7 @@ export class ManualDatasetSelection {
  * @param {object} options
  * @param Client options.registry The Dataset Registry Client to query for datasets.
  * @param string options.query Optional custom SPARQL query to select datasets.
- * @param object options.criteria Optional search criteria to select datasets.
+ * @param SearchCriteria options.criteria Optional search criteria to select datasets.
  */
 export class RegistrySelector {
     registry;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.15.0",
+  "version": "0.15.1",
   "repository": {
-    "url": "git+https://github.com/ldengine/lde.git",
+    "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
   },
   "type": "module",
@@ -24,7 +24,7 @@
   ],
   "dependencies": {
     "@lde/dataset": "0.7.0",
-    "@lde/dataset-registry-client": "0.7.0",
+    "@lde/dataset-registry-client": "0.7.1",
     "@lde/sparql-importer": "0.3.0",
     "@lde/sparql-server": "0.4.10",
     "@rdfjs/types": "^2.0.1",