@lde/pipeline 0.2.0

package/dist/analyzer.d.ts

@@ -0,0 +1,18 @@
+import { DataEmittingStep, NotSupported } from './step.js';
+import { Dataset } from '@lde/dataset';
+import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+export interface Args {
+    identifier: string;
+    query: string;
+    fetcher?: SparqlEndpointFetcher;
+}
+export declare class SparqlQueryAnalyzer implements DataEmittingStep {
+    readonly identifier: string;
+    private readonly query;
+    private readonly fetcher;
+    constructor({ identifier, query, fetcher }: Args);
+    execute(dataset: Dataset): Promise<NotSupported | (import("readable-stream").Readable & import("@rdfjs/types").Stream<import("@rdfjs/types").Quad>)>;
+    static fromFile(filename: string): Promise<SparqlQueryAnalyzer>;
+}
+export declare function fromFile(filename: string): Promise<string>;
+//# sourceMappingURL=analyzer.d.ts.map

package/dist/analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../src/analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,MAAM,WAAW,IAAI;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED,qBAAa,mBAAoB,YAAW,gBAAgB;IAC1D,SAAgB,UAAU,SAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAEb,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,IAAI;IAM1C,OAAO,CAAC,OAAO,EAAE,OAAO;WAqBV,QAAQ,CAAC,QAAQ,EAAE,MAAM;CAM9C;AAED,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,mBAE9C"}

package/dist/analyzer.js

@@ -0,0 +1,34 @@
+import { NotSupported } from './step.js';
+import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+export class SparqlQueryAnalyzer {
+    identifier;
+    query;
+    fetcher;
+    constructor({ identifier, query, fetcher }) {
+        this.identifier = identifier;
+        this.query = query;
+        this.fetcher = fetcher ?? new SparqlEndpointFetcher();
+    }
+    async execute(dataset) {
+        const distribution = dataset.getSparqlDistribution();
+        if (null === distribution) {
+            return new NotSupported('No SPARQL distribution available');
+        }
+        const query = this.query
+            .replace('#subjectFilter#', distribution.subjectFilter ?? '')
+            .replace('?dataset', `<${dataset.iri}>`)
+            .replace('#namedGraph#', distribution.namedGraph ? `FROM <${distribution.namedGraph}>` : '');
+        return await this.fetcher.fetchTriples(distribution.accessUrl.toString(), query);
+    }
+    static async fromFile(filename) {
+        return new SparqlQueryAnalyzer({
+            identifier: filename,
+            query: await fromFile(filename),
+        });
+    }
+}
+export async function fromFile(filename) {
+    return (await readFile(resolve(filename))).toString();
+}

package/dist/import.d.ts

@@ -0,0 +1,30 @@
+import { Dataset } from '@lde/dataset';
+import { Failure, Finishable, NotSupported, SingleStep, Success } from './step.js';
+import { Importer } from '@lde/sparql-importer';
+import { SparqlServer } from '@lde/sparql-server';
+/**
+ * A pipeline step that imports a database using an {@link Importer} and makes
+ * the import available at a local SPARQL endpoint.
+ */
+export declare class Import implements SingleStep, Finishable {
+    readonly identifier = "import";
+    private readonly importer;
+    private readonly server;
+    private readonly forceImport;
+    /**
+     * Create a Pipeline ImportStep.
+     *
+     * @param {object} args
+     * @param args.importer A concrete importer that will import the distribution if needed.
+     * @param args.server SPARQL server that will be started to serve the imported data.
+     * @param args.forceImport Whether to force an import even if the dataset already has a SPARQL distribution.
+     */
+    constructor({ importer, server, forceImport, }: {
+        importer: Importer;
+        server: SparqlServer;
+        forceImport?: boolean;
+    });
+    execute(dataset: Dataset): Promise<NotSupported | Failure | Success>;
+    finish(): Promise<void>;
+}
+//# sourceMappingURL=import.d.ts.map

package/dist/import.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../src/import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,OAAO,EACP,UAAU,EACV,YAAY,EACZ,UAAU,EACV,OAAO,EACR,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,QAAQ,EAGT,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD;;;GAGG;AACH,qBAAa,MAAO,YAAW,UAAU,EAAE,UAAU;IACnD,SAAgB,UAAU,YAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IAEtC;;;;;;;OAOG;gBACS,EACV,QAAQ,EACR,MAAM,EACN,WAAW,GACZ,EAAE;QACD,QAAQ,EAAE,QAAQ,CAAC;QACnB,MAAM,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAMY,OAAO,CAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC;IAuB/B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAGrC"}

package/dist/import.js

@@ -0,0 +1,44 @@
+import { Distribution } from '@lde/dataset';
+import { Failure, NotSupported, Success, } from './step.js';
+import { ImportFailed, NotSupported as ImporterNotSupported, } from '@lde/sparql-importer';
+/**
+ * A pipeline step that imports a database using an {@link Importer} and makes
+ * the import available at a local SPARQL endpoint.
+ */
+export class Import {
+    identifier = 'import';
+    importer;
+    server;
+    forceImport;
+    /**
+     * Create a Pipeline ImportStep.
+     *
+     * @param {object} args
+     * @param args.importer A concrete importer that will import the distribution if needed.
+     * @param args.server SPARQL server that will be started to serve the imported data.
+     * @param args.forceImport Whether to force an import even if the dataset already has a SPARQL distribution.
+     */
+    constructor({ importer, server, forceImport, }) {
+        this.importer = importer;
+        this.server = server;
+        this.forceImport = forceImport ?? false;
+    }
+    async execute(dataset) {
+        if (dataset.getSparqlDistribution()?.isValid && !this.forceImport) {
+            return new NotSupported('A valid SPARQL distribution is available so no import needed');
+        }
+        const result = await this.importer.import(dataset);
+        if (result instanceof ImporterNotSupported) {
+            return new NotSupported('No download distribution available');
+        }
+        if (result instanceof ImportFailed) {
+            return new Failure(result.distribution, result.error);
+        }
+        await this.server.start();
+        dataset.distributions.push(Distribution.sparql(this.server.queryEndpoint));
+        return new Success(dataset, result.distribution);
+    }
+    async finish() {
+        await this.server.stop();
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from './pipeline.js';
+export * from './selector.js';
+export * from './step.js';
+export * from './step/sparqlQuery.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC"}

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from './pipeline.js';
+export * from './selector.js';
+export * from './step.js';
+export * from './step/sparqlQuery.js';

package/dist/pipeline.d.ts

@@ -0,0 +1,11 @@
+import { Selector } from './selector.js';
+import { Step } from './step.js';
+export declare class Pipeline {
+    private readonly config;
+    constructor(config: {
+        selector: Selector;
+        steps: Step[];
+    });
+    run(): Promise<void>;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAc,IAAI,EAAE,MAAM,WAAW,CAAC;AAI7C,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE;IAE7D,GAAG;CAgCjB"}

package/dist/pipeline.js

@@ -0,0 +1,41 @@
+import { NotSupported } from './step.js';
+import { Readable } from 'node:stream';
+export class Pipeline {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async run() {
+        const datasets = await this.config.selector.select();
+        for await (const dataset of datasets) {
+            for (const step of this.config.steps) {
+                const result = await step.execute(dataset);
+                if (result instanceof NotSupported) {
+                    console.error(result);
+                }
+                else if (result instanceof Readable) {
+                    const promise = new Promise((resolve, reject) => {
+                        result.on('data', (data) => {
+                            // TODO: pipe to writers.
+                            console.log('Data:', data);
+                        });
+                        result.on('error', (error) => {
+                            console.error('rejecting');
+                            reject(error);
+                        });
+                        result.on('end', resolve);
+                    });
+                    await promise;
+                }
+            }
+            for (const step of this.config.steps) {
+                if (isFinishable(step)) {
+                    await step.finish();
+                }
+            }
+        }
+    }
+}
+const isFinishable = (step) => {
+    return typeof step.finish === 'function';
+};

package/dist/selector.d.ts

@@ -0,0 +1,40 @@
+import { Dataset } from '@lde/dataset';
+import { Client, Paginator } from '@lde/dataset-registry-client';
+/**
+ * Select {@link Dataset}s for processing in a pipeline.
+ */
+export interface Selector {
+    select(): Promise<Paginator<Dataset>>;
+}
+export declare class ManualDatasetSelection implements Selector {
+    private readonly datasets;
+    constructor(datasets: Dataset[]);
+    select(): Promise<Paginator<Dataset>>;
+}
+/**
+ * Select Datasets from a Dataset Registry.
+ *
+ *
+ *
+ * @example
+ * ```typescript
+ *
+ * ```
+ *
+ * @param {object} options
+ * @param Client options.registry The Dataset Registry Client to query for datasets.
+ * @param string options.query Optional custom SPARQL query to select datasets.
+ * @param object options.criteria Optional search criteria to select datasets.
+ */
+export declare class RegistrySelector implements Selector {
+    private readonly registry;
+    private readonly query?;
+    private readonly criteria?;
+    constructor({ registry, query, criteria, }: {
+        registry: Client;
+        query?: string;
+        criteria?: object;
+    });
+    select(): Promise<Paginator<Dataset>>;
+}
+//# sourceMappingURL=selector.d.ts.map

package/dist/selector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../src/selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;CACvC;AAED,qBAAa,sBAAuB,YAAW,QAAQ;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,OAAO,EAAE;IAE1C,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAG5C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,QAAQ;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;gBAEvB,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB;IAMK,MAAM;CAOb"}

package/dist/selector.js

@@ -0,0 +1,43 @@
+import { Paginator } from '@lde/dataset-registry-client';
+export class ManualDatasetSelection {
+    datasets;
+    constructor(datasets) {
+        this.datasets = datasets;
+    }
+    async select() {
+        return new Paginator(async () => this.datasets, this.datasets.length);
+    }
+}
+/**
+ * Select Datasets from a Dataset Registry.
+ *
+ *
+ *
+ * @example
+ * ```typescript
+ *
+ * ```
+ *
+ * @param {object} options
+ * @param Client options.registry The Dataset Registry Client to query for datasets.
+ * @param string options.query Optional custom SPARQL query to select datasets.
+ * @param object options.criteria Optional search criteria to select datasets.
+ */
+export class RegistrySelector {
+    registry;
+    query;
+    criteria;
+    constructor({ registry, query, criteria, }) {
+        this.registry = registry;
+        this.query = query;
+        this.criteria = criteria;
+    }
+    async select() {
+        if (this.query) {
+            return this.registry.query(this.query);
+        }
+        else {
+            return this.registry.query(this.criteria ?? {});
+        }
+    }
+}

package/dist/step/sparqlQuery.d.ts

@@ -0,0 +1,28 @@
+import { DataEmittingStep, NotSupported } from './../step.js';
+import { Dataset } from '@lde/dataset';
+import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+/**
+ * Arguments for the SparqlQuery step.
+ *
+ * @param identifier Unique identifier for the step.
+ * @param query: SPARQL CONSTRUCT query to execute.
+ * @param fetcher Optional SPARQL endpoint fetcher; defaults to SparqlEndpointFetcher.
+ */
+export interface Args {
+    identifier: string;
+    query: string;
+    fetcher?: SparqlEndpointFetcher;
+}
+/**
+ * Executes a SPARQL CONSTRUCT query and emits the resulting
+ */
+export declare class SparqlQuery implements DataEmittingStep {
+    readonly identifier: string;
+    private readonly query;
+    private readonly fetcher;
+    constructor({ identifier, query, fetcher }: Args);
+    execute(dataset: Dataset): Promise<NotSupported | (import("readable-stream").Readable & import("@rdfjs/types").Stream<import("@rdfjs/types").Quad>)>;
+    static fromFile(filename: string): Promise<SparqlQuery>;
+}
+export declare function fromFile(filename: string): Promise<string>;
+//# sourceMappingURL=sparqlQuery.d.ts.map

package/dist/step/sparqlQuery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sparqlQuery.d.ts","sourceRoot":"","sources":["../../src/step/sparqlQuery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAI9D;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;GAEG;AACH,qBAAa,WAAY,YAAW,gBAAgB;IAClD,SAAgB,UAAU,SAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAEb,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,IAAI;IAM1C,OAAO,CAAC,OAAO,EAAE,OAAO;WAqBV,QAAQ,CAAC,QAAQ,EAAE,MAAM;CAM9C;AAED,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,mBAE9C"}

package/dist/step/sparqlQuery.js

@@ -0,0 +1,37 @@
+import { NotSupported } from './../step.js';
+import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+/**
+ * Executes a SPARQL CONSTRUCT query and emits the resulting
+ */
+export class SparqlQuery {
+    identifier;
+    query;
+    fetcher;
+    constructor({ identifier, query, fetcher }) {
+        this.identifier = identifier;
+        this.query = query;
+        this.fetcher = fetcher ?? new SparqlEndpointFetcher();
+    }
+    async execute(dataset) {
+        const distribution = dataset.getSparqlDistribution();
+        if (null === distribution || !distribution.isValid) {
+            return new NotSupported('No SPARQL distribution available');
+        }
+        const query = this.query
+            .replace('#subjectFilter#', distribution.subjectFilter ?? '')
+            .replace('?dataset', `<${dataset.iri}>`)
+            .replace('#namedGraph#', distribution.namedGraph ? `FROM <${distribution.namedGraph}>` : '');
+        return await this.fetcher.fetchTriples(distribution.accessUrl.toString(), query);
+    }
+    static async fromFile(filename) {
+        return new this({
+            identifier: filename,
+            query: await fromFile(filename),
+        });
+    }
+}
+export async function fromFile(filename) {
+    return (await readFile(resolve(filename))).toString();
+}

package/dist/step.d.ts

@@ -0,0 +1,55 @@
+import { Dataset, Distribution } from '@lde/dataset';
+import type { Stream } from '@rdfjs/types';
+interface AbstractStep {
+    readonly identifier: string;
+}
+export type Step = DataEmittingStep | SingleStep;
+/**
+ * A pipeline step that returns a data-emitting stream of RDF quads.
+ * Failure is expressed by emitting an error event; success by the end event.
+ */
+export interface DataEmittingStep extends AbstractStep {
+    execute(dataset: Dataset): Promise<Stream | NotSupported>;
+}
+/**
+ * A pipeline step that executes an operation without emitting data.
+ */
+export interface SingleStep extends AbstractStep {
+    execute(dataset: Dataset): Promise<NotSupported | Failure | Success>;
+}
+export interface Finishable {
+    finish(): Promise<void>;
+}
+/**
+ * A pipeline step failed to run.
+ *
+ * @param distribution The distribution that was processed.
+ * @param message Optional error message.
+ */
+export declare class Failure {
+    readonly distribution: Distribution;
+    readonly message?: string | undefined;
+    constructor(distribution: Distribution, message?: string | undefined);
+}
+/**
+ * A pipeline ran successfully.
+ *
+ * @param dataset: The dataset, with possible modifications, that was processed.
+ * @param distribution The distribution that was processed.
+ */
+export declare class Success {
+    readonly dataset: Dataset;
+    readonly distribution: Distribution;
+    constructor(dataset: Dataset, distribution: Distribution);
+}
+/**
+ * A pipeline step could not be run because the dataset lacks a distribution supported by the step.
+ *
+ * @param message: A message explaining why the step is not supported.
+ */
+export declare class NotSupported {
+    readonly message: string;
+    constructor(message: string);
+}
+export {};
+//# sourceMappingURL=step.d.ts.map

package/dist/step.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../src/step.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,UAAU,YAAY;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,MAAM,IAAI,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;CAC3D;AAED;;GAEG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;aAEA,YAAY,EAAE,YAAY;aAC1B,OAAO,CAAC,EAAE,MAAM;gBADhB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,MAAM,YAAA;CAEnC;AAED;;;;;GAKG;AACH,qBAAa,OAAO;aAEA,OAAO,EAAE,OAAO;aAChB,YAAY,EAAE,YAAY;gBAD1B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY;CAE7C;AAED;;;;GAIG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C"}

package/dist/step.js

@@ -0,0 +1,39 @@
+/**
+ * A pipeline step failed to run.
+ *
+ * @param distribution The distribution that was processed.
+ * @param message Optional error message.
+ */
+export class Failure {
+    distribution;
+    message;
+    constructor(distribution, message) {
+        this.distribution = distribution;
+        this.message = message;
+    }
+}
+/**
+ * A pipeline ran successfully.
+ *
+ * @param dataset: The dataset, with possible modifications, that was processed.
+ * @param distribution The distribution that was processed.
+ */
+export class Success {
+    dataset;
+    distribution;
+    constructor(dataset, distribution) {
+        this.dataset = dataset;
+        this.distribution = distribution;
+    }
+}
+/**
+ * A pipeline step could not be run because the dataset lacks a distribution supported by the step.
+ *
+ * @param message: A message explaining why the step is not supported.
+ */
+export class NotSupported {
+    message;
+    constructor(message) {
+        this.message = message;
+    }
+}

package/dist/writer/fileWriter.d.ts

@@ -0,0 +1,21 @@
+import { Writer } from './../writer.js';
+import { Dataset, RdfFormat } from '@lde/dataset';
+import { Readable } from 'node:stream';
+import { Quad, Stream } from '@rdfjs/types';
+export declare class FileWriter implements Writer {
+    private readonly directory;
+    private readonly datasetToFilename;
+    private readonly format;
+    private readonly prefixes;
+    constructor({ directory, datasetToFilename, format, prefixes, }: {
+        directory?: string;
+        datasetToFilename: (identifier: string, dataset: Dataset, format: RdfFormat) => stringq;
+        format: RdfFormat;
+        prefixes?: {
+            [key: string]: string;
+        };
+    });
+    write(identifier: string, dataset: Dataset, data: Stream<Quad> & Readable): Promise<void>;
+    private writeStream;
+}
+//# sourceMappingURL=fileWriter.d.ts.map

package/dist/writer/fileWriter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,SAAS,EAA4B,MAAM,cAAc,CAAC;AAE5E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKvC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE5C,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAItB;IACZ,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;gBAEzC,EACV,SAAoB,EACpB,iBAG0C,EAC1C,MAA+B,EAC/B,QAAa,GACd,EAAE;QACD,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,iBAAiB,EAAE,CACjB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,SAAS,KACd,OAAO,CAAC;QACb,MAAM,EAAE,SAAS,CAAC;QAClB,QAAQ,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;SAAE,CAAC;KACtC;IAOY,KAAK,CAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ;IAY/B,OAAO,CAAC,WAAW;CASpB"}

package/dist/writer/fileWriter.js

@@ -0,0 +1,29 @@
+import { RdfFormat, rdfFormatToFileExtension } from '@lde/dataset';
+import fs from 'node:fs';
+import path from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import filenamyifyUrl from 'filenamify-url';
+import { StreamWriter } from 'n3';
+export class FileWriter {
+    directory;
+    datasetToFilename;
+    format;
+    prefixes;
+    constructor({ directory = 'output', datasetToFilename = (identifier, dataset, format) => `${filenamyifyUrl(dataset.iri.toString(), {
+        replacement: '-',
+    })}.${rdfFormatToFileExtension(format)}`, format = RdfFormat['N-Triples'], prefixes = {}, }) {
+        this.directory = directory;
+        this.datasetToFilename = datasetToFilename;
+        this.format = format;
+        this.prefixes = prefixes;
+    }
+    async write(identifier, dataset, data) {
+        await pipeline(data, new StreamWriter({
+            prefixes: this.prefixes,
+            format: this.format,
+        }), this.writeStream(identifier, dataset));
+    }
+    writeStream(identifier, dataset) {
+        return fs.createWriteStream(path.join(this.directory, this.datasetToFilename(identifier, dataset, this.format)), { flags: 'a' });
+    }
+}

package/dist/writer/mergeWriter.d.ts

@@ -0,0 +1,12 @@
+import { Writer } from './../writer.js';
+import { Dataset } from '@lde/dataset';
+import { Quad, Stream } from '@rdfjs/types';
+/**
+ * Buffers output from steps and merges it so you get a single output per dataset.
+ */
+export declare class MergeWriter implements Writer {
+    private readonly decorated;
+    constructor(decorated: Writer);
+    write(dataset: Dataset, data: Stream<Quad>): Promise<void>;
+}
+//# sourceMappingURL=mergeWriter.d.ts.map

package/dist/writer/mergeWriter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mergeWriter.d.ts","sourceRoot":"","sources":["../../src/writer/mergeWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE5C;;GAEG;AACH,qBAAa,WAAY,YAAW,MAAM;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAEjC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CACxE"}

package/dist/writer/mergeWriter.js

@@ -0,0 +1,10 @@
+/**
+ * Buffers output from steps and merges it so you get a single output per dataset.
+ */
+export class MergeWriter {
+    decorated;
+    constructor(decorated) {
+        this.decorated = decorated;
+    }
+    async write(dataset, data) { }
+}

package/dist/writer.d.ts

@@ -0,0 +1,6 @@
+import { Quad, Stream } from '@rdfjs/types';
+import { Dataset } from '@lde/dataset';
+export interface Writer {
+    write(identifier: string, dataset: Dataset, data: Stream<Quad>): Promise<void>;
+}
+//# sourceMappingURL=writer.d.ts.map

package/dist/writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../src/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,WAAW,MAAM;IACrB,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB"}

package/dist/writer.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@lde/pipeline",
+  "version": "0.2.0",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "development": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "@lde/dataset": "0.2.0",
+    "@lde/dataset-registry-client": "0.2.0",
+    "@lde/sparql-importer": "0.0.3",
+    "@lde/sparql-server": "0.2.0",
+    "fetch-sparql-endpoint": "^6.0.0",
+    "tslib": "^2.3.0",
+    "@rdfjs/types": "^2.0.1"
+  }
+}