@lde/pipeline-void 0.2.15 → 0.2.16

package/README.md

@@ -5,7 +5,16 @@ VOiD (Vocabulary of Interlinked Datasets) statistical analysis for RDF datasets.
 ## Analyzers
 
 - **SparqlQueryAnalyzer** — Execute SPARQL CONSTRUCT queries with template substitution
-- **PerClassAnalyzer** — Two-phase analyzer that iterates over classes to avoid timeouts
+
+## Per-class stages
+
+Factory functions that create `Stage` instances for per-class analysis.
+Each stage first selects classes from the endpoint, then runs a CONSTRUCT query
+with `?class` bound via VALUES:
+
+- `createDatatypeStage` — per-class datatype partitions
+- `createLanguageStage` — per-class language tags
+- `createObjectClassStage` — per-class object class partitions
 
 ## SPARQL Queries
 
@@ -33,14 +42,30 @@ Generic VOiD analysis queries included:
 ## Usage
 
 ```typescript
-import { SparqlQueryAnalyzer } from '@lde/pipeline-void';
+import {
+  SparqlQueryAnalyzer,
+  Success,
+  createDatatypeStage,
+} from '@lde/pipeline-void';
+import { Distribution } from '@lde/dataset';
 
-// Load a query from file
+// Simple CONSTRUCT query analyzer
 const analyzer = await SparqlQueryAnalyzer.fromFile('triples.rq');
-
-// Execute against a dataset
 const result = await analyzer.execute(dataset);
 if (result instanceof Success) {
   // result.data contains the VOiD statistics as RDF
 }
+
+// Per-class stage (streaming)
+const distribution = Distribution.sparql(new URL('http://example.com/sparql'));
+const stage = await createDatatypeStage(distribution);
+const quads = await stage.run(dataset, distribution);
+```
+
+## Validation
+
+```sh
+npx nx test pipeline-void
+npx nx lint pipeline-void
+npx nx typecheck pipeline-void
 ```

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export { Stage } from '@lde/pipeline';
 export { type Analyzer, BaseAnalyzer, Success, Failure, NotSupported, } from '@lde/pipeline/analyzer';
 export * from './sparqlQueryAnalyzer.js';
 export * from './perClassAnalyzer.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,QAAQ,EACb,YAAY,EACZ,OAAO,EACP,OAAO,EACP,YAAY,GACb,MAAM,wBAAwB,CAAC;AAChC,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EACL,KAAK,QAAQ,EACb,YAAY,EACZ,OAAO,EACP,OAAO,EACP,YAAY,GACb,MAAM,wBAAwB,CAAC;AAChC,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,iBAAiB,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export { Stage } from '@lde/pipeline';
 export { BaseAnalyzer, Success, Failure, NotSupported, } from '@lde/pipeline/analyzer';
 export * from './sparqlQueryAnalyzer.js';
 export * from './perClassAnalyzer.js';

package/dist/perClassAnalyzer.d.ts

@@ -1,51 +1,6 @@
-import { Dataset } from '@lde/dataset';
-import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
-import { BaseAnalyzer, Success, Failure, NotSupported } from '@lde/pipeline/analyzer';
-export interface PerClassAnalyzerOptions {
-    /**
-     * Timeout for SPARQL queries in milliseconds.
-     * @default 300000 (5 minutes)
-     */
-    timeout?: number;
-    /**
-     * Custom SparqlEndpointFetcher instance.
-     */
-    fetcher?: SparqlEndpointFetcher;
-    /**
-     * Maximum number of classes to analyze.
-     * @default 1000
-     */
-    maxClasses?: number;
-}
-/**
- * Two-phase analyzer that first retrieves classes, then runs a query for each class.
- *
- * This approach prevents timeouts and OOM errors on large datasets by splitting
- * the analysis into smaller queries per class.
- *
- * Supports legacy template substitution:
- * - `#subjectFilter#` — replaced with the dataset's subject filter (if any)
- * - `#namedGraph#` — replaced with `FROM <graph>` clause if the distribution has a named graph
- * - `?dataset` — replaced with the dataset IRI
- * - `<#class#>` — replaced with the current class IRI
- */
-export declare class PerClassAnalyzer extends BaseAnalyzer {
-    readonly name: string;
-    private readonly fetcher;
-    private readonly query;
-    private readonly maxClasses;
-    constructor(name: string, query: string, options?: PerClassAnalyzerOptions);
-    /**
-     * Create an analyzer from a query file in the queries directory.
-     *
-     * @param filename Query filename (e.g., 'class-property-datatypes.rq')
-     * @param options Optional analyzer options
-     */
-    static fromFile(filename: string, options?: PerClassAnalyzerOptions): Promise<PerClassAnalyzer>;
-    execute(dataset: Dataset): Promise<Success | Failure | NotSupported>;
-    private getClasses;
-}
-export declare function createDatatypeAnalyzer(options?: PerClassAnalyzerOptions): Promise<PerClassAnalyzer>;
-export declare function createLanguageAnalyzer(options?: PerClassAnalyzerOptions): Promise<PerClassAnalyzer>;
-export declare function createObjectClassAnalyzer(options?: PerClassAnalyzerOptions): Promise<PerClassAnalyzer>;
+import { Distribution } from '@lde/dataset';
+import { Stage } from '@lde/pipeline';
+export declare function createDatatypeStage(distribution: Distribution): Promise<Stage>;
+export declare function createLanguageStage(distribution: Distribution): Promise<Stage>;
+export declare function createObjectClassStage(distribution: Distribution): Promise<Stage>;
 //# sourceMappingURL=perClassAnalyzer.d.ts.map

package/dist/perClassAnalyzer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"perClassAnalyzer.d.ts","sourceRoot":"","sources":["../src/perClassAnalyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAQrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAG9D,OAAO,EACL,YAAY,EACZ,OAAO,EACP,OAAO,EACP,YAAY,EACb,MAAM,wBAAwB,CAAC;AAIhC,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,gBAAiB,SAAQ,YAAY;aAM9B,IAAI,EAAE,MAAM;IAL9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAGlB,IAAI,EAAE,MAAM,EAC5B,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,uBAAuB;IAYnC;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC,gBAAgB,CAAC;IAKf,OAAO,CAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,OAAO,GAAG,OAAO,GAAG,YAAY,CAAC;YAsC9B,UAAU;CAkCzB;AAED,wBAAgB,sBAAsB,CACpC,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAE3B;AAED,wBAAgB,sBAAsB,CACpC,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAE3B;AAED,wBAAgB,yBAAyB,CACvC,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAE3B"}
+{"version":3,"file":"perClassAnalyzer.d.ts","sourceRoot":"","sources":["../src/perClassAnalyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,KAAK,EAIN,MAAM,eAAe,CAAC;AAkDvB,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,KAAK,CAAC,CAEhB;AAED,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,KAAK,CAAC,CAEhB;AAED,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,KAAK,CAAC,CAEhB"}

package/dist/perClassAnalyzer.js

@@ -1,102 +1,46 @@
-import { SparqlConstructExecutor, substituteQueryTemplates, readQueryFile, collect, } from '@lde/pipeline';
-import { Store } from 'n3';
-import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+import { Stage, SparqlSelector, SparqlConstructExecutor, readQueryFile, } from '@lde/pipeline';
 import { resolve, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { BaseAnalyzer, Success, Failure, NotSupported, } from '@lde/pipeline/analyzer';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /**
- * Two-phase analyzer that first retrieves classes, then runs a query for each class.
+ * Create a Stage that first selects classes from the endpoint,
+ * then runs a per-class CONSTRUCT query with `?class` bound via VALUES.
  *
- * This approach prevents timeouts and OOM errors on large datasets by splitting
- * the analysis into smaller queries per class.
- *
- * Supports legacy template substitution:
- * - `#subjectFilter#` — replaced with the dataset's subject filter (if any)
- * - `#namedGraph#` — replaced with `FROM <graph>` clause if the distribution has a named graph
- * - `?dataset` — replaced with the dataset IRI
- * - `<#class#>` — replaced with the current class IRI
+ * Replaces the legacy `PerClassAnalyzer` two-phase loop with streaming.
  */
-export class PerClassAnalyzer extends BaseAnalyzer {
-    name;
-    fetcher;
-    query;
-    maxClasses;
-    constructor(name, query, options) {
-        super();
-        this.name = name;
-        this.query = query;
-        this.fetcher =
-            options?.fetcher ??
-                new SparqlEndpointFetcher({
-                    timeout: options?.timeout ?? 300_000,
-                });
-        this.maxClasses = options?.maxClasses ?? 1000;
-    }
-    /**
-     * Create an analyzer from a query file in the queries directory.
-     *
-     * @param filename Query filename (e.g., 'class-property-datatypes.rq')
-     * @param options Optional analyzer options
-     */
-    static async fromFile(filename, options) {
-        const query = await readQueryFile(resolve(__dirname, 'queries', filename));
-        return new PerClassAnalyzer(filename, query, options);
-    }
-    async execute(dataset) {
-        const sparqlDistribution = dataset.getSparqlDistribution();
-        if (sparqlDistribution === null) {
-            return new NotSupported('No SPARQL distribution available');
-        }
-        const store = new Store();
-        try {
-            // Phase 1: Get all classes.
-            const classes = await this.getClasses(sparqlDistribution, dataset);
-            // Phase 2: Run query for each class.
-            for (const classIri of classes) {
-                const substituted = substituteQueryTemplates(this.query.replaceAll('<#class#>', `<${classIri}>`), sparqlDistribution, dataset);
-                const executor = new SparqlConstructExecutor({
-                    query: substituted,
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    fetcher: this.fetcher,
-                });
-                const stream = await executor.execute(dataset, sparqlDistribution);
-                store.addQuads([...(await collect(stream))]);
-            }
-        }
-        catch (e) {
-            const accessUrl = sparqlDistribution.accessUrl;
-            return new Failure(accessUrl ?? new URL('unknown://'), e instanceof Error ? e.message : undefined);
-        }
-        return new Success(store);
-    }
-    async getClasses(distribution, dataset) {
-        const classQuery = substituteQueryTemplates(`SELECT DISTINCT ?class
-       #namedGraph#
-       WHERE {
-         #subjectFilter#
-         ?s a ?class .
-       }
-       LIMIT ${this.maxClasses}`, distribution, dataset);
-        const bindings = await this.fetcher.fetchBindings(distribution.accessUrl.toString(), classQuery);
-        const classes = [];
-        for await (const binding of bindings) {
-            // Bindings are Record<string, RDF.Term>.
-            const bindingRecord = binding;
-            const classValue = bindingRecord['class'];
-            if (classValue && classValue.termType === 'NamedNode') {
-                classes.push(classValue.value);
-            }
-        }
-        return classes;
-    }
+async function createPerClassStage(queryFilename, distribution) {
+    const rawQuery = await readQueryFile(resolve(__dirname, 'queries', queryFilename));
+    // Pre-process #subjectFilter# before the query is parsed as SPARQL.
+    const subjectFilter = distribution.subjectFilter ?? '';
+    const query = rawQuery.replace('#subjectFilter#', subjectFilter);
+    // Build the selector SELECT query (same substitution for subjectFilter).
+    const fromClause = distribution.namedGraph
+        ? `FROM <${distribution.namedGraph}>`
+        : '';
+    const selectorQuery = [
+        'SELECT DISTINCT ?class',
+        fromClause,
+        `WHERE { ${subjectFilter} ?s a ?class . }`,
+        'LIMIT 1000',
+    ].join('\n');
+    const selector = new SparqlSelector({
+        query: selectorQuery,
+        endpoint: distribution.accessUrl,
+        pageSize: 1000,
+    });
+    const executor = new SparqlConstructExecutor({ query });
+    return new Stage({
+        name: queryFilename,
+        selector,
+        executors: executor,
+    });
 }
-export function createDatatypeAnalyzer(options) {
-    return PerClassAnalyzer.fromFile('class-property-datatypes.rq', options);
+export function createDatatypeStage(distribution) {
+    return createPerClassStage('class-property-datatypes.rq', distribution);
 }
-export function createLanguageAnalyzer(options) {
-    return PerClassAnalyzer.fromFile('class-property-languages.rq', options);
+export function createLanguageStage(distribution) {
+    return createPerClassStage('class-property-languages.rq', distribution);
 }
-export function createObjectClassAnalyzer(options) {
-    return PerClassAnalyzer.fromFile('class-property-object-classes.rq', options);
+export function createObjectClassStage(distribution) {
+    return createPerClassStage('class-property-object-classes.rq', distribution);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline-void",
-  "version": "0.2.15",
+  "version": "0.2.16",
   "description": "VOiD (Vocabulary of Interlinked Datasets) statistical analysis for RDF datasets",
   "repository": {
     "url": "https://github.com/ldengine/lde",