@lde/pipeline 0.40.0 → 0.41.0

package/dist/selector.d.ts

@@ -11,6 +11,30 @@ export declare class ManualDatasetSelection implements DatasetSelector {
     constructor(datasets: Dataset[]);
     select(): Promise<Paginator<Dataset>>;
 }
+/**
+ * Select one {@link Dataset} per RDF file directly inside a local directory, by
+ * {@link Dataset.localFile}: the directory acts as a registry of data dumps
+ * without descriptions. A file whose extension names no RDF format is skipped
+ * with a warning, and subdirectories are ignored.
+ *
+ * Only pass a directory you trust: everything in it gets read.
+ */
+export declare class DirectorySelector implements DatasetSelector {
+    private readonly directory;
+    /** @param directory Path of the directory. */
+    constructor(directory: string);
+    select(): Promise<Paginator<Dataset>>;
+}
+/**
+ * Select the {@link Dataset}s of several selectors in one run, in the order
+ * given, so a run can take datasets from a registry and from local files alike.
+ * Each selector is read one page at a time.
+ */
+export declare class CombinedSelector implements DatasetSelector {
+    private readonly selectors;
+    constructor(selectors: readonly DatasetSelector[]);
+    select(): Promise<Paginator<Dataset>>;
+}
 /**
  * Select Datasets from a Dataset Registry.
  *

package/dist/selector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../src/selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EACL,MAAM,EACN,SAAS,EACT,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;CACvC;AAED,qBAAa,sBAAuB,YAAW,eAAe;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,OAAO,EAAE;IAE1C,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAG5C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAiB;gBAE/B,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,cAAc,CAAC;KAC3B;IAMK,MAAM;CAOb"}
+{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../src/selector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EACL,MAAM,EACN,SAAS,EACT,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAItC;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;CACvC;AAED,qBAAa,sBAAuB,YAAW,eAAe;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,OAAO,EAAE;IAE1C,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAG5C;AAED;;;;;;;GAOG;AACH,qBAAa,iBAAkB,YAAW,eAAe;IAE3C,OAAO,CAAC,QAAQ,CAAC,SAAS;IADtC,8CAA8C;gBACjB,SAAS,EAAE,MAAM;IAExC,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAiB5C;AAED;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,SAAS,eAAe,EAAE;IAE5D,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CAK5C;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAiB;gBAE/B,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,cAAc,CAAC;KAC3B;IAMK,MAAM;CAOb"}

package/dist/selector.js

@@ -1,4 +1,7 @@
+import { Dataset } from '@lde/dataset';
 import { Paginator, } from '@lde/dataset-registry-client';
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
 export class ManualDatasetSelection {
     datasets;
     constructor(datasets) {
@@ -8,6 +11,52 @@ export class ManualDatasetSelection {
         return new Paginator(async () => this.datasets, this.datasets.length);
     }
 }
+/**
+ * Select one {@link Dataset} per RDF file directly inside a local directory, by
+ * {@link Dataset.localFile}: the directory acts as a registry of data dumps
+ * without descriptions. A file whose extension names no RDF format is skipped
+ * with a warning, and subdirectories are ignored.
+ *
+ * Only pass a directory you trust: everything in it gets read.
+ */
+export class DirectorySelector {
+    directory;
+    /** @param directory Path of the directory. */
+    constructor(directory) {
+        this.directory = directory;
+    }
+    async select() {
+        const entries = await readdir(this.directory, { withFileTypes: true });
+        const datasets = entries
+            .filter((entry) => !entry.isDirectory())
+            .map((entry) => join(this.directory, entry.name))
+            .sort()
+            .flatMap((path) => {
+            try {
+                return [Dataset.localFile(path)];
+            }
+            catch (error) {
+                console.warn(`Skipping ${path}: ${error.message}`);
+                return [];
+            }
+        });
+        return new ManualDatasetSelection(datasets).select();
+    }
+}
+/**
+ * Select the {@link Dataset}s of several selectors in one run, in the order
+ * given, so a run can take datasets from a registry and from local files alike.
+ * Each selector is read one page at a time.
+ */
+export class CombinedSelector {
+    selectors;
+    constructor(selectors) {
+        this.selectors = selectors;
+    }
+    async select() {
+        return new ConcatenatedPaginator(await Promise.all(this.selectors.map((selector) => selector.select())));
+    }
+}
 /**
  * Select Datasets from a Dataset Registry.
  *
@@ -41,3 +90,19 @@ export class RegistrySelector {
         }
     }
 }
+/**
+ * Several paginators read as one. Each iteration reads them afresh, so loops
+ * over it never share state, and each paginator still reads one page at a time.
+ */
+class ConcatenatedPaginator extends Paginator {
+    paginators;
+    constructor(paginators) {
+        super(async () => [], paginators.reduce((total, paginator) => total + paginator.total, 0));
+        this.paginators = paginators;
+    }
+    async *[Symbol.asyncIterator]() {
+        for (const paginator of this.paginators) {
+            yield* paginator;
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.40.0",
+  "version": "0.41.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -24,11 +24,11 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "^0.9.0",
-    "@lde/dataset-registry-client": "^0.11.0",
-    "@lde/distribution-health": "^0.5.0",
-    "@lde/distribution-probe": "^0.4.0",
-    "@lde/sparql-importer": "^0.9.0",
+    "@lde/dataset": "^0.10.0",
+    "@lde/dataset-registry-client": "^0.12.0",
+    "@lde/distribution-health": "^0.6.0",
+    "@lde/distribution-probe": "^0.5.0",
+    "@lde/sparql-importer": "^0.10.0",
     "@lde/sparql-server": "^0.6.0",
     "@rdfjs/namespace": "^2.0.1",
     "@rdfjs/types": "^2.0.1",