@lde/search-indexer 0.27.1 → 0.28.0

package/dist/cli.js

@@ -8,7 +8,7 @@ const program = new Command()
     .name('search-indexer')
     .description('Run the object-grain search indexer: select datasets, extract and project each root type per the mounted schema module, and rebuild the Typesense collections. Configuration comes from environment variables; see the README.')
     .version(version)
-    .argument('[datasets...]', 'dataset IRIs to index (overrides the DATASETS environment variable)')
+    .argument('[datasets...]', 'dataset IRIs and file: URLs to index (overrides the DATASETS environment variable)')
     .option('--check', 'validate the configuration and schema module, then exit without indexing');
 program.parse();
 try {

package/dist/composition.d.ts

@@ -3,7 +3,10 @@ import type { RootType, SearchDocument, SearchSchema } from '@lde/search';
 import { type QleverOptions } from '@lde/sparql-qlever';
 import type { Client } from 'typesense';
 import type { IndexerConfig } from './config.js';
-/** The registry-backed dataset selection the configuration describes. */
+/**
+ * The dataset selection the configuration describes: what the registry selects,
+ * followed by the local files and directories it names.
+ */
 export declare function datasetSelectorFrom(config: IndexerConfig): DatasetSelector;
 /**
  * The engine-writer factory the configuration describes: an

package/dist/composition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAIA,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EAEb,MAAM,aAAa,CAAC;AAMrB,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAK1E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAmBxE;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,EACrB,oBAAoB,CAAC,EAAE,aAAa,CAAC,sBAAsB,CAAC,GAC3D,oBAAoB,GAAG,SAAS,CAmClC"}
+{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EAEb,MAAM,aAAa,CAAC;AAMrB,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAiB1E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAmBxE;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,EACrB,oBAAoB,CAAC,EAAE,aAAa,CAAC,sBAAsB,CAAC,GAC3D,oBAAoB,GAAG,SAAS,CAmClC"}

package/dist/composition.js

@@ -1,16 +1,28 @@
 // The composition pieces of createSearchIndexer, each usable on its own so a
 // deployment that needs one bespoke stage keeps the rest of the wiring
 // (Pipeline keeps its own wiring private).
+import { Dataset } from '@lde/dataset';
 import { Client as RegistryClient } from '@lde/dataset-registry-client';
-import { ImportResolver, RegistrySelector, SparqlDistributionResolver, } from '@lde/pipeline';
+import { CombinedSelector, DirectorySelector, ImportResolver, ManualDatasetSelection, RegistrySelector, SparqlDistributionResolver, } from '@lde/pipeline';
 import { BlueGreenRebuild, deriveCollectionName, InPlaceRebuild, } from '@lde/search-typesense';
 import { createQlever } from '@lde/sparql-qlever';
-/** The registry-backed dataset selection the configuration describes. */
+/**
+ * The dataset selection the configuration describes: what the registry selects,
+ * followed by the local files and directories it names.
+ */
 export function datasetSelectorFrom(config) {
-    return new RegistrySelector({
-        registry: new RegistryClient(config.registryEndpoint),
-        criteria: config.datasetCriteria,
-    });
+    const selectors = config.localSources.map((path) => path.endsWith('/')
+        ? new DirectorySelector(path)
+        : new ManualDatasetSelection([Dataset.localFile(path)]));
+    if (config.registryEndpoint) {
+        selectors.unshift(new RegistrySelector({
+            registry: new RegistryClient(config.registryEndpoint),
+            criteria: config.datasetCriteria,
+        }));
+    }
+    return selectors.length === 1
+        ? selectors[0]
+        : new CombinedSelector(selectors);
 }
 /**
  * The engine-writer factory the configuration describes: an

package/dist/config.d.ts

@@ -9,8 +9,9 @@ export interface IndexerConfig {
     /** Path of the mounted schema-declaration module (`SCHEMA_MODULE`). */
     readonly schemaModulePath: string;
     /** SPARQL endpoint of the DCAT dataset registry the selection queries
-     *  (`REGISTRY_ENDPOINT`). */
-    readonly registryEndpoint: URL;
+     *  (`REGISTRY_ENDPOINT`). Undefined when `DATASETS` names only local data, so
+     *  the run has no registry to read. */
+    readonly registryEndpoint?: URL;
     /**
      * Root types extracted from {@link registryEndpoint} instead of from each
      * dataset’s own distribution (`REGISTRY_ROOT_TYPES`, whitespace- or
@@ -24,6 +25,10 @@ export interface IndexerConfig {
      *  search criteria (`DATASET_CRITERIA`), or every dataset when neither is
      *  set. */
     readonly datasetCriteria: SearchCriteria;
+    /** Local data to index besides what the registry selects, from the `file:`
+     *  URLs in `DATASETS`: paths of files, each one dataset, and of directories,
+     *  which end in `/`, with one dataset per RDF file. */
+    readonly localSources: readonly string[];
     /** Where the Typesense engine writes (`TYPESENSE_*`). */
     readonly typesense: TypesenseConnection;
     /** How each type’s collection is rebuilt (`REBUILD_MODE`): update the live

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;iCAC6B;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;;iBAGa;IACb,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,GAAG,CACA;IACE,qEAAqE;IACrE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;iDAG6C;IAC7C,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B,GACD;IACE;8DAC0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACzB,CACJ,CAAC;AAUF;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAsFf"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAInE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;;2CAEuC;IACvC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,GAAG,CAAC;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC;;2DAEuD;IACvD,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;;iBAGa;IACb,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,GAAG,CACA;IACE,qEAAqE;IACrE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;iDAG6C;IAC7C,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B,GACD;IACE;8DAC0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACzB,CACJ,CAAC;AAUF;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAkGf"}

package/dist/config.js

@@ -1,3 +1,6 @@
+import { Dataset } from '@lde/dataset';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
 const QLEVER_MODES = ['docker', 'remote'];
 const IMPORT_STRATEGIES = [
     'sparql',
@@ -43,7 +46,13 @@ export function configFromEnvironment(environment) {
             return new URL('http://invalid.invalid');
         }
     };
-    const registryEndpoint = url('REGISTRY_ENDPOINT', required('REGISTRY_ENDPOINT'));
+    const datasets = names(environment['DATASETS']);
+    const registryEndpoint = datasets.length > 0 && datasets.every(isLocal)
+        ? undefined
+        : url('REGISTRY_ENDPOINT', required('REGISTRY_ENDPOINT'));
+    if (registryEndpoint?.protocol === 'file:') {
+        problems.push('REGISTRY_ENDPOINT must be a SPARQL endpoint: name local files and directories in DATASETS');
+    }
     const protocol = environment['TYPESENSE_PROTOCOL'] ?? 'http';
     if (protocol !== 'http' && protocol !== 'https') {
         problems.push(`TYPESENSE_PROTOCOL must be “http” or “https”, got “${protocol}”`);
@@ -52,14 +61,16 @@ export function configFromEnvironment(environment) {
     if (rebuildMode !== 'in-place' && rebuildMode !== 'blue-green') {
         problems.push(`REBUILD_MODE must be “in-place” or “blue-green”, got “${rebuildMode}”`);
     }
-    const datasetCriteria = criteriaFromEnvironment(environment, problems);
+    const datasetCriteria = criteriaFromEnvironment(environment, datasets, problems);
     const provenance = provenanceFromEnvironment(environment, rebuildMode, problems);
     const qlever = qleverFromEnvironment(environment, problems);
+    const localSources = localSourcesFrom(datasets, qlever, problems);
     const config = {
         schemaModulePath: environment['SCHEMA_MODULE'] ?? '/config/search-schema.mjs',
         registryEndpoint,
         registryRootTypes: names(environment['REGISTRY_ROOT_TYPES']),
         datasetCriteria,
+        localSources,
         typesense: {
             host: required('TYPESENSE_HOST'),
             port: integer('TYPESENSE_PORT', 8108),
@@ -87,17 +98,53 @@ function names(value) {
         ? []
         : value.split(/[\s,]+/).filter((name) => name.length > 0);
 }
-/** `DATASETS` (whitespace-separated IRIs) or `DATASET_CRITERIA` (a JSON
- *  object) – never both; neither selects the whole registry. */
-function criteriaFromEnvironment(environment, problems) {
-    const datasets = environment['DATASETS'];
+/** Whether a `DATASETS` entry names local data rather than a registered dataset.
+ *  Parsed, because a URL scheme is case-insensitive. */
+function isLocal(entry) {
+    return URL.parse(entry)?.protocol === 'file:';
+}
+/** The paths of the `file:` URLs in `DATASETS`. Only QLever can read a file, so
+ *  they require the import path. */
+function localSourcesFrom(datasets, qlever, problems) {
+    const entries = datasets.filter(isLocal);
+    if (entries.length > 0 && !qlever) {
+        problems.push('A file: URL in DATASETS requires QLEVER_IMAGE or QLEVER_MODE=remote: a file must be imported before it can be queried');
+    }
+    const paths = entries.flatMap((entry) => {
+        try {
+            const path = fileURLToPath(entry);
+            if (!path.endsWith('/')) {
+                Dataset.localFile(path);
+            }
+            return [path];
+        }
+        catch (error) {
+            problems.push(`DATASETS contains “${entry}”, which is not a local RDF file or a directory ending in /: ${error.message}`);
+            return [];
+        }
+    });
+    const directories = new Set(paths.filter((path) => path.endsWith('/')));
+    paths.forEach((path, index) => {
+        if (paths.indexOf(path) !== index) {
+            problems.push(`DATASETS lists “${path}” twice`);
+        }
+        else if (!path.endsWith('/') && directories.has(`${dirname(path)}/`)) {
+            problems.push(`DATASETS lists “${path}” and its directory, which already selects it`);
+        }
+    });
+    return paths;
+}
+/** `DATASETS` (whitespace-separated IRIs, of which `file:` URLs are local data
+ *  rather than registered datasets) or `DATASET_CRITERIA` (a JSON object) –
+ *  never both; neither selects the whole registry. */
+function criteriaFromEnvironment(environment, datasets, problems) {
     const criteriaJson = environment['DATASET_CRITERIA'];
-    if (datasets && criteriaJson) {
+    if (environment['DATASETS'] && criteriaJson) {
         problems.push('DATASETS and DATASET_CRITERIA are mutually exclusive; set at most one');
         return {};
     }
-    if (datasets) {
-        const iris = datasets.split(/[\s,]+/).filter((iri) => iri.length > 0);
+    if (environment['DATASETS']) {
+        const iris = datasets.filter((iri) => !isLocal(iri));
         for (const iri of iris) {
             try {
                 new URL(iri);

package/dist/indexer.js

@@ -34,7 +34,7 @@ export async function createSearchIndexer(config, extensions = {}) {
         datasets: datasetSelectorFrom(config),
         distributionResolver: distributionResolverFrom(config, extensions.replaceJsonLdContext),
         writerFor: writerFactoryFrom(client, config),
-        registryTypes: config.registryRootTypes.length > 0
+        registryTypes: config.registryEndpoint && config.registryRootTypes.length > 0
             ? {
                 endpoint: config.registryEndpoint,
                 names: config.registryRootTypes,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-indexer",
-  "version": "0.27.1",
+  "version": "0.28.0",
   "description": "The @lde/search-pipeline indexer as a bootable process and prebuilt Docker image: loads a mounted schema-declaration module, dataset selection, Typesense connection and rebuild mode from the environment, then runs the object-grain searchIndexerPipeline once to completion. The composition layer that binds the engine-agnostic @lde/search-pipeline stages to @lde/search-typesense – the write-path counterpart of @lde/search-api-server.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -28,13 +28,14 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset-registry-client": "^0.11.0",
-    "@lde/pipeline": "^0.40.1",
-    "@lde/pipeline-console-reporter": "^0.31.1",
+    "@lde/dataset": "^0.10.0",
+    "@lde/dataset-registry-client": "^0.12.0",
+    "@lde/pipeline": "^0.41.0",
+    "@lde/pipeline-console-reporter": "^0.32.0",
     "@lde/search": "^0.34.0",
-    "@lde/search-pipeline": "^0.37.1",
-    "@lde/search-typesense": "^0.40.1",
-    "@lde/sparql-qlever": "^0.19.0",
+    "@lde/search-pipeline": "^0.38.0",
+    "@lde/search-typesense": "^0.41.0",
+    "@lde/sparql-qlever": "^0.20.0",
     "commander": "^15.0.0",
     "tslib": "^2.3.0",
     "typesense": "^3.1.1"