@lde/search-pipeline 0.15.0 → 0.15.2

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export { searchIndexerPipeline } from './search-indexer-pipeline.js';
 export type { SearchIndexerPipelineOptions } from './search-indexer-pipeline.js';
-export { searchStages, selectByClass } from './search-stages.js';
+export { registrySource, searchStages, selectByClass, } from './search-stages.js';
 export type { SearchStagesOptions, SearchStageType } from './search-stages.js';
 export { extractionQuery, extractionQueryString } from './extraction.js';
 export type { ExtractionOptions } from './extraction.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,YAAY,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACzE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,YAAY,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AACjF,OAAO,EACL,cAAc,EACd,YAAY,EACZ,aAAa,GACd,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACzE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}

package/dist/index.js

@@ -1,4 +1,4 @@
 export { searchIndexerPipeline } from './search-indexer-pipeline.js';
-export { searchStages, selectByClass } from './search-stages.js';
+export { registrySource, searchStages, selectByClass, } from './search-stages.js';
 export { extractionQuery, extractionQueryString } from './extraction.js';
 export { searchIndexWriter } from './search-index-writer.js';

package/dist/search-indexer-pipeline.d.ts

@@ -35,6 +35,25 @@ export interface SearchIndexerPipelineOptions {
      * it.
      */
     writerFor: (searchType: RootType, schema: SearchSchema) => Writer<SearchDocument>;
+    /**
+     * Root types whose triples come from the **dataset registry** rather than
+     * from each dataset’s own distribution – typically the dataset description
+     * itself and what hangs off it, which live in the register because
+     * registering a dataset is submitting a description
+     * ({@link registrySource}).
+     *
+     * Routing is deployment topology, so it is configured here and not declared
+     * on the {@link SearchType}: a type is defined by its `class`, not by where
+     * its triples come from. Name the types by their `name`; an unknown name
+     * throws here, at wiring time, rather than yielding an empty collection at
+     * run time.
+     */
+    registryTypes?: {
+        /** The registry’s SPARQL endpoint – where these types read. */
+        endpoint: URL;
+        /** {@link SearchType.name}s of the root types to source from it. */
+        names: readonly string[];
+    };
     /**
      * Optional per-dataset processing memory: skip a dataset whose source
      * fingerprint and {@link pipelineVersion} both match the stored record.

package/dist/search-indexer-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-indexer-pipeline.d.ts","sourceRoot":"","sources":["../src/search-indexer-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAEL,QAAQ,EACR,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,iDAAiD;AACjD,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;IACtC;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;;;OAOG;IACH,SAAS,EAAE,CACT,UAAU,EAAE,QAAQ,EACpB,MAAM,EAAE,YAAY,KACjB,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,4BAA4B,GACpC,QAAQ,CAAC,mBAAmB,CAAC,CAoB/B"}
+{"version":3,"file":"search-indexer-pipeline.d.ts","sourceRoot":"","sources":["../src/search-indexer-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAEL,QAAQ,EACR,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,iDAAiD;AACjD,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;IACtC;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;;;OAOG;IACH,SAAS,EAAE,CACT,UAAU,EAAE,QAAQ,EACpB,MAAM,EAAE,YAAY,KACjB,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5B;;;;;;;;;;;;OAYG;IACH,aAAa,CAAC,EAAE;QACd,+DAA+D;QAC/D,QAAQ,EAAE,GAAG,CAAC;QACd,oEAAoE;QACpE,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;KAC1B,CAAC;IACF;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,4BAA4B,GACpC,QAAQ,CAAC,mBAAmB,CAAC,CA0B/B"}

package/dist/search-indexer-pipeline.js

@@ -1,6 +1,6 @@
 import { ManualDatasetSelection, Pipeline, } from '@lde/pipeline';
 import { searchIndexWriter } from './search-index-writer.js';
-import { searchStages, selectByClass } from './search-stages.js';
+import { registrySource, searchStages, selectByClass, } from './search-stages.js';
 /**
  * Wire the common **object-grain** search indexer and return the ready-to-run
  * {@link Pipeline}: one projecting stage per {@link RootType} in the schema,
@@ -35,7 +35,11 @@ import { searchStages, selectByClass } from './search-stages.js';
  * owns no capability of its own.
  */
 export function searchIndexerPipeline(options) {
-    const { schema, datasets } = options;
+    const { schema, datasets, registryTypes } = options;
+    const fromRegistry = registryTypeNames(schema, registryTypes?.names);
+    const sourceFor = registryTypes === undefined
+        ? undefined
+        : registrySource(registryTypes.endpoint);
     return new Pipeline({
         datasetSelector: Array.isArray(datasets)
             ? new ManualDatasetSelection(datasets)
@@ -47,6 +51,7 @@ export function searchIndexerPipeline(options) {
                 searchType,
                 rootVariable: 'root',
                 itemSelector: selectByClass(searchType),
+                sourceFor: fromRegistry.has(searchType.name) ? sourceFor : undefined,
             })),
         }),
         writers: searchIndexWriter({ schema, writerFor: options.writerFor }),
@@ -55,3 +60,22 @@ export function searchIndexerPipeline(options) {
         reporter: options.reporter,
     });
 }
+/**
+ * The declared registry-sourced type names, validated against the schema. An
+ * unrecognised name is a configuration mistake that would otherwise be silent –
+ * the type would read the dataset’s distribution, find no dataset description
+ * there, and ship an empty collection – so it throws at wiring time.
+ */
+function registryTypeNames(schema, declaredNames) {
+    const names = new Set(declaredNames ?? []);
+    const declared = new Set([...schema.values()].map((searchType) => searchType.name));
+    const unknown = [...names].filter((name) => !declared.has(name));
+    if (unknown.length > 0) {
+        throw new Error(`Unknown registry root type(s) ${unknown
+            .map((name) => `“${name}”`)
+            .join(', ')}; the schema declares ${[...declared]
+            .map((name) => `“${name}”`)
+            .join(', ')}.`);
+    }
+    return names;
+}

package/dist/search-stages.d.ts

@@ -1,4 +1,4 @@
-import { Stage, type ItemSelector, type StageReaders } from '@lde/pipeline';
+import { Stage, type ItemSelector, type StageOptions, type StageReaders } from '@lde/pipeline';
 import { type RootType, type SearchSchema } from '@lde/search';
 import type { TypedSearchDocument } from './typed-search-document.js';
 /** One root type’s stage in a search pipeline. */
@@ -34,6 +34,13 @@ export interface SearchStageType {
      * readers, or attach transforms.
      */
     readers?: StageReaders;
+    /**
+     * Where this type’s stage reads, when that is not the dataset’s own
+     * distribution – see {@link StageOptions.sourceFor}. {@link registrySource}
+     * builds the one that matters here: a root type described by the **dataset
+     * registry** rather than by the dataset’s data.
+     */
+    sourceFor?: StageOptions<TypedSearchDocument>['sourceFor'];
     /**
      * Roots (and so documents) per batch – the memory bound. Under a root-bound
      * selector it moves memory and request count, never output.
@@ -94,4 +101,44 @@ export declare function searchStages(options: SearchStagesOptions): Stage<TypedS
  * SPARQL reader).
  */
 export declare function selectByClass(searchType: RootType, rootVariable?: string): ItemSelector;
+/**
+ * A {@link SearchStageType.sourceFor} that reads a root type from the **dataset
+ * registry** instead of from the dataset’s own data – the endpoint fixed,
+ * scoped to the graph the dataset in hand names.
+ *
+ * A dataset’s description is governed by a different application profile from
+ * the objects it contains, and it lives in the register, not in the
+ * distribution: registering a dataset *is* submitting a description, so the
+ * register is the one source that covers every dataset a pipeline can select.
+ * Nothing obliges a publisher to ship a description of the dataset in its own
+ * dump.
+ *
+ * The graph scoping is what keeps the stage per-dataset. A register holds every
+ * registration, so an unscoped stage would index the whole catalogue for each
+ * dataset processed; scoped, each pass sees exactly the one registration – and
+ * the roots a `selectByClass` finds inside that graph are that dataset’s own.
+ * It presumes the register names each registration’s graph after the dataset
+ * IRI, which is how a DCAT register that crawls per registration stores it (the
+ * NDE Dataset Register does so for every registration it holds).
+ *
+ * Routing is deployment topology, deliberately kept out of the
+ * {@link SearchType}: a type is defined by its `class`, not by where its
+ * triples come from, so the same declaration serves a deployment that sources
+ * it differently.
+ *
+ * ```ts
+ * searchStages({
+ *   schema,
+ *   types: [...schema.values()].map((searchType) => ({
+ *     searchType,
+ *     rootVariable: 'root',
+ *     itemSelector: selectByClass(searchType),
+ *     sourceFor: registryTypeNames.has(searchType.name)
+ *       ? registrySource(registryEndpoint)
+ *       : undefined,
+ *   })),
+ * });
+ * ```
+ */
+export declare function registrySource(endpoint: URL): NonNullable<SearchStageType['sourceFor']>;
 //# sourceMappingURL=search-stages.d.ts.map

package/dist/search-stages.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,EACL,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,UAAU,EAAE,QAAQ,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,GAC3B,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAwD9B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,SAAS,GACpB,YAAY,CAId"}
+{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,EACL,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,UAAU,EAAE,QAAQ,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,YAAY,CAAC,mBAAmB,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3D;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,GAC3B,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAyD9B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,SAAS,GACpB,YAAY,CAId;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,GAAG,GACZ,WAAW,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAE3C"}

package/dist/search-stages.js

@@ -1,3 +1,4 @@
+import { Distribution } from '@lde/dataset';
 import { SparqlConstructReader, SparqlItemSelector, Stage, } from '@lde/pipeline';
 import { projectRoots } from '@lde/search';
 import { extractionQueryString } from './extraction.js';
@@ -39,6 +40,7 @@ export function searchStages(options) {
             name: searchType.name,
             readers,
             itemSelector: type.itemSelector,
+            sourceFor: type.sourceFor,
             batchSize: type.batchSize,
             maxConcurrency: type.maxConcurrency,
             queueCapacity: type.queueCapacity,
@@ -86,3 +88,45 @@ export function selectByClass(searchType, rootVariable = 'root') {
         query: `SELECT ?${rootVariable} WHERE { ?${rootVariable} a <${searchType.class}> FILTER(!isBlank(?${rootVariable})) }`,
     });
 }
+/**
+ * A {@link SearchStageType.sourceFor} that reads a root type from the **dataset
+ * registry** instead of from the dataset’s own data – the endpoint fixed,
+ * scoped to the graph the dataset in hand names.
+ *
+ * A dataset’s description is governed by a different application profile from
+ * the objects it contains, and it lives in the register, not in the
+ * distribution: registering a dataset *is* submitting a description, so the
+ * register is the one source that covers every dataset a pipeline can select.
+ * Nothing obliges a publisher to ship a description of the dataset in its own
+ * dump.
+ *
+ * The graph scoping is what keeps the stage per-dataset. A register holds every
+ * registration, so an unscoped stage would index the whole catalogue for each
+ * dataset processed; scoped, each pass sees exactly the one registration – and
+ * the roots a `selectByClass` finds inside that graph are that dataset’s own.
+ * It presumes the register names each registration’s graph after the dataset
+ * IRI, which is how a DCAT register that crawls per registration stores it (the
+ * NDE Dataset Register does so for every registration it holds).
+ *
+ * Routing is deployment topology, deliberately kept out of the
+ * {@link SearchType}: a type is defined by its `class`, not by where its
+ * triples come from, so the same declaration serves a deployment that sources
+ * it differently.
+ *
+ * ```ts
+ * searchStages({
+ *   schema,
+ *   types: [...schema.values()].map((searchType) => ({
+ *     searchType,
+ *     rootVariable: 'root',
+ *     itemSelector: selectByClass(searchType),
+ *     sourceFor: registryTypeNames.has(searchType.name)
+ *       ? registrySource(registryEndpoint)
+ *       : undefined,
+ *   })),
+ * });
+ * ```
+ */
+export function registrySource(endpoint) {
+    return (dataset) => Distribution.sparql(endpoint, dataset.iri.toString());
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-pipeline",
-  "version": "0.15.0",
+  "version": "0.15.2",
   "description": "Search indexing as an @lde/pipeline instance: a Writer<Quad> that applies the @lde/search projection to extracted RDF and fans each document out to the transactional engine writer for its type's collection. Glue between the pipeline quad world and the search-engine document world; owns no projection or engine logic itself.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -32,14 +32,14 @@
     "tslib": "^2.3.0"
   },
   "devDependencies": {
-    "@lde/search-typesense": "^0.18.0",
+    "@lde/search-typesense": "^0.18.2",
     "@rdfjs/types": "^2.0.1",
-    "n3": "^2.1.1",
+    "n3": "^2.1.2",
     "testcontainers": "^12.0.3",
     "typesense": "^3.0.6"
   },
   "peerDependencies": {
     "@lde/dataset": "^0.7.9",
-    "@lde/pipeline": "^0.35.4"
+    "@lde/pipeline": "^0.35.6"
   }
 }