@lde/search-pipeline 0.17.0 → 0.18.1

package/dist/search-indexer-pipeline.d.ts

@@ -1,5 +1,5 @@
 import type { Dataset } from '@lde/dataset';
-import { Pipeline, type DatasetSelector, type DistributionResolver, type ProgressReporter, type ProvenanceStore, type Writer } from '@lde/pipeline';
+import { Pipeline, type DatasetSelector, type DistributionResolver, type ProgressReporter, type ProvenanceStore, type QuadTransform, type ReaderContext, type Writer } from '@lde/pipeline';
 import type { RootType, SearchDocument, SearchSchema } from '@lde/search';
 import type { TypedSearchDocument } from './typed-search-document.js';
 /** Options for {@link searchIndexerPipeline}. */
@@ -54,6 +54,21 @@ export interface SearchIndexerPipelineOptions {
         /** {@link SearchType.name}s of the root types to source from it. */
         names: readonly string[];
     };
+    /**
+     * {@link QuadTransform}(s) attached to a root type’s reader, keyed by
+     * {@link SearchType.name} – the extension point a deployment reaches for
+     * when the data needs correcting, enriching or pruning before projection.
+     *
+     * This is what keeps a deployment with one bespoke transform on this
+     * convenience instead of hand-composing a {@link Pipeline}: composing by hand
+     * to reach the reader means restating dataset selection, registry sourcing,
+     * provenance and reporting, and silently losing whichever is forgotten.
+     *
+     * Named by `name` rather than by class IRI, like {@link registryTypes}; a
+     * name the schema does not declare throws at wiring time rather than
+     * attaching nothing and shipping an unenriched collection.
+     */
+    transforms?: Readonly<Record<string, QuadTransform<ReaderContext> | QuadTransform<ReaderContext>[]>>;
     /**
      * Optional per-dataset processing memory: skip a dataset whose source
      * fingerprint and {@link pipelineVersion} both match the stored record.
@@ -99,11 +114,15 @@ export interface SearchIndexerPipelineOptions {
  * await pipeline.run();
  * ```
  *
+ * Domain behaviour on top of the standard wiring is {@link transforms}, so
+ * adding it costs the behaviour rather than this whole composition.
+ *
  * A deployment that needs more – a bespoke root selector (the entry point is a
  * domain fact, not a class), per-stage tuning (`batchSize`, `maxConcurrency`),
  * non-SPARQL readers, or quad-level plugins – composes {@link searchStages},
- * {@link searchIndexWriter} and {@link Pipeline} directly; this convenience
- * owns no capability of its own.
+ * {@link searchIndexWriter} and {@link Pipeline} directly, restating the
+ * registry sourcing, provenance and reporting this function wires; beyond
+ * those, this convenience owns no capability of its own.
  */
 export declare function searchIndexerPipeline(options: SearchIndexerPipelineOptions): Pipeline<TypedSearchDocument>;
 //# sourceMappingURL=search-indexer-pipeline.d.ts.map

package/dist/search-indexer-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-indexer-pipeline.d.ts","sourceRoot":"","sources":["../src/search-indexer-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAEL,QAAQ,EACR,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,iDAAiD;AACjD,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;IACtC;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;;;OAOG;IACH,SAAS,EAAE,CACT,UAAU,EAAE,QAAQ,EACpB,MAAM,EAAE,YAAY,KACjB,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5B;;;;;;;;;;;;OAYG;IACH,aAAa,CAAC,EAAE;QACd,+DAA+D;QAC/D,QAAQ,EAAE,GAAG,CAAC;QACd,oEAAoE;QACpE,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;KAC1B,CAAC;IACF;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,4BAA4B,GACpC,QAAQ,CAAC,mBAAmB,CAAC,CA0B/B"}
+{"version":3,"file":"search-indexer-pipeline.d.ts","sourceRoot":"","sources":["../src/search-indexer-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAEL,QAAQ,EACR,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,iDAAiD;AACjD,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;IACtC;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;;;OAOG;IACH,SAAS,EAAE,CACT,UAAU,EAAE,QAAQ,EACpB,MAAM,EAAE,YAAY,KACjB,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5B;;;;;;;;;;;;OAYG;IACH,aAAa,CAAC,EAAE;QACd,+DAA+D;QAC/D,QAAQ,EAAE,GAAG,CAAC;QACd,oEAAoE;QACpE,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;KAC1B,CAAC;IACF;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,QAAQ,CACnB,MAAM,CACJ,MAAM,EACN,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAC9D,CACF,CAAC;IACF;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,4BAA4B,GACpC,QAAQ,CAAC,mBAAmB,CAAC,CA4B/B"}

package/dist/search-indexer-pipeline.js

@@ -28,15 +28,20 @@ import { registrySource, searchStages, selectByClass, } from './search-stages.js
  * await pipeline.run();
  * ```
  *
+ * Domain behaviour on top of the standard wiring is {@link transforms}, so
+ * adding it costs the behaviour rather than this whole composition.
+ *
  * A deployment that needs more – a bespoke root selector (the entry point is a
  * domain fact, not a class), per-stage tuning (`batchSize`, `maxConcurrency`),
  * non-SPARQL readers, or quad-level plugins – composes {@link searchStages},
- * {@link searchIndexWriter} and {@link Pipeline} directly; this convenience
- * owns no capability of its own.
+ * {@link searchIndexWriter} and {@link Pipeline} directly, restating the
+ * registry sourcing, provenance and reporting this function wires; beyond
+ * those, this convenience owns no capability of its own.
  */
 export function searchIndexerPipeline(options) {
-    const { schema, datasets, registryTypes } = options;
+    const { schema, datasets, registryTypes, transforms } = options;
     const fromRegistry = registryTypeNames(schema, registryTypes?.names);
+    assertDeclaredTypeNames(schema, Object.keys(transforms ?? {}), 'transform');
     const sourceFor = registryTypes === undefined
         ? undefined
         : registrySource(registryTypes.endpoint);
@@ -52,6 +57,7 @@ export function searchIndexerPipeline(options) {
                 rootVariable: 'root',
                 itemSelector: selectByClass(searchType),
                 sourceFor: fromRegistry.has(searchType.name) ? sourceFor : undefined,
+                transform: transforms?.[searchType.name],
             })),
         }),
         writers: searchIndexWriter({ schema, writerFor: options.writerFor }),
@@ -68,14 +74,23 @@ export function searchIndexerPipeline(options) {
  */
 function registryTypeNames(schema, declaredNames) {
     const names = new Set(declaredNames ?? []);
+    assertDeclaredTypeNames(schema, [...names], 'registry root');
+    return names;
+}
+/**
+ * Every name must be one the schema declares. Shared by the options that key
+ * behaviour by {@link SearchType.name}: naming a type the schema does not have
+ * is a configuration mistake whose only other symptom is an option that
+ * silently does nothing.
+ */
+function assertDeclaredTypeNames(schema, names, label) {
     const declared = new Set([...schema.values()].map((searchType) => searchType.name));
-    const unknown = [...names].filter((name) => !declared.has(name));
+    const unknown = names.filter((name) => !declared.has(name));
     if (unknown.length > 0) {
-        throw new Error(`Unknown registry root type(s) ${unknown
+        throw new Error(`Unknown ${label} type(s) ${unknown
             .map((name) => `“${name}”`)
             .join(', ')}; the schema declares ${[...declared]
             .map((name) => `“${name}”`)
             .join(', ')}.`);
     }
-    return names;
 }

package/dist/search-stages.d.ts

@@ -1,4 +1,4 @@
-import { Stage, type ItemSelector, type StageOptions, type StageReaders } from '@lde/pipeline';
+import { Stage, type ItemSelector, type QuadTransform, type ReaderContext, type StageOptions, type StageReaders } from '@lde/pipeline';
 import { type RootType, type SearchSchema } from '@lde/search';
 import type { TypedSearchDocument } from './typed-search-document.js';
 /** One root type’s stage in a search pipeline. */
@@ -30,10 +30,27 @@ export interface SearchStageType {
      * from `searchType` ({@link extractionQuery}), with `rootVariable` as the free
      * subject the batch’s roots bind to – the schema-derived reader the projection
      * is guaranteed to agree with (they share the {@link irAlias} convention).
-     * Supply your own only to read from a non-SPARQL source, merge several
-     * readers, or attach transforms.
+     * Supply your own only to read from a non-SPARQL source or merge several
+     * readers; to enrich the default reader’s quads, use {@link transform}.
      */
     readers?: StageReaders;
+    /**
+     * {@link QuadTransform}(s) attached to the **default** reader – the usual way
+     * to add behaviour to this stage: correct the data, mint a quad the source
+     * does not ship, drop one it should not.
+     *
+     * Attaching here rather than building the reader yourself is what keeps the
+     * reader’s `subjectVariable` and this stage’s {@link rootVariable} in
+     * agreement: nothing cross-checks them, and a mismatch extracts nothing.
+     * Mutually exclusive with {@link readers} – a caller supplying its own
+     * reader(s) attaches transforms to them directly ({@link AttachedReader}),
+     * since only that caller knows which of several readers each belongs to.
+     *
+     * A field a transform fills must still declare a `path`: projection skips a
+     * field with neither a `path` nor a `derive`, so a transform-minted IR Alias
+     * is otherwise never read.
+     */
+    transform?: QuadTransform<ReaderContext> | QuadTransform<ReaderContext>[];
     /**
      * Where this type’s stage reads, when that is not the dataset’s own
      * distribution – see {@link StageOptions.sourceFor}. {@link registrySource}

package/dist/search-stages.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,EACL,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,UAAU,EAAE,QAAQ,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,YAAY,CAAC,mBAAmB,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3D;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,GAC3B,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAiE9B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,SAAS,GACpB,YAAY,CAId;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,GAAG,GACZ,WAAW,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAE3C"}
+{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,EAEL,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAgB,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,UAAU,EAAE,QAAQ,CAAC;IACrB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;IAC1E;;;;;OAKG;IACH,SAAS,CAAC,EAAE,YAAY,CAAC,mBAAmB,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3D;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,GAC3B,KAAK,CAAC,mBAAmB,CAAC,EAAE,CA6E9B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,SAAS,GACpB,YAAY,CAId;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,GAAG,GACZ,WAAW,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAE3C"}

package/dist/search-stages.js

@@ -27,15 +27,25 @@ export function searchStages(options) {
             throw new Error(`Search type “${type.searchType.name}” (class ${type.searchType.class}) is not in the schema; searchStages projects only types the schema declares.`);
         }
         const { rootVariable } = type;
+        if (type.readers !== undefined && type.transform !== undefined) {
+            // Which of the caller’s readers the transform belongs to is knowable only
+            // to that caller, so guessing (all of them? the first?) would be a silent
+            // wrong answer. Attach it as an AttachedReader instead.
+            throw new Error(`Search type “${searchType.name}”: “transform” attaches to the default reader, so it cannot be combined with “readers” – attach the transform to your own reader instead ({ reader, transform }).`);
+        }
         // Default to the Extraction CONSTRUCT generated from the schema, its subject
         // left free for the batch’s VALUES injection. The reader and the projection
-        // then agree by construction: both key off the same IR Aliases.
+        // then agree by construction: both key off the same IR Aliases – which is
+        // also why an attached transform never has to restate the subject variable.
         const readers = type.readers ??
-            new SparqlConstructReader({
-                query: extractionQueryString(searchType, schema, {
-                    subjectVariable: rootVariable,
+            {
+                reader: new SparqlConstructReader({
+                    query: extractionQueryString(searchType, schema, {
+                        subjectVariable: rootVariable,
+                    }),
                 }),
-            });
+                transform: type.transform,
+            };
         return new Stage({
             name: searchType.name,
             readers,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-pipeline",
-  "version": "0.17.0",
+  "version": "0.18.1",
   "description": "Search indexing as an @lde/pipeline instance: a Writer<Quad> that applies the @lde/search projection to extracted RDF and fans each document out to the transactional engine writer for its type's collection. Glue between the pipeline quad world and the search-engine document world; owns no projection or engine logic itself.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -25,14 +25,14 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/search": "^0.17.0",
+    "@lde/search": "^0.18.0",
     "@traqula/generator-sparql-1-1": "^1.1.8",
     "@traqula/parser-sparql-1-1": "^1.1.5",
     "@traqula/rules-sparql-1-1": "^1.1.0",
     "tslib": "^2.3.0"
   },
   "devDependencies": {
-    "@lde/search-typesense": "^0.20.0",
+    "@lde/search-typesense": "^0.21.0",
     "@rdfjs/types": "^2.0.1",
     "n3": "^2.1.2",
     "testcontainers": "^12.0.3",