@lde/search-pipeline 0.7.2 → 0.8.0

package/README.md

@@ -6,7 +6,8 @@ and a search engine’s document world, so indexing reuses the existing spine
 Discovery → Selection → Iteration → change-gate (skip-unchanged) – instead of
 rebuilding it per consumer.
 
-The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transaction-aware.md)):
+The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transaction-aware.md),
+[ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md)):
 
 - [`@lde/search`](../search) owns the engine-agnostic projection (framed
   document + field model) and stays pipeline-free;
@@ -14,18 +15,32 @@ The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transa
   is the engine adapter: a transactional, **single-collection** `Writer`
   consuming projected documents, owning the run lifecycle (Blue/green alias swap
   or In-place sweeps, cross-pod lock);
-- **this package** composes them: `searchIndexWriter` is the one type-changing
-  step (quad → document), shared across engines, and – since one schema declares
-  several root types – it fans each document out to the engine writer for **its**
-  type’s collection ([ADR 9](../../docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md)).
+- **this package** composes them:
+  - `searchStages` builds one projecting `Stage` per root type. Each stage
+    selects its own roots, extracts each root’s quads, and projects the
+    **root-complete batch** into documents paired with their `SearchType`
+    (`TypedSearchDocument`) – so projection happens inside the batch and memory
+    is bounded by `batchSize` roots, not the dataset
+    ([ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md));
+  - `searchIndexWriter` is the pipeline’s single terminal: an engine-agnostic
+    router that dispatches each document to the engine writer for **its**
+    type’s collection ([ADR 9](../../docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md)).
+    It owns no projection and buffers nothing.
+
+So a search pipeline is **one terminal, N stages**: `new Pipeline<TypedSearchDocument>({ datasetSelector, stages: searchStages(...), writers: searchIndexWriter(...) })`.
 
 ## Usage
 
 ```typescript
-import { Pipeline, Stage, SparqlConstructReader } from '@lde/pipeline';
+import { Pipeline, SparqlConstructReader } from '@lde/pipeline';
 import { searchSchema } from '@lde/search';
 import { BlueGreenRebuild } from '@lde/search-typesense';
-import { searchIndexWriter } from '@lde/search-pipeline';
+import {
+  searchStages,
+  selectByClass,
+  searchIndexWriter,
+  type TypedSearchDocument,
+} from '@lde/search-pipeline';
 
 // One schema, several root types: the `datasets` catalog plus the Organization
 // label collection its references resolve against.
@@ -60,14 +75,33 @@ const schema = searchSchema(
   },
 );
 
-const pipeline = new Pipeline({
+const dataset = schema.get('http://www.w3.org/ns/dcat#Dataset')!;
+const organization = schema.get('http://xmlns.com/foaf/0.1/Organization')!;
+
+const pipeline = new Pipeline<TypedSearchDocument>({
   datasetSelector,
-  stages: [
-    new Stage({
-      name: 'extract',
-      readers: new SparqlConstructReader({ query: '…' }),
-    }),
-  ],
+  // One stage per root type: each selects its own roots (here by class –
+  // `selectByClass` is a convenience for the object grain; a deployment writes
+  // its own selector where the entry point is a domain fact) and extracts each
+  // root’s quads. `rootVariable` couples the selector’s projected variable to
+  // what the projection reads; it must not be `dataset` (reserved by the reader).
+  stages: searchStages({
+    schema,
+    types: [
+      {
+        searchType: dataset,
+        rootVariable: 'root',
+        itemSelector: selectByClass(dataset),
+        readers: new SparqlConstructReader({ query: '…' }),
+      },
+      {
+        searchType: organization,
+        rootVariable: 'root',
+        itemSelector: selectByClass(organization),
+        readers: new SparqlConstructReader({ query: '…' }),
+      },
+    ],
+  }),
   // Each type gets its own collection – an independent blue/green rebuild –
   // named by the adapter from the type itself (`Dataset` → `datasets`,
   // `Organization` → `organizations`), so no naming map is passed here and the
@@ -101,14 +135,21 @@ Whatever the writers are named, the engine reading these collections must be
 given the same names (`collections`), or it reads the derived ones and finds an
 empty index.
 
-Each dataset’s extracted CONSTRUCT quads are buffered until the dataset
-completes, then projected once over the **whole** schema (`@lde/search`’s
-`projectGraph`) into one mixed, type-tagged stream, and each document is
-dispatched to the engine run for its type’s collection – before the engine acts
-on the dataset’s completion, so an In-place stale sweep never races its own
-documents. The run lifecycle (run context, per-dataset flush outcome,
-commit/abort) passes through unchanged: the engine writer’s update mode governs,
-and the pipeline never branches on it, nor on how many collections there are.
+Each stage projects its own root-complete batches (`@lde/search`’s
+`projectRoots`) into documents paired with its `SearchType`, and the terminal
+dispatches each to the engine run for its type’s collection as it arrives –
+before the engine acts on the dataset’s completion, so an In-place stale sweep
+never races its own documents. The run lifecycle (run context, per-dataset flush
+outcome, commit/abort) passes through unchanged: the engine writer’s update mode
+governs, and the pipeline never branches on it, nor on how many collections
+there are.
+
+**Root selection is the deployment’s contract.** A selector that does not bind
+the CONSTRUCT’s subject yields a batch that is not root-complete, and projection
+will then silently emit partial documents – a promise `@lde/pipeline` cannot
+check ([ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md)).
+`selectByClass(searchType)` is a convenience for the object grain (where the
+type’s `class` really is the source class), **not** a default.
 
 ## Per-collection isolation
 
@@ -130,7 +171,12 @@ See [ADR 9](../../docs/decisions/0009-route-a-whole-schema-projection-to-per-typ
 
 ## Memory
 
-Memory is bounded by one dataset’s extraction and released at each dataset
-flush – nothing accumulates across datasets. Streaming bounded entity batches
-_within_ one huge dataset needs the two-level iteration (dataset →
-entity-URI batches) and is not implemented yet.
+Memory is bounded by **`batchSize` roots**, not by the input
+([ADR 12](../../docs/decisions/0012-bound-memory-by-the-unit-of-work-not-the-input.md),
+[ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md)).
+Projection runs inside each root-complete batch and streams straight through the
+terminal to the engine run, which holds only its own batch – nothing accumulates
+across a dataset or across datasets, so the same pipeline indexes a 15-quad
+catalog entry and a dataset of millions of objects. The one irreducible atom is a
+single root’s quads: unbounded for a pathological root, and nothing here changes
+that.

package/dist/index.d.ts

@@ -1,3 +1,6 @@
+export { searchStages, selectByClass } from './search-stages.js';
+export type { SearchStagesOptions, SearchStageType } from './search-stages.js';
 export { searchIndexWriter } from './search-index-writer.js';
 export type { SearchIndexWriterOptions } from './search-index-writer.js';
+export type { TypedSearchDocument } from './typed-search-document.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}

package/dist/index.js

@@ -1 +1,2 @@
+export { searchStages, selectByClass } from './search-stages.js';
 export { searchIndexWriter } from './search-index-writer.js';

package/dist/search-index-writer.d.ts

@@ -1,12 +1,12 @@
-import type { Quad } from '@rdfjs/types';
-import type { Writer } from '@lde/pipeline';
-import { type SearchDocument, type SearchSchema, type SearchType } from '@lde/search';
+import { type Writer } from '@lde/pipeline';
+import type { SearchDocument, SearchSchema, SearchType } from '@lde/search';
+import type { TypedSearchDocument } from './typed-search-document.js';
 /** Options for {@link searchIndexWriter}. */
 export interface SearchIndexWriterOptions {
     /**
-     * The declarative schema driving the projection: one {@link SearchType} per
-     * root type, each mapping framed RDF to flat search-document fields. The
-     * writer opens one engine run per type in it.
+     * The declarative schema: one {@link SearchType} per root type. The writer
+     * opens one engine run per type in it and routes each document to its type’s
+     * run. Must be the same schema the stages project through.
      */
     schema: SearchSchema;
     /**
@@ -19,19 +19,22 @@ export interface SearchIndexWriterOptions {
      * multi-collection deployment (the Dataset Register’s `datasets` plus its
      * Organization / Class / TerminologySource label collections) returns a
      * distinct writer per type, each an independent blue/green rebuild with its
-     * own collection, alias and cross-pod lock. The projection is whole-schema
-     * either way; the per-collection fan-out is this writer’s job.
+     * own collection, alias and cross-pod lock. The per-collection fan-out is this
+     * writer’s job.
      */
     writerFor: (searchType: SearchType) => Writer<SearchDocument>;
 }
 /**
- * The projection step of a search-indexing pipeline, as a quad `Writer`, fanned
- * out across a type’s collections. It turns each dataset’s extracted CONSTRUCT
- * quads into engine-agnostic search documents ({@link projectGraph}: frame by
- * root type, then project each node with its type’s declaration) and dispatches
- * each document to the engine run for **its** type. This is the one
- * type-changing step (quad → document), shared across engines – which is why it
- * lives here and not inside an engine adapter.
+ * The single terminal of a search-indexing pipeline: an engine-agnostic router
+ * that fans already-projected documents out across a type’s collections. The
+ * per-type {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md | stages}
+ * project inside the batch and pair each document with its {@link SearchType}
+ * ({@link TypedSearchDocument}); this writer dispatches each document to the
+ * engine run for **its** type by `searchType.class`. It **owns no projection**
+ * (that moved into the stages) and **buffers nothing** (documents stream through
+ * to the run as they arrive) – it is purely the per-collection fan-out
+ * {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md | ADR 9}
+ * made it.
  *
  * Each root type is an independent engine run (its own collection, alias and
  * lock), so the collections commit, sweep and fail in isolation:
@@ -48,12 +51,9 @@ export interface SearchIndexWriterOptions {
  * - `abort` (a run failure, or a partial commit) drops every half-built
  *   collection that has not committed and leaves the live ones untouched.
  *
- * A dataset’s quads are buffered until its flush and projected then, so the
- * documents land before every engine acts on the dataset’s completion (e.g. an
- * In-place stale sweep). Memory is bounded by one dataset’s extraction, and
- * released at each flush – nothing accumulates across datasets. Streaming
- * bounded entity batches *within* one huge dataset needs the two-level
- * iteration (dataset → entity-URI batches) and is not implemented yet.
+ * Memory is bounded by one batch of documents per type, not the dataset:
+ * `write` routes each document to its type’s run through a bounded queue and
+ * never accumulates them.
  */
-export declare function searchIndexWriter(options: SearchIndexWriterOptions): Writer<Quad>;
+export declare function searchIndexWriter(options: SearchIndexWriterOptions): Writer<TypedSearchDocument>;
 //# sourceMappingURL=search-index-writer.d.ts.map

package/dist/search-index-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-index-writer.d.ts","sourceRoot":"","sources":["../src/search-index-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAIV,MAAM,EACP,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,UAAU,EAChB,MAAM,aAAa,CAAC;AAErB,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,SAAS,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,MAAM,CAAC,cAAc,CAAC,CAAC;CAC/D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,wBAAwB,GAChC,MAAM,CAAC,IAAI,CAAC,CA2Id"}
+{"version":3,"file":"search-index-writer.d.ts","sourceRoot":"","sources":["../src/search-index-writer.ts"],"names":[],"mappings":"AACA,OAAO,EAKL,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,SAAS,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,MAAM,CAAC,cAAc,CAAC,CAAC;CAC/D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,wBAAwB,GAChC,MAAM,CAAC,mBAAmB,CAAC,CAoI7B"}

package/dist/search-index-writer.js

@@ -1,12 +1,15 @@
-import { projectGraph, } from '@lde/search';
+import { AsyncQueue, } from '@lde/pipeline';
 /**
- * The projection step of a search-indexing pipeline, as a quad `Writer`, fanned
- * out across a type’s collections. It turns each dataset’s extracted CONSTRUCT
- * quads into engine-agnostic search documents ({@link projectGraph}: frame by
- * root type, then project each node with its type’s declaration) and dispatches
- * each document to the engine run for **its** type. This is the one
- * type-changing step (quad → document), shared across engines – which is why it
- * lives here and not inside an engine adapter.
+ * The single terminal of a search-indexing pipeline: an engine-agnostic router
+ * that fans already-projected documents out across a type’s collections. The
+ * per-type {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md | stages}
+ * project inside the batch and pair each document with its {@link SearchType}
+ * ({@link TypedSearchDocument}); this writer dispatches each document to the
+ * engine run for **its** type by `searchType.class`. It **owns no projection**
+ * (that moved into the stages) and **buffers nothing** (documents stream through
+ * to the run as they arrive) – it is purely the per-collection fan-out
+ * {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md | ADR 9}
+ * made it.
  *
  * Each root type is an independent engine run (its own collection, alias and
  * lock), so the collections commit, sweep and fail in isolation:
@@ -23,17 +26,14 @@ import { projectGraph, } from '@lde/search';
  * - `abort` (a run failure, or a partial commit) drops every half-built
  *   collection that has not committed and leaves the live ones untouched.
  *
- * A dataset’s quads are buffered until its flush and projected then, so the
- * documents land before every engine acts on the dataset’s completion (e.g. an
- * In-place stale sweep). Memory is bounded by one dataset’s extraction, and
- * released at each flush – nothing accumulates across datasets. Streaming
- * bounded entity batches *within* one huge dataset needs the two-level
- * iteration (dataset → entity-URI batches) and is not implemented yet.
+ * Memory is bounded by one batch of documents per type, not the dataset:
+ * `write` routes each document to its type’s run through a bounded queue and
+ * never accumulates them.
  */
 export function searchIndexWriter(options) {
     const { schema, writerFor } = options;
     // One engine writer per root type, built once; each run opens them all. Keyed
-    // by the type IRI, which is also how a projected document names its type.
+    // by the type IRI, which is also how a paired document names its type.
     const writers = new Map([...schema.values()].map((searchType) => [
         searchType.class,
         writerFor(searchType),
@@ -55,54 +55,49 @@ export function searchIndexWriter(options) {
             // The collections that have gone live, so `abort` never re-finalizes one
             // (a committed blue/green rebuild’s abort drops its now-live collection).
             const committed = new Set();
-            // One buffered pass per dataset, keyed by IRI. The pipeline processes
-            // datasets sequentially, but keeping the key explicit makes a write after
-            // another dataset's flush safe too.
-            const passes = new Map();
-            const project = async (pass) => {
-                passes.delete(pass.dataset.iri.toString());
-                if (pass.quads.length === 0) {
-                    return;
-                }
-                // Whole-schema projection into one mixed stream, split by type here so
-                // each type’s documents reach its own collection’s run.
-                const byType = new Map();
-                for await (const { searchType, document } of projectGraph(pass.quads, schema)) {
-                    const documents = byType.get(searchType.class);
-                    if (documents === undefined) {
-                        byType.set(searchType.class, [document]);
-                    }
-                    else {
-                        documents.push(document);
-                    }
-                }
-                // Every yielded type is a schema type, so it has a run; iterate the
-                // runs and write each the documents projected for its type (none, for a
-                // type absent from this pass).
-                for (const [typeIri, run] of runs) {
-                    const documents = byType.get(typeIri);
-                    if (documents !== undefined) {
-                        await run.write(pass.dataset, stream(documents));
-                    }
-                }
-            };
             return {
-                write: async (dataset, quads) => {
-                    const key = dataset.iri.toString();
-                    let pass = passes.get(key);
-                    if (pass === undefined) {
-                        pass = { dataset, quads: [] };
-                        passes.set(key, pass);
+                write: async (dataset, items) => {
+                    // Route each document to its type’s run, streaming. A stage writes
+                    // one type, but the terminal carries no stage identity, so it routes
+                    // per item by its `searchType`. Each type gets one `run.write`, fed a
+                    // bounded queue the run drains concurrently – so memory stays O(batch)
+                    // per type, never O(dataset). Almost always one lane per call.
+                    const lanes = new Map();
+                    const laneFor = (searchType) => {
+                        let lane = lanes.get(searchType.class);
+                        if (lane === undefined) {
+                            const run = runs.get(searchType.class);
+                            if (run === undefined) {
+                                throw new Error(`No engine run for search type “${searchType.name}” (${searchType.class}); it is not in this writer’s schema.`);
+                            }
+                            const queue = new AsyncQueue();
+                            const done = run.write(dataset, queue);
+                            // If the run stops consuming (its write rejects), unblock the
+                            // producer so a full queue cannot deadlock the push loop below.
+                            done.catch((error) => queue.abort(error));
+                            lane = { queue, done };
+                            lanes.set(searchType.class, lane);
+                        }
+                        return lane;
+                    };
+                    try {
+                        for await (const { searchType, document } of items) {
+                            await laneFor(searchType).queue.push(document);
+                        }
+                        for (const lane of lanes.values()) {
+                            lane.queue.close();
+                        }
+                        await Promise.all([...lanes.values()].map((lane) => lane.done));
                     }
-                    for await (const quad of quads) {
-                        pass.quads.push(quad);
+                    catch (error) {
+                        for (const lane of lanes.values()) {
+                            lane.queue.abort(error);
+                        }
+                        await Promise.allSettled([...lanes.values()].map((lane) => lane.done));
+                        throw error;
                     }
                 },
                 flush: async (dataset, outcome) => {
-                    const pass = passes.get(dataset.iri.toString());
-                    if (pass !== undefined) {
-                        await project(pass);
-                    }
                     // Flush every collection independently: one collection’s flush failure
                     // (a rollback, or an In-place stale sweep) must not skip another’s –
                     // the pipeline isolates a flush error per dataset and still commits,
@@ -113,19 +108,12 @@ export function searchIndexWriter(options) {
                     await settleAll(runs.values(), 'flush', (run) => run.flush?.(dataset, outcome));
                 },
                 reset: async (dataset) => {
-                    // Discard the buffered pass so the re-run replaces it, and let every
-                    // collection’s run discard whatever it already holds for the dataset –
-                    // independently, so one collection’s reset failure never leaves
-                    // another holding the discarded pass’s documents into the re-run.
-                    passes.delete(dataset.iri.toString());
+                    // Let every collection’s run discard whatever it already holds for the
+                    // dataset – independently, so one collection’s reset failure never
+                    // leaves another holding the discarded documents into the re-run.
                     await settleAll(runs.values(), 'reset', (run) => run.reset?.(dataset));
                 },
                 commit: async () => {
-                    // Safety net: project passes that were never flushed, so a committed
-                    // run never silently drops written quads.
-                    for (const pass of [...passes.values()]) {
-                        await project(pass);
-                    }
                     // Commit every collection independently, so one failure neither
                     // blocks nor wipes another – the `datasets` index goes live even if a
                     // label collection cannot. Record each collection that went live, so
@@ -162,7 +150,3 @@ async function settleAll(items, verb, operation) {
         throw new AggregateError(failures, `${failures.length} of ${targets.length} search collections failed to ${verb}`);
     }
 }
-/** Yield the given documents as an async iterable, like a streaming source. */
-async function* stream(items) {
-    yield* items;
-}

package/dist/search-stages.d.ts

@@ -0,0 +1,82 @@
+import { Stage, type ItemSelector, type StageReaders } from '@lde/pipeline';
+import { type SearchSchema, type SearchType } from '@lde/search';
+import type { TypedSearchDocument } from './typed-search-document.js';
+/** One root type’s stage in a search pipeline. */
+export interface SearchStageType {
+    /**
+     * The {@link SearchType} this stage projects. Must belong to
+     * {@link SearchStagesOptions.schema} (matched by `class`); the stage projects
+     * with the schema’s own declaration object, so `assertTypeInSchema`’s identity
+     * check inside {@link projectRoots} always holds.
+     */
+    searchType: SearchType;
+    /**
+     * The selector variable that binds this type’s roots – the CONSTRUCT subject
+     * the batch is complete for. Must **not** be `dataset`: `?dataset` is
+     * substituted with the dataset IRI by the SPARQL reader, so a root bound to it
+     * would never reach the projection. {@link selectByClass} defaults to `root`.
+     */
+    rootVariable: string;
+    /**
+     * Selects this type’s roots, one binding of {@link rootVariable} per root.
+     * Root selection is the deployment’s concern; {@link selectByClass} is a
+     * convenience for the object grain, not a default.
+     */
+    itemSelector: ItemSelector;
+    /** Reader(s) that extract each selected root’s quads. */
+    readers: StageReaders;
+    /**
+     * Roots (and so documents) per batch – the memory bound. Under a root-bound
+     * selector it moves memory and request count, never output.
+     * @default 10
+     */
+    batchSize?: number;
+    /** Maximum concurrent in-flight SPARQL queries for this stage. @default 10 */
+    maxConcurrency?: number;
+    /**
+     * Capacity of the bounded queue funnelling this stage’s projected documents
+     * into the write. A projected document is far heavier than a quad, so lower it
+     * where documents are large. @default 128
+     */
+    queueCapacity?: number;
+}
+/** Options for {@link searchStages}. */
+export interface SearchStagesOptions {
+    /**
+     * The declarative schema driving projection: one {@link SearchType} per root
+     * type. Every {@link SearchStageType.searchType} must be a member of it.
+     */
+    schema: SearchSchema;
+    /** One entry per root type to index, each its own stage. */
+    types: readonly SearchStageType[];
+}
+/**
+ * Compose one projecting {@link Stage} per root type – the source side of a
+ * search pipeline. Each stage selects its own roots, extracts each root’s quads,
+ * and projects the root-complete batch into {@link TypedSearchDocument}s
+ * ({@link projectRoots} + the `searchType` pair), which the pipeline’s single
+ * {@link searchIndexWriter} terminal routes to that type’s collection. Projection
+ * happens **inside the batch**, so memory is bounded by `batchSize` roots, never
+ * by the dataset
+ * ([ADR 13](https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md)).
+ *
+ * Wire the result as `new Pipeline<TypedSearchDocument>({ datasetSelector,
+ * stages: searchStages(...), writers: searchIndexWriter(...) })` – one terminal,
+ * N stages.
+ */
+export declare function searchStages(options: SearchStagesOptions): Stage<TypedSearchDocument>[];
+/**
+ * An {@link ItemSelector} that selects every instance of a root type’s source
+ * class: `SELECT ?‹rootVariable› WHERE { ?‹rootVariable› a <class> }`. A
+ * convenience for the **object grain**, where {@link SearchType.class} really is
+ * the source class – **not** a default: root selection is a deployment concern,
+ * and three of the Dataset Register’s four catalog types have no source class at
+ * all (their entry point – “registered, newest registration, has a title” – is a
+ * deployment fact no schema states).
+ *
+ * `rootVariable` defaults to `root` and must match the stage’s
+ * {@link SearchStageType.rootVariable}; it must not be `dataset` (reserved by the
+ * SPARQL reader).
+ */
+export declare function selectByClass(searchType: SearchType, rootVariable?: string): ItemSelector;
+//# sourceMappingURL=search-stages.d.ts.map

package/dist/search-stages.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-stages.d.ts","sourceRoot":"","sources":["../src/search-stages.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,EACL,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAgB,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B;;;;;OAKG;IACH,UAAU,EAAE,UAAU,CAAC;IACvB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B,yDAAyD;IACzD,OAAO,EAAE,YAAY,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,mBAAmB,GAC3B,KAAK,CAAC,mBAAmB,CAAC,EAAE,CA8C9B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,UAAU,EACtB,YAAY,SAAS,GACpB,YAAY,CAId"}

package/dist/search-stages.js

@@ -0,0 +1,72 @@
+import { SparqlItemSelector, Stage, } from '@lde/pipeline';
+import { projectRoots } from '@lde/search';
+/**
+ * Compose one projecting {@link Stage} per root type – the source side of a
+ * search pipeline. Each stage selects its own roots, extracts each root’s quads,
+ * and projects the root-complete batch into {@link TypedSearchDocument}s
+ * ({@link projectRoots} + the `searchType` pair), which the pipeline’s single
+ * {@link searchIndexWriter} terminal routes to that type’s collection. Projection
+ * happens **inside the batch**, so memory is bounded by `batchSize` roots, never
+ * by the dataset
+ * ([ADR 13](https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md)).
+ *
+ * Wire the result as `new Pipeline<TypedSearchDocument>({ datasetSelector,
+ * stages: searchStages(...), writers: searchIndexWriter(...) })` – one terminal,
+ * N stages.
+ */
+export function searchStages(options) {
+    const { schema, types } = options;
+    return types.map((type) => {
+        // Project with the schema’s OWN declaration object, whatever the caller
+        // passed: `assertTypeInSchema` (inside `projectRoots`) is an identity check,
+        // and re-resolving here makes a class-equal lookalike work too.
+        const searchType = schema.get(type.searchType.class);
+        if (searchType === undefined) {
+            throw new Error(`Search type “${type.searchType.name}” (class ${type.searchType.class}) is not in the schema; searchStages projects only types the schema declares.`);
+        }
+        const { rootVariable } = type;
+        return new Stage({
+            name: searchType.name,
+            readers: type.readers,
+            itemSelector: type.itemSelector,
+            batchSize: type.batchSize,
+            maxConcurrency: type.maxConcurrency,
+            queueCapacity: type.queueCapacity,
+            project: async function* (quads, context) {
+                // The batch is root-complete by construction: `context.bindings` are the
+                // selector rows the readers ran with, so these are exactly this batch’s
+                // roots. Project them, then re-attach the type the stage was built for.
+                const roots = context.bindings.map((binding) => {
+                    const term = binding[rootVariable];
+                    if (term === undefined) {
+                        // The selector projected a different variable than the stage reads:
+                        // a config mismatch. Fail loudly rather than deref `undefined`.
+                        throw new Error(`Stage “${searchType.name}”: selector did not bind ?${rootVariable} – the stage’s rootVariable must match the selector’s projected variable.`);
+                    }
+                    return term.value;
+                });
+                for await (const document of projectRoots(quads, roots, schema, searchType)) {
+                    yield { searchType, document };
+                }
+            },
+        });
+    });
+}
+/**
+ * An {@link ItemSelector} that selects every instance of a root type’s source
+ * class: `SELECT ?‹rootVariable› WHERE { ?‹rootVariable› a <class> }`. A
+ * convenience for the **object grain**, where {@link SearchType.class} really is
+ * the source class – **not** a default: root selection is a deployment concern,
+ * and three of the Dataset Register’s four catalog types have no source class at
+ * all (their entry point – “registered, newest registration, has a title” – is a
+ * deployment fact no schema states).
+ *
+ * `rootVariable` defaults to `root` and must match the stage’s
+ * {@link SearchStageType.rootVariable}; it must not be `dataset` (reserved by the
+ * SPARQL reader).
+ */
+export function selectByClass(searchType, rootVariable = 'root') {
+    return new SparqlItemSelector({
+        query: `SELECT ?${rootVariable} WHERE { ?${rootVariable} a <${searchType.class}> }`,
+    });
+}

package/dist/typed-search-document.d.ts

@@ -0,0 +1,23 @@
+import type { SearchDocument, SearchType } from '@lde/search';
+/**
+ * A projected {@link SearchDocument} paired with the {@link SearchType} it was
+ * projected from.
+ *
+ * A search pipeline is N per-type stages writing to one terminal, and
+ * {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md | ADR 9}’s
+ * `searchIndexWriter` receives `write(dataset, items)` with no stage identity –
+ * so the type travels with the item and the writer routes each document to the
+ * engine run for its type by `searchType.class`. A stage mints the pair itself
+ * (it was constructed for one type), so nothing is ever re-derived from the
+ * document.
+ *
+ * It lives in `@lde/search-pipeline`, not `@lde/search`, because it exists only
+ * because a pipeline terminal routes: that is glue, not projection. `@lde/search`
+ * yields a bare {@link SearchDocument} and stays pipeline-free
+ * ([ADR 13](https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md)).
+ */
+export interface TypedSearchDocument {
+    readonly searchType: SearchType;
+    readonly document: SearchDocument;
+}
+//# sourceMappingURL=typed-search-document.d.ts.map

package/dist/typed-search-document.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"typed-search-document.d.ts","sourceRoot":"","sources":["../src/typed-search-document.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9D;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;CACnC"}

package/dist/typed-search-document.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-pipeline",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Search indexing as an @lde/pipeline instance: a Writer<Quad> that applies the @lde/search projection to extracted RDF and fans each document out to the transactional engine writer for its type's collection. Glue between the pipeline quad world and the search-engine document world; owns no projection or engine logic itself.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -25,12 +25,12 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/search": "^0.8.1",
-    "@rdfjs/types": "^2.0.1",
+    "@lde/search": "^0.9.0",
     "tslib": "^2.3.0"
   },
   "devDependencies": {
-    "@lde/search-typesense": "^0.10.2",
+    "@lde/search-typesense": "^0.11.0",
+    "@rdfjs/types": "^2.0.1",
     "n3": "^2.1.1",
     "testcontainers": "^12.0.3",
     "typesense": "^3.0.6"