@lde/search-pipeline 0.12.1 → 0.12.3

package/README.md

@@ -34,6 +34,62 @@ The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transa
 
 So a search pipeline is **one terminal, N stages**: `new Pipeline<TypedSearchDocument>({ datasetSelector, stages: searchStages(...), writers: searchIndexWriter(...) })`.
 
+## Quick start: `searchIndexerPipeline`
+
+For the common **object grain** – index every IRI-identified instance of each
+root type’s source class – `searchIndexerPipeline` wires that whole composition
+and returns the ready-to-run `Pipeline`. It generates one stage per root type
+in the schema, each selecting the type’s non-blank roots by class
+(`selectByClass`; a blank node has no stable document key, so it can never
+become a document), extracting with the schema-generated CONSTRUCT, and routing
+every document to the engine writer for its type’s collection. The consumer
+supplies only the domain (the schema, which datasets) and the deployment shell
+(the engine writer, the SPARQL/import adapter):
+
+```typescript
+import { searchIndexerPipeline } from '@lde/search-pipeline';
+import {
+  FileProvenanceStore,
+  ImportResolver,
+  SparqlDistributionResolver,
+} from '@lde/pipeline';
+import { ConsoleReporter } from '@lde/pipeline-console-reporter';
+import { createQlever } from '@lde/sparql-qlever';
+import { InPlaceRebuild } from '@lde/search-typesense';
+
+const pipeline = searchIndexerPipeline({
+  schema,
+  datasets,
+  // Import each dataset’s data dump into a local QLever and query that.
+  distributionResolver: new ImportResolver(new SparqlDistributionResolver(), {
+    ...createQlever({ mode: 'docker', image: 'adfreiburg/qlever:latest' }),
+    strategy: 'import',
+  }),
+  writerFor: (searchType) => new InPlaceRebuild(typesenseClient, searchType),
+  // Optional: skip datasets whose source and pipeline version are unchanged.
+  provenanceStore: new FileProvenanceStore({ path: 'data/provenance.json' }),
+  pipelineVersion: '1',
+  reporter: new ConsoleReporter(),
+});
+await pipeline.run();
+```
+
+Each role in that wiring maps to a package:
+
+| Role                                   | Package                                                                                                                                                                                                                |
+| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Dataset selection                      | [`@lde/pipeline`](../pipeline) – `Dataset[]` directly, or a `DatasetSelector` such as `RegistrySelector`                                                                                                               |
+| Distribution resolution and import     | [`@lde/pipeline`](../pipeline) `ImportResolver` + [`@lde/sparql-qlever`](../sparql-qlever) `createQlever` (the concrete [`@lde/sparql-server`](../sparql-server)/[`@lde/sparql-importer`](../sparql-importer) adapter) |
+| Root selection, extraction, projection | this package (`searchStages` over [`@lde/search`](../search))                                                                                                                                                          |
+| Engine writer, one collection per type | [`@lde/search-typesense`](../search-typesense) – `InPlaceRebuild` or `BlueGreenRebuild`                                                                                                                                |
+| Provenance (skip-unchanged)            | [`@lde/pipeline`](../pipeline) – `FileProvenanceStore`, or a triplestore-backed store                                                                                                                                  |
+| Reporting                              | [`@lde/pipeline-console-reporter`](../pipeline-console-reporter) `ConsoleReporter`                                                                                                                                     |
+
+A deployment that needs more – a bespoke root selector (where the entry point
+is a domain fact, not a class), per-stage tuning, non-SPARQL readers, or
+quad-level plugins – composes the parts directly, as below; the convenience
+owns no capability of its own.
+
 ## Usage
 
 ```typescript

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+export { searchIndexerPipeline } from './search-indexer-pipeline.js';
+export type { SearchIndexerPipelineOptions } from './search-indexer-pipeline.js';
 export { searchStages, selectByClass } from './search-stages.js';
 export type { SearchStagesOptions, SearchStageType } from './search-stages.js';
 export { extractionQuery, extractionQueryString } from './extraction.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACzE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,YAAY,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACzE,YAAY,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export { searchIndexerPipeline } from './search-indexer-pipeline.js';
 export { searchStages, selectByClass } from './search-stages.js';
 export { extractionQuery, extractionQueryString } from './extraction.js';
 export { searchIndexWriter } from './search-index-writer.js';

package/dist/search-indexer-pipeline.d.ts

@@ -0,0 +1,86 @@
+import type { Dataset } from '@lde/dataset';
+import { Pipeline, type DatasetSelector, type DistributionResolver, type ProgressReporter, type ProvenanceStore, type Writer } from '@lde/pipeline';
+import type { RootType, SearchDocument, SearchSchema } from '@lde/search';
+import type { TypedSearchDocument } from './typed-search-document.js';
+/** Options for {@link searchIndexerPipeline}. */
+export interface SearchIndexerPipelineOptions {
+    /**
+     * The declarative schema driving the indexer: one stage and one engine
+     * collection per {@link RootType} in it. Reference Types are absent from
+     * `schema.values()`, so none ever earns a stage or a collection.
+     */
+    schema: SearchSchema;
+    /**
+     * Which datasets to index – the deployment’s domain fact. Pass the
+     * {@link Dataset}s directly, or a {@link DatasetSelector} (e.g. a
+     * `RegistrySelector`) when selection is dynamic.
+     */
+    datasets: Dataset[] | DatasetSelector;
+    /**
+     * How each dataset’s data becomes queryable – the deployment’s engine
+     * choice. Typically an `ImportResolver` wrapping a
+     * `SparqlDistributionResolver` with a `@lde/sparql-qlever` `createQlever`
+     * import path, so data-dump distributions are imported into a local
+     * endpoint. Defaults to the {@link Pipeline} default (a bare
+     * `SparqlDistributionResolver`), which serves only datasets that publish a
+     * live SPARQL endpoint.
+     */
+    distributionResolver?: DistributionResolver;
+    /**
+     * The engine writer that owns a given root type’s collection – e.g. a
+     * `@lde/search-typesense` `InPlaceRebuild` or `BlueGreenRebuild` bound to
+     * that type. Called once per {@link RootType} in the schema.
+     */
+    writerFor: (searchType: RootType) => Writer<SearchDocument>;
+    /**
+     * Optional per-dataset processing memory: skip a dataset whose source
+     * fingerprint and {@link pipelineVersion} both match the stored record.
+     * Requires {@link pipelineVersion}.
+     */
+    provenanceStore?: ProvenanceStore;
+    /**
+     * Opaque, consumer-declared version of the indexer’s output-affecting logic
+     * (schema, projection). Required when {@link provenanceStore} is set – a
+     * skip-enabled pipeline with no version would silently freeze.
+     */
+    pipelineVersion?: string;
+    /**
+     * Observer(s) of pipeline lifecycle events, e.g. a
+     * `@lde/pipeline-console-reporter` `ConsoleReporter`.
+     */
+    reporter?: ProgressReporter | readonly ProgressReporter[];
+}
+/**
+ * Wire the common **object-grain** search indexer and return the ready-to-run
+ * {@link Pipeline}: one projecting stage per {@link RootType} in the schema,
+ * each selecting the type’s roots by its source class with blank-node subjects
+ * excluded ({@link selectByClass} – a blank node has no stable document key),
+ * extracting with the schema-generated CONSTRUCT, and projecting inside the
+ * batch; plus the single {@link searchIndexWriter} terminal routing each
+ * document to the engine writer for its type’s collection.
+ *
+ * The consumer supplies only the domain and the deployment shell: the schema,
+ * which datasets, the engine writer per type, and (optionally) the
+ * SPARQL/import adapter, provenance and reporting.
+ *
+ * ```ts
+ * const pipeline = searchIndexerPipeline({
+ *   schema,
+ *   datasets,
+ *   distributionResolver: new ImportResolver(new SparqlDistributionResolver(), {
+ *     ...createQlever({ mode: 'docker', image: 'adfreiburg/qlever:latest' }),
+ *     strategy: 'import',
+ *   }),
+ *   writerFor: (searchType) => new InPlaceRebuild(typesenseClient, searchType),
+ * });
+ * await pipeline.run();
+ * ```
+ *
+ * A deployment that needs more – a bespoke root selector (the entry point is a
+ * domain fact, not a class), per-stage tuning (`batchSize`, `maxConcurrency`),
+ * non-SPARQL readers, or quad-level plugins – composes {@link searchStages},
+ * {@link searchIndexWriter} and {@link Pipeline} directly; this convenience
+ * owns no capability of its own.
+ */
+export declare function searchIndexerPipeline(options: SearchIndexerPipelineOptions): Pipeline<TypedSearchDocument>;
+//# sourceMappingURL=search-indexer-pipeline.d.ts.map

package/dist/search-indexer-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-indexer-pipeline.d.ts","sourceRoot":"","sources":["../src/search-indexer-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAEL,QAAQ,EACR,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,iDAAiD;AACjD,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;OAIG;IACH,QAAQ,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;IACtC;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;OAIG;IACH,SAAS,EAAE,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5D;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;CAC3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,4BAA4B,GACpC,QAAQ,CAAC,mBAAmB,CAAC,CAoB/B"}

package/dist/search-indexer-pipeline.js

@@ -0,0 +1,56 @@
+import { ManualDatasetSelection, Pipeline, } from '@lde/pipeline';
+import { searchIndexWriter } from './search-index-writer.js';
+import { searchStages, selectByClass } from './search-stages.js';
+/**
+ * Wire the common **object-grain** search indexer and return the ready-to-run
+ * {@link Pipeline}: one projecting stage per {@link RootType} in the schema,
+ * each selecting the type’s roots by its source class with blank-node subjects
+ * excluded ({@link selectByClass} – a blank node has no stable document key),
+ * extracting with the schema-generated CONSTRUCT, and projecting inside the
+ * batch; plus the single {@link searchIndexWriter} terminal routing each
+ * document to the engine writer for its type’s collection.
+ *
+ * The consumer supplies only the domain and the deployment shell: the schema,
+ * which datasets, the engine writer per type, and (optionally) the
+ * SPARQL/import adapter, provenance and reporting.
+ *
+ * ```ts
+ * const pipeline = searchIndexerPipeline({
+ *   schema,
+ *   datasets,
+ *   distributionResolver: new ImportResolver(new SparqlDistributionResolver(), {
+ *     ...createQlever({ mode: 'docker', image: 'adfreiburg/qlever:latest' }),
+ *     strategy: 'import',
+ *   }),
+ *   writerFor: (searchType) => new InPlaceRebuild(typesenseClient, searchType),
+ * });
+ * await pipeline.run();
+ * ```
+ *
+ * A deployment that needs more – a bespoke root selector (the entry point is a
+ * domain fact, not a class), per-stage tuning (`batchSize`, `maxConcurrency`),
+ * non-SPARQL readers, or quad-level plugins – composes {@link searchStages},
+ * {@link searchIndexWriter} and {@link Pipeline} directly; this convenience
+ * owns no capability of its own.
+ */
+export function searchIndexerPipeline(options) {
+    const { schema, datasets } = options;
+    return new Pipeline({
+        datasetSelector: Array.isArray(datasets)
+            ? new ManualDatasetSelection(datasets)
+            : datasets,
+        distributionResolver: options.distributionResolver,
+        stages: searchStages({
+            schema,
+            types: [...schema.values()].map((searchType) => ({
+                searchType,
+                rootVariable: 'root',
+                itemSelector: selectByClass(searchType),
+            })),
+        }),
+        writers: searchIndexWriter({ schema, writerFor: options.writerFor }),
+        provenanceStore: options.provenanceStore,
+        pipelineVersion: options.pipelineVersion,
+        reporter: options.reporter,
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-pipeline",
-  "version": "0.12.1",
+  "version": "0.12.3",
   "description": "Search indexing as an @lde/pipeline instance: a Writer<Quad> that applies the @lde/search projection to extracted RDF and fans each document out to the transactional engine writer for its type's collection. Glue between the pipeline quad world and the search-engine document world; owns no projection or engine logic itself.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -32,7 +32,7 @@
     "tslib": "^2.3.0"
   },
   "devDependencies": {
-    "@lde/search-typesense": "^0.15.1",
+    "@lde/search-typesense": "^0.15.3",
     "@rdfjs/types": "^2.0.1",
     "n3": "^2.1.1",
     "testcontainers": "^12.0.3",
@@ -40,6 +40,6 @@
   },
   "peerDependencies": {
     "@lde/dataset": "^0.7.8",
-    "@lde/pipeline": "^0.35.0"
+    "@lde/pipeline": "^0.35.2"
   }
 }