@lde/search-indexer 0.5.0 → 0.6.1

package/dist/composition.d.ts

@@ -8,7 +8,8 @@ export declare function datasetSelectorFrom(config: IndexerConfig): DatasetSelec
  * The engine-writer factory the configuration describes: an
  * {@link InPlaceRebuild} or {@link BlueGreenRebuild} per root type, its
  * collection name derived from the type – prefixed when the deployment says
- * so, so the read side must be configured with the same prefix.
+ * so, so the read side must be configured with the same prefix – and its
+ * untagged text stemmed in the configured {@link IndexerConfig.defaultLocale}.
  *
  * The mounted schema the pipeline hands each call travels into the writer: a
  * type that surfaces an inline reference builds its collection from the

package/dist/composition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO1E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAK1E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAcxE;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,GACpB,oBAAoB,GAAG,SAAS,CA0BlC"}
+{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAIA,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO1E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAK1E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAexE;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,GACpB,oBAAoB,GAAG,SAAS,CA0BlC"}

package/dist/composition.js

@@ -1,5 +1,6 @@
-// Internal composition pieces of createSearchIndexer, split out so each can
-// be tested without running a pipeline (Pipeline keeps its wiring private).
+// The composition pieces of createSearchIndexer, each usable on its own so a
+// deployment that needs one bespoke stage keeps the rest of the wiring
+// (Pipeline keeps its own wiring private).
 import { Client as RegistryClient } from '@lde/dataset-registry-client';
 import { ImportResolver, RegistrySelector, SparqlDistributionResolver, } from '@lde/pipeline';
 import { BlueGreenRebuild, deriveCollectionName, InPlaceRebuild, } from '@lde/search-typesense';
@@ -15,7 +16,8 @@ export function datasetSelectorFrom(config) {
  * The engine-writer factory the configuration describes: an
  * {@link InPlaceRebuild} or {@link BlueGreenRebuild} per root type, its
  * collection name derived from the type – prefixed when the deployment says
- * so, so the read side must be configured with the same prefix.
+ * so, so the read side must be configured with the same prefix – and its
+ * untagged text stemmed in the configured {@link IndexerConfig.defaultLocale}.
  *
  * The mounted schema the pipeline hands each call travels into the writer: a
  * type that surfaces an inline reference builds its collection from the
@@ -24,7 +26,8 @@ export function datasetSelectorFrom(config) {
  */
 export function writerFactoryFrom(client, config) {
     return (searchType, schema) => {
-        const options = {
+        const writerOptions = {
+            defaultLocale: config.defaultLocale,
             schema,
             ...(config.collectionPrefix
                 ? {
@@ -33,8 +36,8 @@ export function writerFactoryFrom(client, config) {
                 : {}),
         };
         return config.rebuildMode === 'blue-green'
-            ? new BlueGreenRebuild(client, searchType, options)
-            : new InPlaceRebuild(client, searchType, options);
+            ? new BlueGreenRebuild(client, searchType, writerOptions)
+            : new InPlaceRebuild(client, searchType, writerOptions);
     };
 }
 /**

package/dist/config.d.ts

@@ -32,6 +32,13 @@ export interface IndexerConfig {
     /** Prefix prepended to every derived collection name
      *  (`COLLECTION_PREFIX`), e.g. for a shared multi-tenant engine. */
     readonly collectionPrefix?: string;
+    /**
+     * Stemming language for text that carries no language tag (`DEFAULT_LOCALE`,
+     * e.g. `nl`). Unset, such text is folded but **unstemmed** – no language is
+     * ever assumed – so `fietsen` does not match `fiets`. Text declaring a real
+     * locale always stems in its own language, whatever this says.
+     */
+    readonly defaultLocale?: string;
     /** Per-dataset processing memory: skip unchanged datasets
      *  (`PROVENANCE_FILE` + `PIPELINE_VERSION`). */
     readonly provenance?: ProvenanceConfig;

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;iCAC6B;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;uEAEmE;IACnE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;oCAGgC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAQD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAqFf"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;iCAC6B;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;uEAEmE;IACnE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;oCAGgC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAQD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAsFf"}

package/dist/config.js

@@ -67,6 +67,7 @@ export function configFromEnvironment(environment) {
         },
         rebuildMode: rebuildMode,
         collectionPrefix: environment['COLLECTION_PREFIX'],
+        defaultLocale: environment['DEFAULT_LOCALE'],
         provenance,
         qlever,
     };

package/dist/index.d.ts

@@ -1,4 +1,6 @@
 export { createSearchIndexer } from './indexer.js';
+export type { SearchIndexerExtensions } from './indexer.js';
+export { datasetSelectorFrom, distributionResolverFrom, writerFactoryFrom, } from './composition.js';
 export { configFromEnvironment } from './config.js';
 export type { IndexerConfig, ProvenanceConfig, QleverConfig, TypesenseConnection, } from './config.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,GACpB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,YAAY,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAO5D,OAAO,EACL,mBAAmB,EACnB,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,GACpB,MAAM,aAAa,CAAC"}

package/dist/index.js

@@ -4,4 +4,11 @@
 // `search-indexer` bin (src/cli.ts) and the Docker image are thin wrappers
 // over these.
 export { createSearchIndexer } from './indexer.js';
+// The pieces createSearchIndexer composes. Adding a transform does NOT need
+// these – that is `createSearchIndexer(config, { transforms })`. They are for
+// the genuinely bespoke deployment (a non-SPARQL reader, another engine, a root
+// selector that is not by class), which composes `searchStages`,
+// `searchIndexWriter` and `Pipeline` directly and would otherwise also re-derive
+// dataset selection, the QLever import path and collection naming.
+export { datasetSelectorFrom, distributionResolverFrom, writerFactoryFrom, } from './composition.js';
 export { configFromEnvironment } from './config.js';

package/dist/indexer.d.ts

@@ -1,13 +1,36 @@
-import { type Pipeline } from '@lde/pipeline';
+import { type Pipeline, type QuadTransform, type ReaderContext } from '@lde/pipeline';
 import { type TypedSearchDocument } from '@lde/search-pipeline';
 import type { IndexerConfig } from './config.js';
+/** What a deployment adds to the standard indexer – the domain behaviour no
+ *  configuration can express. */
+export interface SearchIndexerExtensions {
+    /**
+     * {@link QuadTransform}(s) per root type, keyed by `SearchType.name`: correct
+     * the data, mint a quad the source does not ship, drop one it should not.
+     * Attached to the type’s reader, so a transform sees each root’s quads before
+     * projection.
+     *
+     * This is the whole cost of adding behaviour – dataset selection, the QLever
+     * import path, registry-sourced types, collection naming, provenance and
+     * reporting stay wired. A name the schema does not declare throws at boot.
+     *
+     * A field a transform fills must still declare a `path`: projection skips a
+     * field with neither a `path` nor a `derive`.
+     */
+    readonly transforms?: Readonly<Record<string, QuadTransform<ReaderContext> | QuadTransform<ReaderContext>[]>>;
+}
 /**
  * Compose the ready-to-run search indexer from an {@link IndexerConfig}: load
  * and validate the mounted schema module (the same file the served-API image
  * mounts), select datasets from the registry, bind the Typesense rebuild
  * writers, and wire the optional QLever import path and provenance store into
  * `searchIndexerPipeline`. Every misconfiguration – an invalid schema, an
- * underivable collection name – throws here, at boot, never mid-run.
+ * underivable collection name, a transform naming an undeclared type – throws
+ * here, at boot, never mid-run.
+ *
+ * `extensions` carries the domain behaviour a deployment adds on top
+ * ({@link SearchIndexerExtensions}); everything else about a deployment is
+ * configuration, so the bin and the Docker image need only the config.
  */
-export declare function createSearchIndexer(config: IndexerConfig): Promise<Pipeline<TypedSearchDocument>>;
+export declare function createSearchIndexer(config: IndexerConfig, extensions?: SearchIndexerExtensions): Promise<Pipeline<TypedSearchDocument>>;
 //# sourceMappingURL=indexer.d.ts.map

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGnE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CA8BxC"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,QAAQ,EACb,KAAK,aAAa,EAClB,KAAK,aAAa,EACnB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;iCACiC;AACjC,MAAM,WAAW,uBAAuB;IACtC;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAC5B,MAAM,CACJ,MAAM,EACN,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAC9D,CACF,CAAC;CACH;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,aAAa,EACrB,UAAU,GAAE,uBAA4B,GACvC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CA+BxC"}

package/dist/indexer.js

@@ -1,4 +1,4 @@
-import { FileProvenanceStore } from '@lde/pipeline';
+import { FileProvenanceStore, } from '@lde/pipeline';
 import { ConsoleReporter } from '@lde/pipeline-console-reporter';
 import { loadSchemaModule } from '@lde/search/module';
 import { searchIndexerPipeline, } from '@lde/search-pipeline';
@@ -10,9 +10,14 @@ import { datasetSelectorFrom, distributionResolverFrom, writerFactoryFrom, } fro
  * mounts), select datasets from the registry, bind the Typesense rebuild
  * writers, and wire the optional QLever import path and provenance store into
  * `searchIndexerPipeline`. Every misconfiguration – an invalid schema, an
- * underivable collection name – throws here, at boot, never mid-run.
+ * underivable collection name, a transform naming an undeclared type – throws
+ * here, at boot, never mid-run.
+ *
+ * `extensions` carries the domain behaviour a deployment adds on top
+ * ({@link SearchIndexerExtensions}); everything else about a deployment is
+ * configuration, so the bin and the Docker image need only the config.
  */
-export async function createSearchIndexer(config) {
+export async function createSearchIndexer(config, extensions = {}) {
     const { schema } = await loadSchemaModule(config.schemaModulePath);
     const client = new Client({
         nodes: [
@@ -40,5 +45,6 @@ export async function createSearchIndexer(config) {
             : undefined,
         pipelineVersion: config.provenance?.pipelineVersion,
         reporter: new ConsoleReporter(),
+        transforms: extensions.transforms,
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-indexer",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "description": "The @lde/search-pipeline indexer as a bootable process and prebuilt Docker image: loads a mounted schema-declaration module, dataset selection, Typesense connection and rebuild mode from the environment, then runs the object-grain searchIndexerPipeline once to completion. The composition layer that binds the engine-agnostic @lde/search-pipeline stages to @lde/search-typesense – the write-path counterpart of @lde/search-api-server.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -31,9 +31,9 @@
     "@lde/dataset-registry-client": "^0.9.1",
     "@lde/pipeline": "^0.35.6",
     "@lde/pipeline-console-reporter": "^0.26.6",
-    "@lde/search": "^0.17.0",
-    "@lde/search-pipeline": "^0.17.0",
-    "@lde/search-typesense": "^0.20.0",
+    "@lde/search": "^0.18.0",
+    "@lde/search-pipeline": "^0.18.1",
+    "@lde/search-typesense": "^0.21.0",
     "@lde/sparql-qlever": "^0.15.3",
     "commander": "^15.0.0",
     "tslib": "^2.3.0",