@lde/search-indexer 0.2.0 → 0.3.1

package/dist/composition.d.ts

@@ -1,5 +1,5 @@
 import { type DatasetSelector, type DistributionResolver, type Writer } from '@lde/pipeline';
-import type { RootType, SearchDocument } from '@lde/search';
+import type { RootType, SearchDocument, SearchSchema } from '@lde/search';
 import type { Client } from 'typesense';
 import type { IndexerConfig } from './config.js';
 /** The registry-backed dataset selection the configuration describes. */
@@ -9,8 +9,13 @@ export declare function datasetSelectorFrom(config: IndexerConfig): DatasetSelec
  * {@link InPlaceRebuild} or {@link BlueGreenRebuild} per root type, its
  * collection name derived from the type – prefixed when the deployment says
  * so, so the read side must be configured with the same prefix.
+ *
+ * The mounted schema the pipeline hands each call travels into the writer: a
+ * type that surfaces an inline reference builds its collection from the
+ * Reference Type the schema resolves, so the nested fields are declared rather
+ * than silently dropped.
  */
-export declare function writerFactoryFrom(client: Client, config: IndexerConfig): (searchType: RootType) => Writer<SearchDocument>;
+export declare function writerFactoryFrom(client: Client, config: IndexerConfig): (searchType: RootType, schema: SearchSchema) => Writer<SearchDocument>;
 /**
  * The distribution resolver the configuration describes: with `QLEVER_IMAGE`
  * set, an {@link ImportResolver} that imports data dumps into a

package/dist/composition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAO5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAK1E;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC,cAAc,CAAC,CAWlD;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,GACpB,oBAAoB,GAAG,SAAS,CA0BlC"}
+{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO1E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAK1E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAcxE;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,GACpB,oBAAoB,GAAG,SAAS,CA0BlC"}

package/dist/composition.js

@@ -16,14 +16,22 @@ export function datasetSelectorFrom(config) {
  * {@link InPlaceRebuild} or {@link BlueGreenRebuild} per root type, its
  * collection name derived from the type – prefixed when the deployment says
  * so, so the read side must be configured with the same prefix.
+ *
+ * The mounted schema the pipeline hands each call travels into the writer: a
+ * type that surfaces an inline reference builds its collection from the
+ * Reference Type the schema resolves, so the nested fields are declared rather
+ * than silently dropped.
  */
 export function writerFactoryFrom(client, config) {
-    return (searchType) => {
-        const options = config.collectionPrefix
-            ? {
-                name: `${config.collectionPrefix}${deriveCollectionName(searchType)}`,
-            }
-            : {};
+    return (searchType, schema) => {
+        const options = {
+            schema,
+            ...(config.collectionPrefix
+                ? {
+                    name: `${config.collectionPrefix}${deriveCollectionName(searchType)}`,
+                }
+                : {}),
+        };
         return config.rebuildMode === 'blue-green'
             ? new BlueGreenRebuild(client, searchType, options)
             : new InPlaceRebuild(client, searchType, options);

package/dist/config.d.ts

@@ -11,6 +11,15 @@ export interface IndexerConfig {
     /** SPARQL endpoint of the DCAT dataset registry the selection queries
      *  (`REGISTRY_ENDPOINT`). */
     readonly registryEndpoint: URL;
+    /**
+     * Root types extracted from {@link registryEndpoint} instead of from each
+     * dataset’s own distribution (`REGISTRY_ROOT_TYPES`, whitespace- or
+     * comma-separated {@link SearchType.name}s) – the dataset’s own description
+     * and what hangs off it, which live in the register rather than in the data a
+     * publisher ships. Empty when unset: every type reads the distribution, the
+     * behaviour before this setting existed.
+     */
+    readonly registryRootTypes: readonly string[];
     /** Dataset selection within the registry: explicit IRIs (`DATASETS`),
      *  search criteria (`DATASET_CRITERIA`), or every dataset when neither is
      *  set. */

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;iCAC6B;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;uEAEmE;IACnE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;oCAGgC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAQD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAoFf"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;iCAC6B;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;uEAEmE;IACnE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;oCAGgC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAQD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAqFf"}

package/dist/config.js

@@ -57,6 +57,7 @@ export function configFromEnvironment(environment) {
     const config = {
         schemaModulePath: environment['SCHEMA_MODULE'] ?? '/config/search-schema.mjs',
         registryEndpoint,
+        registryRootTypes: names(environment['REGISTRY_ROOT_TYPES']),
         datasetCriteria,
         typesense: {
             host: required('TYPESENSE_HOST'),
@@ -74,6 +75,16 @@ export function configFromEnvironment(environment) {
     }
     return config;
 }
+/**
+ * A whitespace- or comma-separated list variable, as a list. The names
+ * themselves are validated against the mounted schema when the pipeline is
+ * wired – config has no schema to check them against.
+ */
+function names(value) {
+    return value === undefined
+        ? []
+        : value.split(/[\s,]+/).filter((name) => name.length > 0);
+}
 /** `DATASETS` (whitespace-separated IRIs) or `DATASET_CRITERIA` (a JSON
  *  object) – never both; neither selects the whole registry. */
 function criteriaFromEnvironment(environment, problems) {

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGnE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAuBxC"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGnE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CA8BxC"}

package/dist/indexer.js

@@ -29,6 +29,12 @@ export async function createSearchIndexer(config) {
         datasets: datasetSelectorFrom(config),
         distributionResolver: distributionResolverFrom(config),
         writerFor: writerFactoryFrom(client, config),
+        registryTypes: config.registryRootTypes.length > 0
+            ? {
+                endpoint: config.registryEndpoint,
+                names: config.registryRootTypes,
+            }
+            : undefined,
         provenanceStore: config.provenance
             ? new FileProvenanceStore({ path: config.provenance.path })
             : undefined,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-indexer",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "The @lde/search-pipeline indexer as a bootable process and prebuilt Docker image: loads a mounted schema-declaration module, dataset selection, Typesense connection and rebuild mode from the environment, then runs the object-grain searchIndexerPipeline once to completion. The composition layer that binds the engine-agnostic @lde/search-pipeline stages to @lde/search-typesense – the write-path counterpart of @lde/search-api-server.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -29,11 +29,11 @@
   ],
   "dependencies": {
     "@lde/dataset-registry-client": "^0.9.1",
-    "@lde/pipeline": "^0.35.4",
-    "@lde/pipeline-console-reporter": "^0.26.4",
-    "@lde/search": "^0.14.0",
-    "@lde/search-pipeline": "^0.14.0",
-    "@lde/search-typesense": "^0.17.0",
+    "@lde/pipeline": "^0.35.5",
+    "@lde/pipeline-console-reporter": "^0.26.5",
+    "@lde/search": "^0.15.0",
+    "@lde/search-pipeline": "^0.15.1",
+    "@lde/search-typesense": "^0.18.1",
     "@lde/sparql-qlever": "^0.15.3",
     "commander": "^15.0.0",
     "tslib": "^2.3.0",