@lde/search-pipeline 0.1.1

package/README.md

@@ -0,0 +1,79 @@
+# @lde/search-pipeline
+
+Search indexing as an [`@lde/pipeline`](../pipeline) Configurable Pipeline
+instance: this package supplies the glue between the pipeline’s quad world
+and a search engine’s document world, so indexing reuses the existing spine –
+Discovery → Selection → Iteration → change-gate (skip-unchanged) – instead of
+rebuilding it per consumer.
+
+The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transaction-aware.md)):
+
+- [`@lde/search`](../search) owns the engine-agnostic projection (framed
+  document + field model) and stays pipeline-free;
+- `@lde/search-<engine>` (e.g. [`@lde/search-typesense`](../search-typesense))
+  is the engine adapter: a transactional `Writer` consuming projected
+  documents, owning the run lifecycle (Blue/green alias swap or In-place
+  sweeps, cross-pod lock);
+- **this package** composes them: `searchIndexWriter` is the one
+  type-changing step (quad → document), shared across engines.
+
+## Usage
+
+```typescript
+import { Pipeline, Stage, SparqlConstructReader } from '@lde/pipeline';
+import { searchSchema } from '@lde/search';
+import { BlueGreenRebuild } from '@lde/search-typesense';
+import { searchIndexWriter } from '@lde/search-pipeline';
+
+const schema = searchSchema({
+  name: 'dataset',
+  type: 'http://www.w3.org/ns/dcat#Dataset',
+  fields: [
+    {
+      name: 'title',
+      kind: 'text',
+      path: 'http://purl.org/dc/terms/title',
+      locales: ['und'],
+      output: true,
+      searchable: true,
+    },
+  ],
+});
+
+const pipeline = new Pipeline({
+  datasetSelector,
+  stages: [
+    new Stage({
+      name: 'extract',
+      readers: new SparqlConstructReader({ query: '…' }),
+    }),
+  ],
+  writers: searchIndexWriter({
+    schema,
+    writer: new BlueGreenRebuild(
+      typesenseClient,
+      schema.get('http://www.w3.org/ns/dcat#Dataset')!,
+      {
+        name: 'datasets',
+      },
+    ),
+  }),
+});
+
+await pipeline.run();
+```
+
+Each dataset’s extracted CONSTRUCT quads are buffered until the dataset
+completes, then framed by root type and projected into flat search documents
+(`@lde/search`’s `projectGraph`), and written to the engine run – before the
+engine acts on the dataset’s completion, so an In-place stale sweep never
+races its own documents. The run lifecycle (run context, per-dataset flush
+outcome, commit/abort) passes through unchanged: the engine writer’s update
+mode governs, and the pipeline never branches on it.
+
+## Memory
+
+Memory is bounded by one dataset’s extraction and released at each dataset
+flush – nothing accumulates across datasets. Streaming bounded entity batches
+_within_ one huge dataset needs the two-level iteration (dataset →
+entity-URI batches) and is not implemented yet.

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { searchIndexWriter } from './search-index-writer.js';
+export type { SearchIndexWriterOptions } from './search-index-writer.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/index.js

@@ -0,0 +1 @@
+export { searchIndexWriter } from './search-index-writer.js';

package/dist/search-index-writer.d.ts

@@ -0,0 +1,35 @@
+import type { Quad } from '@rdfjs/types';
+import type { Writer } from '@lde/pipeline';
+import { type SearchDocument, type SearchSchema } from '@lde/search';
+/** Options for {@link searchIndexWriter}. */
+export interface SearchIndexWriterOptions {
+    /**
+     * The declarative schema driving the projection: one {@link SearchType} per
+     * root type, each mapping framed RDF to flat search-document fields.
+     */
+    schema: SearchSchema;
+    /**
+     * The engine adapter the projected documents are written to – e.g.
+     * `@lde/search-typesense`’s `BlueGreenRebuild` or `InPlaceRebuild`. The
+     * run lifecycle (context, per-dataset flush outcome, commit/abort) is
+     * forwarded unchanged, so the engine writer’s update mode governs.
+     */
+    writer: Writer<SearchDocument>;
+}
+/**
+ * The projection step of a search-indexing pipeline, as a quad `Writer`: it
+ * turns each dataset’s extracted CONSTRUCT quads into engine-agnostic search
+ * documents ({@link projectGraph}: frame by root type, then project each node
+ * with its type’s declaration) and feeds them to the engine writer. This is
+ * the one type-changing step (quad → document), shared across engines –
+ * which is why it lives here and not inside an engine adapter.
+ *
+ * A dataset’s quads are buffered until its flush and projected then, so the
+ * documents land before the engine acts on the dataset’s completion (e.g. an
+ * In-place stale sweep). Memory is bounded by one dataset’s extraction, and
+ * released at each flush – nothing accumulates across datasets. Streaming
+ * bounded entity batches *within* one huge dataset needs the two-level
+ * iteration (dataset → entity-URI batches) and is not implemented yet.
+ */
+export declare function searchIndexWriter(options: SearchIndexWriterOptions): Writer<Quad>;
+//# sourceMappingURL=search-index-writer.d.ts.map

package/dist/search-index-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index-writer.d.ts","sourceRoot":"","sources":["../src/search-index-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAIV,MAAM,EACP,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,YAAY,EAClB,MAAM,aAAa,CAAC;AAErB,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC;;;OAGG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;OAKG;IACH,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;CAChC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,wBAAwB,GAChC,MAAM,CAAC,IAAI,CAAC,CA8Dd"}

package/dist/search-index-writer.js

@@ -0,0 +1,70 @@
+import { projectGraph, } from '@lde/search';
+/**
+ * The projection step of a search-indexing pipeline, as a quad `Writer`: it
+ * turns each dataset’s extracted CONSTRUCT quads into engine-agnostic search
+ * documents ({@link projectGraph}: frame by root type, then project each node
+ * with its type’s declaration) and feeds them to the engine writer. This is
+ * the one type-changing step (quad → document), shared across engines –
+ * which is why it lives here and not inside an engine adapter.
+ *
+ * A dataset’s quads are buffered until its flush and projected then, so the
+ * documents land before the engine acts on the dataset’s completion (e.g. an
+ * In-place stale sweep). Memory is bounded by one dataset’s extraction, and
+ * released at each flush – nothing accumulates across datasets. Streaming
+ * bounded entity batches *within* one huge dataset needs the two-level
+ * iteration (dataset → entity-URI batches) and is not implemented yet.
+ */
+export function searchIndexWriter(options) {
+    const { schema, writer } = options;
+    return {
+        async openRun(context) {
+            const run = await writer.openRun(context);
+            // One buffered pass per dataset, keyed by IRI. The pipeline processes
+            // datasets sequentially, but keeping the key explicit makes a write
+            // after another dataset's flush safe too.
+            const passes = new Map();
+            const project = async (pass) => {
+                passes.delete(pass.dataset.iri.toString());
+                if (pass.quads.length === 0) {
+                    return;
+                }
+                await run.write(pass.dataset, projectGraph(pass.quads, schema));
+            };
+            return {
+                write: async (dataset, quads) => {
+                    const key = dataset.iri.toString();
+                    let pass = passes.get(key);
+                    if (pass === undefined) {
+                        pass = { dataset, quads: [] };
+                        passes.set(key, pass);
+                    }
+                    for await (const quad of quads) {
+                        pass.quads.push(quad);
+                    }
+                },
+                flush: async (dataset, outcome) => {
+                    const pass = passes.get(dataset.iri.toString());
+                    if (pass !== undefined) {
+                        await project(pass);
+                    }
+                    await run.flush?.(dataset, outcome);
+                },
+                reset: async (dataset) => {
+                    // Discard the buffered pass so the re-run replaces it, and let the
+                    // engine writer discard whatever it already holds for the dataset.
+                    passes.delete(dataset.iri.toString());
+                    await run.reset?.(dataset);
+                },
+                commit: async () => {
+                    // Safety net: project passes that were never flushed, so a
+                    // committed run never silently drops written quads.
+                    for (const pass of [...passes.values()]) {
+                        await project(pass);
+                    }
+                    await run.commit();
+                },
+                abort: (error) => run.abort(error),
+            };
+        },
+    };
+}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@lde/search-pipeline",
+  "version": "0.1.1",
+  "description": "Search indexing as an @lde/pipeline instance: a Writer<Quad> that applies the @lde/search projection to extracted RDF and streams the documents to a transactional search-engine writer. Glue between the pipeline quad world and the search-engine document world; owns no projection or engine logic itself.",
+  "repository": {
+    "url": "git+https://github.com/ldelements/lde.git",
+    "directory": "packages/search-pipeline"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "@lde/search": "^0.3.1",
+    "@rdfjs/types": "^2.0.1",
+    "tslib": "^2.3.0"
+  },
+  "devDependencies": {
+    "n3": "^2.1.0"
+  },
+  "peerDependencies": {
+    "@lde/dataset": "^0.7.8",
+    "@lde/pipeline": "^0.33.3"
+  }
+}