@lde/search-typesense 0.0.0

package/README.md

@@ -0,0 +1,61 @@
+# @lde/search-typesense
+
+[Typesense](https://typesense.org/) engine adapter for RDF-backed search
+pipelines. Engine-specific (Typesense) but domain-agnostic – the caller supplies
+the collection schema and documents.
+
+The engine-agnostic half of the pipeline – framing `CONSTRUCT` quads into a
+JSON-LD IR and projecting that IR into flat documents from a declarative field
+spec – lives in [`@lde/search`](../search). This package consumes those
+documents and writes them to Typesense.
+
+## Indexing
+
+`rebuild` blue/green-rebuilds a search index in one call: it creates a fresh
+versioned collection (`${schema.name}_<timestamp>`), streams the documents into
+it in batches, atomically repoints the `schema.name` alias to it, then drops the
+collection it superseded. The caller passes only the logical index name (as
+`schema.name`) and a stream of documents; the versioned collection and the alias
+are managed for them.
+
+```ts
+import { Client } from 'typesense';
+import { rebuild } from '@lde/search-typesense';
+
+const client = new Client({
+  nodes: [{ host, port, protocol: 'https' }],
+  apiKey,
+});
+
+// `documents` is an async iterable (e.g. a streaming projection); only one
+// batch is held in memory at a time. `rebuild` returns the live collection name
+// and the imported count (or `null` if another rebuild was already running).
+const result = await rebuild(client, schema, documents);
+```
+
+`rebuild` takes a `Client` the caller owns (and reuses for queries), so this
+package adds no connection or document type of its own – any object with an `id`
+is a valid document, including the `SearchDocument`s `@lde/search` produces.
+
+## Concurrency
+
+`rebuild` is **single-flight per index**: it first takes a lock (a marker
+document in a `rebuild_locks` collection, created on demand) via Typesense’s
+atomic create, so concurrent callers across pods never rebuild the same index at
+once. A call made while another rebuild for that index is in flight returns
+`null` instead of a count. This keeps blue/green safe under replication: without
+it, two same-millisecond rebuilds would collide on the versioned collection name
+and one would delete the other’s in-flight build.
+
+Limitations to design around:
+
+- **Advisory, not a strict mutex.** The lock is built on Typesense, not a
+  consensus store. Under a TTL-reclaim race two rebuilds can briefly run at
+  once; this is safe because blue/green is idempotent (worst case: redundant
+  work and a transient orphaned collection).
+- **Single-flight, not coalescing.** A call skipped with `null` is _not_ queued.
+  If you must capture state that changed mid-build, re-trigger after the running
+  rebuild finishes.
+- **Lock TTL.** A rebuild running longer than `lockTtlMs` (default 10 minutes)
+  can be reclaimed by another caller and run concurrently; size the TTL above
+  your longest rebuild.

package/dist/adapter.d.ts

@@ -0,0 +1,48 @@
+import type { Client, CollectionCreateSchema } from 'typesense';
+/**
+ * Blue/green-rebuild the search index `name`.
+ *
+ * 1. create a fresh versioned collection (`${name}_<timestamp>`) from `schema`
+ * 2. stream `documents` into it in batches
+ * 3. atomically repoint the `name` alias to the new collection, then
+ *    drop the collection it superseded. The caller passes only the logical
+ *    index `name`; the versioned collection name and the alias are managed here.
+ *
+ * The rebuild is **single-flight per index**: it first takes a lock (a marker
+ * document in a `rebuild_locks` collection, created on demand) via Typesense’s
+ * atomic create, so concurrent callers across pods never rebuild the same index
+ * at once. This keeps blue/green safe under replication.
+ *
+ * `documents` is an async iterable (e.g. a streaming projection); only one
+ * `batchSize`-sized chunk is held in memory at a time. On any failure before the
+ * swap nothing is repointed, so the live alias never points at a partial build,
+ * and the orphaned half-built collection is dropped.
+ *
+ * @returns the live collection name and the number of documents imported, or
+ * `null` when the rebuild was skipped because another rebuild for the same index
+ * was already running.
+ *
+ * Limitations:
+ * - **Advisory, not a strict mutex.** The lock is built on Typesense, not a
+ *   consensus store. Under a TTL-reclaim race two rebuilds can briefly run at
+ *   once; this is safe because blue/green is idempotent (worst case: redundant
+ *   work and a transient orphaned collection).
+ * - **Single-flight, not coalescing.** A call made while a rebuild is in flight
+ *   is skipped (returns `null`), not queued. If you must capture state that
+ *   changed mid-build, re-trigger after the running rebuild finishes.
+ * - **Lock TTL.** A rebuild that runs longer than `lockTtlMs` (default 10
+ *   minutes) can be reclaimed by another caller and run concurrently; size the
+ *   TTL above your longest rebuild.
+ */
+export declare function rebuild<Document extends {
+    id: string;
+}>(client: Client, schema: CollectionCreateSchema, documents: AsyncIterable<Document>, options?: {
+    /** Documents imported per Typesense request (default 1000). */
+    batchSize?: number;
+    /** A held lock older than this (ms) is reclaimed (default 10 minutes). */
+    lockTtlMs?: number;
+}): Promise<{
+    collection: string;
+    imported: number;
+} | null>;
+//# sourceMappingURL=adapter.d.ts.map

package/dist/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,sBAAsB,EAAkB,MAAM,WAAW,CAAC;AAKhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,OAAO,CAAC,QAAQ,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAC3D,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,sBAAsB,EAC9B,SAAS,EAAE,aAAa,CAAC,QAAQ,CAAC,EAClC,OAAO,GAAE;IACP,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;CACf,GACL,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAmC1D"}

package/dist/adapter.js

@@ -0,0 +1,209 @@
+const LOCK_COLLECTION = 'rebuild_locks';
+const DEFAULT_LOCK_TTL_MS = 10 * 60 * 1000;
+/**
+ * Blue/green-rebuild the search index `name`.
+ *
+ * 1. create a fresh versioned collection (`${name}_<timestamp>`) from `schema`
+ * 2. stream `documents` into it in batches
+ * 3. atomically repoint the `name` alias to the new collection, then
+ *    drop the collection it superseded. The caller passes only the logical
+ *    index `name`; the versioned collection name and the alias are managed here.
+ *
+ * The rebuild is **single-flight per index**: it first takes a lock (a marker
+ * document in a `rebuild_locks` collection, created on demand) via Typesense’s
+ * atomic create, so concurrent callers across pods never rebuild the same index
+ * at once. This keeps blue/green safe under replication.
+ *
+ * `documents` is an async iterable (e.g. a streaming projection); only one
+ * `batchSize`-sized chunk is held in memory at a time. On any failure before the
+ * swap nothing is repointed, so the live alias never points at a partial build,
+ * and the orphaned half-built collection is dropped.
+ *
+ * @returns the live collection name and the number of documents imported, or
+ * `null` when the rebuild was skipped because another rebuild for the same index
+ * was already running.
+ *
+ * Limitations:
+ * - **Advisory, not a strict mutex.** The lock is built on Typesense, not a
+ *   consensus store. Under a TTL-reclaim race two rebuilds can briefly run at
+ *   once; this is safe because blue/green is idempotent (worst case: redundant
+ *   work and a transient orphaned collection).
+ * - **Single-flight, not coalescing.** A call made while a rebuild is in flight
+ *   is skipped (returns `null`), not queued. If you must capture state that
+ *   changed mid-build, re-trigger after the running rebuild finishes.
+ * - **Lock TTL.** A rebuild that runs longer than `lockTtlMs` (default 10
+ *   minutes) can be reclaimed by another caller and run concurrently; size the
+ *   TTL above your longest rebuild.
+ */
+export async function rebuild(client, schema, documents, options = {}) {
+    const { batchSize = 1000, lockTtlMs = DEFAULT_LOCK_TTL_MS } = options;
+    const name = schema.name;
+    if (!(await acquireLock(client, name, lockTtlMs))) {
+        return null;
+    }
+    const collection = `${name}_${Date.now()}`;
+    try {
+        const previous = await aliasTarget(client, name);
+        await client.collections().create({ ...schema, name: collection });
+        let imported;
+        try {
+            imported = await importStreamed(client, collection, documents, batchSize);
+            await client.aliases().upsert(name, { collection_name: collection });
+        }
+        catch (error) {
+            // The build failed before the swap: the live alias is untouched, so just
+            // drop the orphaned half-built collection rather than let it accumulate.
+            await client
+                .collections(collection)
+                .delete()
+                .catch(() => undefined);
+            throw error;
+        }
+        if (previous !== undefined && previous !== collection) {
+            await client
+                .collections(previous)
+                .delete()
+                .catch(() => undefined);
+        }
+        return { collection, imported };
+    }
+    finally {
+        await releaseLock(client, name);
+    }
+}
+/** The collection an alias currently points at, or `undefined` if unset. */
+async function aliasTarget(client, alias) {
+    try {
+        const { collection_name } = await client.aliases(alias).retrieve();
+        return collection_name;
+    }
+    catch (error) {
+        if (httpStatus(error) === 404) {
+            return undefined;
+        }
+        throw error;
+    }
+}
+/** Upsert a stream of documents in `batchSize` chunks; returns the count. */
+async function importStreamed(client, collection, documents, batchSize) {
+    let imported = 0;
+    let batch = [];
+    for await (const document of documents) {
+        batch.push(document);
+        if (batch.length >= batchSize) {
+            await importBatch(client, collection, batch);
+            imported += batch.length;
+            batch = [];
+        }
+    }
+    if (batch.length > 0) {
+        await importBatch(client, collection, batch);
+        imported += batch.length;
+    }
+    return imported;
+}
+/**
+ * Create-or-replace one batch of whole documents, keyed on `id`, throwing if any
+ * individual document fails (Typesense’s bulk import otherwise reports
+ * per-document failures without rejecting).
+ */
+async function importBatch(client, collection, batch) {
+    const results = (await client
+        .collections(collection)
+        .documents()
+        .import(batch, {
+        action: 'upsert',
+        // Collect per-document outcomes instead of throwing the client’s opaque
+        // ImportError, so we can report which documents failed and why.
+        throwOnFail: false,
+    }));
+    const failures = results.filter((result) => !result.success);
+    if (failures.length > 0) {
+        throw new Error(`Typesense upsert into “${collection}” failed for ${failures.length}/${results.length} documents: ${failures
+            .map((failure) => failure.error)
+            .join('; ')}`);
+    }
+}
+/**
+ * Take the per-alias rebuild lock via an atomic create, reclaiming it if the
+ * current holder is older than `ttlMs`. Returns `false` if another caller holds
+ * a fresh lock.
+ */
+async function acquireLock(client, alias, ttlMs) {
+    await ensureLockCollection(client);
+    try {
+        await client
+            .collections(LOCK_COLLECTION)
+            .documents()
+            .create({ id: alias, acquired_at: Date.now() });
+        return true;
+    }
+    catch (error) {
+        if (httpStatus(error) === 409) {
+            return reclaimIfStale(client, alias, ttlMs);
+        }
+        throw error;
+    }
+}
+/** Take over the lock if its holder has not refreshed it within `ttlMs`. */
+async function reclaimIfStale(client, alias, ttlMs) {
+    let held;
+    try {
+        held = (await client
+            .collections(LOCK_COLLECTION)
+            .documents(alias)
+            .retrieve());
+    }
+    catch (error) {
+        // Released between our create and this read — leave it for the next try.
+        if (httpStatus(error) === 404) {
+            return false;
+        }
+        throw error;
+    }
+    if (Date.now() - held.acquired_at <= ttlMs) {
+        return false;
+    }
+    await client
+        .collections(LOCK_COLLECTION)
+        .documents()
+        .upsert({ id: alias, acquired_at: Date.now() });
+    return true;
+}
+/** Release the per-alias lock; a no-op when it is not currently held. */
+async function releaseLock(client, alias) {
+    try {
+        await client.collections(LOCK_COLLECTION).documents(alias).delete();
+    }
+    catch (error) {
+        if (httpStatus(error) !== 404) {
+            throw error;
+        }
+    }
+}
+/** Create the lock collection on demand, tolerating a concurrent creator. */
+async function ensureLockCollection(client) {
+    try {
+        await client.collections(LOCK_COLLECTION).retrieve();
+        return;
+    }
+    catch (error) {
+        if (httpStatus(error) !== 404) {
+            throw error;
+        }
+    }
+    try {
+        await client.collections().create({
+            name: LOCK_COLLECTION,
+            fields: [{ name: 'acquired_at', type: 'int64' }],
+        });
+    }
+    catch (error) {
+        if (httpStatus(error) !== 409) {
+            throw error;
+        }
+    }
+}
+function httpStatus(error) {
+    return error.httpStatus;
+}

package/dist/frame.d.ts

@@ -0,0 +1,28 @@
+import type { Quad } from '@rdfjs/types';
+/**
+ * A flat Typesense document. `id` is required (Typesense uses it as the document
+ * key); every other field is engine-typed scalar data or an array thereof.
+ */
+export type TypesenseDocument = {
+    id: string;
+} & Record<string, unknown>;
+export type FrameFieldType = 'string' | 'string[]' | 'int' | 'float' | 'bool' | 'unixtime';
+/**
+ * Maps one document field to a single RDF predicate on the framed subject.
+ * This is the generic, engine-agnostic half of projection — straight
+ * predicate-to-field mappings with datatype coercion. Domain-specific
+ * derivations (folding, grouping, cross-graph joins) are the consumer’s job.
+ */
+export interface FrameField {
+    readonly field: string;
+    readonly predicate: string;
+    readonly type: FrameFieldType;
+}
+/**
+ * Frame the quads describing one subject into a flat document, pulling each
+ * configured field’s value(s) from its predicate and coercing to the field’s
+ * type. Single-valued fields take the first object; `string[]` collects all.
+ * Predicates with no matching quad are omitted (left to Typesense optionality).
+ */
+export declare function frame(quads: Iterable<Quad>, subject: string, fields: readonly FrameField[]): TypesenseDocument;
+//# sourceMappingURL=frame.d.ts.map

package/dist/frame.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"frame.d.ts","sourceRoot":"","sources":["../src/frame.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEzE,MAAM,MAAM,cAAc,GACtB,QAAQ,GACR,UAAU,GACV,KAAK,GACL,OAAO,GACP,MAAM,GACN,UAAU,CAAC;AAEf;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAgB,KAAK,CACnB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,EACrB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,SAAS,UAAU,EAAE,GAC5B,iBAAiB,CAoBnB"}

package/dist/frame.js

@@ -0,0 +1,42 @@
+/**
+ * Frame the quads describing one subject into a flat document, pulling each
+ * configured field’s value(s) from its predicate and coercing to the field’s
+ * type. Single-valued fields take the first object; `string[]` collects all.
+ * Predicates with no matching quad are omitted (left to Typesense optionality).
+ */
+export function frame(quads, subject, fields) {
+    const objectsByPredicate = new Map();
+    for (const quad of quads) {
+        if (quad.subject.value !== subject) {
+            continue;
+        }
+        const values = objectsByPredicate.get(quad.predicate.value) ?? [];
+        values.push(quad.object.value);
+        objectsByPredicate.set(quad.predicate.value, values);
+    }
+    const document = { id: subject };
+    for (const { field, predicate, type } of fields) {
+        const values = objectsByPredicate.get(predicate);
+        if (values === undefined || values.length === 0) {
+            continue;
+        }
+        document[field] = coerce(values, type);
+    }
+    return document;
+}
+function coerce(values, type) {
+    switch (type) {
+        case 'string':
+            return values[0];
+        case 'string[]':
+            return values;
+        case 'int':
+            return Math.trunc(Number(values[0]));
+        case 'float':
+            return Number(values[0]);
+        case 'bool':
+            return values[0] === 'true' || values[0] === '1';
+        case 'unixtime':
+            return Math.trunc(new Date(values[0]).getTime() / 1000);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { rebuild } from './adapter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1 @@
+export { rebuild } from './adapter.js';

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@lde/search-typesense",
+  "version": "0.0.0",
+  "description": "Generic Typesense engine adapter for RDF-backed search pipelines: collection lifecycle, bulk upsert and blue/green alias swap",
+  "repository": {
+    "url": "git+https://github.com/ldelements/lde.git",
+    "directory": "packages/search-typesense"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "tslib": "^2.3.0",
+    "typesense": "^3.0.6"
+  },
+  "devDependencies": {
+    "testcontainers": "^12.0.1"
+  }
+}