@86d-app/search 0.0.4 → 0.0.13

package/src/admin/endpoints/index.ts

@@ -1,16 +1,22 @@
 import { analyticsEndpoint } from "./analytics";
+import { bulkIndex } from "./bulk-index";
+import { clickAnalyticsEndpoint } from "./click-analytics";
+import { getSettings } from "./get-settings";
 import { indexItem, removeFromIndex } from "./index-manage";
 import { popularEndpoint } from "./popular";
 import { addSynonym, listSynonyms, removeSynonym } from "./synonyms";
 import { zeroResultsEndpoint } from "./zero-results";
 
 export const adminEndpoints = {
+	"/admin/search/settings": getSettings,
 	"/admin/search/analytics": analyticsEndpoint,
 	"/admin/search/popular": popularEndpoint,
 	"/admin/search/zero-results": zeroResultsEndpoint,
+	"/admin/search/clicks": clickAnalyticsEndpoint,
 	"/admin/search/synonyms": listSynonyms,
 	"/admin/search/synonyms/add": addSynonym,
 	"/admin/search/synonyms/:id/delete": removeSynonym,
 	"/admin/search/index": indexItem,
 	"/admin/search/index/remove": removeFromIndex,
+	"/admin/search/index/bulk": bulkIndex,
 };

package/src/admin/endpoints/synonyms.ts

@@ -42,7 +42,7 @@ export const removeSynonym = createAdminEndpoint(
 		const controller = ctx.context.controllers.search as SearchController;
 		const removed = await controller.removeSynonym(ctx.params.id);
 		if (!removed) {
-			throw new Error("Synonym not found");
+			return { error: "Synonym not found", status: 404 };
 		}
 		return { success: true };
 	},

package/src/embedding-provider.ts

@@ -0,0 +1,99 @@
+/**
+ * Embedding provider for AI-powered semantic search.
+ *
+ * Calls OpenAI's text-embedding-3-small model to generate vector embeddings
+ * for indexed items and search queries. When configured, search results are
+ * ranked using a hybrid of lexical scoring and cosine similarity.
+ */
+
+interface EmbeddingResponse {
+	data: Array<{ embedding: number[]; index: number }>;
+	model: string;
+	usage: { prompt_tokens: number; total_tokens: number };
+}
+
+interface EmbeddingErrorResponse {
+	error: { message: string; type: string; code?: string };
+}
+
+export interface EmbeddingProvider {
+	generateEmbedding(text: string): Promise<number[] | null>;
+	generateEmbeddings(texts: string[]): Promise<Array<number[] | null>>;
+}
+
+export class OpenAIEmbeddingProvider implements EmbeddingProvider {
+	private readonly apiKey: string;
+	private readonly model: string;
+	private readonly baseUrl: string;
+
+	constructor(apiKey: string, options?: { model?: string; baseUrl?: string }) {
+		this.apiKey = apiKey;
+		this.model = options?.model ?? "text-embedding-3-small";
+		this.baseUrl = options?.baseUrl ?? "https://api.openai.com/v1";
+	}
+
+	async generateEmbedding(text: string): Promise<number[] | null> {
+		const results = await this.generateEmbeddings([text]);
+		return results[0];
+	}
+
+	async generateEmbeddings(texts: string[]): Promise<Array<number[] | null>> {
+		if (texts.length === 0) return [];
+
+		// Truncate and clean input texts
+		const cleaned = texts.map((t) => t.slice(0, 8000).trim()).filter(Boolean);
+		if (cleaned.length === 0) return texts.map(() => null);
+
+		try {
+			const res = await fetch(`${this.baseUrl}/embeddings`, {
+				method: "POST",
+				headers: {
+					Authorization: `Bearer ${this.apiKey}`,
+					"Content-Type": "application/json",
+				},
+				body: JSON.stringify({
+					input: cleaned,
+					model: this.model,
+				}),
+			});
+
+			if (!res.ok) {
+				const err = (await res.json()) as EmbeddingErrorResponse;
+				console.error(
+					`Embedding API error: ${err.error?.message ?? `HTTP ${res.status}`}`,
+				);
+				return texts.map(() => null);
+			}
+
+			const json = (await res.json()) as EmbeddingResponse;
+			const results: Array<number[] | null> = texts.map(() => null);
+			for (const item of json.data) {
+				if (item.index < cleaned.length) {
+					results[item.index] = item.embedding;
+				}
+			}
+			return results;
+		} catch (err) {
+			console.error("Embedding API request failed:", err);
+			return texts.map(() => null);
+		}
+	}
+}
+
+/**
+ * Cosine similarity between two vectors.
+ * Returns a value between -1 and 1 (1 = identical direction).
+ */
+export function cosineSimilarity(a: number[], b: number[]): number {
+	if (a.length !== b.length || a.length === 0) return 0;
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	for (let i = 0; i < a.length; i++) {
+		dot += a[i] * b[i];
+		normA += a[i] * a[i];
+		normB += b[i] * b[i];
+	}
+	const denom = Math.sqrt(normA) * Math.sqrt(normB);
+	return denom === 0 ? 0 : dot / denom;
+}

package/src/index.ts

@@ -1,35 +1,91 @@
 import type { Module, ModuleConfig, ModuleContext } from "@86d-app/core";
 import { adminEndpoints } from "./admin/endpoints";
+import { OpenAIEmbeddingProvider } from "./embedding-provider";
+import { MeiliSearchProvider } from "./meilisearch-provider";
 import { searchSchema } from "./schema";
 import { createSearchController } from "./service-impl";
 import { storeEndpoints } from "./store/endpoints";
 
+export type { EmbeddingProvider } from "./embedding-provider";
+export { OpenAIEmbeddingProvider } from "./embedding-provider";
+export type { MeiliSearchProvider } from "./meilisearch-provider";
 export type {
+	SearchClick,
 	SearchController,
+	SearchFacets,
+	SearchHighlight,
 	SearchIndexItem,
 	SearchQuery,
 	SearchResult,
+	SearchSortField,
 	SearchSynonym,
 } from "./service";
 
 export interface SearchOptions extends ModuleConfig {
 	/** Maximum number of search results per query */
 	maxResults?: number;
+	/** OpenAI API key for AI-powered semantic search */
+	openaiApiKey?: string;
+	/** OpenRouter API key (alternative to OpenAI) */
+	openrouterApiKey?: string;
+	/** Embedding model name (default: text-embedding-3-small) */
+	embeddingModel?: string;
+	/** MeiliSearch host URL (e.g. http://localhost:7700) */
+	meilisearchHost?: string;
+	/** MeiliSearch API key (master or search key) */
+	meilisearchApiKey?: string;
+	/** MeiliSearch index name (default: "search") */
+	meilisearchIndexUid?: string;
 }
 
 export default function search(options?: SearchOptions): Module {
+	// Create embedding provider from env-var-based API keys
+	let embeddingProvider: OpenAIEmbeddingProvider | undefined;
+	if (options?.openaiApiKey) {
+		embeddingProvider = new OpenAIEmbeddingProvider(options.openaiApiKey, {
+			...(options.embeddingModel ? { model: options.embeddingModel } : {}),
+		});
+	} else if (options?.openrouterApiKey) {
+		embeddingProvider = new OpenAIEmbeddingProvider(options.openrouterApiKey, {
+			model: options.embeddingModel ?? "openai/text-embedding-3-small",
+			baseUrl: "https://openrouter.ai/api/v1" as string,
+		});
+	}
+
+	// Create MeiliSearch provider when host and API key are configured
+	let meiliProvider: MeiliSearchProvider | undefined;
+	if (options?.meilisearchHost && options?.meilisearchApiKey) {
+		meiliProvider = new MeiliSearchProvider(
+			options.meilisearchHost,
+			options.meilisearchApiKey,
+			options.meilisearchIndexUid,
+		);
+	}
+
 	return {
 		id: "search",
-		version: "0.0.1",
+		version: "0.1.0",
 		schema: searchSchema,
 		exports: {
 			read: ["searchIndexCount", "popularTerms"],
 		},
 		events: {
-			emits: ["search.queried", "search.indexed", "search.removed"],
+			emits: [
+				"search.queried",
+				"search.indexed",
+				"search.removed",
+				"search.clicked",
+			],
 		},
 		init: async (ctx: ModuleContext) => {
-			const controller = createSearchController(ctx.data);
+			if (meiliProvider) {
+				void meiliProvider.configureIndex();
+			}
+			const controller = createSearchController(
+				ctx.data,
+				embeddingProvider,
+				meiliProvider,
+			);
 			return { controllers: { search: controller } };
 		},
 		search: { store: "/search/store-search" },
@@ -44,7 +100,7 @@ export default function search(options?: SearchOptions): Module {
 					component: "SearchAnalytics",
 					label: "Search",
 					icon: "MagnifyingGlass",
-					group: "Marketing",
+					group: "System",
 				},
 			],
 		},

package/src/meilisearch-provider.ts

@@ -0,0 +1,239 @@
+/**
+ * MeiliSearch provider for real search engine integration.
+ *
+ * When configured with a MeiliSearch host and API key, the search module
+ * delegates indexing and querying to a dedicated MeiliSearch instance instead
+ * of relying on in-application lexical search. Falls back gracefully when
+ * the MeiliSearch instance is unreachable.
+ */
+
+export interface MeiliSearchDocument {
+	id: string;
+	entityType: string;
+	entityId: string;
+	title: string;
+	body?: string | undefined;
+	tags: string[];
+	url: string;
+	image?: string | undefined;
+	indexedAt: string;
+	[key: string]: unknown;
+}
+
+export interface MeiliSearchHit {
+	id: string;
+	entityType: string;
+	entityId: string;
+	title: string;
+	body?: string;
+	tags: string[];
+	url: string;
+	image?: string;
+	indexedAt: string;
+	_formatted?: {
+		title?: string;
+		body?: string;
+		[key: string]: string | undefined;
+	};
+	_rankingScore?: number;
+	[key: string]: unknown;
+}
+
+export interface MeiliSearchResponse {
+	hits: MeiliSearchHit[];
+	query: string;
+	processingTimeMs: number;
+	estimatedTotalHits?: number;
+	totalHits?: number;
+	facetDistribution?: Record<string, Record<string, number>>;
+}
+
+interface MeiliSearchTaskResponse {
+	taskUid: number;
+	indexUid: string;
+	status: "enqueued" | "processing" | "succeeded" | "failed";
+	type: string;
+	enqueuedAt: string;
+}
+
+interface MeiliSearchError {
+	message: string;
+	code: string;
+	type: string;
+	link?: string;
+}
+
+interface MeiliSearchHealthResponse {
+	status: "available";
+}
+
+interface MeiliSearchStatsResponse {
+	numberOfDocuments: number;
+	isIndexing: boolean;
+	fieldDistribution: Record<string, number>;
+}
+
+export class MeiliSearchProvider {
+	private readonly host: string;
+	private readonly apiKey: string;
+	private readonly indexUid: string;
+
+	constructor(host: string, apiKey: string, indexUid = "search") {
+		// Strip trailing slash from host
+		this.host = host.replace(/\/+$/, "");
+		this.apiKey = apiKey;
+		this.indexUid = indexUid;
+	}
+
+	/**
+	 * Add or replace documents in the MeiliSearch index.
+	 * MeiliSearch uses the `id` field as the primary key by default.
+	 */
+	async addDocuments(
+		documents: MeiliSearchDocument[],
+	): Promise<MeiliSearchTaskResponse> {
+		const res = await this.request(
+			`/indexes/${this.indexUid}/documents`,
+			"POST",
+			documents,
+		);
+		return res as MeiliSearchTaskResponse;
+	}
+
+	/**
+	 * Delete a single document by its ID.
+	 */
+	async deleteDocument(documentId: string): Promise<MeiliSearchTaskResponse> {
+		const res = await this.request(
+			`/indexes/${this.indexUid}/documents/${encodeURIComponent(documentId)}`,
+			"DELETE",
+		);
+		return res as MeiliSearchTaskResponse;
+	}
+
+	/**
+	 * Search the MeiliSearch index.
+	 */
+	async search(
+		query: string,
+		options?: {
+			limit?: number | undefined;
+			offset?: number | undefined;
+			filter?: string | undefined;
+			sort?: string[] | undefined;
+			facets?: string[] | undefined;
+			attributesToHighlight?: string[] | undefined;
+			highlightPreTag?: string | undefined;
+			highlightPostTag?: string | undefined;
+			showRankingScore?: boolean | undefined;
+			matchingStrategy?: "last" | "all" | "frequency" | undefined;
+		},
+	): Promise<MeiliSearchResponse> {
+		const body: Record<string, unknown> = { q: query };
+
+		if (options?.limit !== undefined) body.limit = options.limit;
+		if (options?.offset !== undefined) body.offset = options.offset;
+		if (options?.filter) body.filter = options.filter;
+		if (options?.sort) body.sort = options.sort;
+		if (options?.facets) body.facets = options.facets;
+		if (options?.attributesToHighlight) {
+			body.attributesToHighlight = options.attributesToHighlight;
+		}
+		if (options?.highlightPreTag) {
+			body.highlightPreTag = options.highlightPreTag;
+		}
+		if (options?.highlightPostTag) {
+			body.highlightPostTag = options.highlightPostTag;
+		}
+		if (options?.showRankingScore) body.showRankingScore = true;
+		if (options?.matchingStrategy) {
+			body.matchingStrategy = options.matchingStrategy;
+		}
+
+		const res = await this.request(
+			`/indexes/${this.indexUid}/search`,
+			"POST",
+			body,
+		);
+		return res as MeiliSearchResponse;
+	}
+
+	/**
+	 * Check if the MeiliSearch instance is reachable and healthy.
+	 */
+	async isHealthy(): Promise<boolean> {
+		try {
+			const res = await this.request("/health", "GET");
+			return (res as MeiliSearchHealthResponse).status === "available";
+		} catch {
+			return false;
+		}
+	}
+
+	/**
+	 * Get index statistics (document count, indexing status).
+	 */
+	async getStats(): Promise<MeiliSearchStatsResponse | null> {
+		try {
+			const res = await this.request(`/indexes/${this.indexUid}/stats`, "GET");
+			return res as MeiliSearchStatsResponse;
+		} catch {
+			return null;
+		}
+	}
+
+	/**
+	 * Configure index settings: filterable and sortable attributes.
+	 * Called once on init to ensure MeiliSearch knows which fields to index.
+	 */
+	async configureIndex(): Promise<void> {
+		try {
+			await this.request(`/indexes/${this.indexUid}/settings`, "PATCH", {
+				filterableAttributes: ["entityType", "tags"],
+				sortableAttributes: ["indexedAt", "title"],
+				searchableAttributes: ["title", "body", "tags"],
+			});
+		} catch {
+			// Non-critical — index may already be configured or not yet exist.
+			// MeiliSearch auto-creates indexes on first document addition.
+		}
+	}
+
+	private async request(
+		path: string,
+		method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
+		body?: unknown,
+	): Promise<unknown> {
+		const url = `${this.host}${path}`;
+		const headers: Record<string, string> = {
+			Authorization: `Bearer ${this.apiKey}`,
+		};
+		if (body !== undefined) {
+			headers["Content-Type"] = "application/json";
+		}
+
+		const res = await fetch(url, {
+			method,
+			headers,
+			...(body !== undefined ? { body: JSON.stringify(body) } : {}),
+		});
+
+		if (!res.ok) {
+			let errorMessage = `MeiliSearch ${method} ${path} failed: HTTP ${res.status}`;
+			try {
+				const err = (await res.json()) as MeiliSearchError;
+				if (err.message) {
+					errorMessage = `MeiliSearch error: ${err.message} (${err.code})`;
+				}
+			} catch {
+				// Response body may not be JSON
+			}
+			throw new Error(errorMessage);
+		}
+
+		// DELETE and some endpoints may return 204 No Content
+		if (res.status === 204) return {};
+
+		return res.json();
+	}
+}

package/src/schema.ts

@@ -45,4 +45,19 @@ export const searchSchema = {
 			},
 		},
 	},
+	searchClick: {
+		fields: {
+			id: { type: "string", required: true },
+			queryId: { type: "string", required: true },
+			term: { type: "string", required: true },
+			entityType: { type: "string", required: true },
+			entityId: { type: "string", required: true },
+			position: { type: "number", required: true },
+			clickedAt: {
+				type: "date",
+				required: true,
+				defaultValue: () => new Date(),
+			},
+		},
+	},
 } satisfies ModuleSchema;