@taico/taico 0.5.1 → 0.6.0

package/dist/search/index/search-embeddings.service.js

@@ -0,0 +1,115 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var SearchEmbeddingsService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchEmbeddingsService = void 0;
+const common_1 = require("@nestjs/common");
+const ai_1 = require("ai");
+const anthropic_1 = require("@ai-sdk/anthropic");
+const google_1 = require("@ai-sdk/google");
+const google_vertex_1 = require("@ai-sdk/google-vertex");
+const openai_1 = require("@ai-sdk/openai");
+const openai_compatible_1 = require("@ai-sdk/openai-compatible");
+const xai_1 = require("@ai-sdk/xai");
+const ai_model_selections_service_1 = require("../../ai-providers/ai-model-selections.service");
+const enums_1 = require("../../ai-providers/enums");
+const google_environment_1 = require("../../ai-providers/google-environment");
+const QUERY_CACHE_SIZE = 200;
+let SearchEmbeddingsService = SearchEmbeddingsService_1 = class SearchEmbeddingsService {
+    aiModelSelectionsService;
+    logger = new common_1.Logger(SearchEmbeddingsService_1.name);
+    queryCache = new Map();
+    constructor(aiModelSelectionsService) {
+        this.aiModelSelectionsService = aiModelSelectionsService;
+    }
+    async resolveModel() {
+        try {
+            const selection = await this.aiModelSelectionsService.resolve(enums_1.AiModelJob.EMBEDDINGS);
+            const model = buildEmbeddingModel(selection.provider, selection.modelId);
+            if (!model) {
+                return null;
+            }
+            return {
+                model,
+                key: `${selection.provider.id}:${selection.modelId}`,
+                modelId: selection.modelId,
+            };
+        }
+        catch (error) {
+            this.logger.debug({
+                message: 'No usable embedding model',
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return null;
+        }
+    }
+    async embedDocuments(resolved, texts) {
+        if (texts.length === 0) {
+            return [];
+        }
+        const { embeddings } = await (0, ai_1.embedMany)({ model: resolved.model, values: texts });
+        return embeddings;
+    }
+    async embedQuery(resolved, query, options) {
+        const value = options.instruction
+            ? `Instruct: ${options.instruction}\nQuery: ${query}`
+            : query;
+        const cacheKey = `${resolved.key}\n${value}`;
+        const cached = this.queryCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const { embedding } = await (0, ai_1.embed)({
+            model: resolved.model,
+            value,
+            maxRetries: 0,
+            abortSignal: AbortSignal.timeout(options.timeoutMs),
+        });
+        this.queryCache.set(cacheKey, embedding);
+        if (this.queryCache.size > QUERY_CACHE_SIZE) {
+            this.queryCache.delete(this.queryCache.keys().next().value);
+        }
+        return embedding;
+    }
+};
+exports.SearchEmbeddingsService = SearchEmbeddingsService;
+exports.SearchEmbeddingsService = SearchEmbeddingsService = SearchEmbeddingsService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [ai_model_selections_service_1.AiModelSelectionsService])
+], SearchEmbeddingsService);
+function buildEmbeddingModel(config, modelId) {
+    const apiKey = config.apiKey ?? undefined;
+    const baseURL = config.baseUrl ?? undefined;
+    switch (config.type) {
+        case enums_1.AiProviderType.OPENAI:
+            return (0, openai_1.createOpenAI)({ apiKey, baseURL }).textEmbeddingModel(modelId);
+        case enums_1.AiProviderType.GOOGLE:
+            return (0, google_1.createGoogle)({ apiKey: apiKey ?? (0, google_environment_1.googleApiKeyFromEnv)(), baseURL }).textEmbeddingModel(modelId);
+        case enums_1.AiProviderType.GOOGLE_VERTEX: {
+            const credentials = apiKey ? JSON.parse(apiKey) : undefined;
+            return (0, google_vertex_1.createVertex)({
+                project: config.project ?? (0, google_environment_1.vertexProjectFromEnv)(),
+                location: config.location ?? (0, google_environment_1.vertexLocationFromEnv)(),
+                googleAuthOptions: credentials ? { credentials } : undefined,
+            }).textEmbeddingModel(modelId);
+        }
+        case enums_1.AiProviderType.XAI:
+            return (0, xai_1.createXai)({ apiKey, baseURL }).textEmbeddingModel(modelId);
+        case enums_1.AiProviderType.ANTHROPIC:
+            return (0, anthropic_1.createAnthropic)({ apiKey, baseURL }).textEmbeddingModel(modelId);
+        case enums_1.AiProviderType.OPENAI_COMPATIBLE:
+            if (!baseURL) {
+                return null;
+            }
+            return (0, openai_compatible_1.createOpenAICompatible)({ name: config.type, baseURL, apiKey }).textEmbeddingModel(modelId);
+    }
+}
+//# sourceMappingURL=search-embeddings.service.js.map

package/dist/search/index/search-embeddings.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-embeddings.service.js","sourceRoot":"","sources":["../../../src/search/index/search-embeddings.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,2BAAsD;AACtD,iDAAoD;AACpD,2CAA8C;AAC9C,yDAAqD;AACrD,2CAA8C;AAC9C,iEAAmE;AACnE,qCAAwC;AACxC,gGAAwF;AAExF,oDAAoE;AACpE,8EAI6C;AAS7C,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAItB,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAKL;IAJZ,MAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAC;IAElD,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE1D,YAA6B,wBAAkD;QAAlD,6BAAwB,GAAxB,wBAAwB,CAA0B;IAAG,CAAC;IAGnF,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,OAAO,CAAC,kBAAU,CAAC,UAAU,CAAC,CAAC;YACrF,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO;gBACL,KAAK;gBACL,GAAG,EAAE,GAAG,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,SAAS,CAAC,OAAO,EAAE;gBACpD,OAAO,EAAE,SAAS,CAAC,OAAO;aAC3B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBAChB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,cAAc,CAAC,QAAgC,EAAE,KAAe;QACpE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAA,cAAS,EAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACjF,OAAO,UAAU,CAAC;IACpB,CAAC;IAMD,KAAK,CAAC,UAAU,CACd,QAAgC,EAChC,KAAa,EACb,OAAmD;QAEnD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW;YAC/B,CAAC,CAAC,aAAa,OAAO,CAAC,WAAW,YAAY,KAAK,EAAE;YACrD,CAAC,CAAC,KAAK,CAAC;QACV,MAAM,QAAQ,GAAG,GAAG,QAAQ,CAAC,GAAG,KAAK,KAAK,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAA,UAAK,EAAC;YAChC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,KAAK;YAEL,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;SACpD,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,gBAAgB,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAe,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CACF,CAAA;AAtEY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;qCAM4C,sDAAwB;GALpE,uBAAuB,CAsEnC;AAED,SAAS,mBAAmB,CAAC,MAA8B,EAAE,OAAe;IAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC;IAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,SAAS,CAAC;IAE5C,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,sBAAc,CAAC,MAAM;YACxB,OAAO,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEvE,KAAK,sBAAc,CAAC,MAAM;YACxB,OAAO,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,MAAM,IAAI,IAAA,wCAAmB,GAAE,EAAE,OAAO,EAAE,CAAC,CAAC,kBAAkB,CAC1F,OAAO,CACR,CAAC;QAEJ,KAAK,sBAAc,CAAC,aAAa,CAAC,CAAC,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAA6B,CAAC,CAAC,CAAC,SAAS,CAAC;YACzF,OAAO,IAAA,4BAAY,EAAC;gBAClB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAA,yCAAoB,GAAE;gBACjD,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAA,0CAAqB,GAAE;gBACpD,iBAAiB,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;aAC7D,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAED,KAAK,sBAAc,CAAC,GAAG;YACrB,OAAO,IAAA,eAAS,EAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,sBAAc,CAAC,SAAS;YAC3B,OAAO,IAAA,2BAAe,EAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAE1E,KAAK,sBAAc,CAAC,iBAAiB;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,IAAA,0CAAsB,EAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,kBAAkB,CACtF,OAAO,CACR,CAAC;IACN,CAAC;AACH,CAAC"}

package/dist/search/index/search-excerpt.d.ts

@@ -0,0 +1,21 @@
+import { SearchMatchField, SearchStrategy } from '../enums';
+import { SearchEntityType } from '../search.types';
+export type ExcerptSource = {
+    hit_strategy: SearchStrategy;
+    hit_entity_type: SearchEntityType;
+    hit_entity_id: string;
+    hit_author: string | null;
+    hit_body_length: number;
+    hit_body_start: string;
+    hit_body_matches: boolean | null;
+    headline: string | null;
+    chunk_kind: 'title' | 'body' | null;
+    chunk_text: string | null;
+    primary_body_length: number;
+    primary_body_start: string;
+};
+export declare function pickExcerpt(row: ExcerptSource, maxChars: number): {
+    matchedOn: SearchMatchField;
+    text: string;
+    isComplete: boolean;
+};

package/dist/search/index/search-excerpt.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pickExcerpt = pickExcerpt;
+const enums_1 = require("../enums");
+function pickExcerpt(row, maxChars) {
+    const hitIsComment = row.hit_entity_type === 'comment';
+    const passageMatched = (row.hit_strategy === enums_1.SearchStrategy.SEMANTIC && row.chunk_kind === 'body') ||
+        (row.hit_strategy === enums_1.SearchStrategy.KEYWORD && (hitIsComment || row.hit_body_matches === true));
+    if (!passageMatched) {
+        return {
+            matchedOn: enums_1.SearchMatchField.TITLE,
+            text: clip(row.primary_body_start, maxChars, row.primary_body_length > maxChars),
+            isComplete: row.primary_body_length <= maxChars,
+        };
+    }
+    const matchedOn = hitIsComment ? enums_1.SearchMatchField.COMMENT : enums_1.SearchMatchField.CONTENT;
+    if (row.hit_body_length <= maxChars) {
+        return { matchedOn, text: row.hit_body_start.trim(), isComplete: true };
+    }
+    const passage = row.hit_strategy === enums_1.SearchStrategy.SEMANTIC ? row.chunk_text : row.headline;
+    const text = passage ?? row.hit_body_start;
+    return { matchedOn, text: clip(text, maxChars, text.length > maxChars), isComplete: false };
+}
+function clip(text, maxChars, truncated) {
+    const trimmed = text.trim();
+    if (!truncated) {
+        return trimmed;
+    }
+    const cut = trimmed.slice(0, maxChars);
+    const lastSpace = cut.lastIndexOf(' ');
+    return `${(lastSpace > maxChars / 2 ? cut.slice(0, lastSpace) : cut).trimEnd()} …`;
+}
+//# sourceMappingURL=search-excerpt.js.map

package/dist/search/index/search-excerpt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-excerpt.js","sourceRoot":"","sources":["../../../src/search/index/search-excerpt.ts"],"names":[],"mappings":";;AAwBA,kCAwBC;AAhDD,oCAA4D;AAwB5D,SAAgB,WAAW,CACzB,GAAkB,EAClB,QAAgB;IAEhB,MAAM,YAAY,GAAG,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC;IACvD,MAAM,cAAc,GAClB,CAAC,GAAG,CAAC,YAAY,KAAK,sBAAc,CAAC,QAAQ,IAAI,GAAG,CAAC,UAAU,KAAK,MAAM,CAAC;QAC3E,CAAC,GAAG,CAAC,YAAY,KAAK,sBAAc,CAAC,OAAO,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC;IAEnG,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO;YACL,SAAS,EAAE,wBAAgB,CAAC,KAAK;YACjC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,EAAE,GAAG,CAAC,mBAAmB,GAAG,QAAQ,CAAC;YAChF,UAAU,EAAE,GAAG,CAAC,mBAAmB,IAAI,QAAQ;SAChD,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,wBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAgB,CAAC,OAAO,CAAC;IACrF,IAAI,GAAG,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC1E,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,KAAK,sBAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC7F,MAAM,IAAI,GAAG,OAAO,IAAI,GAAG,CAAC,cAAc,CAAC;IAC3C,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC9F,CAAC;AAED,SAAS,IAAI,CAAC,IAAY,EAAE,QAAgB,EAAE,SAAkB;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,OAAO,GAAG,CAAC,SAAS,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC;AACrF,CAAC"}

package/dist/search/index/search-excerpt.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/search/index/search-excerpt.spec.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const enums_1 = require("../enums");
+const search_excerpt_1 = require("./search-excerpt");
+const longBody = 'Opening sentence of the block. ' + 'More text follows here. '.repeat(40);
+function source(overrides) {
+    return {
+        hit_strategy: enums_1.SearchStrategy.KEYWORD,
+        hit_entity_type: 'context_block',
+        hit_entity_id: 'hit-id',
+        hit_author: null,
+        hit_body_length: longBody.length,
+        hit_body_start: longBody.slice(0, 100),
+        hit_body_matches: false,
+        headline: null,
+        chunk_kind: null,
+        chunk_text: null,
+        primary_body_length: longBody.length,
+        primary_body_start: longBody.slice(0, 100),
+        ...overrides,
+    };
+}
+describe('pickExcerpt', () => {
+    it('shows the start of the item, marked partial, when only the title matched', () => {
+        for (const hit of [
+            { hit_strategy: enums_1.SearchStrategy.FUZZY_TITLE },
+            { hit_strategy: enums_1.SearchStrategy.SEMANTIC, chunk_kind: 'title', chunk_text: 'Title' },
+            { hit_strategy: enums_1.SearchStrategy.KEYWORD, hit_body_matches: false },
+        ]) {
+            const excerpt = (0, search_excerpt_1.pickExcerpt)(source(hit), 100);
+            expect(excerpt.matchedOn).toBe(enums_1.SearchMatchField.TITLE);
+            expect(excerpt.text.startsWith('Opening sentence of the block.')).toBe(true);
+            expect(excerpt.text.endsWith('…')).toBe(true);
+            expect(excerpt.isComplete).toBe(false);
+        }
+    });
+    it('returns a short item whole on a title match', () => {
+        const excerpt = (0, search_excerpt_1.pickExcerpt)(source({
+            hit_strategy: enums_1.SearchStrategy.FUZZY_TITLE,
+            primary_body_length: 12,
+            primary_body_start: 'Short body.\n',
+        }), 100);
+        expect(excerpt).toEqual({ matchedOn: enums_1.SearchMatchField.TITLE, text: 'Short body.', isComplete: true });
+    });
+    it('returns an empty, complete excerpt for an item without text', () => {
+        const excerpt = (0, search_excerpt_1.pickExcerpt)(source({ hit_strategy: enums_1.SearchStrategy.FUZZY_TITLE, primary_body_length: 0, primary_body_start: '' }), 100);
+        expect(excerpt).toEqual({ matchedOn: enums_1.SearchMatchField.TITLE, text: '', isComplete: true });
+    });
+    it('shows the passage that matched by meaning, marked partial', () => {
+        const excerpt = (0, search_excerpt_1.pickExcerpt)(source({
+            hit_strategy: enums_1.SearchStrategy.SEMANTIC,
+            chunk_kind: 'body',
+            chunk_text: '## Rollback\n\nRevert the commit.',
+        }), 100);
+        expect(excerpt).toEqual({
+            matchedOn: enums_1.SearchMatchField.CONTENT,
+            text: '## Rollback\n\nRevert the commit.',
+            isComplete: false,
+        });
+    });
+    it('shows the keyword fragments from a long text', () => {
+        const excerpt = (0, search_excerpt_1.pickExcerpt)(source({ hit_body_matches: true, headline: 'the signing key … rotated nightly' }), 100);
+        expect(excerpt).toEqual({
+            matchedOn: enums_1.SearchMatchField.CONTENT,
+            text: 'the signing key … rotated nightly',
+            isComplete: false,
+        });
+    });
+    it('returns text that fits whole instead of a passage of it', () => {
+        const excerpt = (0, search_excerpt_1.pickExcerpt)(source({
+            hit_strategy: enums_1.SearchStrategy.SEMANTIC,
+            chunk_kind: 'body',
+            chunk_text: 'Second paragraph.',
+            hit_body_length: 34,
+            hit_body_start: 'First paragraph.\n\nSecond paragraph.',
+        }), 100);
+        expect(excerpt).toEqual({
+            matchedOn: enums_1.SearchMatchField.CONTENT,
+            text: 'First paragraph.\n\nSecond paragraph.',
+            isComplete: true,
+        });
+    });
+    it('reports a matching comment as a comment', () => {
+        const excerpt = (0, search_excerpt_1.pickExcerpt)(source({
+            hit_entity_type: 'comment',
+            hit_body_length: 26,
+            hit_body_start: 'Safari dropped the cookie.',
+        }), 100);
+        expect(excerpt).toEqual({
+            matchedOn: enums_1.SearchMatchField.COMMENT,
+            text: 'Safari dropped the cookie.',
+            isComplete: true,
+        });
+    });
+});
+//# sourceMappingURL=search-excerpt.spec.js.map

package/dist/search/index/search-excerpt.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-excerpt.spec.js","sourceRoot":"","sources":["../../../src/search/index/search-excerpt.spec.ts"],"names":[],"mappings":";;AAAA,oCAA4D;AAC5D,qDAA8D;AAE9D,MAAM,QAAQ,GAAG,iCAAiC,GAAG,0BAA0B,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAE3F,SAAS,MAAM,CAAC,SAAiC;IAC/C,OAAO;QACL,YAAY,EAAE,sBAAc,CAAC,OAAO;QACpC,eAAe,EAAE,eAAe;QAChC,aAAa,EAAE,QAAQ;QACvB,UAAU,EAAE,IAAI;QAChB,eAAe,EAAE,QAAQ,CAAC,MAAM;QAChC,cAAc,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACtC,gBAAgB,EAAE,KAAK;QACvB,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,IAAI;QAChB,mBAAmB,EAAE,QAAQ,CAAC,MAAM;QACpC,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAC1C,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,KAAK,MAAM,GAAG,IAAI;YAChB,EAAE,YAAY,EAAE,sBAAc,CAAC,WAAW,EAAE;YAC5C,EAAE,YAAY,EAAE,sBAAc,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAgB,EAAE,UAAU,EAAE,OAAO,EAAE;YAC5F,EAAE,YAAY,EAAE,sBAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE;SAClE,EAAE,CAAC;YACF,MAAM,OAAO,GAAG,IAAA,4BAAW,EAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YAE9C,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,wBAAgB,CAAC,KAAK,CAAC,CAAC;YACvD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,OAAO,GAAG,IAAA,4BAAW,EACzB,MAAM,CAAC;YACL,YAAY,EAAE,sBAAc,CAAC,WAAW;YACxC,mBAAmB,EAAE,EAAE;YACvB,kBAAkB,EAAE,eAAe;SACpC,CAAC,EACF,GAAG,CACJ,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,wBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,OAAO,GAAG,IAAA,4BAAW,EACzB,MAAM,CAAC,EAAE,YAAY,EAAE,sBAAc,CAAC,WAAW,EAAE,mBAAmB,EAAE,CAAC,EAAE,kBAAkB,EAAE,EAAE,EAAE,CAAC,EACpG,GAAG,CACJ,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,wBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,OAAO,GAAG,IAAA,4BAAW,EACzB,MAAM,CAAC;YACL,YAAY,EAAE,sBAAc,CAAC,QAAQ;YACrC,UAAU,EAAE,MAAM;YAClB,UAAU,EAAE,mCAAmC;SAChD,CAAC,EACF,GAAG,CACJ,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,SAAS,EAAE,wBAAgB,CAAC,OAAO;YACnC,IAAI,EAAE,mCAAmC;YACzC,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,OAAO,GAAG,IAAA,4BAAW,EACzB,MAAM,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,mCAAmC,EAAE,CAAC,EACjF,GAAG,CACJ,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,SAAS,EAAE,wBAAgB,CAAC,OAAO;YACnC,IAAI,EAAE,mCAAmC;YACzC,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,OAAO,GAAG,IAAA,4BAAW,EACzB,MAAM,CAAC;YACL,YAAY,EAAE,sBAAc,CAAC,QAAQ;YACrC,UAAU,EAAE,MAAM;YAClB,UAAU,EAAE,mBAAmB;YAC/B,eAAe,EAAE,EAAE;YACnB,cAAc,EAAE,uCAAuC;SACxD,CAAC,EACF,GAAG,CACJ,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,SAAS,EAAE,wBAAgB,CAAC,OAAO;YACnC,IAAI,EAAE,uCAAuC;YAC7C,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,OAAO,GAAG,IAAA,4BAAW,EACzB,MAAM,CAAC;YACL,eAAe,EAAE,SAAS;YAC1B,eAAe,EAAE,EAAE;YACnB,cAAc,EAAE,4BAA4B;SAC7C,CAAC,EACF,GAAG,CACJ,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;YACtB,SAAS,EAAE,wBAAgB,CAAC,OAAO;YACnC,IAAI,EAAE,4BAA4B;YAClC,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/search/index/search-index-admin.service.d.ts

@@ -0,0 +1,34 @@
+import { DataSource } from 'typeorm';
+import { SearchEntityType } from '../search.types';
+import { SearchEmbedderService } from './search-embedder.service';
+import { SearchEmbeddingsService } from './search-embeddings.service';
+import { SearchIndexerService } from './search-indexer.service';
+import { SearchSettingsService } from './search-settings.service';
+import { SearchVectorStoreService } from './search-vector-store.service';
+export type SearchIndexStatus = {
+    semanticActive: boolean;
+    semanticInactiveReason: string | null;
+    embeddingModelId: string | null;
+    documents: number;
+    embedded: number;
+    pending: number;
+    failed: number;
+    lastError: string | null;
+    byType: Array<{
+        type: SearchEntityType;
+        documents: number;
+        embedded: number;
+    }>;
+};
+export declare class SearchIndexAdminService {
+    private readonly dataSource;
+    private readonly settingsService;
+    private readonly vectorStore;
+    private readonly embeddings;
+    private readonly indexer;
+    private readonly embedder;
+    private readonly logger;
+    constructor(dataSource: DataSource, settingsService: SearchSettingsService, vectorStore: SearchVectorStoreService, embeddings: SearchEmbeddingsService, indexer: SearchIndexerService, embedder: SearchEmbedderService);
+    getStatus(): Promise<SearchIndexStatus>;
+    rebuild(): Promise<SearchIndexStatus>;
+}

package/dist/search/index/search-index-admin.service.js

@@ -0,0 +1,107 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var SearchIndexAdminService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchIndexAdminService = void 0;
+const common_1 = require("@nestjs/common");
+const typeorm_1 = require("typeorm");
+const chunk_plan_1 = require("./chunk-plan");
+const search_embedder_service_1 = require("./search-embedder.service");
+const search_embeddings_service_1 = require("./search-embeddings.service");
+const search_indexer_service_1 = require("./search-indexer.service");
+const search_settings_service_1 = require("./search-settings.service");
+const search_vector_store_service_1 = require("./search-vector-store.service");
+let SearchIndexAdminService = SearchIndexAdminService_1 = class SearchIndexAdminService {
+    dataSource;
+    settingsService;
+    vectorStore;
+    embeddings;
+    indexer;
+    embedder;
+    logger = new common_1.Logger(SearchIndexAdminService_1.name);
+    constructor(dataSource, settingsService, vectorStore, embeddings, indexer, embedder) {
+        this.dataSource = dataSource;
+        this.settingsService = settingsService;
+        this.vectorStore = vectorStore;
+        this.embeddings = embeddings;
+        this.indexer = indexer;
+        this.embedder = embedder;
+    }
+    async getStatus() {
+        const settings = await this.settingsService.get();
+        const storageAvailable = await this.vectorStore.isAvailable();
+        const model = await this.embeddings.resolveModel();
+        let semanticInactiveReason = null;
+        if (!settings.semanticEnabled) {
+            semanticInactiveReason = 'Semantic search is turned off in the search settings.';
+        }
+        else if (!storageAvailable) {
+            semanticInactiveReason =
+                this.vectorStore.getUnavailableReason() ?? 'The embeddings table is not available.';
+        }
+        else if (!model) {
+            semanticInactiveReason =
+                'No usable embedding model is selected. Choose one for the Embeddings job in the model settings.';
+        }
+        const semanticActive = semanticInactiveReason === null;
+        const configKey = semanticActive && model ? (0, chunk_plan_1.embeddingConfigKey)(model.key, settings) : null;
+        const rows = await this.dataSource.query(`SELECT "entity_type",
+              count(*) AS documents,
+              count(*) FILTER (WHERE "embedded_fingerprint" = md5("content_hash" || $1)) AS embedded,
+              count(*) FILTER (
+                WHERE "embed_error" IS NOT NULL
+                  AND "embedded_fingerprint" IS DISTINCT FROM md5("content_hash" || $1)
+              ) AS failed
+       FROM "search_documents"
+       GROUP BY "entity_type"
+       ORDER BY "entity_type"`, [configKey]);
+        const lastError = configKey
+            ? await this.dataSource.query(`SELECT "embed_error" FROM "search_documents"
+           WHERE "embed_error" IS NOT NULL
+             AND "embedded_fingerprint" IS DISTINCT FROM md5("content_hash" || $1)
+           ORDER BY "embed_retry_at" DESC NULLS LAST LIMIT 1`, [configKey])
+            : [];
+        const total = (key) => rows.reduce((sum, row) => sum + Number(row[key]), 0);
+        return {
+            semanticActive,
+            semanticInactiveReason,
+            embeddingModelId: model?.modelId ?? null,
+            documents: total('documents'),
+            embedded: total('embedded'),
+            pending: semanticActive ? total('documents') - total('embedded') : 0,
+            failed: semanticActive ? total('failed') : 0,
+            lastError: lastError[0]?.embed_error ?? null,
+            byType: rows.map((row) => ({
+                type: row.entity_type,
+                documents: Number(row.documents),
+                embedded: Number(row.embedded),
+            })),
+        };
+    }
+    async rebuild() {
+        this.logger.log({ message: 'Rebuilding the search index' });
+        await this.dataSource.query(`DELETE FROM "search_documents"`);
+        await this.indexer.syncAll();
+        this.embedder.wake();
+        return this.getStatus();
+    }
+};
+exports.SearchIndexAdminService = SearchIndexAdminService;
+exports.SearchIndexAdminService = SearchIndexAdminService = SearchIndexAdminService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [typeorm_1.DataSource,
+        search_settings_service_1.SearchSettingsService,
+        search_vector_store_service_1.SearchVectorStoreService,
+        search_embeddings_service_1.SearchEmbeddingsService,
+        search_indexer_service_1.SearchIndexerService,
+        search_embedder_service_1.SearchEmbedderService])
+], SearchIndexAdminService);
+//# sourceMappingURL=search-index-admin.service.js.map

package/dist/search/index/search-index-admin.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index-admin.service.js","sourceRoot":"","sources":["../../../src/search/index/search-index-admin.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,qCAAqC;AAErC,6CAAkD;AAClD,uEAAkE;AAClE,2EAAsE;AACtE,qEAAgE;AAChE,uEAAkE;AAClE,+EAAyE;AAwBlE,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAIf;IACA;IACA;IACA;IACA;IACA;IARF,MAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAC;IAEnE,YACmB,UAAsB,EACtB,eAAsC,EACtC,WAAqC,EACrC,UAAmC,EACnC,OAA6B,EAC7B,QAA+B;QAL/B,eAAU,GAAV,UAAU,CAAY;QACtB,oBAAe,GAAf,eAAe,CAAuB;QACtC,gBAAW,GAAX,WAAW,CAA0B;QACrC,eAAU,GAAV,UAAU,CAAyB;QACnC,YAAO,GAAP,OAAO,CAAsB;QAC7B,aAAQ,GAAR,QAAQ,CAAuB;IAC/C,CAAC;IAEJ,KAAK,CAAC,SAAS;QACb,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;QAEnD,IAAI,sBAAsB,GAAkB,IAAI,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC;YAC9B,sBAAsB,GAAG,uDAAuD,CAAC;QACnF,CAAC;aAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7B,sBAAsB;gBACpB,IAAI,CAAC,WAAW,CAAC,oBAAoB,EAAE,IAAI,wCAAwC,CAAC;QACxF,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAClB,sBAAsB;gBACpB,iGAAiG,CAAC;QACtG,CAAC;QACD,MAAM,cAAc,GAAG,sBAAsB,KAAK,IAAI,CAAC;QAGvD,MAAM,SAAS,GAAG,cAAc,IAAI,KAAK,CAAC,CAAC,CAAC,IAAA,+BAAkB,EAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3F,MAAM,IAAI,GAKL,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAC9B;;;;;;;;;8BASwB,EACxB,CAAC,SAAS,CAAC,CACZ,CAAC;QACF,MAAM,SAAS,GAAmC,SAAS;YACzD,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CACzB;;;6DAGmD,EACnD,CAAC,SAAS,CAAC,CACZ;YACH,CAAC,CAAC,EAAE,CAAC;QAEP,MAAM,KAAK,GAAG,CAAC,GAAwC,EAAE,EAAE,CACzD,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvD,OAAO;YACL,cAAc;YACd,sBAAsB;YACtB,gBAAgB,EAAE,KAAK,EAAE,OAAO,IAAI,IAAI;YACxC,SAAS,EAAE,KAAK,CAAC,WAAW,CAAC;YAC7B,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YAC3B,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACpE,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,IAAI;YAC5C,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACzB,IAAI,EAAE,GAAG,CAAC,WAAW;gBACrB,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;gBAChC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;aAC/B,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAOD,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,6BAA6B,EAAE,CAAC,CAAC;QAE5D,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;CACF,CAAA;AA5FY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;qCAKoB,oBAAU;QACL,+CAAqB;QACzB,sDAAwB;QACzB,mDAAuB;QAC1B,6CAAoB;QACnB,+CAAqB;GATvC,uBAAuB,CA4FnC"}

package/dist/search/index/search-indexer.service.d.ts

@@ -0,0 +1,29 @@
+import { OnApplicationBootstrap } from '@nestjs/common';
+import { DataSource } from 'typeorm';
+import { BlockCreatedEvent, BlockDeletedEvent, BlockUpdatedEvent } from "../../context/events/context.events";
+import { CommentAddedEvent, TaskCreatedEvent, TaskDeletedEvent, TaskUpdatedEvent } from "../../tasks/events/tasks.events";
+import { ThreadCreatedEvent, ThreadDeletedEvent, ThreadTitleUpdatedEvent, ThreadUpdatedEvent } from "../../threads/events/threads.events";
+import { SearchResultType } from '../enums';
+import { SearchEmbedderService } from './search-embedder.service';
+export declare class SearchIndexerService implements OnApplicationBootstrap {
+    private readonly dataSource;
+    private readonly embedder;
+    private readonly logger;
+    private sweeping;
+    constructor(dataSource: DataSource, embedder: SearchEmbedderService);
+    onApplicationBootstrap(): Promise<void>;
+    sweep(): Promise<void>;
+    syncAll(): Promise<number>;
+    syncResult(resultType: `${SearchResultType}`, resultId: string | null): Promise<void>;
+    private syncProjection;
+    onTaskChanged(event: TaskCreatedEvent | TaskUpdatedEvent): Promise<void>;
+    onTaskDeleted(event: TaskDeletedEvent): Promise<void>;
+    onCommentAdded(event: CommentAddedEvent): Promise<void>;
+    onBlockChanged(event: BlockCreatedEvent | BlockUpdatedEvent): Promise<void>;
+    onThreadChanged(event: ThreadCreatedEvent | ThreadUpdatedEvent): Promise<void>;
+    onThreadTitleUpdated(event: ThreadTitleUpdatedEvent): Promise<void>;
+    onThreadDeleted(event: ThreadDeletedEvent): Promise<void>;
+    private syncThreadOfStateBlock;
+    onAgentChanged(): Promise<void>;
+    onBlockDeleted(event: BlockDeletedEvent): Promise<void>;
+}