@taico/taico 0.5.1 → 0.6.0

package/dist/search/index/index-search.service.js

@@ -0,0 +1,322 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var IndexSearchService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexSearchService = void 0;
+const common_1 = require("@nestjs/common");
+const typeorm_1 = require("typeorm");
+const enums_1 = require("../enums");
+const search_embeddings_service_1 = require("./search-embeddings.service");
+const search_excerpt_1 = require("./search-excerpt");
+const search_schema_1 = require("./search-schema");
+const search_settings_service_1 = require("./search-settings.service");
+const search_vector_store_service_1 = require("./search-vector-store.service");
+const MAX_QUERY_CHARS = 1000;
+const MAX_LIMIT = 100;
+const PASSAGES_PER_CANDIDATE = 3;
+const MAX_LOOSE_TERMS = 6;
+const HEADLINE_INPUT_CHARS = 100_000;
+let IndexSearchService = IndexSearchService_1 = class IndexSearchService {
+    dataSource;
+    settingsService;
+    vectorStore;
+    embeddings;
+    logger = new common_1.Logger(IndexSearchService_1.name);
+    constructor(dataSource, settingsService, vectorStore, embeddings) {
+        this.dataSource = dataSource;
+        this.settingsService = settingsService;
+        this.vectorStore = vectorStore;
+        this.embeddings = embeddings;
+    }
+    async search(input) {
+        const query = input.query.trim().slice(0, MAX_QUERY_CHARS);
+        if (!query) {
+            return [];
+        }
+        const settings = await this.settingsService.get();
+        const limit = Math.min(Math.max(Math.trunc(input.limit ?? settings.defaultLimit), 1), MAX_LIMIT);
+        const types = input.types?.length ? [...input.types] : Object.values(enums_1.SearchResultType);
+        const startedAt = Date.now();
+        const vector = await this.embedQuery(query, settings);
+        const embeddedAt = Date.now();
+        const strategies = [];
+        if (settings.keywordWeight > 0)
+            strategies.push(enums_1.SearchStrategy.KEYWORD);
+        if (settings.fuzzyTitleWeight > 0)
+            strategies.push(enums_1.SearchStrategy.FUZZY_TITLE);
+        if (vector && settings.semanticWeight > 0)
+            strategies.push(enums_1.SearchStrategy.SEMANTIC);
+        if (strategies.length === 0) {
+            return [];
+        }
+        const rows = await this.runQuery({ query, types, limit, settings, strategies, vector });
+        const results = rows.map((row) => toResult(row, settings, strategies));
+        this.logger.log({
+            message: 'Search completed',
+            query,
+            types: input.types ?? 'all',
+            strategies,
+            resultCount: results.length,
+            embedMs: embeddedAt - startedAt,
+            queryMs: Date.now() - embeddedAt,
+        });
+        return results;
+    }
+    async embedQuery(query, settings) {
+        if (!settings.semanticEnabled || settings.semanticWeight <= 0) {
+            return null;
+        }
+        try {
+            if (!(await this.vectorStore.isAvailable())) {
+                return null;
+            }
+            const model = await this.embeddings.resolveModel();
+            if (!model) {
+                return null;
+            }
+            const embedding = await this.embeddings.embedQuery(model, query, {
+                instruction: settings.queryInstruction,
+                timeoutMs: settings.queryEmbeddingTimeoutMs,
+            });
+            return {
+                literal: `[${embedding.join(',')}]`,
+                dimensions: embedding.length,
+                modelKey: model.key,
+            };
+        }
+        catch (error) {
+            this.logger.warn({
+                message: 'Query embedding failed; searching without semantic matches',
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return null;
+        }
+    }
+    async runQuery(input) {
+        const { query, types, limit, settings, strategies, vector } = input;
+        const parameters = [];
+        const bind = (value) => {
+            parameters.push(value);
+            return `$${parameters.length}`;
+        };
+        const q = bind(query);
+        const typeList = `${bind(types)}::text[]`;
+        const candidates = `${bind(settings.candidatesPerStrategy)}::int`;
+        const rrfK = `${bind(settings.rrfK)}::float8`;
+        const weight = (strategy) => `${bind(strategyWeights(settings)[strategy])}::float8`;
+        const lists = [];
+        const hits = [];
+        if (strategies.includes(enums_1.SearchStrategy.KEYWORD)) {
+            const minDensity = `${bind(settings.keywordMinDensity)}::float4`;
+            const keywordList = (name, tsquery, condition) => `
+        ${name} AS (
+          SELECT d."id" AS doc_id,
+                 row_number() OVER (ORDER BY ts_rank_cd(d."tsv", ${tsquery}, 1) DESC, d."id") AS rank
+          FROM "search_documents" d, tsq
+          WHERE ${condition}
+            AND d."tsv" @@ ${tsquery} AND d."result_type" = ANY(${typeList})
+            AND ts_rank_cd(d."tsv", ${tsquery}) >= ${minDensity}
+          ORDER BY rank LIMIT ${candidates}
+        )`;
+            lists.push(keywordList('strict_keyword', 'tsq.strict_q', 'TRUE'), keywordList('loose_keyword', 'tsq.loose_q', 'NOT EXISTS (SELECT 1 FROM strict_keyword)'));
+            hits.push(`SELECT doc_id, 'keyword' AS strategy, rank, NULL::uuid AS chunk_id, ${weight(enums_1.SearchStrategy.KEYWORD)} AS weight FROM strict_keyword`, `SELECT doc_id, 'keyword', rank, NULL::uuid, ${weight(enums_1.SearchStrategy.KEYWORD)} FROM loose_keyword`);
+        }
+        if (strategies.includes(enums_1.SearchStrategy.FUZZY_TITLE)) {
+            lists.push(`
+        fuzzy_title AS (
+          SELECT d."id" AS doc_id,
+                 row_number() OVER (ORDER BY word_similarity(${q}, d."search_title") DESC, d."id") AS rank
+          FROM "search_documents" d
+          WHERE ${q} <% d."search_title" AND d."result_type" = ANY(${typeList})
+          ORDER BY rank LIMIT ${candidates}
+        )`);
+            hits.push(`SELECT doc_id, 'fuzzy_title', rank, NULL::uuid, ${weight(enums_1.SearchStrategy.FUZZY_TITLE)} FROM fuzzy_title`);
+        }
+        const semantic = strategies.includes(enums_1.SearchStrategy.SEMANTIC) && vector !== null;
+        if (semantic) {
+            const dimensions = Math.trunc(vector.dimensions);
+            const embedding = `${bind(vector.literal)}::vector(${dimensions})`;
+            lists.push(`
+        nearest AS (
+          SELECT c."id" AS chunk_id, c."document_id" AS doc_id,
+                 c."embedding"::vector(${dimensions}) <=> ${embedding} AS distance
+          FROM "search_chunks" c
+          JOIN "search_documents" d ON d."id" = c."document_id"
+          WHERE c."dimensions" = ${dimensions} AND c."model" = ${bind(vector.modelKey)}
+            AND d."result_type" = ANY(${typeList})
+          ORDER BY distance
+          LIMIT ${candidates} * ${PASSAGES_PER_CANDIDATE}
+        ),
+        best_passage AS (
+          SELECT DISTINCT ON (doc_id) doc_id, chunk_id, distance
+          FROM nearest
+          WHERE distance <= ${bind(1 - settings.semanticMinSimilarity)}::float8
+            -- With a strong match at the top, passages far behind it are noise.
+            AND distance <= (SELECT min(distance) FROM nearest) + ${bind(settings.semanticMaxGap)}::float8
+          ORDER BY doc_id, distance
+        ),
+        semantic AS (
+          SELECT doc_id, chunk_id, row_number() OVER (ORDER BY distance, doc_id) AS rank
+          FROM best_passage
+          ORDER BY rank LIMIT ${candidates}
+        )`);
+            hits.push(`SELECT doc_id, 'semantic', rank, chunk_id, ${weight(enums_1.SearchStrategy.SEMANTIC)} FROM semantic`);
+        }
+        const boost = `${bind(settings.exactTitleBoost)}::float8`;
+        const excerptChars = `${bind(settings.excerptMaxChars)}::int`;
+        const tieBreak = (alias) => `(${alias}.ranks->>'semantic')::int ASC NULLS LAST`;
+        const anyQuery = `CASE WHEN tsq.loose_q IS NULL THEN tsq.strict_q ELSE tsq.strict_q || tsq.loose_q END`;
+        const headlineOptions = `'MaxFragments=3, MinWords=15, MaxWords=60, FragmentDelimiter=" … ", StartSel="", StopSel=""'`;
+        const sql = `
+      WITH terms AS (
+        -- One query per word, so a word can be left out. The text form of a tsquery is already quoted.
+        SELECT DISTINCT plainto_tsquery('${search_schema_1.SEARCH_TEXT_CONFIG}', w)::text AS term
+        FROM regexp_split_to_table(${q}, '\\s+') AS w
+        WHERE numnode(plainto_tsquery('${search_schema_1.SEARCH_TEXT_CONFIG}', w)) > 0
+      ),
+      tsq AS (
+        SELECT s.strict_q,
+               -- "All but one term". Only for a handful of plain terms: with one term there is
+               -- nothing to drop, a long sentence is a question for semantic search, and a
+               -- negated term (-word) only makes sense when every term must hold.
+               CASE WHEN (SELECT count(*) FROM terms) BETWEEN 2 AND ${MAX_LOOSE_TERMS}
+                         AND s.strict_q::text NOT LIKE '%!%'
+                    THEN (
+                      SELECT string_agg(
+                        '(' || (SELECT string_agg('(' || kept.term || ')', ' & ')
+                                FROM terms kept WHERE kept.term <> dropped.term) || ')',
+                        ' | ')
+                      FROM terms dropped
+                    )::tsquery
+               END AS loose_q
+        FROM (SELECT websearch_to_tsquery('${search_schema_1.SEARCH_TEXT_CONFIG}', ${q}) AS strict_q) s
+      ),
+      ${lists.join(',')},
+      hits AS (
+        ${hits.join('\n        UNION ALL\n        ')}
+      ),
+      -- A task found through three comments is one result, placed by its best hit in each list.
+      result_hits AS (
+        SELECT d."result_type", d."result_id", h.strategy, h.weight, min(h.rank) AS rank
+        FROM hits h JOIN "search_documents" d ON d."id" = h.doc_id
+        GROUP BY d."result_type", d."result_id", h.strategy, h.weight
+      ),
+      fused AS (
+        SELECT "result_type", "result_id",
+               sum(weight / (${rrfK} + rank)) AS fused,
+               array_agg(strategy ORDER BY strategy) AS matched_by,
+               jsonb_object_agg(strategy, rank) AS ranks
+        FROM result_hits
+        GROUP BY "result_type", "result_id"
+      ),
+      ranked AS (
+        SELECT f."result_type", f."result_id", f.matched_by, f.ranks, p."id" AS primary_doc_id,
+               p."title", p."meta",
+               (f.fused + CASE
+                  WHEN lower(p."search_title") = lower(${q}) THEN ${boost}
+                  WHEN starts_with(lower(p."search_title"), lower(${q})) THEN ${boost} / 2
+                  ELSE 0 END)
+               * CASE WHEN f."result_type" = 'task' AND p."meta"->>'status' = 'DONE'
+                      THEN ${bind(settings.doneTaskMultiplier)}::float8 ELSE 1 END AS score
+        FROM fused f
+        -- The result's own row: a comment's row has a different entity type.
+        JOIN "search_documents" p
+          ON p."entity_type" = f."result_type" AND p."entity_id" = f."result_id"
+        ORDER BY score DESC, ${tieBreak('f')}, p."title", f."result_id"
+        LIMIT ${bind(limit)}::int
+      ),
+      -- The single hit that did most to place the result decides what the excerpt shows.
+      best_hit AS (
+        SELECT DISTINCT ON (d."result_type", d."result_id")
+               d."result_type", d."result_id", h.strategy, h.doc_id, h.chunk_id
+        FROM hits h JOIN "search_documents" d ON d."id" = h.doc_id
+        ORDER BY d."result_type", d."result_id",
+                 h.weight / (${rrfK} + h.rank) DESC, (h.strategy = 'semantic') DESC
+      )
+      SELECT r."result_type", r."result_id", r."title", r."meta", r.score, r.matched_by, r.ranks,
+             b.strategy AS hit_strategy,
+             hd."entity_type" AS hit_entity_type, hd."entity_id" AS hit_entity_id,
+             hd."meta"->>'author' AS hit_author,
+             char_length(hd."body") AS hit_body_length,
+             left(hd."body", ${excerptChars}) AS hit_body_start,
+             CASE WHEN b.strategy = 'keyword'
+                  THEN to_tsvector('${search_schema_1.SEARCH_TEXT_CONFIG}', left(hd."body", ${HEADLINE_INPUT_CHARS})) @@ ${anyQuery}
+             END AS hit_body_matches,
+             CASE WHEN b.strategy = 'keyword' AND char_length(hd."body") > ${excerptChars}
+                  THEN ts_headline('${search_schema_1.SEARCH_TEXT_CONFIG}', left(hd."body", ${HEADLINE_INPUT_CHARS}), ${anyQuery}, ${headlineOptions})
+             END AS headline,
+             ${semantic ? `c."kind"` : 'NULL'} AS chunk_kind,
+             ${semantic ? `c."text"` : 'NULL'} AS chunk_text,
+             char_length(p."body") AS primary_body_length,
+             left(p."body", ${excerptChars}) AS primary_body_start
+      FROM ranked r
+      JOIN best_hit b ON b."result_type" = r."result_type" AND b."result_id" = r."result_id"
+      JOIN "search_documents" hd ON hd."id" = b.doc_id
+      JOIN "search_documents" p ON p."id" = r.primary_doc_id
+      ${semantic ? `LEFT JOIN "search_chunks" c ON c."id" = b.chunk_id` : ''}
+      CROSS JOIN tsq
+      ORDER BY r.score DESC, ${tieBreak('r')}, r."title", r."result_id"`;
+        return this.dataSource.transaction(async (manager) => {
+            await manager.query(`SELECT set_config('pg_trgm.word_similarity_threshold', $1, true)`, [
+                String(settings.fuzzyMinSimilarity),
+            ]);
+            if (semantic) {
+                const passages = settings.candidatesPerStrategy * PASSAGES_PER_CANDIDATE;
+                await manager.query(`SELECT set_config('hnsw.ef_search', $1, true)`, [
+                    String(Math.min(Math.max(passages, 40), 1000)),
+                ]);
+                if (this.vectorStore.supportsIterativeScan()) {
+                    await manager.query(`SELECT set_config('hnsw.iterative_scan', 'relaxed_order', true)`);
+                }
+            }
+            return manager.query(sql, parameters);
+        });
+    }
+};
+exports.IndexSearchService = IndexSearchService;
+exports.IndexSearchService = IndexSearchService = IndexSearchService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [typeorm_1.DataSource,
+        search_settings_service_1.SearchSettingsService,
+        search_vector_store_service_1.SearchVectorStoreService,
+        search_embeddings_service_1.SearchEmbeddingsService])
+], IndexSearchService);
+function strategyWeights(settings) {
+    return {
+        [enums_1.SearchStrategy.KEYWORD]: settings.keywordWeight,
+        [enums_1.SearchStrategy.FUZZY_TITLE]: settings.fuzzyTitleWeight,
+        [enums_1.SearchStrategy.SEMANTIC]: settings.semanticWeight,
+    };
+}
+function maxScore(settings, strategies) {
+    const weights = strategyWeights(settings);
+    const fused = strategies.reduce((total, strategy) => total + weights[strategy] / (settings.rrfK + 1), 0);
+    return fused + settings.exactTitleBoost;
+}
+function toResult(row, settings, strategies) {
+    const excerpt = (0, search_excerpt_1.pickExcerpt)(row, settings.excerptMaxChars);
+    const isComment = excerpt.matchedOn === enums_1.SearchMatchField.COMMENT;
+    return {
+        type: row.result_type,
+        id: row.result_id,
+        title: row.title,
+        score: Math.min(1, Math.max(0, Number(row.score) / maxScore(settings, strategies))),
+        matchedBy: [...new Set(row.matched_by)],
+        ranks: row.ranks,
+        matchedOn: excerpt.matchedOn,
+        excerpt: excerpt.text,
+        excerptIsComplete: excerpt.isComplete,
+        commentId: isComment ? row.hit_entity_id : null,
+        commentAuthor: isComment ? row.hit_author : null,
+        meta: row.meta ?? {},
+    };
+}
+//# sourceMappingURL=index-search.service.js.map

package/dist/search/index/index-search.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-search.service.js","sourceRoot":"","sources":["../../../src/search/index/index-search.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,qCAAqC;AACrC,oCAA8E;AAE9E,2EAAsE;AACtE,qDAA8D;AAC9D,mDAAqD;AAErD,uEAAkE;AAClE,+EAAyE;AAEzE,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAEjC,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,oBAAoB,GAAG,OAAO,CAAC;AA2B9B,IAAM,kBAAkB,0BAAxB,MAAM,kBAAkB;IAIV;IACA;IACA;IACA;IANF,MAAM,GAAG,IAAI,eAAM,CAAC,oBAAkB,CAAC,IAAI,CAAC,CAAC;IAE9D,YACmB,UAAsB,EACtB,eAAsC,EACtC,WAAqC,EACrC,UAAmC;QAHnC,eAAU,GAAV,UAAU,CAAY;QACtB,oBAAe,GAAf,eAAe,CAAuB;QACtC,gBAAW,GAAX,WAAW,CAA0B;QACrC,eAAU,GAAV,UAAU,CAAyB;IACnD,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,KAAuB;QAClC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACjG,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,wBAAgB,CAAC,CAAC;QAEvF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,MAAM,UAAU,GAAqB,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,aAAa,GAAG,CAAC;YAAE,UAAU,CAAC,IAAI,CAAC,sBAAc,CAAC,OAAO,CAAC,CAAC;QACxE,IAAI,QAAQ,CAAC,gBAAgB,GAAG,CAAC;YAAE,UAAU,CAAC,IAAI,CAAC,sBAAc,CAAC,WAAW,CAAC,CAAC;QAC/E,IAAI,MAAM,IAAI,QAAQ,CAAC,cAAc,GAAG,CAAC;YAAE,UAAU,CAAC,IAAI,CAAC,sBAAc,CAAC,QAAQ,CAAC,CAAC;QACpF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;QACxF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;QAEvE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YACd,OAAO,EAAE,kBAAkB;YAC3B,KAAK;YACL,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,KAAK;YAC3B,UAAU;YACV,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,OAAO,EAAE,UAAU,GAAG,SAAS;YAC/B,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;SACjC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAGO,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,QAAwB;QAC9D,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,cAAc,IAAI,CAAC,EAAE,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBAC5C,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;YACnD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE;gBAC/D,WAAW,EAAE,QAAQ,CAAC,gBAAgB;gBACtC,SAAS,EAAE,QAAQ,CAAC,uBAAuB;aAC5C,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;gBACnC,UAAU,EAAE,SAAS,CAAC,MAAM;gBAC5B,QAAQ,EAAE,KAAK,CAAC,GAAG;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACf,OAAO,EAAE,4DAA4D;gBACrE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,KAOtB;QACC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QACpE,MAAM,UAAU,GAAc,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE;YACtC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACjC,CAAC,CAAC;QAEF,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QAC1C,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC,OAAO,CAAC;QAClE,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC;QAC9C,MAAM,MAAM,GAAG,CAAC,QAAwB,EAAU,EAAE,CAClD,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC;QAEzD,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,UAAU,CAAC,QAAQ,CAAC,sBAAc,CAAC,OAAO,CAAC,EAAE,CAAC;YAIhD,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,UAAU,CAAC;YACjE,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,OAAe,EAAE,SAAiB,EAAE,EAAE,CAAC;UACtE,IAAI;;mEAEqD,OAAO;;kBAExD,SAAS;6BACE,OAAO,8BAA8B,QAAQ;sCACpC,OAAO,QAAQ,UAAU;gCAC/B,UAAU;UAChC,CAAC;YACL,KAAK,CAAC,IAAI,CACR,WAAW,CAAC,gBAAgB,EAAE,cAAc,EAAE,MAAM,CAAC,EACrD,WAAW,CAAC,eAAe,EAAE,aAAa,EAAE,2CAA2C,CAAC,CACzF,CAAC;YACF,IAAI,CAAC,IAAI,CACP,uEAAuE,MAAM,CAAC,sBAAc,CAAC,OAAO,CAAC,gCAAgC,EACrI,+CAA+C,MAAM,CAAC,sBAAc,CAAC,OAAO,CAAC,qBAAqB,CACnG,CAAC;QACJ,CAAC;QAED,IAAI,UAAU,CAAC,QAAQ,CAAC,sBAAc,CAAC,WAAW,CAAC,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC;;;+DAG8C,CAAC;;kBAE9C,CAAC,kDAAkD,QAAQ;gCAC7C,UAAU;UAChC,CAAC,CAAC;YACN,IAAI,CAAC,IAAI,CACP,mDAAmD,MAAM,CAAC,sBAAc,CAAC,WAAW,CAAC,mBAAmB,CACzG,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,sBAAc,CAAC,QAAQ,CAAC,IAAI,MAAM,KAAK,IAAI,CAAC;QACjF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,SAAS,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,UAAU,GAAG,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC;;;yCAGwB,UAAU,SAAS,SAAS;;;mCAGlC,UAAU,oBAAoB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;wCAC9C,QAAQ;;kBAE9B,UAAU,MAAM,sBAAsB;;;;;8BAK1B,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,qBAAqB,CAAC;;oEAEF,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;;;;;;gCAMjE,UAAU;UAChC,CAAC,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,8CAA8C,MAAM,CAAC,sBAAc,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QAC3G,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,UAAU,CAAC;QAC1D,MAAM,YAAY,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC;QAE9D,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,KAAK,0CAA0C,CAAC;QAExF,MAAM,QAAQ,GAAG,sFAAsF,CAAC;QACxG,MAAM,eAAe,GAAG,8FAA8F,CAAC;QAEvH,MAAM,GAAG,GAAG;;;2CAG2B,kCAAkB;qCACxB,CAAC;yCACG,kCAAkB;;;;;;;sEAOW,eAAe;;;;;;;;;;6CAUxC,kCAAkB,MAAM,CAAC;;QAE9D,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;;UAEb,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC;;;;;;;;;;+BAUrB,IAAI;;;;;;;;;;yDAUsB,CAAC,UAAU,KAAK;oEACL,CAAC,WAAW,KAAK;;;6BAGxD,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC;;;;;+BAK/B,QAAQ,CAAC,GAAG,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC;;;;;;;;+BAQI,IAAI;;;;;;;+BAOJ,YAAY;;sCAEL,kCAAkB,sBAAsB,oBAAoB,SAAS,QAAQ;;6EAEtC,YAAY;sCACnD,kCAAkB,sBAAsB,oBAAoB,MAAM,QAAQ,KAAK,eAAe;;eAErH,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM;eAC9B,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM;;8BAEf,YAAY;;;;;QAKlC,QAAQ,CAAC,CAAC,CAAC,oDAAoD,CAAC,CAAC,CAAC,EAAE;;+BAE7C,QAAQ,CAAC,GAAG,CAAC,4BAA4B,CAAC;QAErE,OAAO,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAEnD,MAAM,OAAO,CAAC,KAAK,CAAC,kEAAkE,EAAE;gBACtF,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC;aACpC,CAAC,CAAC;YACH,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,QAAQ,GAAG,QAAQ,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACzE,MAAM,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE;oBACnE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;iBAC/C,CAAC,CAAC;gBACH,IAAI,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,CAAC;oBAE7C,MAAM,OAAO,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAC;gBACzF,CAAC;YACH,CAAC;YACD,OAAO,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;CACF,CAAA;AAjSY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;qCAKoB,oBAAU;QACL,+CAAqB;QACzB,sDAAwB;QACzB,mDAAuB;GAP3C,kBAAkB,CAiS9B;AAED,SAAS,eAAe,CAAC,QAAwB;IAC/C,OAAO;QACL,CAAC,sBAAc,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,aAAa;QAChD,CAAC,sBAAc,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,gBAAgB;QACvD,CAAC,sBAAc,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,cAAc;KACnD,CAAC;AACJ,CAAC;AAGD,SAAS,QAAQ,CAAC,QAAwB,EAAE,UAA4B;IACtE,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAC7B,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,EACpE,CAAC,CACF,CAAC;IACF,OAAO,KAAK,GAAG,QAAQ,CAAC,eAAe,CAAC;AAC1C,CAAC;AAED,SAAS,QAAQ,CACf,GAAc,EACd,QAAwB,EACxB,UAA4B;IAE5B,MAAM,OAAO,GAAG,IAAA,4BAAW,EAAC,GAAG,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,KAAK,wBAAgB,CAAC,OAAO,CAAC;IACjE,OAAO;QACL,IAAI,EAAE,GAAG,CAAC,WAAW;QACrB,EAAE,EAAE,GAAG,CAAC,SAAS;QACjB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;QACnF,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvC,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,OAAO,EAAE,OAAO,CAAC,IAAI;QACrB,iBAAiB,EAAE,OAAO,CAAC,UAAU;QACrC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI;QAC/C,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;QAChD,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;KACrB,CAAC;AACJ,CAAC"}

package/dist/search/index/search-embedder.service.d.ts

@@ -0,0 +1,23 @@
+import { OnModuleDestroy } from '@nestjs/common';
+import { DataSource } from 'typeorm';
+import { SearchEmbeddingsService } from './search-embeddings.service';
+import { SearchSettingsService } from './search-settings.service';
+import { SearchVectorStoreService } from './search-vector-store.service';
+export declare class SearchEmbedderService implements OnModuleDestroy {
+    private readonly dataSource;
+    private readonly settingsService;
+    private readonly vectorStore;
+    private readonly embeddings;
+    private readonly logger;
+    private running;
+    private wakeRequested;
+    private stopped;
+    private wakeTimer;
+    constructor(dataSource: DataSource, settingsService: SearchSettingsService, vectorStore: SearchVectorStoreService, embeddings: SearchEmbeddingsService);
+    onModuleDestroy(): void;
+    wake(): void;
+    run(): Promise<void>;
+    private processBatch;
+    private embedDocuments;
+    private recordFailure;
+}

package/dist/search/index/search-embedder.service.js

@@ -0,0 +1,251 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var SearchEmbedderService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchEmbedderService = void 0;
+const common_1 = require("@nestjs/common");
+const schedule_1 = require("@nestjs/schedule");
+const typeorm_1 = require("typeorm");
+const chunk_plan_1 = require("./chunk-plan");
+const search_embeddings_service_1 = require("./search-embeddings.service");
+const search_settings_service_1 = require("./search-settings.service");
+const search_vector_store_service_1 = require("./search-vector-store.service");
+const TICK_INTERVAL_MS = 30_000;
+const DOCUMENTS_PER_BATCH = 20;
+const TEXTS_PER_REQUEST = 64;
+const CLAIM_MINUTES = 5;
+const MAX_RETRY_MINUTES = 60;
+let SearchEmbedderService = SearchEmbedderService_1 = class SearchEmbedderService {
+    dataSource;
+    settingsService;
+    vectorStore;
+    embeddings;
+    logger = new common_1.Logger(SearchEmbedderService_1.name);
+    running = false;
+    wakeRequested = false;
+    stopped = false;
+    wakeTimer = null;
+    constructor(dataSource, settingsService, vectorStore, embeddings) {
+        this.dataSource = dataSource;
+        this.settingsService = settingsService;
+        this.vectorStore = vectorStore;
+        this.embeddings = embeddings;
+    }
+    onModuleDestroy() {
+        this.stopped = true;
+        if (this.wakeTimer) {
+            clearTimeout(this.wakeTimer);
+        }
+    }
+    wake() {
+        if (this.stopped) {
+            return;
+        }
+        if (this.running) {
+            this.wakeRequested = true;
+            return;
+        }
+        this.wakeTimer ??= setTimeout(() => {
+            this.wakeTimer = null;
+            void this.run();
+        }, 0);
+    }
+    async run() {
+        if (this.running || this.stopped) {
+            return;
+        }
+        this.running = true;
+        try {
+            do {
+                this.wakeRequested = false;
+                while (!this.stopped && (await this.processBatch()) > 0) {
+                }
+            } while (this.wakeRequested && !this.stopped);
+        }
+        catch (error) {
+            this.logger.error({ message: 'Search embedding run failed', error: errorMessage(error) });
+        }
+        finally {
+            this.running = false;
+        }
+    }
+    async processBatch() {
+        const settings = await this.settingsService.get();
+        if (!settings.semanticEnabled || !(await this.vectorStore.isAvailable())) {
+            return 0;
+        }
+        const model = await this.embeddings.resolveModel();
+        if (!model) {
+            return 0;
+        }
+        const configKey = (0, chunk_plan_1.embeddingConfigKey)(model.key, settings);
+        const documents = await this.dataSource.query(`WITH claimed AS (
+         UPDATE "search_documents" SET "embed_claimed_until" = now() + interval '${CLAIM_MINUTES} minutes'
+         WHERE "id" IN (
+           SELECT "id" FROM "search_documents"
+           WHERE "embedded_fingerprint" IS DISTINCT FROM md5("content_hash" || $1)
+             AND ("embed_retry_at" IS NULL OR "embed_retry_at" <= now())
+             AND ("embed_claimed_until" IS NULL OR "embed_claimed_until" <= now())
+           ORDER BY "indexed_at"
+           LIMIT ${DOCUMENTS_PER_BATCH}
+           FOR UPDATE SKIP LOCKED
+         )
+         RETURNING "id", "entity_type", "title", "search_title", "body", "content_hash"
+       )
+       SELECT * FROM claimed`, [configKey]);
+        if (documents.length === 0) {
+            return 0;
+        }
+        try {
+            await this.embedDocuments(documents, model, settings, configKey);
+            return documents.length;
+        }
+        catch (error) {
+            if (documents.length === 1) {
+                await this.recordFailure(documents, errorMessage(error));
+                return 0;
+            }
+        }
+        let embedded = 0;
+        let failuresInARow = 0;
+        for (const [index, document] of documents.entries()) {
+            try {
+                await this.embedDocuments([document], model, settings, configKey);
+                embedded += 1;
+                failuresInARow = 0;
+            }
+            catch (error) {
+                failuresInARow += 1;
+                const failed = failuresInARow >= 3 ? documents.slice(index) : [document];
+                await this.recordFailure(failed, errorMessage(error));
+                if (failed.length > 1) {
+                    break;
+                }
+            }
+        }
+        return embedded;
+    }
+    async embedDocuments(documents, model, settings, configKey) {
+        const existing = await this.dataSource.query(`SELECT "id", "document_id", "text_hash", "model" FROM "search_chunks" WHERE "document_id" = ANY($1::uuid[])`, [documents.map((document) => document.id)]);
+        const plans = documents.map((document) => {
+            const reusable = new Map();
+            const stale = [];
+            for (const chunk of existing) {
+                if (chunk.document_id !== document.id) {
+                    continue;
+                }
+                if (chunk.model === model.key) {
+                    reusable.set(chunk.text_hash, [...(reusable.get(chunk.text_hash) ?? []), chunk.id]);
+                }
+                else {
+                    stale.push(chunk.id);
+                }
+            }
+            const kept = [];
+            const fresh = [];
+            for (const chunk of (0, chunk_plan_1.planChunks)(document, settings)) {
+                const id = reusable.get(chunk.textHash)?.pop();
+                if (id) {
+                    kept.push({ id, chunk });
+                }
+                else {
+                    fresh.push(chunk);
+                }
+            }
+            const unused = [...reusable.values()].flat();
+            return { document, kept, fresh, remove: [...stale, ...unused] };
+        });
+        const texts = plans.flatMap((plan) => plan.fresh.map((chunk) => chunk.embedText));
+        const vectors = [];
+        for (let offset = 0; offset < texts.length; offset += TEXTS_PER_REQUEST) {
+            vectors.push(...(await this.embeddings.embedDocuments(model, texts.slice(offset, offset + TEXTS_PER_REQUEST))));
+        }
+        if (vectors.length > 0) {
+            await this.vectorStore.ensureAnnIndex(vectors[0].length);
+        }
+        let next = 0;
+        for (const plan of plans) {
+            const freshVectors = vectors.slice(next, next + plan.fresh.length);
+            next += plan.fresh.length;
+            await this.dataSource.transaction(async (manager) => {
+                if (plan.remove.length > 0) {
+                    await manager.query(`DELETE FROM "search_chunks" WHERE "id" = ANY($1::uuid[])`, [
+                        plan.remove,
+                    ]);
+                }
+                if (plan.kept.length > 0) {
+                    await manager.query(`UPDATE "search_chunks" c SET "kind" = v.kind, "ordinal" = v.ordinal, "text" = v.text
+             FROM unnest($1::uuid[], $2::text[], $3::int[], $4::text[]) AS v(id, kind, ordinal, text)
+             WHERE c."id" = v.id`, [
+                        plan.kept.map((entry) => entry.id),
+                        plan.kept.map((entry) => entry.chunk.kind),
+                        plan.kept.map((entry) => entry.chunk.ordinal),
+                        plan.kept.map((entry) => entry.chunk.text),
+                    ]);
+                }
+                if (plan.fresh.length > 0) {
+                    await manager.query(`INSERT INTO "search_chunks" ("document_id", "kind", "ordinal", "text", "text_hash", "model", "dimensions", "embedding")
+             SELECT $1::uuid, v.kind, v.ordinal, v.text, v.text_hash, $2, vector_dims(v.embedding::vector), v.embedding::vector
+             FROM unnest($3::text[], $4::int[], $5::text[], $6::text[], $7::text[]) AS v(kind, ordinal, text, text_hash, embedding)`, [
+                        plan.document.id,
+                        model.key,
+                        plan.fresh.map((chunk) => chunk.kind),
+                        plan.fresh.map((chunk) => chunk.ordinal),
+                        plan.fresh.map((chunk) => chunk.text),
+                        plan.fresh.map((chunk) => chunk.textHash),
+                        freshVectors.map((vector) => `[${vector.join(',')}]`),
+                    ]);
+                }
+                await manager.query(`UPDATE "search_documents"
+           SET "embedded_fingerprint" = md5($2 || $3), "embed_attempts" = 0, "embed_error" = NULL,
+               "embed_retry_at" = NULL, "embed_claimed_until" = NULL
+           WHERE "id" = $1`, [plan.document.id, plan.document.content_hash, configKey]);
+            });
+        }
+        this.logger.log({
+            message: 'Embedded search index rows',
+            documents: documents.length,
+            embedded: texts.length,
+            reused: plans.reduce((total, plan) => total + plan.kept.length, 0),
+        });
+    }
+    async recordFailure(documents, message) {
+        this.logger.warn({
+            message: 'Embedding search index rows failed; they will be retried',
+            documents: documents.length,
+            error: message,
+        });
+        await this.dataSource.query(`UPDATE "search_documents"
+       SET "embed_attempts" = "embed_attempts" + 1,
+           "embed_error" = $2,
+           "embed_claimed_until" = NULL,
+           "embed_retry_at" = now() + least(power(2, "embed_attempts"), ${MAX_RETRY_MINUTES}) * interval '1 minute'
+       WHERE "id" = ANY($1::uuid[])`, [documents.map((document) => document.id), message.slice(0, 1000)]);
+    }
+};
+exports.SearchEmbedderService = SearchEmbedderService;
+__decorate([
+    (0, schedule_1.Interval)(TICK_INTERVAL_MS),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], SearchEmbedderService.prototype, "run", null);
+exports.SearchEmbedderService = SearchEmbedderService = SearchEmbedderService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [typeorm_1.DataSource,
+        search_settings_service_1.SearchSettingsService,
+        search_vector_store_service_1.SearchVectorStoreService,
+        search_embeddings_service_1.SearchEmbeddingsService])
+], SearchEmbedderService);
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=search-embedder.service.js.map

package/dist/search/index/search-embedder.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-embedder.service.js","sourceRoot":"","sources":["../../../src/search/index/search-embedder.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAqE;AACrE,+CAA4C;AAC5C,qCAAqC;AAErC,6CAA4E;AAC5E,2EAA8F;AAE9F,uEAAkE;AAClE,+EAAyE;AAEzE,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAkBtB,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IAQb;IACA;IACA;IACA;IAVF,MAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAC;IACzD,OAAO,GAAG,KAAK,CAAC;IAChB,aAAa,GAAG,KAAK,CAAC;IACtB,OAAO,GAAG,KAAK,CAAC;IAChB,SAAS,GAA0B,IAAI,CAAC;IAEhD,YACmB,UAAsB,EACtB,eAAsC,EACtC,WAAqC,EACrC,UAAmC;QAHnC,eAAU,GAAV,UAAU,CAAY;QACtB,oBAAe,GAAf,eAAe,CAAuB;QACtC,gBAAW,GAAX,WAAW,CAA0B;QACrC,eAAU,GAAV,UAAU,CAAyB;IACnD,CAAC;IAEJ,eAAe;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAGD,IAAI;QACF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,KAAK,UAAU,CAAC,GAAG,EAAE;YACjC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;QAClB,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC;IAGK,AAAN,KAAK,CAAC,GAAG;QACP,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBAE1D,CAAC;YACH,CAAC,QAAQ,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,6BAA6B,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAGO,KAAK,CAAC,YAAY;QACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzE,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,SAAS,GAAG,IAAA,+BAAkB,EAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAsB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAC9D;mFAC6E,aAAa;;;;;;;mBAO7E,mBAAmB;;;;;6BAKT,EACvB,CAAC,SAAS,CAAC,CACZ,CAAC;QACF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;YACjE,OAAO,SAAS,CAAC,MAAM,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QAGD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAClE,QAAQ,IAAI,CAAC,CAAC;gBACd,cAAc,GAAG,CAAC,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,cAAc,IAAI,CAAC,CAAC;gBAEpB,MAAM,MAAM,GAAG,cAAc,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACzE,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;gBACtD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,SAA4B,EAC5B,KAA6B,EAC7B,QAAwB,EACxB,SAAiB;QAEjB,MAAM,QAAQ,GAAmD,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAC1F,6GAA6G,EAC7G,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAC3C,CAAC;QAIF,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC7C,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACtC,SAAS;gBACX,CAAC;gBACD,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,GAAG,EAAE,CAAC;oBAC9B,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBACtF,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAA+C,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAmB,EAAE,CAAC;YACjC,KAAK,MAAM,KAAK,IAAI,IAAA,uBAAU,EAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACnD,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC;gBAC/C,IAAI,EAAE,EAAE,CAAC;oBACP,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3B,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QAClF,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACxE,OAAO,CAAC,IAAI,CACV,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CACtC,KAAK,EACL,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,iBAAiB,CAAC,CAChD,CAAC,CACH,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACnE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAE1B,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAClD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,MAAM,OAAO,CAAC,KAAK,CAAC,0DAA0D,EAAE;wBAC9E,IAAI,CAAC,MAAM;qBACZ,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,OAAO,CAAC,KAAK,CACjB;;iCAEqB,EACrB;wBACE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;wBAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;wBAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;wBAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;qBAC3C,CACF,CAAC;gBACJ,CAAC;gBACD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,OAAO,CAAC,KAAK,CACjB;;oIAEwH,EACxH;wBACE,IAAI,CAAC,QAAQ,CAAC,EAAE;wBAChB,KAAK,CAAC,GAAG;wBACT,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;wBACrC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;wBACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;wBACrC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC;wBACzC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;qBACtD,CACF,CAAC;gBACJ,CAAC;gBAGD,MAAM,OAAO,CAAC,KAAK,CACjB;;;2BAGiB,EACjB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,CAC1D,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YACd,OAAO,EAAE,4BAA4B;YACrC,SAAS,EAAE,SAAS,CAAC,MAAM;YAC3B,QAAQ,EAAE,KAAK,CAAC,MAAM;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAGO,KAAK,CAAC,aAAa,CAAC,SAA4B,EAAE,OAAe;QACvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,OAAO,EAAE,0DAA0D;YACnE,SAAS,EAAE,SAAS,CAAC,MAAM;YAC3B,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CACzB;;;;0EAIoE,iBAAiB;oCACvD,EAC9B,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC;CACF,CAAA;AA3PY,sDAAqB;AAqC1B;IADL,IAAA,mBAAQ,EAAC,gBAAgB,CAAC;;;;gDAkB1B;gCAtDU,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCASoB,oBAAU;QACL,+CAAqB;QACzB,sDAAwB;QACzB,mDAAuB;GAX3C,qBAAqB,CA2PjC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}

package/dist/search/index/search-embeddings.service.d.ts

@@ -0,0 +1,19 @@
+import { EmbeddingModel } from 'ai';
+import { AiModelSelectionsService } from "../../ai-providers/ai-model-selections.service";
+export type ResolvedEmbeddingModel = {
+    model: EmbeddingModel;
+    key: string;
+    modelId: string;
+};
+export declare class SearchEmbeddingsService {
+    private readonly aiModelSelectionsService;
+    private readonly logger;
+    private readonly queryCache;
+    constructor(aiModelSelectionsService: AiModelSelectionsService);
+    resolveModel(): Promise<ResolvedEmbeddingModel | null>;
+    embedDocuments(resolved: ResolvedEmbeddingModel, texts: string[]): Promise<number[][]>;
+    embedQuery(resolved: ResolvedEmbeddingModel, query: string, options: {
+        instruction: string;
+        timeoutMs: number;
+    }): Promise<number[]>;
+}