@astro-minimax/ai 0.9.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +108 -18
- package/dist/cache/global-cache.d.ts +6 -2
- package/dist/cache/global-cache.d.ts.map +1 -1
- package/dist/cache/global-cache.js +24 -9
- package/dist/cache/index.d.ts +7 -6
- package/dist/cache/index.d.ts.map +1 -1
- package/dist/cache/index.js +12 -4
- package/dist/cache/injection-cache.d.ts +36 -0
- package/dist/cache/injection-cache.d.ts.map +1 -0
- package/dist/cache/injection-cache.js +90 -0
- package/dist/cache/kv-adapter.d.ts.map +1 -1
- package/dist/cache/kv-adapter.js +2 -1
- package/dist/cache/memory-adapter.d.ts.map +1 -1
- package/dist/cache/memory-adapter.js +2 -1
- package/dist/cache/response-cache.d.ts +10 -5
- package/dist/cache/response-cache.d.ts.map +1 -1
- package/dist/cache/response-cache.js +18 -6
- package/dist/components/AIChatContainer.d.ts +2 -2
- package/dist/components/AIChatContainer.d.ts.map +1 -1
- package/dist/components/AIChatContainer.js +8 -920
- package/dist/components/ChatInput.d.ts +15 -0
- package/dist/components/ChatInput.d.ts.map +1 -0
- package/dist/components/ChatInput.js +72 -0
- package/dist/components/ChatPanel.d.ts +1 -1
- package/dist/components/ChatPanel.d.ts.map +1 -1
- package/dist/components/ChatPanel.js +210 -672
- package/dist/components/CodeBlock.d.ts +31 -0
- package/dist/components/CodeBlock.d.ts.map +1 -0
- package/dist/components/CodeBlock.js +143 -0
- package/dist/components/MarkmapBlock.d.ts +4 -0
- package/dist/components/MarkmapBlock.d.ts.map +1 -0
- package/dist/components/MarkmapBlock.js +180 -0
- package/dist/components/MermaidBlock.d.ts +4 -0
- package/dist/components/MermaidBlock.d.ts.map +1 -0
- package/dist/components/MermaidBlock.js +193 -0
- package/dist/components/MessageBubble.d.ts +21 -0
- package/dist/components/MessageBubble.d.ts.map +1 -0
- package/dist/components/MessageBubble.js +233 -0
- package/dist/components/ReasoningBlock.d.ts +6 -0
- package/dist/components/ReasoningBlock.d.ts.map +1 -0
- package/dist/components/ReasoningBlock.js +11 -0
- package/dist/components/RichText.d.ts +41 -0
- package/dist/components/RichText.d.ts.map +1 -0
- package/dist/components/RichText.js +202 -0
- package/dist/components/VizShared.d.ts +57 -0
- package/dist/components/VizShared.d.ts.map +1 -0
- package/dist/components/VizShared.js +233 -0
- package/dist/components/tool-auto-continue.d.ts +5 -0
- package/dist/components/tool-auto-continue.d.ts.map +1 -0
- package/dist/components/tool-auto-continue.js +33 -0
- package/dist/constants.d.ts +61 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +72 -0
- package/dist/data/index.d.ts +4 -3
- package/dist/data/index.d.ts.map +1 -1
- package/dist/data/index.js +4 -10
- package/dist/data/knowledge-types.d.ts +8 -0
- package/dist/data/knowledge-types.d.ts.map +1 -0
- package/dist/data/knowledge-types.js +14 -0
- package/dist/data/metadata-loader.d.ts +4 -28
- package/dist/data/metadata-loader.d.ts.map +1 -1
- package/dist/data/metadata-loader.js +11 -34
- package/dist/data/types.d.ts +17 -2
- package/dist/data/types.d.ts.map +1 -1
- package/dist/extensions/index.d.ts +5 -0
- package/dist/extensions/index.d.ts.map +1 -0
- package/dist/extensions/index.js +24 -0
- package/dist/extensions/injector.d.ts +14 -0
- package/dist/extensions/injector.d.ts.map +1 -0
- package/dist/extensions/injector.js +146 -0
- package/dist/extensions/loader.d.ts +5 -0
- package/dist/extensions/loader.d.ts.map +1 -0
- package/dist/extensions/loader.js +45 -0
- package/dist/extensions/registry.d.ts +4 -0
- package/dist/extensions/registry.d.ts.map +1 -0
- package/dist/extensions/registry.js +144 -0
- package/dist/extensions/types.d.ts +126 -0
- package/dist/extensions/types.d.ts.map +1 -0
- package/dist/extensions/types.js +0 -0
- package/dist/fact-registry/prompt-injector.d.ts +1 -1
- package/dist/fact-registry/prompt-injector.d.ts.map +1 -1
- package/dist/fact-registry/prompt-injector.js +2 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/intelligence/citation-guard.d.ts +2 -13
- package/dist/intelligence/citation-guard.d.ts.map +1 -1
- package/dist/intelligence/citation-guard.js +52 -23
- package/dist/intelligence/evidence-analysis.d.ts +24 -16
- package/dist/intelligence/evidence-analysis.d.ts.map +1 -1
- package/dist/intelligence/evidence-analysis.js +118 -20
- package/dist/intelligence/evidence-budget.d.ts +13 -0
- package/dist/intelligence/evidence-budget.d.ts.map +1 -0
- package/dist/intelligence/evidence-budget.js +49 -0
- package/dist/intelligence/index.d.ts +10 -4
- package/dist/intelligence/index.d.ts.map +1 -1
- package/dist/intelligence/index.js +27 -3
- package/dist/intelligence/keyword-extract.d.ts +1 -1
- package/dist/intelligence/keyword-extract.d.ts.map +1 -1
- package/dist/intelligence/keyword-extract.js +5 -9
- package/dist/intelligence/request-interpretation.d.ts +40 -0
- package/dist/intelligence/request-interpretation.d.ts.map +1 -0
- package/dist/intelligence/request-interpretation.js +71 -0
- package/dist/intelligence/response-templates.d.ts +1 -0
- package/dist/intelligence/response-templates.d.ts.map +1 -1
- package/dist/intelligence/response-templates.js +13 -0
- package/dist/prompt/dynamic-layer.d.ts +1 -5
- package/dist/prompt/dynamic-layer.d.ts.map +1 -1
- package/dist/prompt/dynamic-layer.js +145 -9
- package/dist/prompt/prompt-builder.d.ts +1 -1
- package/dist/prompt/prompt-builder.d.ts.map +1 -1
- package/dist/prompt/prompt-builder.js +5 -1
- package/dist/prompt/semi-static-layer.d.ts +1 -1
- package/dist/prompt/semi-static-layer.d.ts.map +1 -1
- package/dist/prompt/semi-static-layer.js +22 -12
- package/dist/prompt/static-layer.d.ts.map +1 -1
- package/dist/prompt/static-layer.js +37 -4
- package/dist/prompt/types.d.ts +9 -4
- package/dist/prompt/types.d.ts.map +1 -1
- package/dist/provider-manager/base.d.ts +5 -1
- package/dist/provider-manager/base.d.ts.map +1 -1
- package/dist/provider-manager/base.js +22 -2
- package/dist/provider-manager/config.d.ts.map +1 -1
- package/dist/provider-manager/config.js +3 -2
- package/dist/provider-manager/index.d.ts +1 -1
- package/dist/provider-manager/index.d.ts.map +1 -1
- package/dist/provider-manager/index.js +1 -2
- package/dist/provider-manager/manager.d.ts +10 -1
- package/dist/provider-manager/manager.d.ts.map +1 -1
- package/dist/provider-manager/manager.js +26 -10
- package/dist/provider-manager/openai.d.ts +2 -2
- package/dist/provider-manager/openai.d.ts.map +1 -1
- package/dist/provider-manager/openai.js +19 -4
- package/dist/provider-manager/types.d.ts +18 -38
- package/dist/provider-manager/types.d.ts.map +1 -1
- package/dist/provider-manager/workers.d.ts +2 -2
- package/dist/provider-manager/workers.d.ts.map +1 -1
- package/dist/provider-manager/workers.js +15 -4
- package/dist/query/followup.d.ts +7 -0
- package/dist/query/followup.d.ts.map +1 -0
- package/dist/query/followup.js +46 -0
- package/dist/query/intent.d.ts +6 -0
- package/dist/query/intent.d.ts.map +1 -0
- package/dist/query/intent.js +137 -0
- package/dist/query/types.d.ts +8 -0
- package/dist/query/types.d.ts.map +1 -0
- package/dist/query/types.js +0 -0
- package/dist/search/hybrid-search.d.ts +111 -0
- package/dist/search/hybrid-search.d.ts.map +1 -0
- package/dist/search/hybrid-search.js +326 -0
- package/dist/search/index.d.ts +11 -9
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +46 -10
- package/dist/search/scoring.d.ts +18 -0
- package/dist/search/scoring.d.ts.map +1 -0
- package/dist/search/{search-utils.js → scoring.js} +14 -27
- package/dist/search/search-api.d.ts +16 -1
- package/dist/search/search-api.d.ts.map +1 -1
- package/dist/search/search-api.js +118 -15
- package/dist/search/search-index.d.ts +2 -2
- package/dist/search/search-index.d.ts.map +1 -1
- package/dist/search/search-index.js +4 -2
- package/dist/search/session-cache.d.ts +4 -10
- package/dist/search/session-cache.d.ts.map +1 -1
- package/dist/search/session-cache.js +12 -45
- package/dist/search/types.d.ts +28 -0
- package/dist/search/types.d.ts.map +1 -1
- package/dist/search/vector-reranker.d.ts +3 -3
- package/dist/search/vector-reranker.d.ts.map +1 -1
- package/dist/search/vector-reranker.js +14 -2
- package/dist/server/chat-handler.d.ts +86 -1
- package/dist/server/chat-handler.d.ts.map +1 -1
- package/dist/server/chat-handler.js +835 -401
- package/dist/server/chat-message-utils.d.ts +6 -0
- package/dist/server/chat-message-utils.d.ts.map +1 -0
- package/dist/server/chat-message-utils.js +40 -0
- package/dist/server/chat-utils.d.ts +30 -0
- package/dist/server/chat-utils.d.ts.map +1 -0
- package/dist/server/chat-utils.js +88 -0
- package/dist/server/dev-server.js +238 -101
- package/dist/server/env-config.d.ts +22 -0
- package/dist/server/env-config.d.ts.map +1 -0
- package/dist/server/env-config.js +25 -0
- package/dist/server/errors.d.ts +1 -0
- package/dist/server/errors.d.ts.map +1 -1
- package/dist/server/errors.js +14 -7
- package/dist/server/index.d.ts +2 -4
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +4 -25
- package/dist/server/metadata-init.d.ts +10 -5
- package/dist/server/metadata-init.d.ts.map +1 -1
- package/dist/server/metadata-init.js +78 -34
- package/dist/server/notify.d.ts +12 -11
- package/dist/server/notify.d.ts.map +1 -1
- package/dist/server/notify.js +46 -48
- package/dist/server/prompt-runtime.d.ts +60 -0
- package/dist/server/prompt-runtime.d.ts.map +1 -0
- package/dist/server/prompt-runtime.js +284 -0
- package/dist/server/stream-helpers.d.ts +30 -16
- package/dist/server/stream-helpers.d.ts.map +1 -1
- package/dist/server/stream-helpers.js +152 -15
- package/dist/server/types.d.ts +47 -12
- package/dist/server/types.d.ts.map +1 -1
- package/dist/structured-output/generator.d.ts +6 -0
- package/dist/structured-output/generator.d.ts.map +1 -0
- package/dist/structured-output/generator.js +164 -0
- package/dist/structured-output/index.d.ts +4 -0
- package/dist/structured-output/index.d.ts.map +1 -0
- package/dist/structured-output/index.js +6 -0
- package/dist/structured-output/schemas/evidence.d.ts +88 -0
- package/dist/structured-output/schemas/evidence.d.ts.map +1 -0
- package/dist/structured-output/schemas/evidence.js +65 -0
- package/dist/structured-output/types.d.ts +69 -0
- package/dist/structured-output/types.d.ts.map +1 -0
- package/dist/structured-output/types.js +0 -0
- package/dist/tools/action-tools.d.ts +63 -0
- package/dist/tools/action-tools.d.ts.map +1 -0
- package/dist/tools/action-tools.js +158 -0
- package/dist/tools/index.d.ts +2 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +30 -0
- package/dist/utils/i18n.d.ts +1 -1
- package/dist/utils/i18n.d.ts.map +1 -1
- package/dist/utils/i18n.js +1 -1
- package/dist/utils/logger.d.ts +11 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +36 -0
- package/dist/utils/text.d.ts +11 -0
- package/dist/utils/text.d.ts.map +1 -0
- package/dist/utils/text.js +87 -0
- package/dist/utils/url.d.ts +19 -0
- package/dist/utils/url.d.ts.map +1 -0
- package/dist/utils/url.js +13 -0
- package/package.json +46 -12
- package/dist/intelligence/intent-detect.d.ts +0 -40
- package/dist/intelligence/intent-detect.d.ts.map +0 -1
- package/dist/intelligence/intent-detect.js +0 -93
- package/dist/providers/index.d.ts +0 -2
- package/dist/providers/index.d.ts.map +0 -1
- package/dist/providers/index.js +0 -5
- package/dist/search/search-utils.d.ts +0 -47
- package/dist/search/search-utils.d.ts.map +0 -1
- package/dist/stream/index.d.ts +0 -3
- package/dist/stream/index.d.ts.map +0 -1
- package/dist/stream/index.js +0 -8
- package/dist/stream/mock-stream.d.ts +0 -12
- package/dist/stream/mock-stream.d.ts.map +0 -1
- package/dist/stream/mock-stream.js +0 -26
- package/dist/stream/response.d.ts +0 -10
- package/dist/stream/response.d.ts.map +0 -1
- package/dist/stream/response.js +0 -21
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type ArticleChunk, type ArticleWithChunks } from "./hybrid-search.js";
|
|
2
|
+
import type { SearchDocument, ArticleContext, ProjectContext } from "./types.js";
|
|
2
3
|
/**
|
|
3
4
|
* Initializes the article search index from the provided documents.
|
|
4
5
|
* Should be called once at startup (e.g., when the edge function initializes).
|
|
5
6
|
*/
|
|
6
7
|
export declare function initArticleIndex(documents: SearchDocument[]): void;
|
|
7
8
|
export declare function initProjectIndex(documents: SearchDocument[]): void;
|
|
9
|
+
export declare function initArticleChunks(chunksData: Record<string, ArticleChunk[]>): void;
|
|
10
|
+
export declare function hasArticleChunks(): boolean;
|
|
11
|
+
export declare function getArticleChunks(postId: string): ArticleChunk[] | undefined;
|
|
8
12
|
/**
|
|
9
13
|
* Searches for articles related to the query.
|
|
10
14
|
* Returns enriched ArticleContext objects ready for prompt injection.
|
|
@@ -12,7 +16,18 @@ export declare function initProjectIndex(documents: SearchDocument[]): void;
|
|
|
12
16
|
export declare function searchArticles(query: string, options?: {
|
|
13
17
|
enableDeepContent?: boolean;
|
|
14
18
|
siteUrl?: string;
|
|
19
|
+
enableRRF?: boolean;
|
|
20
|
+
sessionId?: string;
|
|
15
21
|
}): ArticleContext[];
|
|
22
|
+
/**
|
|
23
|
+
* Searches for relevant chunks within articles.
|
|
24
|
+
* Used for paragraph-level retrieval and injection.
|
|
25
|
+
*/
|
|
26
|
+
export declare function searchArticleChunks(query: string, articles: ArticleWithChunks[], topK?: number): Array<{
|
|
27
|
+
article: ArticleWithChunks;
|
|
28
|
+
chunk: ArticleChunk;
|
|
29
|
+
score: number;
|
|
30
|
+
}>;
|
|
16
31
|
/**
|
|
17
32
|
* Searches for projects related to the query.
|
|
18
33
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-api.d.ts","sourceRoot":"","sources":["../../src/search/search-api.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"search-api.d.ts","sourceRoot":"","sources":["../../src/search/search-api.ts"],"names":[],"mappings":"AAQA,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EACV,cAAc,EAGd,cAAc,EACd,cAAc,EACf,MAAM,YAAY,CAAC;AAkBpB;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,IAAI,CAElE;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,IAAI,CAElE;AAED,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,GACzC,IAAI,CASN;AAED,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,GAAG,SAAS,CAE3E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IACP,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACf,GACL,cAAc,EAAE,CAwGlB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,iBAAiB,EAAE,EAC7B,IAAI,GAAE,MAAW,GAChB,KAAK,CAAC;IAAE,OAAO,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAG3E;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GACjC,cAAc,EAAE,CAelB;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EACpD,OAAO,EAAE,CAAC,EAAE,EACZ,SAAS,EAAE,CAAC,EAAE,GACb,CAAC,EAAE,CAUL"}
|
|
@@ -1,19 +1,49 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
scoreDocument,
|
|
3
|
+
filterLowRelevance,
|
|
4
|
+
pickAnchorTerms
|
|
5
|
+
} from "./scoring.js";
|
|
6
|
+
import { tokenize, normalizeText } from "../utils/text.js";
|
|
2
7
|
import { buildSearchIndex, getIDFMapForIndex } from "./search-index.js";
|
|
3
8
|
import { hasVectorIndex, rerankWithVectors } from "./vector-reranker.js";
|
|
9
|
+
import {
|
|
10
|
+
hybridSearch,
|
|
11
|
+
searchChunks
|
|
12
|
+
} from "./hybrid-search.js";
|
|
13
|
+
import { safeJoinUrl } from "../utils/url.js";
|
|
4
14
|
let articleIndex = null;
|
|
5
15
|
let projectIndex = null;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
const
|
|
16
|
+
let articleChunks = /* @__PURE__ */ new Map();
|
|
17
|
+
import { SEARCH } from "../constants.js";
|
|
18
|
+
import { createLogger } from "../utils/logger.js";
|
|
19
|
+
const log = createLogger("search");
|
|
20
|
+
const ARTICLE_LIMIT = SEARCH.ARTICLE_LIMIT;
|
|
21
|
+
const ARTICLE_LIMIT_BROAD = SEARCH.ARTICLE_LIMIT_BROAD;
|
|
22
|
+
const PROJECT_LIMIT = SEARCH.PROJECT_LIMIT;
|
|
23
|
+
const DEEP_CONTENT_SCORE_THRESHOLD = SEARCH.DEEP_CONTENT_SCORE_THRESHOLD;
|
|
24
|
+
const DEEP_CONTENT_MAX_LENGTH = SEARCH.DEEP_CONTENT_MAX_LENGTH;
|
|
11
25
|
function initArticleIndex(documents) {
|
|
12
26
|
articleIndex = buildSearchIndex(documents);
|
|
13
27
|
}
|
|
14
28
|
function initProjectIndex(documents) {
|
|
15
29
|
projectIndex = buildSearchIndex(documents);
|
|
16
30
|
}
|
|
31
|
+
function initArticleChunks(chunksData) {
|
|
32
|
+
articleChunks = new Map(Object.entries(chunksData));
|
|
33
|
+
const totalChunks = [...articleChunks.values()].reduce(
|
|
34
|
+
(sum, c) => sum + c.length,
|
|
35
|
+
0
|
|
36
|
+
);
|
|
37
|
+
log.info(
|
|
38
|
+
`Loaded chunks: ${articleChunks.size} articles, ${totalChunks} total chunks`
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
function hasArticleChunks() {
|
|
42
|
+
return articleChunks.size > 0;
|
|
43
|
+
}
|
|
44
|
+
function getArticleChunks(postId) {
|
|
45
|
+
return articleChunks.get(postId);
|
|
46
|
+
}
|
|
17
47
|
function searchArticles(query, options = {}) {
|
|
18
48
|
if (!query.trim() || !articleIndex) return [];
|
|
19
49
|
const tokens = tokenize(query);
|
|
@@ -21,41 +51,88 @@ function searchArticles(query, options = {}) {
|
|
|
21
51
|
const limit = tokens.length <= 2 ? ARTICLE_LIMIT_BROAD : ARTICLE_LIMIT;
|
|
22
52
|
const rawResults = scoreDocs(articleIndex, tokens, limit * 2);
|
|
23
53
|
const filtered = applyAnchorFilter(rawResults, query, tokens);
|
|
24
|
-
const deduplicated = filterLowRelevance(
|
|
25
|
-
|
|
54
|
+
const deduplicated = filterLowRelevance(
|
|
55
|
+
filtered.length > 0 ? filtered : rawResults
|
|
56
|
+
);
|
|
57
|
+
const purityFiltered = applyPurityFilter(query, deduplicated);
|
|
58
|
+
const results = purityFiltered.slice(0, limit);
|
|
59
|
+
log.debug(
|
|
60
|
+
`searchArticles: query="${query}", tokens=${tokens.length}, raw=${rawResults.length}, anchor=${filtered.length}, dedup=${deduplicated.length}, purity=${purityFiltered.length}, final=${results.length}, limit=${limit}`
|
|
61
|
+
);
|
|
26
62
|
const topScore = results[0]?.score ?? 0;
|
|
27
63
|
const secondScore = results[1]?.score ?? 0;
|
|
28
64
|
const isDeepHit = options.enableDeepContent && topScore >= DEEP_CONTENT_SCORE_THRESHOLD && topScore > secondScore * 1.5;
|
|
29
65
|
let articles = results.map((result, index) => {
|
|
30
|
-
const
|
|
31
|
-
const
|
|
66
|
+
const url = safeJoinUrl(options.siteUrl ?? "", result.url);
|
|
67
|
+
const chunks = articleChunks.get(result.id);
|
|
32
68
|
const fullContent = isDeepHit && index === 0 && result.content ? result.content.slice(0, DEEP_CONTENT_MAX_LENGTH) : void 0;
|
|
33
69
|
return {
|
|
70
|
+
id: result.id,
|
|
34
71
|
title: result.title,
|
|
35
72
|
url,
|
|
73
|
+
lang: result.lang,
|
|
36
74
|
summary: result.summary ?? result.excerpt,
|
|
37
75
|
keyPoints: result.keyPoints,
|
|
38
76
|
categories: result.categories,
|
|
39
77
|
dateTime: result.dateTime,
|
|
40
78
|
fullContent,
|
|
41
|
-
score: result.score
|
|
79
|
+
score: result.score,
|
|
80
|
+
readingTime: result.readingTime,
|
|
81
|
+
chunks
|
|
42
82
|
};
|
|
43
83
|
});
|
|
44
|
-
if (hasVectorIndex() && articles.length > 1) {
|
|
84
|
+
if (options.enableRRF && hasVectorIndex() && articles.length > 1) {
|
|
85
|
+
const before = articles.slice(0, 5).map((a) => `${a.title}:${(a.score ?? 0).toFixed(3)}`).join(" | ");
|
|
86
|
+
const vectorResults = articles.map((a) => ({ ...a, score: a.score || 0 }));
|
|
87
|
+
const hybridResults = hybridSearch(query, articles, vectorResults, {
|
|
88
|
+
topK: limit
|
|
89
|
+
});
|
|
90
|
+
const chunksMap = new Map(articles.map((a) => [a.url, a.chunks]));
|
|
91
|
+
const articleMetaMap = new Map(
|
|
92
|
+
articles.map((a) => [a.url, { id: a.id ?? a.url, lang: a.lang ?? "" }])
|
|
93
|
+
);
|
|
94
|
+
articles = hybridResults.map((h) => ({
|
|
95
|
+
id: articleMetaMap.get(h.url)?.id ?? h.url,
|
|
96
|
+
title: h.title,
|
|
97
|
+
url: h.url,
|
|
98
|
+
lang: articleMetaMap.get(h.url)?.lang ?? "",
|
|
99
|
+
summary: h.summary ?? "",
|
|
100
|
+
keyPoints: h.keyPoints ?? [],
|
|
101
|
+
categories: h.categories ?? [],
|
|
102
|
+
dateTime: h.dateTime ?? 0,
|
|
103
|
+
fullContent: h.fullContent,
|
|
104
|
+
score: h.score ?? 0,
|
|
105
|
+
readingTime: h.readingTime,
|
|
106
|
+
chunks: chunksMap.get(h.url),
|
|
107
|
+
rrfScore: h.rrfScore,
|
|
108
|
+
bm25Rank: h.bm25Rank,
|
|
109
|
+
vectorRank: h.vectorRank
|
|
110
|
+
}));
|
|
111
|
+
log.debug(
|
|
112
|
+
`searchArticles: hybrid rerank changed top results from [${before}] to [${articles.slice(0, 5).map((a) => `${a.title}:${(a.score ?? 0).toFixed(3)}`).join(" | ")}]`
|
|
113
|
+
);
|
|
114
|
+
} else if (hasVectorIndex() && articles.length > 1) {
|
|
115
|
+
const before = articles.slice(0, 5).map((a) => `${a.title}:${(a.score ?? 0).toFixed(3)}`).join(" | ");
|
|
45
116
|
articles = rerankWithVectors(query, articles);
|
|
117
|
+
log.debug(
|
|
118
|
+
`searchArticles: vector rerank changed top results from [${before}] to [${articles.slice(0, 5).map((a) => `${a.title}:${(a.score ?? 0).toFixed(3)}`).join(" | ")}]`
|
|
119
|
+
);
|
|
46
120
|
}
|
|
47
121
|
return articles;
|
|
48
122
|
}
|
|
123
|
+
function searchArticleChunks(query, articles, topK = 10) {
|
|
124
|
+
if (!query.trim() || !articles.length) return [];
|
|
125
|
+
return searchChunks(query, articles, topK);
|
|
126
|
+
}
|
|
49
127
|
function searchProjects(query, options = {}) {
|
|
50
128
|
if (!query.trim() || !projectIndex) return [];
|
|
51
129
|
const tokens = tokenize(query);
|
|
52
130
|
if (!tokens.length) return [];
|
|
53
131
|
const rawResults = scoreDocs(projectIndex, tokens, PROJECT_LIMIT * 2);
|
|
54
132
|
if (!rawResults.length) return [];
|
|
55
|
-
const baseUrl = options.siteUrl ?? "";
|
|
56
133
|
return rawResults.slice(0, PROJECT_LIMIT).map((r) => ({
|
|
57
134
|
name: r.title,
|
|
58
|
-
url:
|
|
135
|
+
url: safeJoinUrl(options.siteUrl ?? "", r.url),
|
|
59
136
|
description: r.excerpt || r.content.slice(0, 200),
|
|
60
137
|
score: r.score
|
|
61
138
|
}));
|
|
@@ -80,15 +157,41 @@ function applyAnchorFilter(results, query, tokens) {
|
|
|
80
157
|
const anchorTerms = pickAnchorTerms(query, results, 2, 2);
|
|
81
158
|
if (!anchorTerms.length) return results;
|
|
82
159
|
const strict = results.filter((r) => {
|
|
83
|
-
const text = normalizeText(
|
|
160
|
+
const text = normalizeText(
|
|
161
|
+
[r.title, ...r.keyPoints, ...r.categories].join(" ")
|
|
162
|
+
);
|
|
84
163
|
return anchorTerms.some((term) => text.includes(term));
|
|
85
164
|
});
|
|
86
165
|
return strict.length > 0 ? strict : results;
|
|
87
166
|
}
|
|
167
|
+
function applyPurityFilter(query, results) {
|
|
168
|
+
if (results.length <= 3) return results;
|
|
169
|
+
const queryTokens = tokenize(query);
|
|
170
|
+
if (queryTokens.length < 2) return results;
|
|
171
|
+
const anchorTerms = pickAnchorTerms(query, results, 3, 2);
|
|
172
|
+
if (!anchorTerms.length) return results;
|
|
173
|
+
const topScore = results[0]?.score ?? 0;
|
|
174
|
+
const filtered = results.filter((result, index) => {
|
|
175
|
+
if (index === 0) return true;
|
|
176
|
+
const primaryText = normalizeText(
|
|
177
|
+
[result.title, ...result.keyPoints, ...result.categories].join(" ")
|
|
178
|
+
);
|
|
179
|
+
const titleText = normalizeText(result.title);
|
|
180
|
+
const anchorHits = anchorTerms.filter((term) => primaryText.includes(term));
|
|
181
|
+
if (anchorHits.length > 0) return true;
|
|
182
|
+
const relativeScore = topScore > 0 ? result.score / topScore : 0;
|
|
183
|
+
return titleText.length > 0 && relativeScore >= 0.82;
|
|
184
|
+
});
|
|
185
|
+
return filtered.length >= 2 ? filtered : results;
|
|
186
|
+
}
|
|
88
187
|
export {
|
|
188
|
+
getArticleChunks,
|
|
189
|
+
hasArticleChunks,
|
|
190
|
+
initArticleChunks,
|
|
89
191
|
initArticleIndex,
|
|
90
192
|
initProjectIndex,
|
|
91
193
|
mergeResults,
|
|
194
|
+
searchArticleChunks,
|
|
92
195
|
searchArticles,
|
|
93
196
|
searchProjects
|
|
94
197
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type IDFMap } from
|
|
2
|
-
import type { SearchDocument, IndexedDocument } from
|
|
1
|
+
import { type IDFMap } from "./idf.js";
|
|
2
|
+
import type { SearchDocument, IndexedDocument } from "./types.js";
|
|
3
3
|
/**
|
|
4
4
|
* Builds an in-memory inverted index from a list of documents
|
|
5
5
|
* and computes IDF weights across the corpus.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-index.d.ts","sourceRoot":"","sources":["../../src/search/search-index.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAIlE;;;;;;GAMG;AACH,wBAAgB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"search-index.d.ts","sourceRoot":"","sources":["../../src/search/search-index.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACpD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAIlE;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,cAAc,EAAE,GAC1B,eAAe,EAAE,CAUnB;AAED,wBAAgB,iBAAiB,IAAI,MAAM,GAAG,IAAI,CAEjD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { normalizeText } from "
|
|
1
|
+
import { normalizeText } from "../utils/text.js";
|
|
2
2
|
import { buildIDFMap } from "./idf.js";
|
|
3
3
|
let cachedIDFMap = null;
|
|
4
4
|
function buildSearchIndex(documents) {
|
|
@@ -24,7 +24,9 @@ function buildDocumentTokens(doc) {
|
|
|
24
24
|
...doc.tags,
|
|
25
25
|
doc.summary ?? ""
|
|
26
26
|
];
|
|
27
|
-
return [
|
|
27
|
+
return [
|
|
28
|
+
...new Set(parts.map(normalizeText).join(" ").split(/\s+/).filter(Boolean))
|
|
29
|
+
];
|
|
28
30
|
}
|
|
29
31
|
export {
|
|
30
32
|
buildSearchIndex,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { CacheAdapter } from
|
|
2
|
-
import type { CachedSearchContext } from
|
|
3
|
-
export { type CachedSearchContext,
|
|
4
|
-
export declare const SESSION_CACHE_TTL_SECONDS
|
|
1
|
+
import type { CacheAdapter } from "../cache/types.js";
|
|
2
|
+
import type { CachedSearchContext } from "./types.js";
|
|
3
|
+
export { type CachedSearchContext, } from "./types.js";
|
|
4
|
+
export declare const SESSION_CACHE_TTL_SECONDS: 600;
|
|
5
5
|
export declare const SESSION_CACHE_TTL_MS: number;
|
|
6
6
|
export declare function getSessionCacheKey(req: Request): string | null;
|
|
7
7
|
export declare function setCacheAdapter(cache: CacheAdapter): void;
|
|
@@ -10,10 +10,4 @@ export declare function getCachedContext(key: string, cache?: CacheAdapter): Pro
|
|
|
10
10
|
export declare function setCachedContext(key: string, ctx: CachedSearchContext, cache?: CacheAdapter): Promise<void>;
|
|
11
11
|
export declare function deleteCachedContext(key: string, cache?: CacheAdapter): Promise<boolean>;
|
|
12
12
|
export declare function cleanupCache(_now: number): void;
|
|
13
|
-
/** @deprecated Use getCachedContext instead */
|
|
14
|
-
export declare function getCachedContextSync(key: string): CachedSearchContext | undefined;
|
|
15
|
-
/** @deprecated Use setCachedContext instead */
|
|
16
|
-
export declare function setCachedContextSync(key: string, ctx: CachedSearchContext): void;
|
|
17
|
-
/** @deprecated Use cleanupCache instead (no-op) */
|
|
18
|
-
export declare function cleanupCacheLegacy(now: number): void;
|
|
19
13
|
//# sourceMappingURL=session-cache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-cache.d.ts","sourceRoot":"","sources":["../../src/search/session-cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"session-cache.d.ts","sourceRoot":"","sources":["../../src/search/session-cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EACV,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAIpB,OAAO,EACL,KAAK,mBAAmB,GACzB,MAAM,YAAY,CAAC;AAOpB,eAAO,MAAM,yBAAyB,KAAoB,CAAC;AAC3D,eAAO,MAAM,oBAAoB,QAAmC,CAAC;AAcrE,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAM9D;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAEzD;AAED,wBAAgB,eAAe,IAAI,YAAY,CAE9C;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,YAAY,GACnB,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAO1C;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,mBAAmB,EACxB,KAAK,CAAC,EAAE,YAAY,GACnB,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,mBAAmB,CACvC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,YAAY,GACnB,OAAO,CAAC,OAAO,CAAC,CAIlB;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAG/C"}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { MemoryCacheAdapter } from "../cache/memory-adapter.js";
|
|
2
|
+
import { createLogger } from "../utils/logger.js";
|
|
3
|
+
import { CACHE } from "../constants.js";
|
|
4
|
+
const log = createLogger("session-cache");
|
|
2
5
|
const SESSION_ID_PATTERN = /^[a-z0-9][a-z0-9_-]{7,63}$/i;
|
|
3
|
-
const SESSION_CACHE_TTL_SECONDS =
|
|
6
|
+
const SESSION_CACHE_TTL_SECONDS = CACHE.SESSION_TTL;
|
|
4
7
|
const SESSION_CACHE_TTL_MS = SESSION_CACHE_TTL_SECONDS * 1e3;
|
|
5
8
|
let defaultCache = null;
|
|
6
9
|
function getDefaultCache() {
|
|
@@ -28,69 +31,33 @@ function getCacheAdapter() {
|
|
|
28
31
|
async function getCachedContext(key, cache) {
|
|
29
32
|
const adapter = cache ?? getDefaultCache();
|
|
30
33
|
const entry = await adapter.get(key);
|
|
34
|
+
log.debug(
|
|
35
|
+
`getCachedContext: key=${key}, hit=${Boolean(entry?.value)}, adapter=${adapter.name}`
|
|
36
|
+
);
|
|
31
37
|
return entry?.value;
|
|
32
38
|
}
|
|
33
39
|
async function setCachedContext(key, ctx, cache) {
|
|
34
40
|
const adapter = cache ?? getDefaultCache();
|
|
35
41
|
await adapter.set(key, ctx, { ttl: SESSION_CACHE_TTL_SECONDS });
|
|
42
|
+
log.debug(
|
|
43
|
+
`setCachedContext: key=${key}, articles=${ctx.articles.length}, projects=${ctx.projects.length}, adapter=${adapter.name}`
|
|
44
|
+
);
|
|
36
45
|
}
|
|
37
46
|
async function deleteCachedContext(key, cache) {
|
|
38
47
|
const adapter = cache ?? getDefaultCache();
|
|
48
|
+
log.debug(`deleteCachedContext: key=${key}, adapter=${adapter.name}`);
|
|
39
49
|
return adapter.delete(key);
|
|
40
50
|
}
|
|
41
51
|
function cleanupCache(_now) {
|
|
42
52
|
}
|
|
43
|
-
const legacyCache = /* @__PURE__ */ new Map();
|
|
44
|
-
const LEGACY_TTL_MS = 10 * 60 * 1e3;
|
|
45
|
-
const MAX_CACHE_SIZE = 400;
|
|
46
|
-
function getCachedContextSync(key) {
|
|
47
|
-
const entry = legacyCache.get(key);
|
|
48
|
-
if (!entry) return void 0;
|
|
49
|
-
if (Date.now() - entry.updatedAt > LEGACY_TTL_MS) {
|
|
50
|
-
legacyCache.delete(key);
|
|
51
|
-
return void 0;
|
|
52
|
-
}
|
|
53
|
-
return entry;
|
|
54
|
-
}
|
|
55
|
-
function setCachedContextSync(key, ctx) {
|
|
56
|
-
legacyCache.set(key, ctx);
|
|
57
|
-
if (legacyCache.size > MAX_CACHE_SIZE) {
|
|
58
|
-
const overflow = legacyCache.size - MAX_CACHE_SIZE;
|
|
59
|
-
const keys = legacyCache.keys();
|
|
60
|
-
for (let i = 0; i < overflow; i++) {
|
|
61
|
-
const next = keys.next();
|
|
62
|
-
if (next.done) break;
|
|
63
|
-
legacyCache.delete(next.value);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
function cleanupCacheLegacy(now) {
|
|
68
|
-
for (const [key, value] of legacyCache) {
|
|
69
|
-
if (now - value.updatedAt > LEGACY_TTL_MS) {
|
|
70
|
-
legacyCache.delete(key);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
if (legacyCache.size > MAX_CACHE_SIZE) {
|
|
74
|
-
const overflow = legacyCache.size - MAX_CACHE_SIZE;
|
|
75
|
-
const keys = legacyCache.keys();
|
|
76
|
-
for (let i = 0; i < overflow; i++) {
|
|
77
|
-
const next = keys.next();
|
|
78
|
-
if (next.done) break;
|
|
79
|
-
legacyCache.delete(next.value);
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
53
|
export {
|
|
84
54
|
SESSION_CACHE_TTL_MS,
|
|
85
55
|
SESSION_CACHE_TTL_SECONDS,
|
|
86
56
|
cleanupCache,
|
|
87
|
-
cleanupCacheLegacy,
|
|
88
57
|
deleteCachedContext,
|
|
89
58
|
getCacheAdapter,
|
|
90
59
|
getCachedContext,
|
|
91
|
-
getCachedContextSync,
|
|
92
60
|
getSessionCacheKey,
|
|
93
61
|
setCacheAdapter,
|
|
94
|
-
setCachedContext
|
|
95
|
-
setCachedContextSync
|
|
62
|
+
setCachedContext
|
|
96
63
|
};
|
package/dist/search/types.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export interface SearchDocument {
|
|
|
10
10
|
dateTime: number;
|
|
11
11
|
lang: string;
|
|
12
12
|
summary?: string;
|
|
13
|
+
readingTime?: number;
|
|
13
14
|
}
|
|
14
15
|
export interface IndexedDocument extends SearchDocument {
|
|
15
16
|
/** Normalized token list for fast lookup */
|
|
@@ -18,15 +19,42 @@ export interface IndexedDocument extends SearchDocument {
|
|
|
18
19
|
export interface SearchResult extends SearchDocument {
|
|
19
20
|
score: number;
|
|
20
21
|
}
|
|
22
|
+
export interface ArticleChunk {
|
|
23
|
+
id: string;
|
|
24
|
+
postId: string;
|
|
25
|
+
heading: string;
|
|
26
|
+
content: string;
|
|
27
|
+
position: number;
|
|
28
|
+
tokenCount: number;
|
|
29
|
+
headers: Record<string, string>;
|
|
30
|
+
}
|
|
21
31
|
export interface ArticleContext {
|
|
32
|
+
id?: string;
|
|
22
33
|
title: string;
|
|
23
34
|
url: string;
|
|
35
|
+
lang?: string;
|
|
24
36
|
summary?: string;
|
|
25
37
|
keyPoints: string[];
|
|
26
38
|
categories: string[];
|
|
27
39
|
dateTime: number;
|
|
28
40
|
fullContent?: string;
|
|
29
41
|
score?: number;
|
|
42
|
+
readingTime?: number;
|
|
43
|
+
chunks?: ArticleChunk[];
|
|
44
|
+
rrfScore?: number;
|
|
45
|
+
bm25Rank?: number;
|
|
46
|
+
vectorRank?: number;
|
|
47
|
+
}
|
|
48
|
+
export interface SourceSelection {
|
|
49
|
+
title: string;
|
|
50
|
+
url?: string;
|
|
51
|
+
lang?: string;
|
|
52
|
+
reason: "chunk" | "evidence" | "article-context" | "retrieval-fallback" | "cache";
|
|
53
|
+
score?: number;
|
|
54
|
+
chunkId?: string;
|
|
55
|
+
heading?: string;
|
|
56
|
+
snippet?: string;
|
|
57
|
+
matchTerms?: string[];
|
|
30
58
|
}
|
|
31
59
|
export interface ProjectContext {
|
|
32
60
|
name: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/search/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/search/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD,4CAA4C;IAC5C,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,YAAa,SAAQ,cAAc;IAClD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EACF,OAAO,GACP,UAAU,GACV,iBAAiB,GACjB,oBAAoB,GACpB,OAAO,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* uses cosine similarity to refine the ranking of search results.
|
|
6
6
|
* Gracefully degrades to a no-op when no vector index is loaded.
|
|
7
7
|
*/
|
|
8
|
-
import type { ArticleContext } from
|
|
8
|
+
import type { ArticleContext } from "./types.js";
|
|
9
9
|
export interface VectorChunk {
|
|
10
10
|
postId: string;
|
|
11
11
|
title: string;
|
|
@@ -16,7 +16,7 @@ export interface VectorChunk {
|
|
|
16
16
|
}
|
|
17
17
|
export interface VectorIndex {
|
|
18
18
|
version: number;
|
|
19
|
-
method:
|
|
19
|
+
method: "tfidf" | "openai";
|
|
20
20
|
createdAt: string;
|
|
21
21
|
vocabulary?: string[];
|
|
22
22
|
chunks: VectorChunk[];
|
|
@@ -34,5 +34,5 @@ export declare function hasVectorIndex(): boolean;
|
|
|
34
34
|
*
|
|
35
35
|
* @param alpha - Blend factor (0 = ignore vectors, 1 = vectors only). Default 0.3
|
|
36
36
|
*/
|
|
37
|
-
export declare function rerankWithVectors<T extends Pick<ArticleContext,
|
|
37
|
+
export declare function rerankWithVectors<T extends Pick<ArticleContext, "url" | "score">>(query: string, candidates: T[], alpha?: number): T[];
|
|
38
38
|
//# sourceMappingURL=vector-reranker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-reranker.d.ts","sourceRoot":"","sources":["../../src/search/vector-reranker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,GAAG,QAAQ,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAOD,wBAAgB,eAAe,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,IAAI,CAO9D;AAED,wBAAgB,gBAAgB,IAAI,IAAI,CAGvC;AAED,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAID;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,
|
|
1
|
+
{"version":3,"file":"vector-reranker.d.ts","sourceRoot":"","sources":["../../src/search/vector-reranker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,GAAG,QAAQ,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAOD,wBAAgB,eAAe,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,IAAI,CAO9D;AAED,wBAAgB,gBAAgB,IAAI,IAAI,CAGvC;AAED,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAID;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,CAAC,SAAS,IAAI,CAAC,cAAc,EAAE,KAAK,GAAG,OAAO,CAAC,EAC/C,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,SAAM,GAAG,CAAC,EAAE,CA+ClD"}
|
|
@@ -18,7 +18,11 @@ function rerankWithVectors(query, candidates, alpha = 0.3) {
|
|
|
18
18
|
if (!loadedIndex || !idfCache || !loadedIndex.vocabulary) {
|
|
19
19
|
return candidates;
|
|
20
20
|
}
|
|
21
|
-
const queryVector = computeQueryVector(
|
|
21
|
+
const queryVector = computeQueryVector(
|
|
22
|
+
query,
|
|
23
|
+
loadedIndex.vocabulary,
|
|
24
|
+
idfCache
|
|
25
|
+
);
|
|
22
26
|
if (!queryVector) return candidates;
|
|
23
27
|
const articleScores = /* @__PURE__ */ new Map();
|
|
24
28
|
for (const chunk of loadedIndex.chunks) {
|
|
@@ -38,7 +42,13 @@ function rerankWithVectors(query, candidates, alpha = 0.3) {
|
|
|
38
42
|
const blended = originalNorm * (1 - alpha) + vectorScore * alpha;
|
|
39
43
|
return { ...article, score: blended };
|
|
40
44
|
});
|
|
41
|
-
|
|
45
|
+
const sorted = reranked.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
46
|
+
log.debug(
|
|
47
|
+
`rerankWithVectors: candidates=${candidates.length}, alpha=${alpha}, top=${sorted.slice(0, 5).map(
|
|
48
|
+
(item) => `${extractSlugFromUrl(item.url)}:${(item.score ?? 0).toFixed(3)}`
|
|
49
|
+
).join(" | ")}`
|
|
50
|
+
);
|
|
51
|
+
return sorted;
|
|
42
52
|
}
|
|
43
53
|
function cosineSimilarity(a, b) {
|
|
44
54
|
if (a.length !== b.length) return 0;
|
|
@@ -95,6 +105,8 @@ function extractSlugFromUrl(url) {
|
|
|
95
105
|
if (match) return `${match[1]}/${match[2]}`;
|
|
96
106
|
return path.replace(/^\/|\/$/g, "");
|
|
97
107
|
}
|
|
108
|
+
import { createLogger } from "../utils/logger.js";
|
|
109
|
+
const log = createLogger("vector-reranker");
|
|
98
110
|
export {
|
|
99
111
|
clearVectorIndex,
|
|
100
112
|
hasVectorIndex,
|
|
@@ -1,3 +1,88 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { resolveSearchInterpretation } from "../intelligence/index.js";
|
|
2
|
+
import { searchArticles, searchProjects } from "../search/index.js";
|
|
3
|
+
import type { PublicQuestionType } from "../cache/global-cache.js";
|
|
4
|
+
import type { SourceSelection } from "../search/types.js";
|
|
5
|
+
import type { ChatHandlerOptions, ChatContext } from "./types.js";
|
|
2
6
|
export declare function handleChatRequest(options: ChatHandlerOptions): Promise<Response>;
|
|
7
|
+
interface SearchPhaseResult {
|
|
8
|
+
searchQuery: string;
|
|
9
|
+
relatedArticles: ReturnType<typeof searchArticles>;
|
|
10
|
+
relatedProjects: ReturnType<typeof searchProjects>;
|
|
11
|
+
selectedSources?: SourceSelection[];
|
|
12
|
+
budget: ReturnType<typeof resolveSearchInterpretation>["budget"];
|
|
13
|
+
interpretation: ReturnType<typeof resolveSearchInterpretation>["interpretation"];
|
|
14
|
+
}
|
|
15
|
+
export declare function resolveSearchAnswerShaping(query: string): Pick<SearchPhaseResult, "budget" | "interpretation">;
|
|
16
|
+
export declare function rankArticlesForQuery(query: string, articles: ReturnType<typeof searchArticles>): ReturnType<typeof searchArticles>;
|
|
17
|
+
interface CurrentArticleBoostOptions {
|
|
18
|
+
articleSlug?: string;
|
|
19
|
+
context?: ChatContext;
|
|
20
|
+
}
|
|
21
|
+
export declare function extractQuotedCandidate(text: string): string;
|
|
22
|
+
export declare function isLikelyQuotedArticleQuery(text: string): boolean;
|
|
23
|
+
export declare function isCrossArticleIntent(text: string): boolean;
|
|
24
|
+
export declare function rerankArticlesForCurrentArticleQuote(query: string, articles: ReturnType<typeof searchArticles>, options?: CurrentArticleBoostOptions): ReturnType<typeof searchArticles>;
|
|
25
|
+
export declare function shapeArticlesForQuery(query: string, articles: ReturnType<typeof searchArticles>, options?: CurrentArticleBoostOptions): {
|
|
26
|
+
interpretation: ReturnType<typeof resolveSearchInterpretation>["interpretation"];
|
|
27
|
+
budget: ReturnType<typeof resolveSearchInterpretation>["budget"];
|
|
28
|
+
articles: ReturnType<typeof searchArticles>;
|
|
29
|
+
};
|
|
30
|
+
export declare function rerankArticlesForCodeAnchors(query: string, articles: ReturnType<typeof searchArticles>): ReturnType<typeof searchArticles>;
|
|
31
|
+
export declare function shouldPersistAuthoritativeSources(sources: SourceSelection[]): boolean;
|
|
32
|
+
export declare function shapeCachedSearchForQuery(args: {
|
|
33
|
+
query: string;
|
|
34
|
+
articles: ReturnType<typeof searchArticles>;
|
|
35
|
+
projects: ReturnType<typeof searchProjects>;
|
|
36
|
+
}): {
|
|
37
|
+
interpretation: ReturnType<typeof resolveSearchInterpretation>["interpretation"];
|
|
38
|
+
budget: ReturnType<typeof resolveSearchInterpretation>["budget"];
|
|
39
|
+
articles: ReturnType<typeof searchArticles>;
|
|
40
|
+
projects: ReturnType<typeof searchProjects>;
|
|
41
|
+
};
|
|
42
|
+
export declare function shouldPersistResponseCacheEntry(args: {
|
|
43
|
+
enabled: boolean;
|
|
44
|
+
success: boolean;
|
|
45
|
+
responseText: string;
|
|
46
|
+
sources: SourceSelection[];
|
|
47
|
+
}): boolean;
|
|
48
|
+
type PublicQuestionLike = {
|
|
49
|
+
type: PublicQuestionType;
|
|
50
|
+
needsContext: boolean;
|
|
51
|
+
};
|
|
52
|
+
export declare function shouldUsePublicQuestionCaches(args: {
|
|
53
|
+
publicQuestion: PublicQuestionLike | null;
|
|
54
|
+
articleSlug?: string;
|
|
55
|
+
}): boolean;
|
|
56
|
+
export declare function buildPublicCacheContext(args: {
|
|
57
|
+
articleSlug?: string;
|
|
58
|
+
lang: string;
|
|
59
|
+
latestText: string;
|
|
60
|
+
}): {
|
|
61
|
+
articleSlug?: string;
|
|
62
|
+
lang?: string;
|
|
63
|
+
queryKey?: string;
|
|
64
|
+
};
|
|
65
|
+
export declare function shapePublicCacheBranch(args: {
|
|
66
|
+
publicQuestion: PublicQuestionLike | null;
|
|
67
|
+
articleSlug?: string;
|
|
68
|
+
lang: string;
|
|
69
|
+
latestText: string;
|
|
70
|
+
}): {
|
|
71
|
+
enabled: boolean;
|
|
72
|
+
context: {
|
|
73
|
+
articleSlug?: string;
|
|
74
|
+
lang?: string;
|
|
75
|
+
queryKey?: string;
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
export declare function buildFinalSources(args: {
|
|
79
|
+
relatedArticles: ReturnType<typeof searchArticles>;
|
|
80
|
+
selectedSources: SourceSelection[];
|
|
81
|
+
query: string;
|
|
82
|
+
lang: string;
|
|
83
|
+
max: number;
|
|
84
|
+
articleSlug?: string;
|
|
85
|
+
context?: ChatContext;
|
|
86
|
+
}): SourceSelection[];
|
|
87
|
+
export {};
|
|
3
88
|
//# sourceMappingURL=chat-handler.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat-handler.d.ts","sourceRoot":"","sources":["../../src/server/chat-handler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chat-handler.d.ts","sourceRoot":"","sources":["../../src/server/chat-handler.ts"],"names":[],"mappings":"AAMA,OAAO,EAcL,2BAA2B,EAC5B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,cAAc,EACd,cAAc,EAMf,MAAM,oBAAoB,CAAC;AAiC5B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,KAAK,EACV,kBAAkB,EAElB,WAAW,EAEZ,MAAM,YAAY,CAAC;AAoCpB,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,QAAQ,CAAC,CAoEnB;AA8HD,UAAU,iBAAiB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;IACnD,eAAe,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;IACnD,eAAe,CAAC,EAAE,eAAe,EAAE,CAAC;IACpC,MAAM,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,QAAQ,CAAC,CAAC;IACjE,cAAc,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,gBAAgB,CAAC,CAAC;CAClF;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAC7D,iBAAiB,EACjB,QAAQ,GAAG,gBAAgB,CAC5B,CAMA;AAED,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,GAC1C,UAAU,CAAC,OAAO,cAAc,CAAC,CAGnC;AAED,UAAU,0BAA0B;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAoBD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAU3D;AAED,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAIhE;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAO1D;AA4BD,wBAAgB,oCAAoC,CAClD,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,EAC3C,OAAO,GAAE,0BAA+B,GACvC,UAAU,CAAC,OAAO,cAAc,CAAC,CA6BnC;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,EAC3C,OAAO,GAAE,0BAA+B,GACvC;IACD,cAAc,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACjF,MAAM,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,QAAQ,CAAC,CAAC;IACjE,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;CAC7C,CAiBA;AAED,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,GAC1C,UAAU,CAAC,OAAO,cAAc,CAAC,CAcnC;AA8DD,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,eAAe,EAAE,GACzB,OAAO,CAOT;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;IAC5C,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;CAC7C,GAAG;IACF,cAAc,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACjF,MAAM,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,QAAQ,CAAC,CAAC;IACjE,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;IAC5C,QAAQ,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;CAC7C,CAQA;AAED,wBAAgB,+BAA+B,CAAC,IAAI,EAAE;IACpD,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B,GAAG,OAAO,CAOV;AAED,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,kBAAkB,CAAC;IACzB,YAAY,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,wBAAgB,6BAA6B,CAAC,IAAI,EAAE;IAClD,cAAc,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAEV;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAM7D;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE;IAC3C,cAAc,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG;IACF,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACrE,CAYA;AAskBD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,eAAe,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;IACnD,eAAe,EAAE,eAAe,EAAE,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB,GAAG,eAAe,EAAE,CAyGpB"}
|