@qvac/core 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/dist-lib/lib/api/chat.d.ts +3 -1
  2. package/dist-lib/lib/api/chat.js +6 -2
  3. package/dist-lib/lib/api/index.d.ts +2 -0
  4. package/dist-lib/lib/api/index.js +3 -1
  5. package/dist-lib/lib/api/inference-calls.d.ts +18 -0
  6. package/dist-lib/lib/api/inference-calls.js +15 -0
  7. package/dist-lib/lib/api/rag.js +10 -2
  8. package/dist-lib/lib/assistant/harness.d.ts +1 -0
  9. package/dist-lib/lib/assistant/history.d.ts +10 -1
  10. package/dist-lib/lib/assistant/history.js +36 -1
  11. package/dist-lib/lib/assistant/message.d.ts +2 -0
  12. package/dist-lib/lib/assistant/message.js +11 -0
  13. package/dist-lib/lib/assistant/skill-encode.js +27 -1
  14. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  15. package/dist-lib/lib/assistant-tree.js +11 -2
  16. package/dist-lib/lib/assistant.d.ts +19 -3
  17. package/dist-lib/lib/assistant.js +148 -23
  18. package/dist-lib/lib/core/chunk.js +5 -0
  19. package/dist-lib/lib/core/files/rag/build.d.ts +1 -2
  20. package/dist-lib/lib/core/files/rag/build.js +1 -2
  21. package/dist-lib/lib/core/files/rag/extract/office-collabora.d.ts +1 -1
  22. package/dist-lib/lib/core/files/rag/index.d.ts +4 -24
  23. package/dist-lib/lib/core/files/rag/index.js +3 -17
  24. package/dist-lib/lib/core/files/rag/load.d.ts +1 -2
  25. package/dist-lib/lib/core/files/rag/load.js +1 -2
  26. package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
  27. package/dist-lib/lib/core/files/rag/produce.js +1 -3
  28. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -1
  29. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -4
  30. package/dist-lib/lib/core/files/rag/similarity.d.ts +5 -0
  31. package/dist-lib/lib/core/files/rag/similarity.js +6 -0
  32. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  33. package/dist-lib/lib/core/inference-calls-handoff.d.ts +20 -0
  34. package/dist-lib/lib/core/inference-calls-handoff.js +37 -0
  35. package/dist-lib/lib/core/inference-calls-receiver.d.ts +14 -0
  36. package/dist-lib/lib/core/inference-calls-receiver.js +39 -0
  37. package/dist-lib/lib/core/notification.js +5 -4
  38. package/dist-lib/lib/core/skill-secrets.js +3 -6
  39. package/dist-lib/lib/core.d.ts +9 -1
  40. package/dist-lib/lib/core.js +129 -13
  41. package/dist-lib/lib/credential-pusher.d.ts +2 -3
  42. package/dist-lib/lib/credential-pusher.js +67 -39
  43. package/dist-lib/lib/inference-calls.d.ts +40 -0
  44. package/dist-lib/lib/inference-calls.js +266 -0
  45. package/dist-lib/lib/integrations/index.d.ts +1 -2
  46. package/dist-lib/lib/integrations/index.js +0 -1
  47. package/dist-lib/lib/integrations/setup-routes.js +0 -15
  48. package/dist-lib/lib/oauth/flow-core.js +1 -1
  49. package/dist-lib/lib/oauth/flow.d.ts +4 -2
  50. package/dist-lib/lib/oauth/flow.js +3 -4
  51. package/dist-lib/lib/oauth/host.d.ts +3 -1
  52. package/dist-lib/lib/oauth/host.js +3 -1
  53. package/dist-lib/lib/oauth/keys.d.ts +21 -20
  54. package/dist-lib/lib/oauth/keys.js +59 -85
  55. package/dist-lib/lib/oauth/loopback.d.ts +2 -2
  56. package/dist-lib/lib/oauth/loopback.js +32 -19
  57. package/dist-lib/lib/oauth/loopback.stub.d.ts +2 -2
  58. package/dist-lib/lib/oauth/loopback.stub.js +2 -2
  59. package/dist-lib/lib/oauth/providers.d.ts +3 -5
  60. package/dist-lib/lib/oauth/providers.js +20 -67
  61. package/dist-lib/lib/oauth/redirect-types.d.ts +1 -0
  62. package/dist-lib/lib/oauth/registry.d.ts +10 -0
  63. package/dist-lib/lib/oauth/registry.js +25 -0
  64. package/dist-lib/lib/oauth/resolve.js +1 -1
  65. package/dist-lib/lib/oauth/token-exchange-core.js +2 -1
  66. package/dist-lib/lib/oauth/types.d.ts +1 -1
  67. package/dist-lib/lib/oauth.client.d.ts +3 -1
  68. package/dist-lib/lib/oauth.client.js +3 -1
  69. package/dist-lib/lib/oauth.d.ts +4 -7
  70. package/dist-lib/lib/oauth.js +4 -7
  71. package/dist-lib/lib/skill-types.d.ts +12 -0
  72. package/dist-lib/schema/content.js +5 -1
  73. package/dist-lib/schema/engine.js +63 -2
  74. package/dist-lib/schema/qvac.js +4 -2
  75. package/dist-lib/spec/bind.js +3 -0
  76. package/dist-lib/spec/calls.js +1 -0
  77. package/dist-lib/spec/capabilities.d.ts +2 -0
  78. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  79. package/dist-lib/spec/hrpc/index.js +15 -3
  80. package/dist-lib/spec/hrpc/messages.js +284 -52
  81. package/dist-lib/spec/hyperschema/index.js +284 -52
  82. package/dist-lib/spec/hyperschema/schema.json +210 -1
  83. package/dist-lib/spec/hyperschema/types.d.ts +51 -1
  84. package/dist-lib/spec/local/hyperdb/messages.js +64 -128
  85. package/dist-lib/spec/local/hyperschema/index.js +32 -64
  86. package/dist-lib/spec/local/hyperschema/schema.json +16 -0
  87. package/dist-lib/spec/mesh/hyperdb/messages.js +176 -311
  88. package/dist-lib/spec/mesh/hyperdispatch/messages.js +132 -236
  89. package/dist-lib/spec/mesh/hyperschema/index.js +132 -236
  90. package/dist-lib/spec/mesh/hyperschema/schema.json +70 -1
  91. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  92. package/package.json +15 -18
  93. package/tui/app.mjs +98 -155
  94. package/tui/lib/knowledge-import.mjs +1 -1
  95. package/tui/lib/save-file.mjs +1 -1
  96. package/.claude/skills/asana/SKILL.md +0 -86
  97. package/.claude/skills/compaction/SKILL.md +0 -136
  98. package/.claude/skills/device-mesh-debug/SKILL.md +0 -328
  99. package/.claude/skills/gh-stack/SKILL.md +0 -181
  100. package/.claude/skills/gh-stack/references/commands.md +0 -177
  101. package/.claude/skills/gh-stack/references/stack-design.md +0 -97
  102. package/.claude/skills/gh-stack/references/troubleshooting.md +0 -159
  103. package/.claude/skills/js-house-style/SKILL.md +0 -671
  104. package/.claude/skills/pr-create/SKILL.md +0 -243
  105. package/.claude/skills/pr-review/SKILL.md +0 -428
  106. package/.claude/skills/release/SKILL.md +0 -122
  107. package/CLAUDE.md +0 -48
  108. package/dist-lib/lib/core/files/rag/artifact/codec.d.ts +0 -3
  109. package/dist-lib/lib/core/files/rag/artifact/codec.js +0 -172
  110. package/dist-lib/lib/core/files/rag/artifact/id.d.ts +0 -10
  111. package/dist-lib/lib/core/files/rag/artifact/id.js +0 -39
  112. package/dist-lib/lib/core/files/rag/artifact/types.d.ts +0 -24
  113. package/dist-lib/lib/core/files/rag/artifact/types.js +0 -41
  114. package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts +0 -12
  115. package/dist-lib/lib/core/files/rag/chunk/chunking.js +0 -117
  116. package/dist-lib/lib/core/files/rag/chunk/metadata.d.ts +0 -30
  117. package/dist-lib/lib/core/files/rag/chunk/metadata.js +0 -89
  118. package/dist-lib/lib/core/files/rag/chunk/tokenizer.d.ts +0 -7
  119. package/dist-lib/lib/core/files/rag/chunk/tokenizer.js +0 -18
  120. package/dist-lib/lib/core/files/rag/extract/csv-text.d.ts +0 -1
  121. package/dist-lib/lib/core/files/rag/extract/csv-text.js +0 -117
  122. package/dist-lib/lib/core/files/rag/extract/document-types.d.ts +0 -22
  123. package/dist-lib/lib/core/files/rag/extract/document-types.js +0 -152
  124. package/dist-lib/lib/core/files/rag/extract/extract.d.ts +0 -17
  125. package/dist-lib/lib/core/files/rag/extract/extract.js +0 -81
  126. package/dist-lib/lib/core/files/rag/extract/html-table-text.d.ts +0 -1
  127. package/dist-lib/lib/core/files/rag/extract/html-table-text.js +0 -125
  128. package/dist-lib/lib/core/files/rag/extract/html-text.d.ts +0 -1
  129. package/dist-lib/lib/core/files/rag/extract/html-text.js +0 -132
  130. package/dist-lib/lib/core/files/rag/extract/pdf-platform.d.ts +0 -17
  131. package/dist-lib/lib/core/files/rag/extract/pdf-platform.js +0 -141
  132. package/dist-lib/lib/core/files/rag/extract/pdf-text.d.ts +0 -12
  133. package/dist-lib/lib/core/files/rag/extract/pdf-text.js +0 -43
  134. package/dist-lib/lib/core/files/rag/extract/rtf-text.d.ts +0 -1
  135. package/dist-lib/lib/core/files/rag/extract/rtf-text.js +0 -15
  136. package/dist-lib/lib/core/files/rag/extract/text-decoding.d.ts +0 -1
  137. package/dist-lib/lib/core/files/rag/extract/text-decoding.js +0 -91
  138. package/dist-lib/lib/core/files/rag/extract/types.d.ts +0 -36
  139. package/dist-lib/lib/core/files/rag/extract/types.js +0 -12
  140. package/dist-lib/lib/core/files/rag/extract/utf.d.ts +0 -2
  141. package/dist-lib/lib/core/files/rag/extract/utf.js +0 -18
  142. package/dist-lib/lib/core/files/rag/retrieval/citation-match.d.ts +0 -2
  143. package/dist-lib/lib/core/files/rag/retrieval/citation-match.js +0 -16
  144. package/dist-lib/lib/core/files/rag/retrieval/coverage.d.ts +0 -4
  145. package/dist-lib/lib/core/files/rag/retrieval/coverage.js +0 -48
  146. package/dist-lib/lib/core/files/rag/retrieval/diversity.d.ts +0 -10
  147. package/dist-lib/lib/core/files/rag/retrieval/diversity.js +0 -37
  148. package/dist-lib/lib/core/files/rag/retrieval/floor.d.ts +0 -5
  149. package/dist-lib/lib/core/files/rag/retrieval/floor.js +0 -21
  150. package/dist-lib/lib/core/files/rag/retrieval/format.d.ts +0 -6
  151. package/dist-lib/lib/core/files/rag/retrieval/format.js +0 -21
  152. package/dist-lib/lib/core/files/rag/retrieval/knowledge.d.ts +0 -2
  153. package/dist-lib/lib/core/files/rag/retrieval/knowledge.js +0 -54
  154. package/dist-lib/lib/core/files/rag/retrieval/notices.d.ts +0 -16
  155. package/dist-lib/lib/core/files/rag/retrieval/notices.js +0 -27
  156. package/dist-lib/lib/core/files/rag/retrieval/scoring.d.ts +0 -8
  157. package/dist-lib/lib/core/files/rag/retrieval/scoring.js +0 -82
  158. package/dist-lib/lib/core/files/rag/retrieval/search.d.ts +0 -3
  159. package/dist-lib/lib/core/files/rag/retrieval/search.js +0 -49
  160. package/dist-lib/lib/core/files/rag/retrieval/selection.d.ts +0 -11
  161. package/dist-lib/lib/core/files/rag/retrieval/selection.js +0 -93
  162. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.d.ts +0 -6
  163. package/dist-lib/lib/core/files/rag/retrieval/source-tag-filter.js +0 -79
  164. package/dist-lib/lib/core/files/rag/retrieval/sources.d.ts +0 -13
  165. package/dist-lib/lib/core/files/rag/retrieval/sources.js +0 -20
  166. package/dist-lib/lib/core/files/rag/retrieval/target.d.ts +0 -13
  167. package/dist-lib/lib/core/files/rag/retrieval/target.js +0 -89
  168. package/dist-lib/lib/core/files/rag/retrieval/terms.d.ts +0 -8
  169. package/dist-lib/lib/core/files/rag/retrieval/terms.js +0 -70
  170. package/dist-lib/lib/core/files/rag/retrieval/types.d.ts +0 -29
  171. package/dist-lib/lib/core/files/rag/retrieval/types.js +0 -1
  172. package/dist-lib/lib/core/files/rag/retrieval/web-sources.d.ts +0 -8
  173. package/dist-lib/lib/core/files/rag/retrieval/web-sources.js +0 -41
  174. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.d.ts +0 -1
  175. package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js +0 -96
  176. package/dist-lib/lib/core/files/rag/vector/memory.d.ts +0 -15
  177. package/dist-lib/lib/core/files/rag/vector/memory.js +0 -135
  178. package/dist-lib/lib/core/files/rag/vector/types.d.ts +0 -28
  179. package/dist-lib/lib/core/files/rag/vector/types.js +0 -1
  180. package/dist-lib/lib/integrations/oauth-providers.d.ts +0 -14
  181. package/dist-lib/lib/integrations/oauth-providers.js +0 -116
  182. package/dist-lib/lib/oauth/asana.d.ts +0 -35
  183. package/dist-lib/lib/oauth/asana.js +0 -71
  184. package/dist-lib/lib/oauth/google.d.ts +0 -37
  185. package/dist-lib/lib/oauth/google.js +0 -77
  186. package/dist-lib/lib/oauth/notion.d.ts +0 -34
  187. package/dist-lib/lib/oauth/notion.js +0 -82
  188. package/dist-lib/lib/oauth/spotify.d.ts +0 -32
  189. package/dist-lib/lib/oauth/spotify.js +0 -74
  190. package/dist-lib/lib/oauth/verify.d.ts +0 -8
  191. package/dist-lib/lib/oauth/verify.js +0 -58
@@ -1,21 +0,0 @@
1
- import '../../../../logger.js';
2
- import { parseRagMetadata } from '../chunk/metadata.js';
3
- export const RAG_MIN_SIMILARITY = 0.4;
4
- export function applySimilarityFloor(hits, targetDocumentId) {
5
- if (targetDocumentId)
6
- return hits;
7
- return hits.filter((hit) => hit.score >= RAG_MIN_SIMILARITY);
8
- }
9
- export function logSimilarityScores(candidates, kept) {
10
- if (candidates.length === 0) {
11
- logger.debug('rag', 'semantic search returned 0 hits');
12
- return;
13
- }
14
- const keptIds = new Set(kept.map((hit) => hit.id));
15
- for (const [index, hit] of candidates.entries()) {
16
- const source = parseRagMetadata(hit.metadataJson)?.title ?? hit.id;
17
- const percent = (hit.score * 100).toFixed(1);
18
- const status = keptIds.has(hit.id) ? 'kept' : 'dropped';
19
- logger.debug('rag', `similarity #${index + 1} ${percent}% (${status}) - ${source}`);
20
- }
21
- }
@@ -1,6 +0,0 @@
1
- type FormatHit = {
2
- readonly content: string;
3
- readonly metadataJson: string | null;
4
- };
5
- export declare function formatRagContext(hits: ReadonlyArray<FormatHit>, availableFiles?: ReadonlyArray<string>): string;
6
- export {};
@@ -1,21 +0,0 @@
1
- import { parseRagMetadata } from '../chunk/metadata.js';
2
- export function formatRagContext(hits, availableFiles = []) {
3
- const manifest = availableFiles.length > 0 ? [`Available files in this chat: ${availableFiles.join(', ')}`] : [];
4
- return [
5
- 'The following knowledge excerpts are the primary source for answering the user.',
6
- 'Each excerpt is tagged with its source file name and chunk index in a [Source N: filename, ...] label.',
7
- 'The app renders source attribution separately. Do not echo the [Source N: ...] labels. When attribution helps, name only the files you used naturally in prose.',
8
- ...manifest,
9
- 'If the user references a specific file or document by name, prioritise excerpts whose source matches that file.',
10
- 'If the user references a file that has no excerpts below, tell the user that file is not in the retrieved knowledge - do not invent an answer from other files.',
11
- 'If the excerpts collectively do not contain the answer, say the knowledge base did not contain it.',
12
- ...hits.flatMap((hit, index) => ['', formatHitLabel(hit, index), hit.content])
13
- ].join('\n');
14
- }
15
- function formatHitLabel(hit, index) {
16
- const metadata = hit.metadataJson ? parseRagMetadata(hit.metadataJson) : null;
17
- const title = metadata?.title ?? 'document';
18
- const chunk = metadata ? `chunk ${metadata.chunkIndex}` : `chunk ${index}`;
19
- const attachment = metadata?.attachmentId ? `, attachmentId=${metadata.attachmentId}` : '';
20
- return `[Source ${index + 1}: ${title}, ${chunk}${attachment}]`;
21
- }
@@ -1,2 +0,0 @@
1
- import type { RagKnowledgeQuery, RagKnowledgeResult, RagRetrievalDeps } from './types.ts';
2
- export declare function retrieveRagKnowledge(query: RagKnowledgeQuery, deps: RagRetrievalDeps): Promise<RagKnowledgeResult>;
@@ -1,54 +0,0 @@
1
- import { applySimilarityFloor, logSimilarityScores } from './floor.js';
2
- import { searchRagChunks } from './search.js';
3
- import { DEFAULT_CHUNK_BUDGET_TOKENS, MAX_SELECTED_CHUNKS } from './selection.js';
4
- import { resolveTargetDocument } from './target.js';
5
- import { isWholeDocumentQuery } from './whole-document-query.js';
6
- export async function retrieveRagKnowledge(query, deps) {
7
- if (query.scopeDocumentIds?.length) {
8
- const hits = await searchRagChunks({
9
- query: query.query,
10
- chatId: query.chatId,
11
- liveSourceIds: query.liveSourceIds,
12
- coverDocuments: query.scopeDocumentIds.length > 1,
13
- ...scopedShape(query.query, query.scopeDocumentIds)
14
- }, deps);
15
- logSimilarityScores(hits, hits);
16
- return { hits, targetDocumentId: null, searchedCount: hits.length };
17
- }
18
- const targetDocumentId = resolveTargetDocument(query.query, query.documents);
19
- if (!targetDocumentId && isWholeDocumentQuery(query.query)) {
20
- const hits = await searchRagChunks({
21
- query: query.query,
22
- chatId: query.chatId,
23
- liveSourceIds: query.liveSourceIds,
24
- documentIds: query.documents.map((document) => document.documentId),
25
- coverage: true,
26
- chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS
27
- }, deps);
28
- return { hits, targetDocumentId, searchedCount: hits.length };
29
- }
30
- const { hits: searched, scoped } = await searchKnowledge(query, deps, targetDocumentId);
31
- const hits = applySimilarityFloor(searched, scoped ? targetDocumentId : null);
32
- logSimilarityScores(searched, hits);
33
- return { hits, targetDocumentId, searchedCount: searched.length };
34
- }
35
- async function searchKnowledge(query, deps, targetDocumentId) {
36
- const base = { query: query.query, chatId: query.chatId, liveSourceIds: query.liveSourceIds };
37
- if (!targetDocumentId) {
38
- const hits = await searchRagChunks({ ...base, documentFirst: true, chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS }, deps);
39
- return { hits, scoped: false };
40
- }
41
- const scopedHits = await searchRagChunks({ ...base, ...scopedShape(query.query, [targetDocumentId]) }, deps);
42
- if (scopedHits.length > 0)
43
- return { hits: scopedHits, scoped: true };
44
- const hits = await searchRagChunks({ ...base, documentFirst: true, chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS }, deps);
45
- return { hits, scoped: false };
46
- }
47
- function scopedShape(query, documentIds) {
48
- return {
49
- documentIds,
50
- chunkBudgetTokens: DEFAULT_CHUNK_BUDGET_TOKENS,
51
- ...(documentIds.length === 1 ? { perDocChunkCap: MAX_SELECTED_CHUNKS } : {}),
52
- ...(isWholeDocumentQuery(query) ? { coverage: true } : { documentFirst: true })
53
- };
54
- }
@@ -1,16 +0,0 @@
1
- export declare const RAG_FAILED_NOTICE = "Knowledge search failed and was not used.";
2
- export declare const RAG_EMPTY_NOTICE = "Knowledge is enabled, but no relevant excerpts were found.";
3
- export declare const RAG_UNAVAILABLE_NOTICE = "Knowledge is still being indexed and was not used yet.";
4
- export declare const RAG_KNOWLEDGE_OFF_NOTICE = "Knowledge is off for this agent, so the attached document was not used. Enable Knowledge to let the agent read it.";
5
- export declare const RAG_TRUNCATED_NOTICE = "Only the beginning of the attached document was used. Enable Knowledge to let the agent search all of it.";
6
- export declare function ragUnreadableNotice(files: ReadonlyArray<string>): string;
7
- export declare function chooseRagNotice(input: {
8
- readonly errorMessage?: string | null;
9
- readonly hitCount: number;
10
- readonly searchedCount: number;
11
- readonly indexEmpty: boolean;
12
- readonly hasKnowledge: boolean;
13
- readonly pendingReferenced?: boolean;
14
- readonly failedFiles?: ReadonlyArray<string>;
15
- readonly pendingFiles?: ReadonlyArray<string>;
16
- }): string | null;
@@ -1,27 +0,0 @@
1
- export const RAG_FAILED_NOTICE = 'Knowledge search failed and was not used.';
2
- export const RAG_EMPTY_NOTICE = 'Knowledge is enabled, but no relevant excerpts were found.';
3
- export const RAG_UNAVAILABLE_NOTICE = 'Knowledge is still being indexed and was not used yet.';
4
- export const RAG_KNOWLEDGE_OFF_NOTICE = 'Knowledge is off for this agent, so the attached document was not used. Enable Knowledge to let the agent read it.';
5
- export const RAG_TRUNCATED_NOTICE = 'Only the beginning of the attached document was used. Enable Knowledge to let the agent search all of it.';
6
- export function ragUnreadableNotice(files) {
7
- const plural = files.length > 1;
8
- return `Attached file${plural ? 's' : ''} could not be read and ${plural ? 'were' : 'was'} not used: ${files.join(', ')}.`;
9
- }
10
- export function chooseRagNotice(input) {
11
- if (input.errorMessage)
12
- return RAG_FAILED_NOTICE;
13
- if (input.pendingReferenced)
14
- return RAG_UNAVAILABLE_NOTICE;
15
- if (input.failedFiles?.length)
16
- return ragUnreadableNotice(input.failedFiles);
17
- if (input.pendingFiles?.length) {
18
- return `Files not yet available: ${input.pendingFiles.join(', ')}.`;
19
- }
20
- if (input.hitCount > 0)
21
- return null;
22
- if (input.searchedCount > 0)
23
- return RAG_EMPTY_NOTICE;
24
- if (input.indexEmpty && input.hasKnowledge)
25
- return RAG_UNAVAILABLE_NOTICE;
26
- return RAG_EMPTY_NOTICE;
27
- }
@@ -1,8 +0,0 @@
1
- import type { VectorHit } from '../vector/types.ts';
2
- export type ScoredDocument = {
3
- readonly documentId: string;
4
- readonly title: string;
5
- readonly score: number;
6
- readonly hits: ReadonlyArray<VectorHit>;
7
- };
8
- export declare function scoreDocuments(hits: ReadonlyArray<VectorHit>, query: string): ReadonlyArray<ScoredDocument>;
@@ -1,82 +0,0 @@
1
- import { normalize } from 'asenion';
2
- import { parseRagMetadata } from '../chunk/metadata.js';
3
- import { extractSalientTerms, isMeaningfulToken } from './terms.js';
4
- const SEM_MAX_WEIGHT = 0.7;
5
- const SEM_MEAN_WEIGHT = 0.3;
6
- const SEM_TOP_K = 3;
7
- const LEX_PHRASE_WEIGHT = 1;
8
- const LEX_ENTITY_WEIGHT = 1;
9
- const LEX_TOKEN_OVERLAP_WEIGHT = 0.5;
10
- const TITLE_EXACT_SCORE = 1;
11
- const TITLE_TOKEN_SCORE = 0.6;
12
- const DOC_SEMANTIC_WEIGHT = 1;
13
- const DOC_LEXICAL_WEIGHT = 0.6;
14
- const DOC_TITLE_WEIGHT = 0.8;
15
- const ALL_ENTITIES_BOOST = 0.5;
16
- export function scoreDocuments(hits, query) {
17
- const terms = extractSalientTerms(query);
18
- const scored = [];
19
- for (const [documentId, group] of groupHitsByDocument(hits)) {
20
- const docHits = [...group.hits].sort((left, right) => right.score - left.score);
21
- const contentNorm = normalize(docHits.map((hit) => hit.content).join(' '));
22
- const titleNorm = normalize(group.title);
23
- const allEntities = terms.entities.length > 0 &&
24
- terms.entities.every((entity) => contentNorm.includes(entity) || titleNorm.includes(entity));
25
- const score = DOC_SEMANTIC_WEIGHT * semanticScore(docHits) +
26
- DOC_LEXICAL_WEIGHT * lexicalScore(terms, contentNorm, titleNorm) +
27
- DOC_TITLE_WEIGHT * titleScore(terms, query, group.title) +
28
- (allEntities ? ALL_ENTITIES_BOOST : 0);
29
- scored.push({ documentId, title: group.title, score, hits: docHits });
30
- }
31
- return scored.sort((left, right) => right.score - left.score);
32
- }
33
- function groupHitsByDocument(hits) {
34
- const groups = new Map();
35
- for (const hit of hits) {
36
- const metadata = parseRagMetadata(hit.metadataJson);
37
- if (!metadata)
38
- continue;
39
- const group = groups.get(metadata.documentId) ?? { title: metadata.title, hits: [] };
40
- group.hits.push(hit);
41
- groups.set(metadata.documentId, group);
42
- }
43
- return groups;
44
- }
45
- function semanticScore(docHits) {
46
- if (docHits.length === 0)
47
- return 0;
48
- const max = docHits[0]?.score ?? 0;
49
- const top = docHits.slice(0, SEM_TOP_K);
50
- const mean = top.reduce((sum, hit) => sum + hit.score, 0) / top.length;
51
- return SEM_MAX_WEIGHT * max + SEM_MEAN_WEIGHT * mean;
52
- }
53
- function lexicalScore(terms, contentNorm, titleNorm) {
54
- const phrase = terms.quotedPhrases.some((value) => contentNorm.includes(value) || titleNorm.includes(value))
55
- ? 1
56
- : 0;
57
- const entityFraction = terms.entities.length === 0
58
- ? 0
59
- : terms.entities.filter((entity) => contentNorm.includes(entity) || titleNorm.includes(entity)).length / terms.entities.length;
60
- const tokenOverlap = terms.tokens.length === 0 ? 0 : overlapCount(terms.tokens, contentNorm) / terms.tokens.length;
61
- return (LEX_PHRASE_WEIGHT * phrase +
62
- LEX_ENTITY_WEIGHT * entityFraction +
63
- LEX_TOKEN_OVERLAP_WEIGHT * tokenOverlap);
64
- }
65
- function titleScore(terms, query, title) {
66
- const stemTokens = normalize(title.replace(/\.[A-Za-z0-9]+$/, ''))
67
- .split(' ')
68
- .filter(isMeaningfulToken);
69
- if (stemTokens.length === 0)
70
- return 0;
71
- const stem = stemTokens.join(' ');
72
- if (normalize(query).includes(stem))
73
- return TITLE_EXACT_SCORE;
74
- const titleTokens = new Set(stemTokens);
75
- const tokenHit = terms.tokens.some((token) => titleTokens.has(token));
76
- const entityHit = terms.entities.some((entity) => stem.includes(entity));
77
- return tokenHit || entityHit ? TITLE_TOKEN_SCORE : 0;
78
- }
79
- function overlapCount(tokens, contentNorm) {
80
- const contentTokens = new Set(contentNorm.split(' '));
81
- return tokens.filter((token) => contentTokens.has(token)).length;
82
- }
@@ -1,3 +0,0 @@
1
- import type { VectorHit } from '../vector/types.ts';
2
- import type { RagRetrievalDeps, RagRetrievalQuery } from './types.ts';
3
- export declare function searchRagChunks(query: RagRetrievalQuery, deps: RagRetrievalDeps): Promise<ReadonlyArray<VectorHit>>;
@@ -1,49 +0,0 @@
1
- import { selectCoverage } from './coverage.js';
2
- import { applyPerSourceCap, scopeFilterFor } from './diversity.js';
3
- import { scoreDocuments } from './scoring.js';
4
- import { selectChunks, selectScopedChunks } from './selection.js';
5
- const DEFAULT_LIMIT = 5;
6
- export async function searchRagChunks(query, deps) {
7
- const trimmed = query.query.trim();
8
- if (!trimmed)
9
- return [];
10
- if (deps.index.info().chunks === 0)
11
- return [];
12
- const embedding = await embedQuery(deps, trimmed);
13
- const total = deps.index.info().chunks;
14
- if (total === 0)
15
- return [];
16
- const hits = await deps.index.search(embedding, { limit: total, filter: scopeFilterFor(query) });
17
- if (query.coverage) {
18
- return selectCoverage(hits, {
19
- ...(query.chunkBudgetTokens && query.chunkBudgetTokens > 0
20
- ? { budgetTokens: query.chunkBudgetTokens }
21
- : {})
22
- });
23
- }
24
- if (query.coverDocuments) {
25
- return selectScopedChunks(scoreDocuments(hits, query.query), query.chunkBudgetTokens);
26
- }
27
- if (useDocumentFirst(query))
28
- return documentFirstSelect(query, hits);
29
- return applyPerSourceCap(hits, query.limit ?? DEFAULT_LIMIT);
30
- }
31
- async function embedQuery(deps, text) {
32
- const rows = await deps.embed([text]);
33
- const embedding = rows[0];
34
- if (!embedding)
35
- throw new Error('rag retrieval: the embedder returned no query embedding');
36
- return embedding;
37
- }
38
- function useDocumentFirst(query) {
39
- return query.documentFirst === true && Boolean(query.chatId) && !query.attachmentIds?.length;
40
- }
41
- function documentFirstSelect(query, hits) {
42
- const ranked = scoreDocuments(hits, query.query);
43
- return selectChunks(ranked, {
44
- ...(query.chunkBudgetTokens && query.chunkBudgetTokens > 0
45
- ? { chunkBudgetTokens: query.chunkBudgetTokens }
46
- : {}),
47
- ...(query.perDocChunkCap ? { perDocChunkCap: query.perDocChunkCap } : {})
48
- });
49
- }
@@ -1,11 +0,0 @@
1
- import type { VectorHit } from '../vector/types.ts';
2
- import type { ScoredDocument } from './scoring.ts';
3
- export declare const DEFAULT_CHUNK_BUDGET_TOKENS = 3072;
4
- export declare const MAX_SELECTED_CHUNKS: number;
5
- export type SelectChunksOptions = {
6
- readonly maxDocs?: number;
7
- readonly chunkBudgetTokens?: number;
8
- readonly perDocChunkCap?: number;
9
- };
10
- export declare function selectChunks(ranked: ReadonlyArray<ScoredDocument>, options?: SelectChunksOptions): ReadonlyArray<VectorHit>;
11
- export declare function selectScopedChunks(ranked: ReadonlyArray<ScoredDocument>, budgetTokens?: number): ReadonlyArray<VectorHit>;
@@ -1,93 +0,0 @@
1
- import { parseRagMetadata } from '../chunk/metadata.js';
2
- import { estimateRagTokens, tokenizeForRag } from '../chunk/tokenizer.js';
3
- const DEFAULT_MAX_DOCS = 8;
4
- const DEFAULT_PER_DOC_CHUNK_CAP = 4;
5
- export const DEFAULT_CHUNK_BUDGET_TOKENS = 3072;
6
- export const MAX_SELECTED_CHUNKS = DEFAULT_MAX_DOCS * DEFAULT_PER_DOC_CHUNK_CAP;
7
- export function selectChunks(ranked, options = {}) {
8
- const topDocs = ranked.slice(0, options.maxDocs ?? DEFAULT_MAX_DOCS);
9
- if (topDocs.length === 0)
10
- return [];
11
- const state = {
12
- topDocs,
13
- budget: options.chunkBudgetTokens ?? DEFAULT_CHUNK_BUDGET_TOKENS,
14
- perDocCap: options.perDocChunkCap ?? DEFAULT_PER_DOC_CHUNK_CAP,
15
- used: new Set(),
16
- perDocCount: new Map(),
17
- selected: [],
18
- usedTokens: 0
19
- };
20
- seedCoverage(state);
21
- fillBudget(state);
22
- return state.selected
23
- .sort((left, right) => left.docRank - right.docRank || left.chunkIndex - right.chunkIndex)
24
- .map((entry) => entry.hit);
25
- }
26
- export function selectScopedChunks(ranked, budgetTokens = DEFAULT_CHUNK_BUDGET_TOKENS) {
27
- const share = Math.floor(budgetTokens / ranked.length);
28
- const selected = [];
29
- for (const document of ranked) {
30
- let remaining = share;
31
- let count = 0;
32
- for (const hit of document.hits) {
33
- if (remaining === 0 || count === DEFAULT_PER_DOC_CHUNK_CAP)
34
- break;
35
- const tokens = tokenizeForRag(hit.content);
36
- if (tokens.length > remaining) {
37
- if (count > 0)
38
- break;
39
- selected.push({
40
- ...hit,
41
- content: hit.content.slice(0, tokens[remaining - 1].end),
42
- metadataJson: JSON.stringify({
43
- ...parseRagMetadata(hit.metadataJson),
44
- tokenCount: remaining
45
- })
46
- });
47
- break;
48
- }
49
- selected.push(hit);
50
- remaining -= tokens.length;
51
- count++;
52
- }
53
- }
54
- return selected;
55
- }
56
- function seedCoverage(state) {
57
- state.topDocs.forEach((doc, rank) => {
58
- const best = doc.hits[0];
59
- if (!best || state.used.has(best.id))
60
- return;
61
- if (rank === 0 || state.usedTokens + hitTokens(best) <= state.budget) {
62
- take(state, rank, best);
63
- }
64
- });
65
- }
66
- function fillBudget(state) {
67
- let added = true;
68
- while (added) {
69
- added = false;
70
- state.topDocs.forEach((doc, rank) => {
71
- if ((state.perDocCount.get(doc.documentId) ?? 0) >= state.perDocCap)
72
- return;
73
- const next = doc.hits.find((hit) => !state.used.has(hit.id));
74
- if (next && state.usedTokens + hitTokens(next) <= state.budget) {
75
- take(state, rank, next);
76
- added = true;
77
- }
78
- });
79
- }
80
- }
81
- function take(state, docRank, hit) {
82
- state.used.add(hit.id);
83
- const documentId = state.topDocs[docRank]?.documentId ?? '';
84
- state.perDocCount.set(documentId, (state.perDocCount.get(documentId) ?? 0) + 1);
85
- state.usedTokens += hitTokens(hit);
86
- state.selected.push({ docRank, chunkIndex: chunkIndexOf(hit), hit });
87
- }
88
- function hitTokens(hit) {
89
- return parseRagMetadata(hit.metadataJson)?.tokenCount ?? estimateRagTokens(hit.content);
90
- }
91
- function chunkIndexOf(hit) {
92
- return parseRagMetadata(hit.metadataJson)?.chunkIndex ?? 0;
93
- }
@@ -1,6 +0,0 @@
1
- export type RagSourceTagFilter = {
2
- readonly add: (sourceNames: ReadonlyArray<string>) => void;
3
- readonly push: (text: string) => string;
4
- readonly flush: () => string;
5
- };
6
- export declare function createRagSourceTagFilter(): RagSourceTagFilter;
@@ -1,79 +0,0 @@
1
- const SOURCE_TAG_START = /\[\s*sources?(?:\s+\d+)?\s*:/i;
2
- const POSSIBLE_SOURCE_TAG_START = /^\[\s*(?:s(?:o(?:u(?:r(?:c(?:e(?:s)?)?)?)?)?)?)?(?:\s+\d*)?\s*$/i;
3
- export function createRagSourceTagFilter() {
4
- const state = { buffer: '', hasSources: false };
5
- return {
6
- add(sourceNames) {
7
- if (state.hasSources)
8
- return;
9
- for (const name of sourceNames) {
10
- if (name.trim().length > 0) {
11
- state.hasSources = true;
12
- return;
13
- }
14
- }
15
- },
16
- push(text) {
17
- state.buffer += text;
18
- return drain(state);
19
- },
20
- flush() {
21
- const start = SOURCE_TAG_START.exec(state.buffer);
22
- if (!start) {
23
- const output = state.buffer;
24
- state.buffer = '';
25
- return output;
26
- }
27
- const prefix = state.buffer.slice(0, start.index);
28
- const tag = state.buffer.slice(start.index);
29
- state.buffer = '';
30
- if (state.hasSources)
31
- return prefix.trimEnd();
32
- return prefix + tag;
33
- }
34
- };
35
- }
36
- function drain(state) {
37
- let output = '';
38
- for (;;) {
39
- const start = SOURCE_TAG_START.exec(state.buffer);
40
- if (!start)
41
- return output + emitNonTagText(state);
42
- const end = state.buffer.indexOf(']', start.index);
43
- if (end === -1) {
44
- const holdStart = trailingWhitespaceStart(state.buffer, start.index);
45
- output += state.buffer.slice(0, holdStart);
46
- state.buffer = state.buffer.slice(holdStart);
47
- return output;
48
- }
49
- const prefix = state.buffer.slice(0, start.index);
50
- const tag = state.buffer.slice(start.index, end + 1);
51
- state.buffer = state.buffer.slice(end + 1);
52
- if (state.hasSources)
53
- output += prefix.trimEnd();
54
- else
55
- output += prefix + tag;
56
- }
57
- }
58
- function emitNonTagText(state) {
59
- const possibleStart = possibleSourceTagStart(state.buffer);
60
- if (possibleStart === -1) {
61
- const output = state.buffer;
62
- state.buffer = '';
63
- return output;
64
- }
65
- const holdStart = trailingWhitespaceStart(state.buffer, possibleStart);
66
- const output = state.buffer.slice(0, holdStart);
67
- state.buffer = state.buffer.slice(holdStart);
68
- return output;
69
- }
70
- function possibleSourceTagStart(text) {
71
- const start = text.lastIndexOf('[');
72
- return start !== -1 && POSSIBLE_SOURCE_TAG_START.test(text.slice(start)) ? start : -1;
73
- }
74
- function trailingWhitespaceStart(text, end) {
75
- let start = end;
76
- while (start > 0 && /\s/.test(text[start - 1] ?? ''))
77
- start--;
78
- return start;
79
- }
@@ -1,13 +0,0 @@
1
- import { RAG_SOURCE_EXCERPT_CHARS } from '../../../../../schema/qvac.ts';
2
- import type { VectorHit } from '../vector/types.ts';
3
- export { RAG_SOURCE_EXCERPT_CHARS };
4
- export type RagSource = {
5
- readonly documentId: string | null;
6
- readonly title: string;
7
- readonly chunkIndex: number;
8
- readonly excerpt: string;
9
- readonly attachmentId: string | null;
10
- readonly artifactId: string | null;
11
- readonly score: number;
12
- };
13
- export declare function sourcesFromHits(hits: ReadonlyArray<VectorHit>): RagSource[];
@@ -1,20 +0,0 @@
1
- import { MAX_RAG_SOURCES_PER_CHUNK, RAG_SOURCE_EXCERPT_CHARS } from '../../../../../schema/qvac.js';
2
- import { parseRagMetadata } from '../chunk/metadata.js';
3
- export { RAG_SOURCE_EXCERPT_CHARS };
4
- export function sourcesFromHits(hits) {
5
- return hits.slice(0, MAX_RAG_SOURCES_PER_CHUNK).map((hit, index) => {
6
- const metadata = parseRagMetadata(hit.metadataJson);
7
- const chunkIndex = metadata && Number.isSafeInteger(metadata.chunkIndex) && metadata.chunkIndex >= 0
8
- ? metadata.chunkIndex
9
- : index;
10
- return {
11
- documentId: metadata?.documentId ?? null,
12
- title: metadata?.title ?? 'document',
13
- chunkIndex,
14
- excerpt: hit.content.slice(0, RAG_SOURCE_EXCERPT_CHARS),
15
- attachmentId: metadata?.attachmentId ?? null,
16
- artifactId: metadata?.artifactId ?? null,
17
- score: hit.score
18
- };
19
- });
20
- }
@@ -1,13 +0,0 @@
1
- export type RagDocumentRef = {
2
- readonly documentId: string;
3
- readonly title: string;
4
- };
5
- export declare function resolveTargetDocument(query: string, documents: ReadonlyArray<RagDocumentRef>): string | null;
6
- export declare function referencedDocuments<T extends {
7
- readonly title: string;
8
- }>(query: string, documents: ReadonlyArray<T>): ReadonlyArray<T>;
9
- export declare function referencesPendingFile(query: string, documents: ReadonlyArray<{
10
- readonly title: string;
11
- readonly pending: boolean;
12
- }>): boolean;
13
- export declare function containsFileName(haystack: string, name: string): boolean;
@@ -1,89 +0,0 @@
1
- import { has } from 'asenion';
2
- const TYPE_KEYWORDS = [
3
- ['pdf', ['pdf']],
4
- ['html', ['html', 'htm']],
5
- ['csv', ['csv']],
6
- ['spreadsheet', ['csv', 'xlsx', 'xls']],
7
- ['json', ['json']],
8
- ['markdown', ['md', 'markdown']],
9
- ['xml', ['xml']],
10
- ['yaml', ['yaml', 'yml']]
11
- ];
12
- const MIN_STEM_LENGTH = 4;
13
- const STEM_STOPWORDS = new Set([
14
- 'report',
15
- 'summary',
16
- 'notes',
17
- 'note',
18
- 'data',
19
- 'test',
20
- 'tests',
21
- 'file',
22
- 'files',
23
- 'document',
24
- 'documents',
25
- 'draft',
26
- 'final',
27
- 'image',
28
- 'images',
29
- 'photo',
30
- 'doc',
31
- 'docs',
32
- 'untitled',
33
- 'example',
34
- 'sample',
35
- 'output',
36
- 'input',
37
- 'readme',
38
- 'index',
39
- 'main',
40
- 'temp',
41
- 'guide'
42
- ]);
43
- export function resolveTargetDocument(query, documents) {
44
- const named = documents.filter((document) => mentionsName(query.toLowerCase(), document.title));
45
- const matches = named.length ? named : referencedDocuments(query, documents);
46
- return matches.length === 1 ? matches[0].documentId : null;
47
- }
48
- export function referencedDocuments(query, documents) {
49
- const haystack = query.toLowerCase();
50
- return documents.filter((document) => mentionsName(haystack, document.title) || referencesFileType(haystack, document.title));
51
- }
52
- function mentionsName(haystack, title) {
53
- const name = title.toLowerCase();
54
- if (containsFileName(haystack, name))
55
- return true;
56
- const stem = name.replace(/\.[^.]+$/, '');
57
- if (stem.length < MIN_STEM_LENGTH || STEM_STOPWORDS.has(stem) || !looksDistinctive(stem)) {
58
- return false;
59
- }
60
- return has(haystack, stem);
61
- }
62
- export function referencesPendingFile(query, documents) {
63
- return referencedDocuments(query, documents).some((document) => document.pending);
64
- }
65
- function referencesFileType(haystack, title) {
66
- const ext = title.toLowerCase().match(/\.([^.]+)$/)?.[1];
67
- if (!ext)
68
- return false;
69
- return TYPE_KEYWORDS.some(([word, exts]) => exts.includes(ext) && referencesTypeWord(haystack, word));
70
- }
71
- function referencesTypeWord(haystack, word) {
72
- const w = word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
73
- for (const m of haystack.matchAll(new RegExp(`(^|[^a-z0-9._-])${w}(?=$|[^a-z0-9._-]|\\.(?:$|[^a-z0-9._-]))`, 'g'))) {
74
- const before = haystack.slice(0, (m.index ?? 0) + m[1].length);
75
- const after = haystack.slice((m.index ?? 0) + m[0].length);
76
- const asFormat = /(^|[^a-z0-9])(?:in|to|into|as)\s+(?:an?\s+)?$/.test(before) ||
77
- /^\s+format(?![a-z0-9])/.test(after);
78
- if (!asFormat)
79
- return true;
80
- }
81
- return false;
82
- }
83
- function looksDistinctive(stem) {
84
- return /[0-9_\-. ]/.test(stem);
85
- }
86
- export function containsFileName(haystack, name) {
87
- const escaped = name.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
88
- return new RegExp(`(^|[^a-z0-9._-])${escaped}(?=$|[^a-z0-9._-]|\\.(?:$|[^a-z0-9._-]))`).test(haystack.toLowerCase());
89
- }
@@ -1,8 +0,0 @@
1
- export type SalientTerms = {
2
- readonly tokens: ReadonlyArray<string>;
3
- readonly quotedPhrases: ReadonlyArray<string>;
4
- readonly entities: ReadonlyArray<string>;
5
- };
6
- export declare function isMeaningfulToken(token: string): boolean;
7
- export declare function extractSalientTerms(query: string): SalientTerms;
8
- export declare function uniqueStrings(values: ReadonlyArray<string>): ReadonlyArray<string>;