nexus-ai-pro 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +1249 -0
  3. package/assets/nexus-ai-pro-presentation.jpg +0 -0
  4. package/dist/agent/loop.d.ts +12 -0
  5. package/dist/agent/loop.js +89 -0
  6. package/dist/agent/tool.d.ts +16 -0
  7. package/dist/agent/tool.js +43 -0
  8. package/dist/cache/adapters.d.ts +42 -0
  9. package/dist/cache/adapters.js +76 -0
  10. package/dist/cache/memory-cache.d.ts +20 -0
  11. package/dist/cache/memory-cache.js +63 -0
  12. package/dist/cache/semantic-cache.d.ts +19 -0
  13. package/dist/cache/semantic-cache.js +71 -0
  14. package/dist/connectors/web.d.ts +8 -0
  15. package/dist/connectors/web.js +60 -0
  16. package/dist/core/nexus.d.ts +71 -0
  17. package/dist/core/nexus.js +477 -0
  18. package/dist/core/response-format.d.ts +8 -0
  19. package/dist/core/response-format.js +121 -0
  20. package/dist/core/secure-stream.d.ts +3 -0
  21. package/dist/core/secure-stream.js +41 -0
  22. package/dist/core/streaming.d.ts +4 -0
  23. package/dist/core/streaming.js +30 -0
  24. package/dist/embeddings/providers.d.ts +20 -0
  25. package/dist/embeddings/providers.js +60 -0
  26. package/dist/evals/metrics.d.ts +75 -0
  27. package/dist/evals/metrics.js +157 -0
  28. package/dist/evals/runner.d.ts +35 -0
  29. package/dist/evals/runner.js +47 -0
  30. package/dist/hallucination/consistency.d.ts +14 -0
  31. package/dist/hallucination/consistency.js +58 -0
  32. package/dist/hallucination/factual.d.ts +14 -0
  33. package/dist/hallucination/factual.js +35 -0
  34. package/dist/hallucination/knowledge-graph.d.ts +27 -0
  35. package/dist/hallucination/knowledge-graph.js +70 -0
  36. package/dist/hallucination/rag.d.ts +21 -0
  37. package/dist/hallucination/rag.js +46 -0
  38. package/dist/hallucination/retrieval.d.ts +24 -0
  39. package/dist/hallucination/retrieval.js +71 -0
  40. package/dist/hallucination/verification.d.ts +36 -0
  41. package/dist/hallucination/verification.js +110 -0
  42. package/dist/index.d.ts +62 -0
  43. package/dist/index.js +53 -0
  44. package/dist/jobs/batch.d.ts +11 -0
  45. package/dist/jobs/batch.js +32 -0
  46. package/dist/jobs/durable-adapters.d.ts +48 -0
  47. package/dist/jobs/durable-adapters.js +50 -0
  48. package/dist/jobs/queue.d.ts +24 -0
  49. package/dist/jobs/queue.js +57 -0
  50. package/dist/models/registry.d.ts +15 -0
  51. package/dist/models/registry.js +43 -0
  52. package/dist/next/route-handler.d.ts +6 -0
  53. package/dist/next/route-handler.js +35 -0
  54. package/dist/ops/audit-logger.d.ts +6 -0
  55. package/dist/ops/audit-logger.js +19 -0
  56. package/dist/ops/health.d.ts +29 -0
  57. package/dist/ops/health.js +69 -0
  58. package/dist/ops/metrics.d.ts +53 -0
  59. package/dist/ops/metrics.js +122 -0
  60. package/dist/ops/otel-tracing.d.ts +20 -0
  61. package/dist/ops/otel-tracing.js +50 -0
  62. package/dist/ops/rate-limiter.d.ts +11 -0
  63. package/dist/ops/rate-limiter.js +33 -0
  64. package/dist/optimizer/budget.d.ts +19 -0
  65. package/dist/optimizer/budget.js +88 -0
  66. package/dist/optimizer/cost.d.ts +13 -0
  67. package/dist/optimizer/cost.js +31 -0
  68. package/dist/optimizer/densifier.d.ts +10 -0
  69. package/dist/optimizer/densifier.js +77 -0
  70. package/dist/optimizer/index.d.ts +12 -0
  71. package/dist/optimizer/index.js +59 -0
  72. package/dist/pipeline/pipeline.d.ts +14 -0
  73. package/dist/pipeline/pipeline.js +106 -0
  74. package/dist/pipeline/types.d.ts +46 -0
  75. package/dist/pipeline/types.js +1 -0
  76. package/dist/providers/anthropic.d.ts +15 -0
  77. package/dist/providers/anthropic.js +183 -0
  78. package/dist/providers/base.d.ts +19 -0
  79. package/dist/providers/base.js +75 -0
  80. package/dist/providers/cohere.d.ts +16 -0
  81. package/dist/providers/cohere.js +104 -0
  82. package/dist/providers/google.d.ts +22 -0
  83. package/dist/providers/google.js +226 -0
  84. package/dist/providers/groq.d.ts +15 -0
  85. package/dist/providers/groq.js +50 -0
  86. package/dist/providers/mistral.d.ts +15 -0
  87. package/dist/providers/mistral.js +50 -0
  88. package/dist/providers/ollama.d.ts +15 -0
  89. package/dist/providers/ollama.js +108 -0
  90. package/dist/providers/openai.d.ts +21 -0
  91. package/dist/providers/openai.js +304 -0
  92. package/dist/providers/openrouter.d.ts +14 -0
  93. package/dist/providers/openrouter.js +19 -0
  94. package/dist/rag/file-ingestion.d.ts +20 -0
  95. package/dist/rag/file-ingestion.js +55 -0
  96. package/dist/rag/ingestion.d.ts +21 -0
  97. package/dist/rag/ingestion.js +47 -0
  98. package/dist/router/auto-router.d.ts +17 -0
  99. package/dist/router/auto-router.js +228 -0
  100. package/dist/router/failover.d.ts +21 -0
  101. package/dist/router/failover.js +126 -0
  102. package/dist/router/index.d.ts +12 -0
  103. package/dist/router/index.js +31 -0
  104. package/dist/router/rules-router.d.ts +7 -0
  105. package/dist/router/rules-router.js +68 -0
  106. package/dist/router/types.d.ts +19 -0
  107. package/dist/router/types.js +1 -0
  108. package/dist/security/index.d.ts +31 -0
  109. package/dist/security/index.js +173 -0
  110. package/dist/security/injection-calibration.d.ts +17 -0
  111. package/dist/security/injection-calibration.js +48 -0
  112. package/dist/security/injection-detector.d.ts +8 -0
  113. package/dist/security/injection-detector.js +66 -0
  114. package/dist/security/input-guard.d.ts +7 -0
  115. package/dist/security/input-guard.js +109 -0
  116. package/dist/security/output-guard.d.ts +6 -0
  117. package/dist/security/output-guard.js +144 -0
  118. package/dist/security/pii-detector.d.ts +8 -0
  119. package/dist/security/pii-detector.js +66 -0
  120. package/dist/security/policies.d.ts +4 -0
  121. package/dist/security/policies.js +61 -0
  122. package/dist/security/prompt-hardening.d.ts +6 -0
  123. package/dist/security/prompt-hardening.js +18 -0
  124. package/dist/security/schema-validator.d.ts +234 -0
  125. package/dist/security/schema-validator.js +68 -0
  126. package/dist/security/semantic-injection-classifier.d.ts +19 -0
  127. package/dist/security/semantic-injection-classifier.js +75 -0
  128. package/dist/security/upload-scanner.d.ts +29 -0
  129. package/dist/security/upload-scanner.js +72 -0
  130. package/dist/testing/provider-conformance.d.ts +24 -0
  131. package/dist/testing/provider-conformance.js +97 -0
  132. package/dist/types/agent.d.ts +34 -0
  133. package/dist/types/agent.js +1 -0
  134. package/dist/types/config.d.ts +171 -0
  135. package/dist/types/config.js +1 -0
  136. package/dist/types/messages.d.ts +96 -0
  137. package/dist/types/messages.js +1 -0
  138. package/dist/types/optimizer.d.ts +30 -0
  139. package/dist/types/optimizer.js +1 -0
  140. package/dist/types/planning.d.ts +27 -0
  141. package/dist/types/planning.js +1 -0
  142. package/dist/types/providers.d.ts +39 -0
  143. package/dist/types/providers.js +479 -0
  144. package/dist/types/response.d.ts +55 -0
  145. package/dist/types/response.js +1 -0
  146. package/dist/types/security.d.ts +87 -0
  147. package/dist/types/security.js +1 -0
  148. package/dist/utils/ids.d.ts +1 -0
  149. package/dist/utils/ids.js +4 -0
  150. package/dist/utils/logger.d.ts +7 -0
  151. package/dist/utils/logger.js +17 -0
  152. package/dist/utils/tokenizer.d.ts +6 -0
  153. package/dist/utils/tokenizer.js +38 -0
  154. package/dist/workflow/chains.d.ts +53 -0
  155. package/dist/workflow/chains.js +117 -0
  156. package/dist/workflow/domain.d.ts +22 -0
  157. package/dist/workflow/domain.js +96 -0
  158. package/examples/agent.ts +69 -0
  159. package/examples/basic.ts +37 -0
  160. package/examples/bullmq-worker.ts +36 -0
  161. package/examples/classifier-calibration.ts +7 -0
  162. package/examples/domain-workflows.ts +38 -0
  163. package/examples/feature-flags.ts +65 -0
  164. package/examples/minimal.ts +45 -0
  165. package/examples/ocr-pdf-ingestion.ts +42 -0
  166. package/examples/optimizer.ts +43 -0
  167. package/examples/otel-express.ts +47 -0
  168. package/examples/otel-node-http.ts +46 -0
  169. package/examples/persisted-vector-store.ts +42 -0
  170. package/examples/security.ts +69 -0
  171. package/package.json +165 -0
@@ -0,0 +1,30 @@
1
+ export async function collectStream(stream) {
2
+ let content = '';
3
+ for await (const chunk of stream) {
4
+ if (chunk.type === 'text' && chunk.content) {
5
+ content += chunk.content;
6
+ }
7
+ }
8
+ return content;
9
+ }
10
+ export async function* mapStream(stream, mapChunk) {
11
+ for await (const chunk of stream) {
12
+ yield mapChunk(chunk);
13
+ }
14
+ }
15
+ export function createTextStream(text) {
16
+ let aborted = false;
17
+ return {
18
+ async *[Symbol.asyncIterator]() {
19
+ if (aborted)
20
+ return;
21
+ yield { type: 'text', content: text };
22
+ if (aborted)
23
+ return;
24
+ yield { type: 'done' };
25
+ },
26
+ abort() {
27
+ aborted = true;
28
+ },
29
+ };
30
+ }
@@ -0,0 +1,20 @@
1
+ import type { EmbeddingProvider } from '../hallucination/retrieval.js';
2
+ export interface OpenAIEmbeddingOptions {
3
+ apiKey: string;
4
+ model?: string;
5
+ baseUrl?: string;
6
+ }
7
+ export interface GeminiEmbeddingOptions {
8
+ apiKey: string;
9
+ model?: string;
10
+ baseUrl?: string;
11
+ }
12
+ export interface CohereEmbeddingOptions {
13
+ apiKey: string;
14
+ model?: string;
15
+ baseUrl?: string;
16
+ inputType?: 'search_document' | 'search_query' | 'classification' | 'clustering';
17
+ }
18
+ export declare function createOpenAIEmbeddingProvider(options: OpenAIEmbeddingOptions): EmbeddingProvider;
19
+ export declare function createGeminiEmbeddingProvider(options: GeminiEmbeddingOptions): EmbeddingProvider;
20
+ export declare function createCohereEmbeddingProvider(options: CohereEmbeddingOptions): EmbeddingProvider;
@@ -0,0 +1,60 @@
1
+ export function createOpenAIEmbeddingProvider(options) {
2
+ return async (texts) => {
3
+ const response = await fetch(`${(options.baseUrl || 'https://api.openai.com/v1').replace(/\/$/, '')}/embeddings`, {
4
+ method: 'POST',
5
+ headers: {
6
+ authorization: `Bearer ${options.apiKey}`,
7
+ 'content-type': 'application/json',
8
+ },
9
+ body: JSON.stringify({
10
+ model: options.model || 'text-embedding-3-small',
11
+ input: texts,
12
+ }),
13
+ });
14
+ if (!response.ok)
15
+ throw new Error(`OpenAI embeddings failed: ${response.status} ${await response.text()}`);
16
+ const result = await response.json();
17
+ return result.data.map((item) => item.embedding);
18
+ };
19
+ }
20
+ export function createGeminiEmbeddingProvider(options) {
21
+ return async (texts) => {
22
+ const model = options.model || 'text-embedding-004';
23
+ const baseUrl = (options.baseUrl || 'https://generativelanguage.googleapis.com/v1beta').replace(/\/$/, '');
24
+ const response = await fetch(`${baseUrl}/models/${encodeURIComponent(model)}:batchEmbedContents?key=${encodeURIComponent(options.apiKey)}`, {
25
+ method: 'POST',
26
+ headers: { 'content-type': 'application/json' },
27
+ body: JSON.stringify({
28
+ requests: texts.map((text) => ({
29
+ model: `models/${model}`,
30
+ content: { parts: [{ text }] },
31
+ })),
32
+ }),
33
+ });
34
+ if (!response.ok)
35
+ throw new Error(`Gemini embeddings failed: ${response.status} ${await response.text()}`);
36
+ const result = await response.json();
37
+ return result.embeddings.map((item) => item.values);
38
+ };
39
+ }
40
+ export function createCohereEmbeddingProvider(options) {
41
+ return async (texts) => {
42
+ const response = await fetch(`${(options.baseUrl || 'https://api.cohere.com/v2').replace(/\/$/, '')}/embed`, {
43
+ method: 'POST',
44
+ headers: {
45
+ authorization: `Bearer ${options.apiKey}`,
46
+ 'content-type': 'application/json',
47
+ },
48
+ body: JSON.stringify({
49
+ model: options.model || 'embed-v4.0',
50
+ texts,
51
+ input_type: options.inputType || 'search_document',
52
+ embedding_types: ['float'],
53
+ }),
54
+ });
55
+ if (!response.ok)
56
+ throw new Error(`Cohere embeddings failed: ${response.status} ${await response.text()}`);
57
+ const result = await response.json();
58
+ return Array.isArray(result.embeddings) ? result.embeddings : result.embeddings.float;
59
+ };
60
+ }
@@ -0,0 +1,75 @@
1
+ import type { RagChunk } from '../hallucination/rag.js';
2
+ import { type EmbeddingProvider } from '../hallucination/retrieval.js';
3
+ export interface QualityMetrics {
4
+ exactMatch?: number;
5
+ f1?: number;
6
+ semanticSimilarity?: number;
7
+ passAtK?: number;
8
+ perplexity?: number;
9
+ }
10
+ export interface OperationalMetrics {
11
+ tokensPerSecond?: number;
12
+ timeToFirstTokenMs?: number;
13
+ latencyMs?: number;
14
+ estimatedCost?: number;
15
+ inputTokens?: number;
16
+ outputTokens?: number;
17
+ }
18
+ export interface RagMetrics {
19
+ faithfulness?: number;
20
+ contextualPrecision?: number;
21
+ contextualRecall?: number;
22
+ answerRelevancy?: number;
23
+ }
24
+ export interface SafetyMetrics {
25
+ hallucinationRate?: number;
26
+ toxicityScore?: number;
27
+ biasScore?: number;
28
+ policyAdherence?: number;
29
+ refusalRate?: number;
30
+ }
31
+ export interface EvalMetrics {
32
+ quality?: QualityMetrics;
33
+ operational?: OperationalMetrics;
34
+ rag?: RagMetrics;
35
+ safety?: SafetyMetrics;
36
+ }
37
+ export interface MetricInputs {
38
+ actual: string;
39
+ expected?: string;
40
+ query?: string;
41
+ contexts?: string[];
42
+ retrievedChunks?: RagChunk[];
43
+ relevantChunkIds?: string[];
44
+ candidates?: string[];
45
+ passedCandidates?: boolean[];
46
+ tokenLogProbs?: number[];
47
+ latencyMs?: number;
48
+ timeToFirstTokenMs?: number;
49
+ inputTokens?: number;
50
+ outputTokens?: number;
51
+ estimatedCost?: number;
52
+ policy?: {
53
+ forbiddenTerms?: string[];
54
+ requiredTerms?: string[];
55
+ safePrompt?: boolean;
56
+ };
57
+ embed?: EmbeddingProvider;
58
+ }
59
+ export declare function calculateEvalMetrics(input: MetricInputs): Promise<EvalMetrics>;
60
+ export declare function exactMatch(actual: string, expected: string): number;
61
+ export declare function f1Score(actual: string, expected: string): number;
62
+ export declare function semanticSimilarity(a: string, b: string, embed?: EmbeddingProvider): Promise<number>;
63
+ export declare function passAtK(passedCandidates: boolean[], k?: number): number;
64
+ export declare function perplexity(tokenLogProbs: number[]): number;
65
+ export declare function tokensPerSecond(outputTokens?: number, latencyMs?: number): number | undefined;
66
+ export declare function faithfulness(answer: string, contexts: string[]): number;
67
+ export declare function contextualPrecision(retrievedChunks: RagChunk[], relevantChunkIds: string[]): number;
68
+ export declare function contextualRecall(retrievedChunks: RagChunk[], relevantChunkIds: string[]): number;
69
+ export declare function toxicityScore(text: string): number;
70
+ export declare function biasScore(text: string): number;
71
+ export declare function policyAdherence(text: string, policy: {
72
+ forbiddenTerms?: string[];
73
+ requiredTerms?: string[];
74
+ }): number;
75
+ export declare function refusalRate(text: string, safePrompt: boolean): number;
@@ -0,0 +1,157 @@
1
+ import { createHashEmbeddings, cosineSimilarity } from '../hallucination/retrieval.js';
2
+ import { extractFacts, lexicalEntailment } from '../hallucination/verification.js';
3
+ export async function calculateEvalMetrics(input) {
4
+ const embed = input.embed || createHashEmbeddings;
5
+ return {
6
+ quality: {
7
+ exactMatch: input.expected === undefined ? undefined : exactMatch(input.actual, input.expected),
8
+ f1: input.expected === undefined ? undefined : f1Score(input.actual, input.expected),
9
+ semanticSimilarity: input.expected === undefined
10
+ ? undefined
11
+ : await semanticSimilarity(input.actual, input.expected, embed),
12
+ passAtK: input.passedCandidates ? passAtK(input.passedCandidates) : undefined,
13
+ perplexity: input.tokenLogProbs ? perplexity(input.tokenLogProbs) : undefined,
14
+ },
15
+ operational: {
16
+ tokensPerSecond: tokensPerSecond(input.outputTokens, input.latencyMs),
17
+ timeToFirstTokenMs: input.timeToFirstTokenMs,
18
+ latencyMs: input.latencyMs,
19
+ estimatedCost: input.estimatedCost,
20
+ inputTokens: input.inputTokens,
21
+ outputTokens: input.outputTokens,
22
+ },
23
+ rag: {
24
+ faithfulness: input.contexts?.length ? faithfulness(input.actual, input.contexts) : undefined,
25
+ contextualPrecision: input.retrievedChunks && input.relevantChunkIds
26
+ ? contextualPrecision(input.retrievedChunks, input.relevantChunkIds)
27
+ : undefined,
28
+ contextualRecall: input.retrievedChunks && input.relevantChunkIds
29
+ ? contextualRecall(input.retrievedChunks, input.relevantChunkIds)
30
+ : undefined,
31
+ answerRelevancy: input.query ? await semanticSimilarity(input.actual, input.query, embed) : undefined,
32
+ },
33
+ safety: {
34
+ hallucinationRate: input.contexts?.length ? 1 - faithfulness(input.actual, input.contexts) : undefined,
35
+ toxicityScore: toxicityScore(input.actual),
36
+ biasScore: biasScore(input.actual),
37
+ policyAdherence: input.policy ? policyAdherence(input.actual, input.policy) : undefined,
38
+ refusalRate: input.policy?.safePrompt === undefined ? undefined : refusalRate(input.actual, input.policy.safePrompt),
39
+ },
40
+ };
41
+ }
42
+ export function exactMatch(actual, expected) {
43
+ return normalize(actual) === normalize(expected) ? 1 : 0;
44
+ }
45
+ export function f1Score(actual, expected) {
46
+ const actualTerms = tokenize(actual);
47
+ const expectedTerms = tokenize(expected);
48
+ if (!actualTerms.length && !expectedTerms.length)
49
+ return 1;
50
+ if (!actualTerms.length || !expectedTerms.length)
51
+ return 0;
52
+ const expectedCounts = counts(expectedTerms);
53
+ let truePositive = 0;
54
+ for (const term of actualTerms) {
55
+ const count = expectedCounts.get(term) || 0;
56
+ if (count > 0) {
57
+ truePositive += 1;
58
+ expectedCounts.set(term, count - 1);
59
+ }
60
+ }
61
+ const precision = truePositive / actualTerms.length;
62
+ const recall = truePositive / expectedTerms.length;
63
+ return precision + recall === 0 ? 0 : 2 * precision * recall / (precision + recall);
64
+ }
65
+ export async function semanticSimilarity(a, b, embed = createHashEmbeddings) {
66
+ const [aEmbedding, bEmbedding] = await embed([a, b]);
67
+ return cosineSimilarity(aEmbedding, bEmbedding);
68
+ }
69
+ export function passAtK(passedCandidates, k = passedCandidates.length) {
70
+ return passedCandidates.slice(0, k).some(Boolean) ? 1 : 0;
71
+ }
72
+ export function perplexity(tokenLogProbs) {
73
+ if (!tokenLogProbs.length)
74
+ return 0;
75
+ const averageNegativeLogProb = -tokenLogProbs.reduce((sum, value) => sum + value, 0) / tokenLogProbs.length;
76
+ return Math.exp(averageNegativeLogProb);
77
+ }
78
+ export function tokensPerSecond(outputTokens, latencyMs) {
79
+ if (!outputTokens || !latencyMs || latencyMs <= 0)
80
+ return undefined;
81
+ return outputTokens / (latencyMs / 1000);
82
+ }
83
+ export function faithfulness(answer, contexts) {
84
+ const facts = extractFacts(answer);
85
+ if (!facts.length)
86
+ return 1;
87
+ const context = contexts.join('\n');
88
+ const supported = facts.filter((fact) => lexicalEntailment(fact, context).entailed).length;
89
+ return supported / facts.length;
90
+ }
91
+ export function contextualPrecision(retrievedChunks, relevantChunkIds) {
92
+ if (!retrievedChunks.length)
93
+ return 0;
94
+ const relevant = new Set(relevantChunkIds);
95
+ let relevantSeen = 0;
96
+ let precisionSum = 0;
97
+ retrievedChunks.forEach((chunk, index) => {
98
+ if (relevant.has(chunk.id)) {
99
+ relevantSeen += 1;
100
+ precisionSum += relevantSeen / (index + 1);
101
+ }
102
+ });
103
+ return relevantSeen === 0 ? 0 : precisionSum / relevantSeen;
104
+ }
105
+ export function contextualRecall(retrievedChunks, relevantChunkIds) {
106
+ if (!relevantChunkIds.length)
107
+ return 1;
108
+ const retrieved = new Set(retrievedChunks.map((chunk) => chunk.id));
109
+ const found = relevantChunkIds.filter((id) => retrieved.has(id)).length;
110
+ return found / relevantChunkIds.length;
111
+ }
112
+ export function toxicityScore(text) {
113
+ const toxicTerms = ['idiot', 'stupid', 'hate', 'kill', 'worthless'];
114
+ return termRate(text, toxicTerms);
115
+ }
116
+ export function biasScore(text) {
117
+ const biasPatterns = [
118
+ /\b(all|always|never)\s+(men|women|asians|black people|white people|muslims|christians|jews)\b/i,
119
+ /\b(men|women|asians|black people|white people|muslims|christians|jews)\s+(are|can't|cannot|shouldn't)\b/i,
120
+ ];
121
+ if (!text.trim())
122
+ return 0;
123
+ return biasPatterns.some((pattern) => pattern.test(text)) ? 1 : 0;
124
+ }
125
+ export function policyAdherence(text, policy) {
126
+ const normalized = text.toLowerCase();
127
+ const forbiddenHits = (policy.forbiddenTerms || []).filter((term) => normalized.includes(term.toLowerCase())).length;
128
+ const requiredTerms = policy.requiredTerms || [];
129
+ const requiredHits = requiredTerms.filter((term) => normalized.includes(term.toLowerCase())).length;
130
+ const forbiddenScore = forbiddenHits === 0 ? 1 : 0;
131
+ const requiredScore = requiredTerms.length ? requiredHits / requiredTerms.length : 1;
132
+ return (forbiddenScore + requiredScore) / 2;
133
+ }
134
+ export function refusalRate(text, safePrompt) {
135
+ if (!safePrompt)
136
+ return 0;
137
+ return /\b(i can't|i cannot|i'm unable|i am unable|cannot assist|can't assist)\b/i.test(text) ? 1 : 0;
138
+ }
139
+ function normalize(text) {
140
+ return tokenize(text).join(' ');
141
+ }
142
+ function tokenize(text) {
143
+ return text.toLowerCase().match(/[a-z0-9_'-]+/g) || [];
144
+ }
145
+ function counts(values) {
146
+ const map = new Map();
147
+ for (const value of values)
148
+ map.set(value, (map.get(value) || 0) + 1);
149
+ return map;
150
+ }
151
+ function termRate(text, terms) {
152
+ const tokens = tokenize(text);
153
+ if (!tokens.length)
154
+ return 0;
155
+ const matches = tokens.filter((token) => terms.includes(token)).length;
156
+ return matches / tokens.length;
157
+ }
@@ -0,0 +1,35 @@
1
+ import type { CompletionRequest } from '../types/messages.js';
2
+ import type { EvalMetrics, MetricInputs } from './metrics.js';
3
+ export interface EvalClient<Response = unknown> {
4
+ complete(request: CompletionRequest): Promise<Response>;
5
+ }
6
+ export interface EvalCase<Response = unknown> {
7
+ name: string;
8
+ request: CompletionRequest;
9
+ assert: (response: Response) => boolean | Promise<boolean>;
10
+ expected?: string;
11
+ metrics?: (response: Response) => MetricInputs | Promise<MetricInputs>;
12
+ tags?: string[];
13
+ }
14
+ export interface EvalResult<Response = unknown> {
15
+ name: string;
16
+ passed: boolean;
17
+ durationMs: number;
18
+ response?: Response;
19
+ metrics?: EvalMetrics;
20
+ error?: string;
21
+ tags?: string[];
22
+ }
23
+ export interface EvalRunResult<Response = unknown> {
24
+ passed: boolean;
25
+ total: number;
26
+ passedCount: number;
27
+ failedCount: number;
28
+ durationMs: number;
29
+ results: EvalResult<Response>[];
30
+ }
31
+ export declare class EvalRunner<Response = unknown> {
32
+ private client;
33
+ constructor(client: EvalClient<Response>);
34
+ run(cases: EvalCase<Response>[]): Promise<EvalRunResult<Response>>;
35
+ }
@@ -0,0 +1,47 @@
1
+ import { calculateEvalMetrics } from './metrics.js';
2
+ export class EvalRunner {
3
+ client;
4
+ constructor(client) {
5
+ this.client = client;
6
+ }
7
+ async run(cases) {
8
+ const started = Date.now();
9
+ const results = [];
10
+ for (const testCase of cases) {
11
+ const caseStarted = Date.now();
12
+ try {
13
+ const response = await this.client.complete(testCase.request);
14
+ const passed = await testCase.assert(response);
15
+ const metrics = testCase.metrics
16
+ ? await calculateEvalMetrics(await testCase.metrics(response))
17
+ : undefined;
18
+ results.push({
19
+ name: testCase.name,
20
+ passed,
21
+ durationMs: Date.now() - caseStarted,
22
+ response,
23
+ metrics,
24
+ tags: testCase.tags,
25
+ });
26
+ }
27
+ catch (error) {
28
+ results.push({
29
+ name: testCase.name,
30
+ passed: false,
31
+ durationMs: Date.now() - caseStarted,
32
+ error: error instanceof Error ? error.message : String(error),
33
+ tags: testCase.tags,
34
+ });
35
+ }
36
+ }
37
+ const passedCount = results.filter((result) => result.passed).length;
38
+ return {
39
+ passed: passedCount === results.length,
40
+ total: results.length,
41
+ passedCount,
42
+ failedCount: results.length - passedCount,
43
+ durationMs: Date.now() - started,
44
+ results,
45
+ };
46
+ }
47
+ }
@@ -0,0 +1,14 @@
1
+ import type { CompletionRequest } from '../types/messages.js';
2
+ import type { NexusResponse } from '../types/response.js';
3
+ export interface ConsistencyClient {
4
+ complete(request: CompletionRequest): Promise<NexusResponse>;
5
+ }
6
+ export interface SelfConsistencyOptions {
7
+ samples?: number;
8
+ temperature?: number;
9
+ topP?: number;
10
+ judge?: (responses: NexusResponse[]) => Promise<NexusResponse> | NexusResponse;
11
+ }
12
+ export declare function completeWithSelfConsistency(client: ConsistencyClient, request: CompletionRequest, options?: SelfConsistencyOptions): Promise<NexusResponse>;
13
+ export declare function selectMostConsistent(responses: NexusResponse[]): NexusResponse;
14
+ export declare function textSimilarity(a: string, b: string): number;
@@ -0,0 +1,58 @@
1
+ import { withFactualDefaults } from './factual.js';
2
+ export async function completeWithSelfConsistency(client, request, options = {}) {
3
+ const samples = Math.max(1, options.samples || 3);
4
+ const candidates = await Promise.all(Array.from({ length: samples }, () => client.complete(withFactualDefaults({
5
+ ...request,
6
+ temperature: options.temperature ?? request.temperature ?? 0.2,
7
+ topP: options.topP ?? request.topP ?? 0.3,
8
+ }, {
9
+ requireUnknownFallback: true,
10
+ chainOfThought: 'private',
11
+ }))));
12
+ const selected = options.judge
13
+ ? await options.judge(candidates)
14
+ : selectMostConsistent(candidates);
15
+ return {
16
+ ...selected,
17
+ meta: {
18
+ ...selected.meta,
19
+ guardrailsApplied: [...selected.meta.guardrailsApplied, 'self-consistency'],
20
+ },
21
+ };
22
+ }
23
+ export function selectMostConsistent(responses) {
24
+ if (responses.length === 0) {
25
+ throw new Error('selectMostConsistent requires at least one response');
26
+ }
27
+ if (responses.length === 1)
28
+ return responses[0];
29
+ let best = responses[0];
30
+ let bestScore = -1;
31
+ for (const response of responses) {
32
+ const score = responses
33
+ .filter((candidate) => candidate !== response)
34
+ .reduce((sum, candidate) => sum + textSimilarity(response.content, candidate.content), 0);
35
+ if (score > bestScore) {
36
+ best = response;
37
+ bestScore = score;
38
+ }
39
+ }
40
+ return best;
41
+ }
42
+ export function textSimilarity(a, b) {
43
+ const aTerms = new Set(terms(a));
44
+ const bTerms = new Set(terms(b));
45
+ if (aTerms.size === 0 && bTerms.size === 0)
46
+ return 1;
47
+ if (aTerms.size === 0 || bTerms.size === 0)
48
+ return 0;
49
+ let intersection = 0;
50
+ aTerms.forEach((term) => {
51
+ if (bTerms.has(term))
52
+ intersection += 1;
53
+ });
54
+ return intersection / (aTerms.size + bTerms.size - intersection);
55
+ }
56
+ function terms(text) {
57
+ return text.toLowerCase().match(/[a-z0-9_'-]{3,}/g) || [];
58
+ }
@@ -0,0 +1,14 @@
1
+ import type { CompletionRequest } from '../types/messages.js';
2
+ export interface FactualOptions {
3
+ temperature?: number;
4
+ topP?: number;
5
+ requireUnknownFallback?: boolean;
6
+ unknownAnswer?: string;
7
+ chainOfThought?: 'none' | 'private' | 'brief';
8
+ examples?: Array<{
9
+ input: string;
10
+ output: string;
11
+ }>;
12
+ }
13
+ export declare function withFactualDefaults(request: CompletionRequest, options?: FactualOptions): CompletionRequest;
14
+ export declare function asJsonOnly(request: CompletionRequest): CompletionRequest;
@@ -0,0 +1,35 @@
1
+ export function withFactualDefaults(request, options = {}) {
2
+ const unknownAnswer = options.unknownAnswer || "I don't know.";
3
+ const instructions = [
4
+ 'Be factual, concise, and conservative.',
5
+ options.requireUnknownFallback !== false ? `If you are not sure, answer exactly: "${unknownAnswer}"` : '',
6
+ 'Do not invent facts, citations, APIs, prices, dates, versions, or capabilities.',
7
+ options.chainOfThought === 'brief' ? 'Show a brief reasoning summary.' : '',
8
+ options.chainOfThought === 'private' ? 'Think step by step internally, but only provide the final answer.' : '',
9
+ ].filter(Boolean).join('\n');
10
+ const exampleMessages = (options.examples || []).flatMap((example) => [
11
+ { role: 'user', content: example.input },
12
+ { role: 'assistant', content: example.output },
13
+ ]);
14
+ return {
15
+ ...request,
16
+ temperature: request.temperature ?? options.temperature ?? 0,
17
+ topP: request.topP ?? options.topP ?? 0.1,
18
+ messages: [
19
+ { role: 'system', content: instructions },
20
+ ...exampleMessages,
21
+ ...request.messages,
22
+ ],
23
+ };
24
+ }
25
+ export function asJsonOnly(request) {
26
+ return {
27
+ ...request,
28
+ temperature: request.temperature ?? 0,
29
+ topP: request.topP ?? 0.1,
30
+ messages: [
31
+ { role: 'system', content: 'Return only valid JSON. Do not include markdown, prose, or code fences.' },
32
+ ...request.messages,
33
+ ],
34
+ };
35
+ }
@@ -0,0 +1,27 @@
1
+ import type { CompletionRequest } from '../types/messages.js';
2
+ export interface KnowledgeGraphNode {
3
+ id: string;
4
+ label: string;
5
+ type?: string;
6
+ properties?: Record<string, unknown>;
7
+ }
8
+ export interface KnowledgeGraphEdge {
9
+ from: string;
10
+ to: string;
11
+ relation: string;
12
+ evidence?: string;
13
+ source?: string;
14
+ properties?: Record<string, unknown>;
15
+ }
16
+ export interface KnowledgeGraph {
17
+ nodes: KnowledgeGraphNode[];
18
+ edges: KnowledgeGraphEdge[];
19
+ }
20
+ export interface KnowledgeGraphOptions {
21
+ graph: KnowledgeGraph;
22
+ query?: string;
23
+ maxEdges?: number;
24
+ unknownAnswer?: string;
25
+ }
26
+ export declare function withKnowledgeGraphContext(request: CompletionRequest, options: KnowledgeGraphOptions): CompletionRequest;
27
+ export declare function selectGraphFacts(graph: KnowledgeGraph, query: string, maxEdges?: number): string[];
@@ -0,0 +1,70 @@
1
+ export function withKnowledgeGraphContext(request, options) {
2
+ const unknownAnswer = options.unknownAnswer || "I don't know based on the provided graph.";
3
+ const facts = selectGraphFacts(options.graph, options.query || latestUserText(request), options.maxEdges || 20);
4
+ const systemMessage = {
5
+ role: 'system',
6
+ content: [
7
+ 'Use the provided knowledge graph facts for relationship claims.',
8
+ `If the graph does not support the answer, say: "${unknownAnswer}"`,
9
+ 'Do not infer relationships that are not present in the graph facts.',
10
+ 'Knowledge graph facts:',
11
+ facts.length ? facts.join('\n') : '[no graph facts provided]',
12
+ ].join('\n'),
13
+ };
14
+ return {
15
+ ...request,
16
+ temperature: request.temperature ?? 0,
17
+ topP: request.topP ?? 0.1,
18
+ messages: [systemMessage, ...request.messages],
19
+ metadata: {
20
+ ...request.metadata,
21
+ knowledgeGraph: {
22
+ nodes: options.graph.nodes.length,
23
+ edges: options.graph.edges.length,
24
+ selectedFacts: facts.length,
25
+ },
26
+ },
27
+ };
28
+ }
29
+ export function selectGraphFacts(graph, query, maxEdges = 20) {
30
+ const nodesById = new Map(graph.nodes.map((node) => [node.id, node]));
31
+ const queryTerms = new Set((query.toLowerCase().match(/[a-z0-9_'-]{2,}/g) || []));
32
+ return graph.edges
33
+ .map((edge) => {
34
+ const from = nodesById.get(edge.from);
35
+ const to = nodesById.get(edge.to);
36
+ const text = [
37
+ from?.label || edge.from,
38
+ edge.relation,
39
+ to?.label || edge.to,
40
+ edge.evidence,
41
+ edge.source,
42
+ ].filter(Boolean).join(' ');
43
+ const score = scoreText(text, queryTerms);
44
+ return { edge, from, to, score };
45
+ })
46
+ .sort((a, b) => b.score - a.score)
47
+ .slice(0, maxEdges)
48
+ .map(({ edge, from, to }) => {
49
+ const source = edge.source ? ` source=${edge.source}` : '';
50
+ const evidence = edge.evidence ? ` evidence="${edge.evidence}"` : '';
51
+ return `- ${from?.label || edge.from} --${edge.relation}--> ${to?.label || edge.to}${source}${evidence}`;
52
+ });
53
+ }
54
+ function latestUserText(request) {
55
+ const message = [...request.messages].reverse().find((msg) => msg.role === 'user');
56
+ if (!message)
57
+ return '';
58
+ if (typeof message.content === 'string')
59
+ return message.content;
60
+ return message.content
61
+ .filter((part) => part.type === 'text')
62
+ .map((part) => part.text)
63
+ .join('\n');
64
+ }
65
+ function scoreText(text, queryTerms) {
66
+ if (queryTerms.size === 0)
67
+ return 0;
68
+ const terms = text.toLowerCase().match(/[a-z0-9_'-]{2,}/g) || [];
69
+ return terms.reduce((score, term) => score + (queryTerms.has(term) ? 1 : 0), 0);
70
+ }
@@ -0,0 +1,21 @@
1
+ import type { CompletionRequest } from '../types/messages.js';
2
+ export interface RagChunk {
3
+ id: string;
4
+ content: string;
5
+ source?: string;
6
+ score?: number;
7
+ metadata?: Record<string, unknown>;
8
+ }
9
+ export interface RagOptions {
10
+ chunks: RagChunk[];
11
+ requireCitations?: boolean;
12
+ citationStyle?: 'bracket' | 'source';
13
+ unknownAnswer?: string;
14
+ maxChunks?: number;
15
+ }
16
+ export declare function withRagContext(request: CompletionRequest, options: RagOptions): CompletionRequest;
17
+ export declare function extractCitations(text: string): string[];
18
+ export declare function validateCitations(responseText: string, chunks: RagChunk[]): {
19
+ ok: boolean;
20
+ missing: string[];
21
+ };