nexus-ai-pro 1.19.0 → 1.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/API_STABILITY.md +11 -0
  2. package/CHANGELOG.md +37 -1
  3. package/README.md +14 -29
  4. package/dist/cli/db.js +12 -1
  5. package/dist/hallucination/retrieval.d.ts +31 -2
  6. package/dist/hallucination/retrieval.js +31 -14
  7. package/dist/index.d.ts +1 -1
  8. package/dist/index.js +1 -1
  9. package/dist/ops/health.js +2 -0
  10. package/dist/ops/metrics.js +2 -0
  11. package/dist/optimizer/index.js +2 -0
  12. package/dist/postgres/index.d.ts +7 -3
  13. package/dist/postgres/index.js +8 -0
  14. package/dist/postgres/vectors.d.ts +46 -0
  15. package/dist/postgres/vectors.js +100 -0
  16. package/dist/providers/google.js +2 -0
  17. package/dist/providers/ollama.js +2 -0
  18. package/dist/rag/ingestion.d.ts +2 -0
  19. package/dist/rag/ingestion.js +1 -0
  20. package/dist/rag/qdrant.d.ts +65 -0
  21. package/dist/rag/qdrant.js +126 -0
  22. package/dist/security/index.js +5 -0
  23. package/dist/utils/deprecation.d.ts +12 -0
  24. package/dist/utils/deprecation.js +17 -0
  25. package/dist-cjs/cli/db.js +12 -1
  26. package/dist-cjs/hallucination/retrieval.d.ts +31 -2
  27. package/dist-cjs/hallucination/retrieval.js +32 -14
  28. package/dist-cjs/index.d.ts +1 -1
  29. package/dist-cjs/index.js +5 -3
  30. package/dist-cjs/ops/health.js +2 -0
  31. package/dist-cjs/ops/metrics.js +2 -0
  32. package/dist-cjs/optimizer/index.js +2 -0
  33. package/dist-cjs/postgres/index.d.ts +7 -3
  34. package/dist-cjs/postgres/index.js +11 -1
  35. package/dist-cjs/postgres/vectors.d.ts +46 -0
  36. package/dist-cjs/postgres/vectors.js +105 -0
  37. package/dist-cjs/providers/google.js +2 -0
  38. package/dist-cjs/providers/ollama.js +2 -0
  39. package/dist-cjs/rag/ingestion.d.ts +2 -0
  40. package/dist-cjs/rag/ingestion.js +7 -0
  41. package/dist-cjs/rag/qdrant.d.ts +65 -0
  42. package/dist-cjs/rag/qdrant.js +131 -0
  43. package/dist-cjs/security/index.js +5 -0
  44. package/dist-cjs/utils/deprecation.d.ts +12 -0
  45. package/dist-cjs/utils/deprecation.js +21 -0
  46. package/package.json +27 -1
package/API_STABILITY.md CHANGED
@@ -455,6 +455,17 @@ with nexus-ai-pro and declares the core versions it works with as a peer depende
455
455
  - The access rules are a guarantee: loopback binding by default, a token on every request, the token
456
456
  in a header for every change, and loopback hosts only unless others are allowed.
457
457
 
458
+ ## Retrieval stores stage (1.20.0)
459
+
460
+ `PostgresVectorStore` (`nexus-ai-pro/postgres/vectors`) and `QdrantVectorStore` (`nexus-ai-pro/rag/qdrant`)
461
+ are experimental in production readiness. The `VectorStore` contract they share with
462
+ `MemoryVectorStore` follows the 1.x rules: a minor release may add optional methods and options, not
463
+ remove or reshape these.
464
+
465
+ - The pgvector table layout may gain columns in a minor release; `migrate()` adds what is missing.
466
+ - A Qdrant point id is derived from the chunk id and is stable across releases, so a collection
467
+ written by one version is read by the next.
468
+
458
469
  ## Deprecation process
459
470
 
460
471
  Deprecated APIs are marked with `@deprecated` in declarations and described in the changelog. Removals
package/CHANGELOG.md CHANGED
@@ -4,6 +4,41 @@ Notable changes to this project are recorded here. The format follows [Keep a Ch
4
4
 
5
5
  ## [Unreleased]
6
6
 
7
+ ## [1.20.0] - 2026-09-28
8
+
9
+ Retrieval that scales past one process. Deprecated options now say so at run time, and every guide
10
+ explains every export it lists.
11
+
12
+ ### Added
13
+
14
+ - **Vector stores for retrieval.** `VectorStore` is a shared contract — `add()`, which replaces a chunk
15
+ whose id already exists, `search()`, `searchVector()` for a vector computed elsewhere, `delete()`,
16
+ and an exact-match metadata `filter` — with three implementations checked by one contract test:
17
+ - `PostgresVectorStore` on `nexus-ai-pro/postgres/vectors`, ranking with pgvector in the database,
18
+ with an HNSW index by default or an exact scan. `vectorStoreMigration()` gives its schema, and
19
+ `postgresMigration()` and `nexus db sql` include it when `vectors` is named.
20
+ - `QdrantVectorStore` on `nexus-ai-pro/rag/qdrant`, through Qdrant's REST API with `fetch` and Web
21
+ Crypto, so no Qdrant client is a dependency and it runs on edge runtimes. `migrate()` creates the
22
+ collection and indexes filter fields; failures raise `QdrantError`.
23
+ - `MemoryVectorStore`, which gains `searchVector()`, `delete()`, and the filter.
24
+ - Retrieval is reachable from `nexus-ai-pro/rag` — `MemoryVectorStore`, `createHashEmbeddings()`,
25
+ `cosineSimilarity()`, `normalizeVector()`, `matchesMetadata()`, and the types — so an application
26
+ that ingests and searches no longer loads the root import.
27
+ - **Deprecated options warn once per process** through Node's `DeprecationWarning` channel, with a
28
+ `NEXUS_DEP_*` code each, so `--no-deprecation` silences them and `--throw-deprecation` fails a CI
29
+ job on them: `GoogleProviderConfig.projectId`, `OllamaProviderConfig.timeout`,
30
+ `MetricsConfig.prometheus`, `HealthConfig.latencyHalfLife`, `DensificationConfig.preserveMarkdown`,
31
+ `InjectionDetectionConfig.sensitivity`, and `ToolPolicyConfig.requiresApproval`.
32
+
33
+ ### Changed
34
+
35
+ - `MemoryVectorStore.add()` replaces a chunk whose id already exists instead of storing a second
36
+ copy, so re-ingesting a document no longer duplicates its passages in search results.
37
+ - The guide check enforces that every export is explained in a guide's own text, not only named in
38
+ its generated reference; the five remaining guides — operations, embeddings, evaluation, images,
39
+ and realtime — now are. The README's limitations and production notes moved into the guides they
40
+ belong to.
41
+
7
42
  ## [1.19.0] - 2026-09-28
8
43
 
9
44
  A studio to look at it all. `nexus-ai-pro-studio` is a separate package — so the core install never
@@ -1008,7 +1043,8 @@ shared policy vocabulary is in place for a future release that revisits this.
1008
1043
  - URL fetching rejects unsafe private-network targets and limits response reads.
1009
1044
  - CLI and audit findings no longer disclose detected secret values.
1010
1045
 
1011
- [Unreleased]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.19.0...HEAD
1046
+ [Unreleased]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.20.0...HEAD
1047
+ [1.20.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.19.0...v1.20.0
1012
1048
  [1.19.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.18.0...v1.19.0
1013
1049
  [1.18.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.17.0...v1.18.0
1014
1050
  [1.17.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.16.0...v1.17.0
package/README.md CHANGED
@@ -197,37 +197,22 @@ Shipped so far:
197
197
  - queryable traces, an evaluation platform, and a CI gate on top of it;
198
198
  - record and replay of provider traffic;
199
199
  - prompt versioning: typed templates, content versions, gated promotion, and serving through outages;
200
+ - a self-hosted agent server, and a local studio as a separate package;
200
201
  - per-entry-point size budgets.
201
202
 
202
- Next is a self-hosted agent server. The image family leaves experimental once recorded live
203
- conformance passes on all three backends.
204
-
205
- ## Known Limitations
206
-
207
- - Voice, telephony, images, local models, and custom providers are optional layers; enable only what
208
- you use.
209
- - Audio and video preprocessing is limited, and provider modality support differs.
210
- - Bundled model metadata is a routing and estimation convenience, not a pricing contract.
211
- - Process-memory cache, queues, and asset storage are not enough for a distributed deployment; use
212
- the Redis, BullMQ, filesystem, or S3 adapters instead.
213
- - Default hash embeddings suit tests and demos, not strong semantic search. Register a real
214
- embedding provider for production retrieval.
215
- - NLI verification is an interface; bring a specialized verifier for high-confidence entailment.
216
- - Guardrails reduce risk but do not replace application authorization, provider-side moderation, or
217
- human review of high-impact actions.
218
- - Realtime sessions are not yet routed through the metrics, audit, and rate-limit path that every
219
- other family uses.
220
- - The image adapters are verified against documented wire shapes; recorded live conformance for all
221
- three backends is still to come, which is why the family is experimental.
222
-
223
- ## Production Notes
224
-
225
- - Use direct or rules-based routing when model choice is already known.
226
- - Keep semantic cache, semantic security, and custom hooks off latency-sensitive routes unless needed.
227
- - Disable response traces with `pipeline.includeTraceInResponse = false`.
228
- - Use shared adapters for distributed apps instead of process memory.
229
- - Treat bundled model pricing and context metadata as defaults, not financial truth.
230
- - Keep server-side authorization and provider moderation around high-risk workflows.
203
+ Next: durable functional workflows and step-level recovery, then retrieval and integration breadth,
204
+ a shared team studio, and self-managed deployment at scale, before 2.0.0 consolidates. The image
205
+ family leaves experimental once recorded live conformance passes on all three backends.
206
+
207
+ ## Before Production
208
+
209
+ Each guide ends with its limitations. The ones worth reading first: the
210
+ [client guide](https://github.com/mkhitar-abrahamyan/nexus-ai/blob/main/docs/core.md#in-production) on latency, traces,
211
+ and shared adapters; the [security guide](https://github.com/mkhitar-abrahamyan/nexus-ai/blob/main/docs/security.md#limitations)
212
+ on what guardrails do not replace; and the
213
+ [grounding guide](https://github.com/mkhitar-abrahamyan/nexus-ai/blob/main/docs/grounding.md#limitations) on embeddings
214
+ and verification. Voice, telephony, images, realtime, local models, and custom providers are optional
215
+ layers; enable only what you use.
231
216
 
232
217
  ## Donations
233
218
 
package/dist/cli/db.js CHANGED
@@ -1,6 +1,14 @@
1
1
  import { postgresMigration } from '../postgres/index.js';
2
2
  import { CliUsageError, listFlag, numberFlag } from './args.js';
3
- const ADAPTERS = ['operations', 'store', 'traces', 'evaluation', 'circuits', 'prompts'];
3
+ const ADAPTERS = [
4
+ 'operations',
5
+ 'store',
6
+ 'traces',
7
+ 'evaluation',
8
+ 'circuits',
9
+ 'prompts',
10
+ 'vectors',
11
+ ];
4
12
  export async function runDbCommand(subcommand, { flags }) {
5
13
  if (subcommand !== 'sql')
6
14
  throw new CliUsageError(`Unknown db command "${subcommand ?? ''}". Use sql.`);
@@ -10,6 +18,9 @@ export async function runDbCommand(subcommand, { flags }) {
10
18
  throw new CliUsageError(`Unknown adapter ${unknown.join(', ')}. Choose from ${ADAPTERS.join(', ')}.`);
11
19
  }
12
20
  const vectorDimensions = numberFlag(flags, 'vector-dimensions');
21
+ if (adapters?.includes('vectors') && vectorDimensions === undefined) {
22
+ throw new CliUsageError('The vectors adapter needs --vector-dimensions.');
23
+ }
13
24
  process.stdout.write(postgresMigration({
14
25
  ...(adapters ? { adapters } : {}),
15
26
  ...(vectorDimensions === undefined ? {} : { vectorDimensions }),
@@ -17,24 +17,52 @@ export interface VectorSearchOptions {
17
17
  topK?: number;
18
18
  /** Lowest similarity returned. Defaults to 0. */
19
19
  minScore?: number;
20
+ /**
21
+ * Exact matches on top-level `metadata` fields, such as `{ tenant: 'acme' }`. Every field must
22
+ * match. Values are strings, numbers, or booleans, which every adapter can filter on natively.
23
+ */
24
+ filter?: Record<string, string | number | boolean>;
25
+ }
26
+ /**
27
+ * Where retrieval chunks live and how they are searched. `MemoryVectorStore`, `PostgresVectorStore`,
28
+ * and `QdrantVectorStore` implement it, and pass the same contract tests.
29
+ *
30
+ * Adding a chunk whose id already exists replaces it, so re-ingesting a document never duplicates
31
+ * its passages.
32
+ */
33
+ export interface VectorStore {
34
+ /** Adds chunks, or replaces the ones whose id already exists, embedding those without a vector. */
35
+ add(documents: VectorDocument[]): Promise<void>;
36
+ /** The chunks most similar to a query text, best first. */
37
+ search(query: string, options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
38
+ /** The chunks most similar to a vector computed elsewhere, best first. */
39
+ searchVector(vector: number[], options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
40
+ /** Removes chunks by id. An id that is not stored is ignored. */
41
+ delete(ids: readonly string[]): Promise<void>;
20
42
  }
21
43
  /**
22
44
  * Chunks and their vectors in process memory, searched by cosine similarity. Defaults to hashed
23
45
  * term vectors, which need no provider.
24
46
  */
25
- export declare class MemoryVectorStore {
47
+ export declare class MemoryVectorStore implements VectorStore {
26
48
  private documents;
27
49
  private embed;
28
50
  constructor(embed?: EmbeddingProvider);
29
- /** Adds chunks, embedding those without a vector in one batch. */
51
+ /** Adds chunks, or replaces those whose id exists, embedding those without a vector in one batch. */
30
52
  add(documents: VectorDocument[]): Promise<void>;
31
53
  /** The chunks most similar to a query, best first. */
32
54
  search(query: string, options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
55
+ /** The chunks most similar to a vector, best first. */
56
+ searchVector(vector: number[], options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
57
+ /** Removes chunks by id. */
58
+ delete(ids: readonly string[]): Promise<void>;
33
59
  /** Removes every chunk. */
34
60
  clear(): void;
35
61
  /** Chunks held. */
36
62
  size(): number;
37
63
  }
64
+ /** Whether a chunk's metadata satisfies an exact-match filter. Every adapter filters the same way. */
65
+ export declare function matchesMetadata(metadata: Record<string, unknown> | undefined, filter: VectorSearchOptions['filter']): boolean;
38
66
  /**
39
67
  * Hashed term-count vectors, normalized to unit length. Deterministic and free, for tests and a
40
68
  * store without a provider.
@@ -42,4 +70,5 @@ export declare class MemoryVectorStore {
42
70
  export declare function createHashEmbeddings(texts: string[], dimensions?: number): number[][];
43
71
  /** Cosine similarity of two unit-length vectors, as their dot product. */
44
72
  export declare function cosineSimilarity(a: number[], b: number[]): number;
73
+ /** Scales a vector to unit length, so a dot product is its cosine similarity. A zero vector stays zero. */
45
74
  export declare function normalizeVector(vector: number[]): number[];
@@ -1,5 +1,5 @@
1
1
  export class MemoryVectorStore {
2
- documents = [];
2
+ documents = new Map();
3
3
  embed;
4
4
  constructor(embed = createHashEmbeddings) {
5
5
  this.embed = embed;
@@ -10,29 +10,46 @@ export class MemoryVectorStore {
10
10
  let generatedIndex = 0;
11
11
  for (const doc of documents) {
12
12
  const embedding = doc.embedding || generated[generatedIndex++];
13
- this.documents.push({ ...doc, embedding: normalizeVector(embedding) });
13
+ this.documents.set(doc.id, { ...doc, embedding: normalizeVector(embedding) });
14
14
  }
15
15
  }
16
16
  async search(query, options = {}) {
17
+ const [queryEmbedding] = await this.embed([query]);
18
+ return this.searchVector(queryEmbedding, options);
19
+ }
20
+ async searchVector(vector, options = {}) {
17
21
  const topK = options.topK || 5;
18
22
  const minScore = options.minScore ?? 0;
19
- const [queryEmbedding] = await this.embed([query]);
20
- const normalizedQuery = normalizeVector(queryEmbedding);
21
- return this.documents
22
- .map(({ embedding, ...doc }) => ({
23
- ...doc,
24
- score: cosineSimilarity(normalizedQuery, embedding),
25
- }))
26
- .filter((doc) => doc.score >= minScore)
27
- .sort((a, b) => b.score - a.score)
28
- .slice(0, topK);
23
+ const normalizedQuery = normalizeVector(vector);
24
+ const results = [];
25
+ for (const { embedding, ...doc } of this.documents.values()) {
26
+ if (!matchesMetadata(doc.metadata, options.filter))
27
+ continue;
28
+ const score = cosineSimilarity(normalizedQuery, embedding);
29
+ if (score >= minScore)
30
+ results.push({ ...doc, score });
31
+ }
32
+ return results.sort((a, b) => b.score - a.score).slice(0, topK);
33
+ }
34
+ async delete(ids) {
35
+ for (const id of ids)
36
+ this.documents.delete(id);
29
37
  }
30
38
  clear() {
31
- this.documents = [];
39
+ this.documents.clear();
32
40
  }
33
41
  size() {
34
- return this.documents.length;
42
+ return this.documents.size;
43
+ }
44
+ }
45
+ export function matchesMetadata(metadata, filter) {
46
+ if (!filter)
47
+ return true;
48
+ for (const [key, expected] of Object.entries(filter)) {
49
+ if (metadata?.[key] !== expected)
50
+ return false;
35
51
  }
52
+ return true;
36
53
  }
37
54
  export function createHashEmbeddings(texts, dimensions = 384) {
38
55
  return texts.map((text) => {
package/dist/index.d.ts CHANGED
@@ -38,7 +38,7 @@ export { PipelineRunner, createPipelineContext, } from './pipeline/pipeline.js';
38
38
  export type { PipelineConfig, PipelineContext, PipelineHookName, PipelineHooksConfig, PipelineMiddleware, PipelineStep, PipelineStepName, PipelineTrace, PipelineTraceStep, } from './pipeline/types.js';
39
39
  export { withFactualDefaults, asJsonOnly, type FactualOptions, } from './hallucination/factual.js';
40
40
  export { withRagContext, extractCitations, validateCitations, type RagChunk, type RagOptions, } from './hallucination/rag.js';
41
- export { MemoryVectorStore, createHashEmbeddings, cosineSimilarity, type EmbeddingProvider, type VectorDocument, type VectorSearchOptions, type VectorSearchResult, } from './hallucination/retrieval.js';
41
+ export { MemoryVectorStore, createHashEmbeddings, cosineSimilarity, matchesMetadata, normalizeVector, type EmbeddingProvider, type VectorDocument, type VectorSearchOptions, type VectorSearchResult, type VectorStore, } from './hallucination/retrieval.js';
42
42
  export { withKnowledgeGraphContext, selectGraphFacts, type KnowledgeGraph, type KnowledgeGraphEdge, type KnowledgeGraphNode, type KnowledgeGraphOptions, } from './hallucination/knowledge-graph.js';
43
43
  export { completeVerified, verifyAgainstContext, extractFacts, lexicalEntailment, type NliVerifier, type VerificationClient, type VerificationFact, type VerificationOptions, type VerificationReport, } from './hallucination/verification.js';
44
44
  export { completeWithSelfConsistency, selectMostConsistent, textSimilarity, type ConsistencyClient, type SelfConsistencyOptions, } from './hallucination/consistency.js';
package/dist/index.js CHANGED
@@ -37,7 +37,7 @@ export { MemoryRateLimitStore, RedisRateLimitStore, } from './ops/rate-limit-ada
37
37
  export { PipelineRunner, createPipelineContext, } from './pipeline/pipeline.js';
38
38
  export { withFactualDefaults, asJsonOnly, } from './hallucination/factual.js';
39
39
  export { withRagContext, extractCitations, validateCitations, } from './hallucination/rag.js';
40
- export { MemoryVectorStore, createHashEmbeddings, cosineSimilarity, } from './hallucination/retrieval.js';
40
+ export { MemoryVectorStore, createHashEmbeddings, cosineSimilarity, matchesMetadata, normalizeVector, } from './hallucination/retrieval.js';
41
41
  export { withKnowledgeGraphContext, selectGraphFacts, } from './hallucination/knowledge-graph.js';
42
42
  export { completeVerified, verifyAgainstContext, extractFacts, lexicalEntailment, } from './hallucination/verification.js';
43
43
  export { completeWithSelfConsistency, selectMostConsistent, textSimilarity, } from './hallucination/consistency.js';
@@ -1,8 +1,10 @@
1
+ import { warnUnreadOption } from '../utils/deprecation.js';
1
2
  export class ProviderHealthMonitor {
2
3
  config;
3
4
  states = new Map();
4
5
  constructor(config = {}) {
5
6
  this.config = config;
7
+ warnUnreadOption(config.latencyHalfLife !== undefined, 'NEXUS_DEP_HEALTH_LATENCY_HALF_LIFE', 'HealthConfig.latencyHalfLife');
6
8
  }
7
9
  recordSuccess(providerName, latencyMs) {
8
10
  if (!this.config.enabled)
@@ -1,3 +1,4 @@
1
+ import { warnUnreadOption } from '../utils/deprecation.js';
1
2
  export class InMemoryMetrics {
2
3
  counters = new Map();
3
4
  histograms = new Map();
@@ -86,6 +87,7 @@ export class MetricsCollector {
86
87
  this.config = config;
87
88
  this.sink = config.sink || this.memory;
88
89
  this.prefix = config.prefix || 'nexus_ai';
90
+ warnUnreadOption(config.prometheus !== undefined, 'NEXUS_DEP_METRICS_PROMETHEUS', 'MetricsConfig.prometheus');
89
91
  }
90
92
  async recordRequest(labels) {
91
93
  if (!this.config.enabled)
@@ -1,6 +1,7 @@
1
1
  import { Tokenizer } from '../utils/tokenizer.js';
2
2
  import { PromptDensifier } from './densifier.js';
3
3
  import { BudgetEnforcer } from './budget.js';
4
+ import { warnUnreadOption } from '../utils/deprecation.js';
4
5
  export class TokenOptimizer {
5
6
  config;
6
7
  tokenizer = new Tokenizer();
@@ -8,6 +9,7 @@ export class TokenOptimizer {
8
9
  budget = new BudgetEnforcer(this.tokenizer);
9
10
  constructor(config = {}) {
10
11
  this.config = config;
12
+ warnUnreadOption(config.densification?.preserveMarkdown !== undefined, 'NEXUS_DEP_DENSIFICATION_PRESERVE_MARKDOWN', 'DensificationConfig.preserveMarkdown');
11
13
  }
12
14
  optimize(request) {
13
15
  if (this.config.enabled === false) {
@@ -5,13 +5,17 @@ export { operationStoreMigration, PostgresOperationStore, type PostgresOperation
5
5
  export { PostgresPromptStore, type PostgresPromptStoreOptions, promptStoreMigration } from './prompts.js';
6
6
  export { PostgresStore, type PostgresStoreOptions, storeMigration } from './store.js';
7
7
  export { PostgresTraceStore, type PostgresTraceStoreOptions, traceStoreMigration } from './traces.js';
8
+ export { PostgresVectorStore, type PostgresVectorStoreOptions, vectorStoreMigration } from './vectors.js';
8
9
  /** The Postgres adapters a migration can include. */
9
- export type PostgresAdapter = 'operations' | 'store' | 'traces' | 'evaluation' | 'circuits' | 'prompts';
10
+ export type PostgresAdapter = 'operations' | 'store' | 'traces' | 'evaluation' | 'circuits' | 'prompts' | 'vectors';
10
11
  /** Options for `postgresMigration()`. */
11
12
  export interface PostgresMigrationOptions {
12
- /** Adapters to include. Defaults to all of them. */
13
+ /**
14
+ * Adapters to include. Defaults to every one but `vectors`, which needs pgvector and
15
+ * `vectorDimensions`, so it is included only when named.
16
+ */
13
17
  adapters?: readonly PostgresAdapter[];
14
- /** Enables pgvector for the long-term store. */
18
+ /** Enables pgvector for the long-term store, and sets the width of the `vectors` table. */
15
19
  vectorDimensions?: number;
16
20
  }
17
21
  /**
@@ -4,6 +4,7 @@ import { operationStoreMigration } from './operations.js';
4
4
  import { promptStoreMigration } from './prompts.js';
5
5
  import { storeMigration } from './store.js';
6
6
  import { traceStoreMigration } from './traces.js';
7
+ import { vectorStoreMigration } from './vectors.js';
7
8
  export { circuitStoreMigration, PostgresCircuitStateStore, } from './circuits.js';
8
9
  export { fromPostgresJs } from './client.js';
9
10
  export { evaluationStoreMigration, PostgresDatasetStore, PostgresExperimentStore, } from './evaluate.js';
@@ -11,6 +12,7 @@ export { operationStoreMigration, PostgresOperationStore } from './operations.js
11
12
  export { PostgresPromptStore, promptStoreMigration } from './prompts.js';
12
13
  export { PostgresStore, storeMigration } from './store.js';
13
14
  export { PostgresTraceStore, traceStoreMigration } from './traces.js';
15
+ export { PostgresVectorStore, vectorStoreMigration } from './vectors.js';
14
16
  export function postgresMigration(options = {}) {
15
17
  const adapters = new Set(options.adapters ?? ['operations', 'store', 'traces', 'evaluation', 'circuits', 'prompts']);
16
18
  const statements = [
@@ -22,6 +24,12 @@ export function postgresMigration(options = {}) {
22
24
  ...(adapters.has('evaluation') ? evaluationStoreMigration() : []),
23
25
  ...(adapters.has('circuits') ? circuitStoreMigration() : []),
24
26
  ...(adapters.has('prompts') ? promptStoreMigration() : []),
27
+ ...(adapters.has('vectors') ? vectorMigrationFor(options.vectorDimensions) : []),
25
28
  ];
26
29
  return `${statements.join(';\n\n')};\n`;
27
30
  }
31
+ function vectorMigrationFor(dimensions) {
32
+ if (dimensions === undefined)
33
+ throw new RangeError('The vectors adapter needs vectorDimensions');
34
+ return vectorStoreMigration({ dimensions });
35
+ }
@@ -0,0 +1,46 @@
1
+ import { type EmbeddingProvider, type VectorDocument, type VectorSearchOptions, type VectorSearchResult, type VectorStore } from '../hallucination/retrieval.js';
2
+ import { type PostgresLikeClient } from './client.js';
3
+ /** Options for the pgvector store. */
4
+ export interface PostgresVectorStoreOptions {
5
+ /** Width of every vector. Must match the embedding function's output. */
6
+ dimensions: number;
7
+ /** Embeds chunks that arrive without a vector, and search queries. Defaults to hashed term vectors. */
8
+ embed?: EmbeddingProvider;
9
+ /** Table name, optionally schema-qualified. Defaults to `nexus_vectors`. */
10
+ table?: string;
11
+ /**
12
+ * The approximate index created by `migrate()`. `hnsw` (the default) answers a search without
13
+ * scanning every row, at the cost of slower writes; `none` scans exactly, which suits a few
14
+ * thousand chunks.
15
+ */
16
+ index?: 'hnsw' | 'none';
17
+ }
18
+ /** The schema, as statements: the pgvector extension, the table, and its index. */
19
+ export declare function vectorStoreMigration(options: Omit<PostgresVectorStoreOptions, 'embed'>): string[];
20
+ /**
21
+ * Retrieval chunks in Postgres with pgvector, ranked by cosine similarity in the database.
22
+ *
23
+ * The same contract as `MemoryVectorStore`, checked by the same tests. Only the nearest chunks leave
24
+ * the database, so a search costs the same whether the table holds a thousand chunks or ten million.
25
+ * Adding a chunk whose id exists replaces it, and a metadata filter runs in SQL.
26
+ */
27
+ export declare class PostgresVectorStore implements VectorStore {
28
+ private readonly client;
29
+ private readonly options;
30
+ private readonly table;
31
+ private readonly embed;
32
+ constructor(client: PostgresLikeClient, options: PostgresVectorStoreOptions);
33
+ /** Creates the extension, table, and index. Never runs implicitly. */
34
+ migrate(): Promise<void>;
35
+ /** Adds chunks, or replaces those whose id exists, embedding those without a vector in one batch. */
36
+ add(documents: VectorDocument[]): Promise<void>;
37
+ /** The chunks most similar to a query, best first. */
38
+ search(query: string, options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
39
+ /** The chunks most similar to a vector, best first. */
40
+ searchVector(vector: number[], options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
41
+ /** Removes chunks by id. */
42
+ delete(ids: readonly string[]): Promise<void>;
43
+ /** Chunks stored. */
44
+ size(): Promise<number>;
45
+ private assertWidth;
46
+ }
@@ -0,0 +1,100 @@
1
+ import { createHashEmbeddings, } from '../hallucination/retrieval.js';
2
+ import { fromJson, quoteDerived, quoteTable, runStatements } from './client.js';
3
+ import { SqlParams, textArray } from './sql.js';
4
+ const DEFAULT_TABLE = 'nexus_vectors';
5
+ export function vectorStoreMigration(options) {
6
+ const dimensions = options.dimensions;
7
+ if (!(Number.isInteger(dimensions) && dimensions > 0))
8
+ throw new RangeError('dimensions must be a positive integer');
9
+ const name = options.table ?? DEFAULT_TABLE;
10
+ const table = quoteTable(name);
11
+ return [
12
+ 'CREATE EXTENSION IF NOT EXISTS vector',
13
+ `CREATE TABLE IF NOT EXISTS ${table} (
14
+ id text PRIMARY KEY,
15
+ content text NOT NULL,
16
+ source text,
17
+ metadata jsonb,
18
+ embedding vector(${dimensions}) NOT NULL
19
+ )`,
20
+ ...(options.index === 'none'
21
+ ? []
22
+ : [
23
+ `CREATE INDEX IF NOT EXISTS ${quoteDerived(name, 'embedding')} ON ${table} USING hnsw (embedding vector_cosine_ops)`,
24
+ ]),
25
+ ];
26
+ }
27
+ export class PostgresVectorStore {
28
+ client;
29
+ options;
30
+ table;
31
+ embed;
32
+ constructor(client, options) {
33
+ this.client = client;
34
+ this.options = options;
35
+ if (!(Number.isInteger(options.dimensions) && options.dimensions > 0)) {
36
+ throw new RangeError('dimensions must be a positive integer');
37
+ }
38
+ this.table = quoteTable(options.table ?? DEFAULT_TABLE);
39
+ this.embed = options.embed ?? ((texts) => createHashEmbeddings(texts, options.dimensions));
40
+ }
41
+ async migrate() {
42
+ await runStatements(this.client, vectorStoreMigration(this.options));
43
+ }
44
+ async add(documents) {
45
+ if (documents.length === 0)
46
+ return;
47
+ const missing = documents.filter((doc) => !doc.embedding).map((doc) => doc.content);
48
+ const generated = missing.length ? await this.embed(missing) : [];
49
+ let generatedIndex = 0;
50
+ const params = new SqlParams();
51
+ const rows = documents.map((doc) => {
52
+ const embedding = doc.embedding ?? generated[generatedIndex++];
53
+ this.assertWidth(embedding);
54
+ return `(${params.add(doc.id)}, ${params.add(doc.content)}, ${params.add(doc.source ?? null)}, ${params.add(doc.metadata === undefined ? null : JSON.stringify(doc.metadata))}::jsonb, ${params.add(JSON.stringify(embedding))}::vector)`;
55
+ });
56
+ await this.client.query(`INSERT INTO ${this.table} (id, content, source, metadata, embedding) VALUES ${rows.join(', ')}
57
+ ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content, source = EXCLUDED.source,
58
+ metadata = EXCLUDED.metadata, embedding = EXCLUDED.embedding`, params.values);
59
+ }
60
+ async search(query, options = {}) {
61
+ const [vector] = await this.embed([query]);
62
+ return this.searchVector(vector, options);
63
+ }
64
+ async searchVector(vector, options = {}) {
65
+ this.assertWidth(vector);
66
+ const params = new SqlParams();
67
+ const query = params.add(JSON.stringify(vector));
68
+ const conditions = [];
69
+ if (options.filter && Object.keys(options.filter).length > 0) {
70
+ conditions.push(`metadata @> ${params.add(JSON.stringify(options.filter))}::jsonb`);
71
+ }
72
+ const minScore = options.minScore ?? 0;
73
+ conditions.push(`1 - (embedding <=> ${query}::vector) >= ${params.add(minScore)}`);
74
+ const limit = params.add(options.topK || 5);
75
+ const { rows } = await this.client.query(`SELECT id, content, source, metadata::text AS metadata, 1 - (embedding <=> ${query}::vector) AS score
76
+ FROM ${this.table} WHERE ${conditions.join(' AND ')}
77
+ ORDER BY embedding <=> ${query}::vector LIMIT ${limit}`, params.values);
78
+ return rows.map((row) => ({
79
+ id: row.id,
80
+ content: row.content,
81
+ ...(row.source === null ? {} : { source: row.source }),
82
+ ...(row.metadata === null ? {} : { metadata: fromJson(row.metadata) }),
83
+ score: Number(row.score),
84
+ }));
85
+ }
86
+ async delete(ids) {
87
+ if (ids.length === 0)
88
+ return;
89
+ await this.client.query(`DELETE FROM ${this.table} WHERE id = ANY($1::text[])`, [textArray(ids)]);
90
+ }
91
+ async size() {
92
+ const { rows } = await this.client.query(`SELECT count(*)::int AS count FROM ${this.table}`);
93
+ return Number(rows[0]?.count ?? 0);
94
+ }
95
+ assertWidth(vector) {
96
+ if (!vector || vector.length !== this.options.dimensions) {
97
+ throw new RangeError(`A vector has ${vector?.length ?? 0} dimensions; this store was created with ${this.options.dimensions}`);
98
+ }
99
+ }
100
+ }
@@ -1,4 +1,5 @@
1
1
  import { BaseProvider } from './base.js';
2
+ import { warnUnreadOption } from '../utils/deprecation.js';
2
3
  import { buildMeta } from '../core/usage.js';
3
4
  import { generateRequestId } from '../utils/ids.js';
4
5
  import { createProviderHttpError, toNexusProviderError } from './errors.js';
@@ -28,6 +29,7 @@ export class GoogleProvider extends BaseProvider {
28
29
  constructor(config) {
29
30
  super();
30
31
  this.config = config;
32
+ warnUnreadOption(config.projectId !== undefined, 'NEXUS_DEP_GOOGLE_PROJECT_ID', 'GoogleProviderConfig.projectId');
31
33
  }
32
34
  async complete(request) {
33
35
  try {
@@ -1,4 +1,5 @@
1
1
  import { BaseProvider } from './base.js';
2
+ import { warnUnreadOption } from '../utils/deprecation.js';
2
3
  import { buildMeta } from '../core/usage.js';
3
4
  export class OllamaProvider extends BaseProvider {
4
5
  info = { name: 'ollama', isLocal: true };
@@ -7,6 +8,7 @@ export class OllamaProvider extends BaseProvider {
7
8
  constructor(config) {
8
9
  super();
9
10
  this.config = config;
11
+ warnUnreadOption(config.timeout !== undefined, 'NEXUS_DEP_OLLAMA_TIMEOUT', 'OllamaProviderConfig.timeout');
10
12
  }
11
13
  async getClient() {
12
14
  if (!this.client) {
@@ -1,4 +1,6 @@
1
1
  import type { RagChunk } from '../hallucination/rag.js';
2
+ export { MemoryVectorStore, createHashEmbeddings, cosineSimilarity, matchesMetadata, normalizeVector, type EmbeddingProvider, type VectorDocument, type VectorSearchOptions, type VectorSearchResult, type VectorStore, } from '../hallucination/retrieval.js';
3
+ export type { RagChunk } from '../hallucination/rag.js';
2
4
  /** A document to split into chunks. */
3
5
  export interface DocumentSource {
4
6
  /** Its id, used in chunk ids. Defaults to `doc-1`, `doc-2`, and so on. */
@@ -1,3 +1,4 @@
1
+ export { MemoryVectorStore, createHashEmbeddings, cosineSimilarity, matchesMetadata, normalizeVector, } from '../hallucination/retrieval.js';
1
2
  export function ingestDocuments(documents, options = {}) {
2
3
  const chunks = [];
3
4
  const chunkSize = options.chunkSize || 1200;