@ultimat3/ai 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +273 -0
- package/package.json +42 -0
- package/src/budget.ts +233 -0
- package/src/embeddings.ts +107 -0
- package/src/errors.ts +422 -0
- package/src/eval-baseline.ts +140 -0
- package/src/evals.ts +242 -0
- package/src/gateway.ts +203 -0
- package/src/index.ts +187 -0
- package/src/llm.ts +313 -0
- package/src/models.ts +163 -0
- package/src/pg-vector-sql.ts +198 -0
- package/src/pg-vector.ts +179 -0
- package/src/prompt.ts +169 -0
- package/src/provider.ts +405 -0
- package/src/rag.ts +186 -0
- package/src/remote-embedder.ts +143 -0
- package/src/runtime.ts +77 -0
- package/src/scorers.ts +107 -0
- package/src/sse.ts +81 -0
- package/src/tools.ts +116 -0
- package/src/vector-scope.ts +76 -0
- package/src/vector.ts +0 -0
- package/src/wire.ts +283 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// The production embedder: one HTTP client for the `/v1/embeddings` request shape.
|
|
2
|
+
//
|
|
3
|
+
// Anthropic ships no embeddings endpoint, and its documented partner speaks the same
|
|
4
|
+
// `{ model, input[] }` -> `{ data: [{ index, embedding }] }` shape as every other hosted or
|
|
5
|
+
// self-hosted embedder worth pointing at. So there is ONE remote embedder rather than one per
|
|
6
|
+
// vendor: `baseUrl` selects the provider and nothing else changes. A second class per vendor
|
|
7
|
+
// would be a second thing to learn for a difference that does not exist on the wire.
|
|
8
|
+
|
|
9
|
+
import type { Embedder } from './embeddings';
|
|
10
|
+
import { normalize } from './embeddings';
|
|
11
|
+
import { AiKeyMissingError, AiTransportError, EmbedderDimMismatchError } from './errors';
|
|
12
|
+
|
|
13
|
+
const API_KEY_ENV = 'EMBEDDINGS_API_KEY';
|
|
14
|
+
const DEFAULT_BASE_URL = 'https://api.voyageai.com/v1';
|
|
15
|
+
/** Providers cap a batch around 128 inputs; 96 leaves headroom for long texts. */
|
|
16
|
+
const DEFAULT_BATCH_SIZE = 96;
|
|
17
|
+
const DETAIL_LIMIT = 300;
|
|
18
|
+
|
|
19
|
+
export interface RemoteEmbedderInput {
|
|
20
|
+
/** The provider's model id. Doubles as the embedder name, so a store records what wrote it. */
|
|
21
|
+
readonly name: string;
|
|
22
|
+
/**
|
|
23
|
+
* Declared once, checked on EVERY response. A provider that quietly changes width would
|
|
24
|
+
* otherwise poison a store one batch at a time, and cosine similarity does not complain.
|
|
25
|
+
*/
|
|
26
|
+
readonly dimension: number;
|
|
27
|
+
/** Reads `EMBEDDINGS_API_KEY` when omitted. Absent at call time is a labelled throw. */
|
|
28
|
+
readonly apiKey?: string;
|
|
29
|
+
/** Any endpoint speaking the same shape. Defaults to Voyage. */
|
|
30
|
+
readonly baseUrl?: string;
|
|
31
|
+
/** Inputs per request. Larger calls are split; the provider's own cap is not the caller's. */
|
|
32
|
+
readonly batchSize?: number;
|
|
33
|
+
/** Injectable so a test can assert the request body without a network. Defaults to `fetch`. */
|
|
34
|
+
readonly fetch?: typeof fetch;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export class RemoteEmbedder implements Embedder {
|
|
38
|
+
readonly name: string;
|
|
39
|
+
readonly dimension: number;
|
|
40
|
+
private readonly config: RemoteEmbedderInput;
|
|
41
|
+
|
|
42
|
+
constructor(input: RemoteEmbedderInput) {
|
|
43
|
+
this.name = input.name;
|
|
44
|
+
this.dimension = input.dimension;
|
|
45
|
+
this.config = input;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Batches are issued in sequence, not in parallel: a 50k-chunk corpus fanned out at once
|
|
50
|
+
* reproduces the rate limit it is trying to get through, and order must survive either way.
|
|
51
|
+
*/
|
|
52
|
+
async embed(texts: readonly string[]): Promise<readonly Float32Array[]> {
|
|
53
|
+
if (texts.length === 0) return [];
|
|
54
|
+
const size = this.config.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
55
|
+
const vectors: Float32Array[] = [];
|
|
56
|
+
for (let start = 0; start < texts.length; start += size) {
|
|
57
|
+
vectors.push(...(await this.batch(texts.slice(start, start + size))));
|
|
58
|
+
}
|
|
59
|
+
return vectors;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
private async batch(texts: readonly string[]): Promise<readonly Float32Array[]> {
|
|
63
|
+
const apiKey = this.config.apiKey ?? Bun.env[API_KEY_ENV];
|
|
64
|
+
if (apiKey === undefined || apiKey === '') {
|
|
65
|
+
throw new AiKeyMissingError({ provider: this.name, envVar: API_KEY_ENV });
|
|
66
|
+
}
|
|
67
|
+
const doFetch = this.config.fetch ?? fetch;
|
|
68
|
+
const response = await doFetch(`${this.config.baseUrl ?? DEFAULT_BASE_URL}/embeddings`, {
|
|
69
|
+
method: 'POST',
|
|
70
|
+
headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
|
|
71
|
+
body: JSON.stringify({ model: this.name, input: texts }),
|
|
72
|
+
});
|
|
73
|
+
if (!response.ok) {
|
|
74
|
+
throw new AiTransportError({
|
|
75
|
+
provider: this.name,
|
|
76
|
+
status: response.status,
|
|
77
|
+
detail: (await response.text().catch(() => '')).slice(0, DETAIL_LIMIT),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
return this.decode((await response.json()) as unknown, texts.length);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private decode(payload: unknown, expected: number): readonly Float32Array[] {
|
|
84
|
+
const data = asRecord(payload)?.['data'];
|
|
85
|
+
if (!Array.isArray(data) || data.length !== expected) {
|
|
86
|
+
throw this.malformed(`expected ${expected} embeddings, got ${countOf(data)}`);
|
|
87
|
+
}
|
|
88
|
+
// Keyed by the provider's own `index` rather than array position: the two agree today, and
|
|
89
|
+
// a store written in the wrong order is a silent relevance collapse if they ever stop.
|
|
90
|
+
const vectors = new Array<Float32Array | undefined>(expected);
|
|
91
|
+
for (let position = 0; position < data.length; position += 1) {
|
|
92
|
+
const entry = asRecord(data[position]);
|
|
93
|
+
const index = entry?.['index'];
|
|
94
|
+
const slot = typeof index === 'number' ? index : position;
|
|
95
|
+
if (!Number.isInteger(slot) || slot < 0 || slot >= expected) {
|
|
96
|
+
throw this.malformed(`embedding at position ${position} claims index ${String(index)}`);
|
|
97
|
+
}
|
|
98
|
+
vectors[slot] = this.vectorOf(entry?.['embedding'], position);
|
|
99
|
+
}
|
|
100
|
+
const missing = vectors.indexOf(undefined);
|
|
101
|
+
if (missing !== -1) throw this.malformed(`no embedding for input ${missing}`);
|
|
102
|
+
return vectors.filter((vector): vector is Float32Array => vector !== undefined);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* L2 normalised on the way in, like every other embedder here, so `cosine` stays a dot
|
|
107
|
+
* product. Providers already return unit vectors, which makes this a no-op — but "already"
|
|
108
|
+
* is a property of today's provider, not of the interface.
|
|
109
|
+
*/
|
|
110
|
+
private vectorOf(raw: unknown, position: number): Float32Array {
|
|
111
|
+
if (!Array.isArray(raw))
|
|
112
|
+
throw this.malformed(`embedding at position ${position} is not an array`);
|
|
113
|
+
if (raw.length !== this.dimension) {
|
|
114
|
+
throw new EmbedderDimMismatchError({
|
|
115
|
+
embedder: this.name,
|
|
116
|
+
expected: this.dimension,
|
|
117
|
+
received: raw.length,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const vector = new Float32Array(raw.length);
|
|
121
|
+
for (let i = 0; i < raw.length; i += 1) {
|
|
122
|
+
const value: unknown = raw[i];
|
|
123
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
124
|
+
throw this.malformed(`embedding at position ${position} has a non-numeric component`);
|
|
125
|
+
}
|
|
126
|
+
vector[i] = value;
|
|
127
|
+
}
|
|
128
|
+
return normalize(vector);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private malformed(detail: string): AiTransportError {
|
|
132
|
+
return new AiTransportError({ provider: this.name, detail });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
137
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined;
|
|
138
|
+
return value as Record<string, unknown>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function countOf(data: unknown): string {
|
|
142
|
+
return Array.isArray(data) ? String(data.length) : 'no data array';
|
|
143
|
+
}
|
package/src/runtime.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// The ambient AI runtime an `llm()` action calls into: the gateway that serves every model
|
|
2
|
+
// call, the embedder the semantic cache keys on, and one semantic cache per declared scope.
|
|
3
|
+
//
|
|
4
|
+
// Ambient rather than injected, for the same reason the budget ledger is: an `llm()`
|
|
5
|
+
// declaration is evaluated at module scope, long before a provider, an API key or a request
|
|
6
|
+
// exists. Boot installs one runtime, a test installs a fake, and there is no second way for
|
|
7
|
+
// an action to reach a provider — which is what keeps budgets and cost accounting
|
|
8
|
+
// un-bypassable rather than merely recommended.
|
|
9
|
+
|
|
10
|
+
import type { SemanticCache } from '@ultimat3/cache';
|
|
11
|
+
import { createMemorySemanticCache } from '@ultimat3/cache';
|
|
12
|
+
import type { Embedder } from './embeddings';
|
|
13
|
+
import { HashEmbedder } from './embeddings';
|
|
14
|
+
import { AiGatewayMissingError } from './errors';
|
|
15
|
+
import type { Gateway } from './gateway';
|
|
16
|
+
|
|
17
|
+
export interface AiRuntimeInput {
|
|
18
|
+
readonly gateway: Gateway;
|
|
19
|
+
/**
|
|
20
|
+
* Keys the semantic cache. Defaults to `HashEmbedder`: deterministic, offline, and lexical
|
|
21
|
+
* rather than semantic — at the thresholds an LLM cache runs at that is a near-exact-match
|
|
22
|
+
* cache, which is the safe way to be wrong. Install a real embedder to get paraphrase hits.
|
|
23
|
+
*/
|
|
24
|
+
readonly embedder?: Embedder;
|
|
25
|
+
/** One cache per scope key. Defaults to the in-memory driver; pgvector in production. */
|
|
26
|
+
readonly semanticCache?: (scope: string) => SemanticCache;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface AiRuntime {
|
|
30
|
+
readonly gateway: Gateway;
|
|
31
|
+
readonly embedder: Embedder;
|
|
32
|
+
readonly semanticCache: (scope: string) => SemanticCache;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
let runtime: AiRuntime | undefined;
|
|
36
|
+
const caches = new Map<string, SemanticCache>();
|
|
37
|
+
|
|
38
|
+
export function configureAi(input: AiRuntimeInput): void {
|
|
39
|
+
runtime = {
|
|
40
|
+
gateway: input.gateway,
|
|
41
|
+
embedder: input.embedder ?? new HashEmbedder(),
|
|
42
|
+
semanticCache: input.semanticCache ?? (() => createMemorySemanticCache()),
|
|
43
|
+
};
|
|
44
|
+
// A new runtime means a new embedder and a new gateway; vectors from the old one are not
|
|
45
|
+
// comparable to vectors from the new one, and a stale hit would answer the wrong question.
|
|
46
|
+
caches.clear();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The installed gateway. `prompt` names the caller so the miss says what was about to run. */
|
|
50
|
+
export function aiGateway(prompt: string): Gateway {
|
|
51
|
+
if (runtime === undefined) throw new AiGatewayMissingError({ prompt });
|
|
52
|
+
return runtime.gateway;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function aiEmbedder(): Embedder {
|
|
56
|
+
return runtime?.embedder ?? new HashEmbedder();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The cache for one scope. Scopes are separate CACHE INSTANCES, never a filter over a shared
|
|
61
|
+
* one: cosine similarity has no notion of a tenant, so two tenants asking near-identical
|
|
62
|
+
* questions of a shared cache is one tenant reading the other's answer. Partitioning is the
|
|
63
|
+
* only thing that makes that structurally impossible.
|
|
64
|
+
*/
|
|
65
|
+
export function semanticCacheFor(scope: string): SemanticCache {
|
|
66
|
+
const existing = caches.get(scope);
|
|
67
|
+
if (existing !== undefined) return existing;
|
|
68
|
+
const created = runtime?.semanticCache(scope) ?? createMemorySemanticCache();
|
|
69
|
+
caches.set(scope, created);
|
|
70
|
+
return created;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Test-only reset. Module-level state otherwise leaks between test files. */
|
|
74
|
+
export function resetAiRuntime(): void {
|
|
75
|
+
runtime = undefined;
|
|
76
|
+
caches.clear();
|
|
77
|
+
}
|
package/src/scorers.ts
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// What a scorer is, and the built-in ones.
|
|
2
|
+
//
|
|
3
|
+
// A scorer returns 0..1 for one answer. Deterministic scorers are preferred and cheap; a judge
|
|
4
|
+
// is a model call, which means it is itself a measuring instrument that can drift — so its
|
|
5
|
+
// prompt is a versioned artifact and its hash is part of the scorer's name.
|
|
6
|
+
|
|
7
|
+
import type { Gateway } from './gateway';
|
|
8
|
+
import type { Prompt } from './prompt';
|
|
9
|
+
|
|
10
|
+
/** A scorer returns 0..1. Deterministic scorers are preferred; judges are a last resort. */
|
|
11
|
+
export interface Scorer {
|
|
12
|
+
readonly name: string;
|
|
13
|
+
score(input: { output: string; expected?: string }): Promise<number> | number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Scores outside 0..1 are a scorer bug; clamping keeps one from skewing a whole run's mean. */
|
|
17
|
+
export const clampScore = (n: number): number => (n < 0 ? 0 : n > 1 ? 1 : n);
|
|
18
|
+
|
|
19
|
+
/** Exact match after trimming. The strictest and cheapest scorer; prefer it when it fits. */
|
|
20
|
+
export const exact: Scorer = {
|
|
21
|
+
name: 'exact',
|
|
22
|
+
score: ({ output, expected }) => (output.trim() === (expected ?? '').trim() ? 1 : 0),
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** Case-insensitive substring. For "did it mention X" without pinning the phrasing. */
|
|
26
|
+
export const contains: Scorer = {
|
|
27
|
+
name: 'contains',
|
|
28
|
+
score: ({ output, expected }) =>
|
|
29
|
+
expected !== undefined && output.toLowerCase().includes(expected.toLowerCase()) ? 1 : 0,
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** Output parses as JSON. Pairs with a prompt that declares an `output` schema. */
|
|
33
|
+
export const jsonValid: Scorer = {
|
|
34
|
+
name: 'json-valid',
|
|
35
|
+
score: ({ output }) => {
|
|
36
|
+
try {
|
|
37
|
+
JSON.parse(output);
|
|
38
|
+
return 1;
|
|
39
|
+
} catch {
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** Output parses AND satisfies the required keys of `schema`. */
|
|
46
|
+
export function jsonSchemaValid(required: readonly string[]): Scorer {
|
|
47
|
+
return {
|
|
48
|
+
name: 'json-schema-valid',
|
|
49
|
+
score: ({ output }) => {
|
|
50
|
+
let parsed: unknown;
|
|
51
|
+
try {
|
|
52
|
+
parsed = JSON.parse(output);
|
|
53
|
+
} catch {
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
if (typeof parsed !== 'object' || parsed === null) return 0;
|
|
57
|
+
const record = parsed as Record<string, unknown>;
|
|
58
|
+
const present = required.filter((key) => record[key] !== undefined).length;
|
|
59
|
+
return required.length === 0 ? 1 : present / required.length;
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Graded numeric closeness — 1 at exact, 0 at or beyond `tolerance`. */
|
|
65
|
+
export function numericTolerance(tolerance: number): Scorer {
|
|
66
|
+
return {
|
|
67
|
+
name: 'numeric-tolerance',
|
|
68
|
+
score: ({ output, expected }) => {
|
|
69
|
+
const got = Number.parseFloat(output.trim());
|
|
70
|
+
const want = Number.parseFloat((expected ?? '').trim());
|
|
71
|
+
if (Number.isNaN(got) || Number.isNaN(want)) return 0;
|
|
72
|
+
const delta = Math.abs(got - want);
|
|
73
|
+
return delta >= tolerance ? 0 : 1 - delta / tolerance;
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* LLM-as-judge. The judge prompt is itself a versioned artifact, so a judge change is a new
|
|
80
|
+
* hash and every score it produced is re-attributable. A judge whose prompt drifts silently
|
|
81
|
+
* is a measuring instrument that lies.
|
|
82
|
+
*/
|
|
83
|
+
export function llmJudge(input: {
|
|
84
|
+
readonly gateway: Gateway;
|
|
85
|
+
readonly judge: Prompt<{ output: string; expected: string }>;
|
|
86
|
+
readonly maxTokens?: number;
|
|
87
|
+
}): Scorer {
|
|
88
|
+
return {
|
|
89
|
+
name: `llm-judge@${input.judge.hash}`,
|
|
90
|
+
async score({ output, expected }) {
|
|
91
|
+
const generated = await input.gateway.generate({
|
|
92
|
+
messages: [
|
|
93
|
+
{
|
|
94
|
+
role: 'user',
|
|
95
|
+
content: input.judge.render({ output, expected: expected ?? '' }),
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
maxTokens: input.maxTokens ?? 256,
|
|
99
|
+
...(input.judge.system !== undefined ? { system: input.judge.system } : {}),
|
|
100
|
+
...(input.judge.model !== undefined ? { model: input.judge.model } : {}),
|
|
101
|
+
});
|
|
102
|
+
// The judge is asked for a bare 0..1; anything else scores 0 rather than guessing.
|
|
103
|
+
const parsed = Number.parseFloat(generated.text.trim());
|
|
104
|
+
return Number.isNaN(parsed) ? 0 : clampScore(parsed);
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
}
|
package/src/sse.ts
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Single responsibility: framing a Server-Sent Events byte stream into `{ event, data }`
|
|
2
|
+
// frames. Protocol level only — nothing here knows what an Anthropic message looks like.
|
|
3
|
+
//
|
|
4
|
+
// Hand written rather than a dependency: the part of the SSE spec that matters to a provider
|
|
5
|
+
// stream is a boundary search and a field split, and the only interesting property — that a
|
|
6
|
+
// frame may arrive split at any byte offset — is exactly what a library would hide.
|
|
7
|
+
|
|
8
|
+
export interface SseFrame {
|
|
9
|
+
/** The `event:` field, or `message` when the frame omits one — the spec's default. */
|
|
10
|
+
readonly event: string;
|
|
11
|
+
/** Every `data:` line of the frame, joined with `\n` as the spec requires. */
|
|
12
|
+
readonly data: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** `\r\n\r\n`, `\n\n` and `\r\r` all end a frame. A lone trailing `\r` is not yet a boundary. */
|
|
16
|
+
const BOUNDARY = /\r\n\r\n|\n\n|\r\r/;
|
|
17
|
+
const LINE = /\r\n|\n|\r/;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Split what has arrived so far into complete frames plus the unterminated remainder. Pure and
|
|
21
|
+
* synchronous, so a test can feed it a stream chopped at any offset — which is the only thing
|
|
22
|
+
* a network does that a fixture does not.
|
|
23
|
+
*/
|
|
24
|
+
export function decodeSse(buffer: string): { readonly frames: readonly SseFrame[]; rest: string } {
|
|
25
|
+
const frames: SseFrame[] = [];
|
|
26
|
+
let rest = buffer;
|
|
27
|
+
for (;;) {
|
|
28
|
+
const boundary = BOUNDARY.exec(rest);
|
|
29
|
+
if (boundary === null) break;
|
|
30
|
+
const frame = frameOf(rest.slice(0, boundary.index));
|
|
31
|
+
if (frame !== undefined) frames.push(frame);
|
|
32
|
+
rest = rest.slice(boundary.index + boundary[0].length);
|
|
33
|
+
}
|
|
34
|
+
return { frames, rest };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** One frame's block of lines. `undefined` for a heartbeat comment, which carries no data. */
|
|
38
|
+
function frameOf(block: string): SseFrame | undefined {
|
|
39
|
+
let event = 'message';
|
|
40
|
+
const data: string[] = [];
|
|
41
|
+
for (const line of block.split(LINE)) {
|
|
42
|
+
// A line starting with `:` is a comment — providers send them to hold the socket open.
|
|
43
|
+
if (line === '' || line.startsWith(':')) continue;
|
|
44
|
+
const colon = line.indexOf(':');
|
|
45
|
+
const field = colon === -1 ? line : line.slice(0, colon);
|
|
46
|
+
const raw = colon === -1 ? '' : line.slice(colon + 1);
|
|
47
|
+
const value = raw.startsWith(' ') ? raw.slice(1) : raw;
|
|
48
|
+
if (field === 'event') event = value;
|
|
49
|
+
if (field === 'data') data.push(value);
|
|
50
|
+
}
|
|
51
|
+
if (data.length === 0) return undefined;
|
|
52
|
+
return { event, data: data.join('\n') };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Read a response body as SSE frames. The trailing partial buffer IS flushed as a final frame:
|
|
57
|
+
* a stream cut mid-message must fail loudly at the consumer that parses it, and dropping the
|
|
58
|
+
* tail silently would turn a truncated answer into a complete-looking one.
|
|
59
|
+
*/
|
|
60
|
+
export async function* readSse(body: ReadableStream<Uint8Array>): AsyncGenerator<SseFrame> {
|
|
61
|
+
const reader = body.getReader();
|
|
62
|
+
const decoder = new TextDecoder();
|
|
63
|
+
let buffer = '';
|
|
64
|
+
try {
|
|
65
|
+
for (;;) {
|
|
66
|
+
const { done, value } = await reader.read();
|
|
67
|
+
if (done) break;
|
|
68
|
+
buffer += decoder.decode(value, { stream: true });
|
|
69
|
+
const decoded = decodeSse(buffer);
|
|
70
|
+
buffer = decoded.rest;
|
|
71
|
+
for (const frame of decoded.frames) yield frame;
|
|
72
|
+
}
|
|
73
|
+
buffer += decoder.decode();
|
|
74
|
+
const tail = frameOf(buffer);
|
|
75
|
+
if (tail !== undefined) yield tail;
|
|
76
|
+
} finally {
|
|
77
|
+
// Releases the socket when a consumer abandons the stream half way — `for await` runs
|
|
78
|
+
// this through the generator's `return()`, so an early `break` does not leak a connection.
|
|
79
|
+
await reader.cancel().catch(() => undefined);
|
|
80
|
+
}
|
|
81
|
+
}
|
package/src/tools.ts
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// Project framework `action`s into LLM tool definitions.
|
|
2
|
+
//
|
|
3
|
+
// This is the SAME projection @ultimat3/mcp performs, in a different wire format: an
|
|
4
|
+
// in-app agent calling a tool through the gateway and an external agent calling it over
|
|
5
|
+
// MCP both end at `action.run`, so they authorize identically. There is no "LLM
|
|
6
|
+
// permissions" concept in Ultimate, because there is no second authz system.
|
|
7
|
+
//
|
|
8
|
+
// The JSON Schema type and the projectable-primitive shape are declared here rather than
|
|
9
|
+
// imported from @ultimat3/mcp: that package is the same tier, so importing it would be a
|
|
10
|
+
// boundary error. Both packages describe the same structural contract.
|
|
11
|
+
|
|
12
|
+
import type { Actor } from '@ultimat3/core';
|
|
13
|
+
|
|
14
|
+
/** The JSON Schema subset the framework emits for tool arguments. */
|
|
15
|
+
export interface JsonSchema {
|
|
16
|
+
readonly type?: 'object' | 'string' | 'number' | 'integer' | 'boolean' | 'array' | 'null';
|
|
17
|
+
readonly description?: string;
|
|
18
|
+
readonly properties?: Readonly<Record<string, JsonSchema>>;
|
|
19
|
+
readonly required?: readonly string[];
|
|
20
|
+
/** A schema, not just a flag: `@ultimat3/schema` emits one for open records. */
|
|
21
|
+
readonly additionalProperties?: boolean | JsonSchema;
|
|
22
|
+
readonly items?: JsonSchema;
|
|
23
|
+
readonly enum?: readonly (string | number | boolean | null)[];
|
|
24
|
+
readonly default?: unknown;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Anthropic Messages API tool definition shape (`tools[]` on a request). */
|
|
28
|
+
export interface LlmTool {
|
|
29
|
+
readonly name: string;
|
|
30
|
+
readonly description: string;
|
|
31
|
+
readonly input_schema: JsonSchema;
|
|
32
|
+
/**
|
|
33
|
+
* Strict mode guarantees the model's `input` validates against the schema exactly.
|
|
34
|
+
* Requires `additionalProperties: false` plus `required` — which every projected action
|
|
35
|
+
* schema already has, so it is on by default.
|
|
36
|
+
*/
|
|
37
|
+
readonly strict?: boolean;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** A tool call the model asked for, as it arrives in a `tool_use` content block. */
|
|
41
|
+
export interface LlmToolCall {
|
|
42
|
+
readonly id: string;
|
|
43
|
+
readonly name: string;
|
|
44
|
+
readonly input: Record<string, unknown>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The result sent back as a `tool_result` block. `isError` is an expected failure. */
|
|
48
|
+
export interface LlmToolResult {
|
|
49
|
+
readonly toolUseId: string;
|
|
50
|
+
readonly content: string;
|
|
51
|
+
readonly isError?: boolean;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The surface a projectable primitive must expose. Structurally identical to
|
|
56
|
+
* @ultimat3/mcp's `ProjectablePrimitive` — one contract, two wire formats.
|
|
57
|
+
*/
|
|
58
|
+
export interface ProjectableAction {
|
|
59
|
+
readonly name: string;
|
|
60
|
+
readonly description?: string;
|
|
61
|
+
readonly mcp?: { readonly expose?: boolean; readonly description?: string };
|
|
62
|
+
readonly inputJsonSchema?: JsonSchema;
|
|
63
|
+
run(args: { input: unknown; actor: Actor }): Promise<unknown>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const EMPTY_SCHEMA: JsonSchema = { type: 'object', properties: {}, additionalProperties: false };
|
|
67
|
+
|
|
68
|
+
/** One action → one LLM tool definition. Opt-in via `mcp.expose`, same flag as MCP. */
|
|
69
|
+
export function toLlmTool(action: ProjectableAction): LlmTool {
|
|
70
|
+
return {
|
|
71
|
+
name: action.name,
|
|
72
|
+
description:
|
|
73
|
+
action.mcp?.description ?? action.description ?? `Run the "${action.name}" action.`,
|
|
74
|
+
input_schema: action.inputJsonSchema ?? EMPTY_SCHEMA,
|
|
75
|
+
strict: true,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Every exposed action as a tool definition, in stable name order. */
|
|
80
|
+
export function toLlmTools(actions: readonly ProjectableAction[]): readonly LlmTool[] {
|
|
81
|
+
return actions
|
|
82
|
+
.filter((a) => a.mcp?.expose === true)
|
|
83
|
+
.map(toLlmTool)
|
|
84
|
+
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Execute a model-requested tool call against the projected actions. The actor comes from
|
|
89
|
+
* the request context, NOT from the model — a model cannot name the identity it acts as.
|
|
90
|
+
*/
|
|
91
|
+
export async function runLlmToolCall(
|
|
92
|
+
actions: readonly ProjectableAction[],
|
|
93
|
+
call: LlmToolCall,
|
|
94
|
+
actor: Actor,
|
|
95
|
+
): Promise<LlmToolResult> {
|
|
96
|
+
const action = actions.find((a) => a.name === call.name && a.mcp?.expose === true);
|
|
97
|
+
if (action === undefined) {
|
|
98
|
+
return { toolUseId: call.id, content: `unknown tool: ${call.name}`, isError: true };
|
|
99
|
+
}
|
|
100
|
+
try {
|
|
101
|
+
const output = await action.run({ input: call.input, actor });
|
|
102
|
+
return { toolUseId: call.id, content: JSON.stringify(output) };
|
|
103
|
+
} catch (error) {
|
|
104
|
+
// A policy denial is an outcome the model should read and react to, not a crash.
|
|
105
|
+
return { toolUseId: call.id, content: describeFailure(error), isError: true };
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function describeFailure(error: unknown): string {
|
|
110
|
+
if (typeof error !== 'object' || error === null) return 'tool failed';
|
|
111
|
+
const e = error as { code?: unknown; cause?: unknown; fix?: unknown };
|
|
112
|
+
if (typeof e.code !== 'string') return 'tool failed';
|
|
113
|
+
const cause = typeof e.cause === 'string' ? e.cause : 'unknown';
|
|
114
|
+
const fix = typeof e.fix === 'string' ? e.fix : '';
|
|
115
|
+
return fix === '' ? `${e.code}: ${cause}` : `${e.code}: ${cause} (fix: ${fix})`;
|
|
116
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// The security envelope a vector store carries: the tenant it is bound to, and the metadata
|
|
2
|
+
// values a policy leaves visible. Kept separate from the stores because both the in-memory dev
|
|
3
|
+
// store and the Postgres one must enforce the SAME envelope — a leak that only reproduces in
|
|
4
|
+
// production is a leak nobody finds. Deriving a scope may only ever TIGHTEN it: a scope that
|
|
5
|
+
// could be widened from a call site is not a scope, it is a hint.
|
|
6
|
+
|
|
7
|
+
import { VectorScopeWidenedError } from './errors';
|
|
8
|
+
|
|
9
|
+
export interface VectorScope {
|
|
10
|
+
/** Bound tenant. Present ⇒ every read, write and delete carries `tenant = <this>`. */
|
|
11
|
+
readonly tenant?: string | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* A policy projected onto metadata: per key, the exact values that stay visible. Default
|
|
14
|
+
* deny — a row missing the key is invisible, and an empty list matches nothing at all.
|
|
15
|
+
*/
|
|
16
|
+
readonly allow?: Readonly<Record<string, readonly string[]>> | undefined;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The store as constructed: every tenant, every row. The backfill and migration path. */
|
|
20
|
+
export const UNSCOPED: VectorScope = Object.freeze({});
|
|
21
|
+
|
|
22
|
+
/** The tenant column value a row carries when its store had no tenant bound. */
|
|
23
|
+
export const NO_TENANT = '';
|
|
24
|
+
|
|
25
|
+
export function tenantOf(scope: VectorScope): string {
|
|
26
|
+
return scope.tenant ?? NO_TENANT;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Derive a narrower scope. Tenants may be SET once, never changed; allow-lists intersect, so a
|
|
31
|
+
* derived scope can add a key or shrink a list but can never restore a value the parent removed.
|
|
32
|
+
*/
|
|
33
|
+
export function narrowScope(store: string, base: VectorScope, next: VectorScope): VectorScope {
|
|
34
|
+
const tenant = narrowTenant(store, base.tenant, next.tenant);
|
|
35
|
+
const allow = narrowAllow(base.allow, next.allow);
|
|
36
|
+
return {
|
|
37
|
+
...(tenant === undefined ? {} : { tenant }),
|
|
38
|
+
...(allow === undefined ? {} : { allow }),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function narrowTenant(
|
|
43
|
+
store: string,
|
|
44
|
+
base: string | undefined,
|
|
45
|
+
next: string | undefined,
|
|
46
|
+
): string | undefined {
|
|
47
|
+
if (next === undefined) return base;
|
|
48
|
+
if (base === undefined || base === next) return next;
|
|
49
|
+
throw new VectorScopeWidenedError({ store, held: base, requested: next });
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function narrowAllow(
|
|
53
|
+
base: Readonly<Record<string, readonly string[]>> | undefined,
|
|
54
|
+
next: Readonly<Record<string, readonly string[]>> | undefined,
|
|
55
|
+
): Readonly<Record<string, readonly string[]>> | undefined {
|
|
56
|
+
if (next === undefined) return base;
|
|
57
|
+
const merged: Record<string, readonly string[]> = { ...(base ?? {}) };
|
|
58
|
+
for (const [key, values] of Object.entries(next)) {
|
|
59
|
+
const held = merged[key];
|
|
60
|
+
merged[key] = held === undefined ? values : values.filter((value) => held.includes(value));
|
|
61
|
+
}
|
|
62
|
+
return merged;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Whether one stored row survives the scope. The in-memory twin of the SQL conditions. */
|
|
66
|
+
export function scopeAdmits(
|
|
67
|
+
scope: VectorScope,
|
|
68
|
+
tenant: string,
|
|
69
|
+
metadata: Readonly<Record<string, string>>,
|
|
70
|
+
): boolean {
|
|
71
|
+
if (scope.tenant !== undefined && scope.tenant !== tenant) return false;
|
|
72
|
+
return Object.entries(scope.allow ?? {}).every(([key, values]) => {
|
|
73
|
+
const value = metadata[key];
|
|
74
|
+
return value !== undefined && values.includes(value);
|
|
75
|
+
});
|
|
76
|
+
}
|
package/src/vector.ts
ADDED
|
Binary file
|