@ultimat3/ai 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +96 -32
- package/README.md +249 -75
- package/package.json +12 -11
- package/src/agent-facts.ts +9 -2
- package/src/agent-transcript.ts +1 -1
- package/src/agent.ts +43 -25
- package/src/budget.ts +178 -51
- package/src/content-blocks.ts +227 -0
- package/src/content-errors.ts +67 -0
- package/src/echo-provider.ts +102 -0
- package/src/embeddings.ts +40 -21
- package/src/error-body.ts +38 -3
- package/src/errors.ts +19 -11
- package/src/eval-baseline.ts +5 -1
- package/src/eval-errors.ts +3 -1
- package/src/evals.ts +1 -1
- package/src/failure-disclosure.ts +90 -0
- package/src/gateway.ts +73 -37
- package/src/hive-pool.ts +24 -16
- package/src/hive-result.ts +4 -1
- package/src/hive.ts +15 -5
- package/src/index.ts +66 -25
- package/src/llm-cache.ts +4 -3
- package/src/llm.ts +27 -65
- package/src/model-resolve.ts +79 -0
- package/src/models.ts +125 -111
- package/src/openai-body.ts +13 -2
- package/src/openai-messages.ts +86 -15
- package/src/openai-provider.ts +28 -31
- package/src/openai-wire.ts +11 -3
- package/src/pg-vector-sql.ts +11 -7
- package/src/pg-vector.ts +21 -10
- package/src/prompt-fence.ts +128 -0
- package/src/prompt.ts +43 -13
- package/src/provider.ts +82 -117
- package/src/rag.ts +17 -9
- package/src/remote-embedder.ts +15 -41
- package/src/respond.ts +86 -0
- package/src/runtime.ts +12 -14
- package/src/scorers.ts +14 -3
- package/src/tool-idempotency.ts +79 -0
- package/src/tools.ts +52 -39
- package/src/vector-scope.ts +46 -4
- package/src/vector.ts +117 -41
- package/src/wire.ts +7 -2
- package/src/fix-line.evals.ts +0 -35
- package/src/fix-line.ts +0 -27
- package/src/openai-models.ts +0 -84
package/src/scorers.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// is a model call, which means it is itself a measuring instrument that can drift — so its
|
|
5
5
|
// prompt is a versioned artifact and its hash is part of the scorer's name.
|
|
6
6
|
|
|
7
|
-
import { finiteCount } from '@ultimat3/core';
|
|
7
|
+
import { assert, finiteCount, finiteOption } from '@ultimat3/core';
|
|
8
8
|
import type { Gateway } from './gateway';
|
|
9
9
|
import type { Prompt } from './prompt';
|
|
10
10
|
|
|
@@ -81,8 +81,17 @@ export function jsonSchemaValid(required: readonly string[]): Scorer {
|
|
|
81
81
|
};
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* Graded numeric closeness — 1 at exact, 0 at or beyond `tolerance`. `tolerance: 0` is exact match.
|
|
86
|
+
* Screened at declaration: a `NaN` scored every answer `NaN`, a negative one every answer 0.
|
|
87
|
+
*/
|
|
85
88
|
export function numericTolerance(tolerance: number): Scorer {
|
|
89
|
+
finiteOption('numericTolerance', 'tolerance', tolerance);
|
|
90
|
+
assert(
|
|
91
|
+
tolerance >= 0,
|
|
92
|
+
`numericTolerance tolerance is ${String(tolerance)}: a distance cannot be negative, so every answer would score 0`,
|
|
93
|
+
'numericTolerance(0) # 0 is exact match; a positive tolerance grades closeness',
|
|
94
|
+
);
|
|
86
95
|
return {
|
|
87
96
|
name: 'numeric-tolerance',
|
|
88
97
|
score: ({ output, expected }) => {
|
|
@@ -90,6 +99,8 @@ export function numericTolerance(tolerance: number): Scorer {
|
|
|
90
99
|
const want = Number.parseFloat((expected ?? '').trim());
|
|
91
100
|
if (Number.isNaN(got) || Number.isNaN(want)) return 0;
|
|
92
101
|
const delta = Math.abs(got - want);
|
|
102
|
+
// Exact first: at `tolerance: 0`, `0 >= 0` read an exact match as out of range.
|
|
103
|
+
if (delta === 0) return 1;
|
|
93
104
|
return delta >= tolerance ? 0 : 1 - delta / tolerance;
|
|
94
105
|
},
|
|
95
106
|
};
|
|
@@ -121,7 +132,7 @@ export function llmJudge(input: {
|
|
|
121
132
|
maxTokens: finiteCount('llmJudge', 'maxTokens', input.maxTokens ?? 256, 1),
|
|
122
133
|
...(input.judge.system !== undefined ? { system: input.judge.system } : {}),
|
|
123
134
|
...(input.judge.model !== undefined ? { model: input.judge.model } : {}),
|
|
124
|
-
// The judge prompt's hash is this scorer's NAME, and `
|
|
135
|
+
// The judge prompt's hash is this scorer's NAME, and `promptHash` covers `effort` and
|
|
125
136
|
// `thinking` — so dropping either measures with a judge the name does not describe.
|
|
126
137
|
...(input.judge.effort !== undefined ? { effort: input.judge.effort } : {}),
|
|
127
138
|
...(input.judge.thinking !== undefined ? { thinking: input.judge.thinking } : {}),
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// The in-app half of `idempotent: true`: an agent's tool carries the reserved `idempotencyKey`
|
|
2
|
+
// argument exactly as `@ultimat3/mcp`'s does, so an action an external agent may retry safely is
|
|
3
|
+
// one an in-app agent may retry safely too. Restated, not imported — `mcp` is this package's own
|
|
4
|
+
// tier — and held equal to `mcp/src/idempotency-arg.ts` by `scripts/ai-mcp-tool-parity.test.ts`.
|
|
5
|
+
|
|
6
|
+
import { MAX_IDEMPOTENCY_KEY_LENGTH } from '@ultimat3/action';
|
|
7
|
+
import { UltimateError } from '@ultimat3/core';
|
|
8
|
+
import type { JsonSchema } from './tools';
|
|
9
|
+
|
|
10
|
+
/** The reserved argument name on every idempotent action's tool — the one MCP reserves. */
|
|
11
|
+
export const TOOL_IDEMPOTENCY_KEY_ARG = 'idempotencyKey';
|
|
12
|
+
|
|
13
|
+
/** Contract text, byte-identical to MCP's: it is what the model reads on either surface. */
|
|
14
|
+
const DESCRIPTION =
|
|
15
|
+
'Idempotency key. Retrying this call with the same key and the same arguments returns the ' +
|
|
16
|
+
'first result instead of running it again; the same key with different arguments is refused. ' +
|
|
17
|
+
'Any unique string, e.g. a UUID — reuse it only to retry.';
|
|
18
|
+
|
|
19
|
+
const KEY_SCHEMA: JsonSchema = {
|
|
20
|
+
type: 'string',
|
|
21
|
+
description: DESCRIPTION,
|
|
22
|
+
minLength: 1,
|
|
23
|
+
maxLength: MAX_IDEMPOTENCY_KEY_LENGTH,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* MCP's own refusal, raised for the same tool: an agent tool is an MCP-exposed action by
|
|
28
|
+
* construction (`X_AGENT_TOOL_UNEXPOSED`), so this is the projection `@ultimat3/mcp` refuses too,
|
|
29
|
+
* and one code for it on both surfaces is the parity. Never titled here — `mcp` registers it.
|
|
30
|
+
*/
|
|
31
|
+
export class ToolIdempotencyKeyShadowedError extends UltimateError {
|
|
32
|
+
constructor(input: { readonly name: string; readonly argument: string }) {
|
|
33
|
+
super({
|
|
34
|
+
code: 'X_MCP_IDEMPOTENCY_KEY_SHADOWED',
|
|
35
|
+
cause: `${input.name} is idempotent and its input declares "${input.argument}", the MCP argument reserved for the idempotency key`,
|
|
36
|
+
fix: `rename ${input.name}'s "${input.argument}" input field — MCP clients send the Idempotency-Key as the reserved "${input.argument}" argument, and the action receives the rest`,
|
|
37
|
+
meta: { action: input.name, argument: input.argument },
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** `schema` with the optional key argument; a non-object input is left alone, a clash refused. */
|
|
43
|
+
export function withToolIdempotencyKey(name: string, schema: JsonSchema): JsonSchema {
|
|
44
|
+
if (schema.type !== 'object') return schema;
|
|
45
|
+
const properties = schema.properties ?? {};
|
|
46
|
+
if (Object.hasOwn(properties, TOOL_IDEMPOTENCY_KEY_ARG)) {
|
|
47
|
+
throw new ToolIdempotencyKeyShadowedError({ name, argument: TOOL_IDEMPOTENCY_KEY_ARG });
|
|
48
|
+
}
|
|
49
|
+
return { ...schema, properties: { ...properties, [TOOL_IDEMPOTENCY_KEY_ARG]: KEY_SCHEMA } };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The model's arguments split into the action's input and the key. No key is an un-keyed run, as
|
|
54
|
+
* HTTP without the header; a non-string is no key. A blank or over-long one goes to `invoke`, whose
|
|
55
|
+
* `X_IDEMPOTENCY_KEY_INVALID` the model reads as a tool error naming the bound.
|
|
56
|
+
*/
|
|
57
|
+
export function takeToolIdempotencyKey(args: unknown): {
|
|
58
|
+
readonly input: unknown;
|
|
59
|
+
readonly idempotencyKey: string | null;
|
|
60
|
+
} {
|
|
61
|
+
if (typeof args !== 'object' || args === null || Array.isArray(args)) {
|
|
62
|
+
return { input: args, idempotencyKey: null };
|
|
63
|
+
}
|
|
64
|
+
if (!Object.hasOwn(args, TOOL_IDEMPOTENCY_KEY_ARG)) return { input: args, idempotencyKey: null };
|
|
65
|
+
const input: Record<string, unknown> = {};
|
|
66
|
+
let key: unknown;
|
|
67
|
+
for (const [field, value] of Object.entries(args)) {
|
|
68
|
+
if (field === TOOL_IDEMPOTENCY_KEY_ARG) key = value;
|
|
69
|
+
// defineProperty, never `input[field] =`: a `__proto__` field would re-prototype the copy.
|
|
70
|
+
else
|
|
71
|
+
Object.defineProperty(input, field, {
|
|
72
|
+
value,
|
|
73
|
+
writable: true,
|
|
74
|
+
enumerable: true,
|
|
75
|
+
configurable: true,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
return { input, idempotencyKey: typeof key === 'string' ? key : null };
|
|
79
|
+
}
|
package/src/tools.ts
CHANGED
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
// permissions" concept in Ultimate, because there is no second authz system.
|
|
7
7
|
//
|
|
8
8
|
// `run` below is `ProjectableAction`'s — the projection SEAM, which is what carries `invoke`.
|
|
9
|
-
// It is not a member of the action facade: an `action()` is `as`/`
|
|
10
|
-
// `contract` and the callable itself
|
|
9
|
+
// It is not a member of the action facade: an `action()` is `as`/`openapi`/`client`/`job`/
|
|
10
|
+
// `contract` and the callable itself (no `tool` since 25.0.0 — `@ultimat3/mcp`'s `toolFrom` is
|
|
11
|
+
// the MCP projection), and this header claimed `action.run` until 2026-08.
|
|
11
12
|
// `asProjectableAction` is what BUILDS that seam out of a real `action()`, so an app writes
|
|
12
13
|
// `agent({ tools: [publishPost] })` and never a hand-shaped stand-in — the same union
|
|
13
14
|
// @ultimat3/mcp's `ListedPrimitive` accepts, adapted at this package's own edge because the two
|
|
@@ -20,21 +21,17 @@
|
|
|
20
21
|
import type { AnyAction } from '@ultimat3/action';
|
|
21
22
|
import { actionName, invoke, isAction } from '@ultimat3/action';
|
|
22
23
|
import type { Actor } from '@ultimat3/core';
|
|
23
|
-
import { isMcpExposed,
|
|
24
|
-
import {
|
|
24
|
+
import { isMcpExposed, tryUseContext } from '@ultimat3/core';
|
|
25
|
+
import type { WireJsonSchema } from '@ultimat3/schema';
|
|
26
|
+
import { toWireSchema } from '@ultimat3/schema';
|
|
27
|
+
import { discloseFailure } from './failure-disclosure';
|
|
28
|
+
import { takeToolIdempotencyKey, withToolIdempotencyKey } from './tool-idempotency';
|
|
25
29
|
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
readonly required?: readonly string[];
|
|
32
|
-
/** A schema, not just a flag: `@ultimat3/schema` emits one for open records. */
|
|
33
|
-
readonly additionalProperties?: boolean | JsonSchema;
|
|
34
|
-
readonly items?: JsonSchema;
|
|
35
|
-
readonly enum?: readonly (string | number | boolean | null)[];
|
|
36
|
-
readonly default?: unknown;
|
|
37
|
-
}
|
|
30
|
+
/**
|
|
31
|
+
* The JSON Schema subset the framework emits for tool arguments — `@ultimat3/schema`'s
|
|
32
|
+
* `WireJsonSchema`, the one document `toolFrom`, `tools/list` and the model are all handed.
|
|
33
|
+
*/
|
|
34
|
+
export type JsonSchema = WireJsonSchema;
|
|
38
35
|
|
|
39
36
|
/** Anthropic Messages API tool definition shape (`tools[]` on a request). */
|
|
40
37
|
export interface LlmTool {
|
|
@@ -80,7 +77,7 @@ export interface ProjectableAction {
|
|
|
80
77
|
*
|
|
81
78
|
* The real `action()` comes first because it is what an app has. Until 2026-08 this list took
|
|
82
79
|
* `ProjectableAction` alone, which no `action()` structurally satisfies — an action carries
|
|
83
|
-
* `as`/`
|
|
80
|
+
* `as`/`openapi`/`client`/`job`/`contract` and never `run` — so the documented shape
|
|
84
81
|
* `agent({ tools: [publishPost] })` was a `TS2741` and every test in this package hand-built a
|
|
85
82
|
* stand-in, which is why the suite stayed green over an API that did not compile (issue #124).
|
|
86
83
|
* `ProjectableAction` stays in the union for a surface that builds its catalog programmatically
|
|
@@ -89,12 +86,10 @@ export interface ProjectableAction {
|
|
|
89
86
|
export type AgentTool = AnyAction | ProjectableAction;
|
|
90
87
|
|
|
91
88
|
/**
|
|
92
|
-
* Adapt whatever the author listed. The same shape @ultimat3/mcp's `
|
|
93
|
-
* the same `invoke` — an in-app agent and an external MCP client end at one execution path, so one
|
|
94
|
-
* policy decides both.
|
|
95
|
-
*
|
|
96
|
-
* that server's arg validator will hold a call to; this one publishes the tool schema the Messages
|
|
97
|
-
* API reads).
|
|
89
|
+
* Adapt whatever the author listed. The same shape @ultimat3/mcp's `primitiveFromAction` produces,
|
|
90
|
+
* from the same `invoke` — an in-app agent and an external MCP client end at one execution path, so one
|
|
91
|
+
* policy decides both. The execution adapter is not shared code and cannot be — `mcp` is this
|
|
92
|
+
* package's own tier — but the schema is: both publish `@ultimat3/schema`'s `toWireSchema`.
|
|
98
93
|
*
|
|
99
94
|
* `isAction` is structural against @ultimat3/action's PRIVATE declaration store, so a look-alike
|
|
100
95
|
* carrying `kind: 'action'` cannot take the first branch — it falls through as the already
|
|
@@ -103,17 +98,30 @@ export type AgentTool = AnyAction | ProjectableAction;
|
|
|
103
98
|
export function asProjectableAction(listed: AgentTool): ProjectableAction {
|
|
104
99
|
if (!isAction(listed)) return listed;
|
|
105
100
|
const mcp = listed.mcp;
|
|
101
|
+
// Throws `X_ACTION_UNREGISTERED` on an unnamed action rather than offering a tool called `''`:
|
|
102
|
+
// a nameless tool is unaddressable by the model, by `runLlmToolCall` and by the author.
|
|
103
|
+
const name = actionName(listed);
|
|
104
|
+
const wire = toWireSchema(listed.input);
|
|
105
|
+
// `idempotent: true` is honoured here as over MCP and HTTP: the model's key reaches `invoke`,
|
|
106
|
+
// which files it under the action, the caller and the key. Until 25.0.0 this adapter dropped it,
|
|
107
|
+
// so an action an external agent could retry safely was one an in-app agent could not.
|
|
108
|
+
const keyed = listed.describe().idempotent;
|
|
106
109
|
return {
|
|
107
|
-
|
|
108
|
-
// a nameless tool is unaddressable by the model, by `runLlmToolCall` and by the author.
|
|
109
|
-
name: actionName(listed),
|
|
110
|
+
name,
|
|
110
111
|
...(mcp === undefined ? {} : { mcp }),
|
|
111
112
|
...(mcp?.description === undefined ? {} : { description: mcp.description }),
|
|
112
|
-
inputJsonSchema:
|
|
113
|
+
inputJsonSchema: keyed ? withToolIdempotencyKey(name, wire) : wire,
|
|
113
114
|
// The actor rides in on the options and `invoke` swaps it inside the one execution path —
|
|
114
115
|
// the action's own `policy` still decides, and its `input:` still parses what the model sent,
|
|
115
116
|
// which is what drops a `{ actor: 'admin' }` the model invented before any handler sees it.
|
|
116
|
-
|
|
117
|
+
// No `admit`: MCP's is its early gate ahead of ITS argument validation, and this surface has
|
|
118
|
+
// none — `invoke` asks the same policy before it reads the input, which is the same order.
|
|
119
|
+
run: keyed
|
|
120
|
+
? ({ input: args, actor }) => {
|
|
121
|
+
const { input, idempotencyKey } = takeToolIdempotencyKey(args);
|
|
122
|
+
return invoke(listed, input, { surface: 'mcp', actor, idempotencyKey });
|
|
123
|
+
}
|
|
124
|
+
: ({ input, actor }) => invoke(listed, input, { surface: 'mcp', actor }),
|
|
117
125
|
};
|
|
118
126
|
}
|
|
119
127
|
|
|
@@ -169,7 +177,7 @@ export async function runLlmToolCall(
|
|
|
169
177
|
return resultOf(call.id, output);
|
|
170
178
|
} catch (error) {
|
|
171
179
|
// A policy denial is an outcome the model should read and react to, not a crash.
|
|
172
|
-
return { toolUseId: call.id, content: describeFailure(error), isError: true };
|
|
180
|
+
return { toolUseId: call.id, content: describeFailure(error, call.name), isError: true };
|
|
173
181
|
}
|
|
174
182
|
}
|
|
175
183
|
|
|
@@ -198,15 +206,20 @@ function resultOf(toolUseId: string, output: unknown): LlmToolResult {
|
|
|
198
206
|
}
|
|
199
207
|
|
|
200
208
|
/**
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
|
|
204
|
-
|
|
209
|
+
* What the model reads in place of a cause it may not see. Fixed, so nothing of the throw rides on
|
|
210
|
+
* it: the provider keeps every request it is sent, and a cause that reached it cannot be recalled.
|
|
211
|
+
*/
|
|
212
|
+
export const HIDDEN_TOOL_CAUSE =
|
|
213
|
+
'the tool failed inside the server; the details are withheld from this conversation; the server log records its code and where it failed';
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* `discloseFailure`'s verdict, rendered as the `tool_result` line — the same rule a hive member's
|
|
217
|
+
* reason follows, so what an agent may read of a throw is decided once. An uncoded throw says only
|
|
218
|
+
* that it failed; a withheld cause becomes `HIDDEN_TOOL_CAUSE`, and the log has the rest.
|
|
205
219
|
*/
|
|
206
|
-
function describeFailure(error: unknown): string {
|
|
207
|
-
const
|
|
208
|
-
if (code === undefined) return 'tool failed';
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
return fix === '' ? `${code}: ${cause}` : `${code}: ${cause} (fix: ${fix})`;
|
|
220
|
+
function describeFailure(error: unknown, toolName: string): string {
|
|
221
|
+
const shown = discloseFailure(error, tryUseContext()?.logger, `tool ${toolName}`);
|
|
222
|
+
if (shown.code === undefined) return 'tool failed';
|
|
223
|
+
const line = `${shown.code}: ${shown.cause ?? HIDDEN_TOOL_CAUSE}`;
|
|
224
|
+
return shown.fix === undefined ? line : `${line} (fix: ${shown.fix})`;
|
|
212
225
|
}
|
package/src/vector-scope.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// production is a leak nobody finds. Deriving a scope may only ever TIGHTEN it: a scope that
|
|
5
5
|
// could be widened from a call site is not a scope, it is a hint.
|
|
6
6
|
|
|
7
|
+
import { tryUseContext, UltimateError } from '@ultimat3/core';
|
|
7
8
|
import { VectorScopeWidenedError } from './errors';
|
|
8
9
|
|
|
9
10
|
export interface VectorScope {
|
|
@@ -14,16 +15,28 @@ export interface VectorScope {
|
|
|
14
15
|
* deny — a row missing the key is invisible, and an empty list matches nothing at all.
|
|
15
16
|
*/
|
|
16
17
|
readonly allow?: Readonly<Record<string, readonly string[]>> | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* Every tenant, ON PURPOSE — `UNSCOPED`'s mark, and the only way a read with no tenant bound
|
|
20
|
+
* runs inside a request acting for an org. Meaningless once a tenant is bound, so dropped then.
|
|
21
|
+
*/
|
|
22
|
+
readonly crossTenant?: true | undefined;
|
|
17
23
|
}
|
|
18
24
|
|
|
19
|
-
/**
|
|
20
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Every tenant, every row, named on purpose: the backfill and migration path. Pass it as a
|
|
27
|
+
* store's `scope` (or `.scoped(UNSCOPED)`) where a cross-tenant read is the point. A store opened
|
|
28
|
+
* with no scope binds no tenant either, but reading it inside an org's request is refused.
|
|
29
|
+
*/
|
|
30
|
+
export const UNSCOPED: VectorScope = Object.freeze({ crossTenant: true });
|
|
31
|
+
|
|
32
|
+
/** A store's scope when none was given: no tenant, and no cross-tenant opt-in. */
|
|
33
|
+
export const UNBOUND: VectorScope = Object.freeze({});
|
|
21
34
|
|
|
22
35
|
/** The tenant column value a row carries when its store had no tenant bound. */
|
|
23
|
-
export const
|
|
36
|
+
export const NO_VECTOR_TENANT = '';
|
|
24
37
|
|
|
25
38
|
export function tenantOf(scope: VectorScope): string {
|
|
26
|
-
return scope.tenant ??
|
|
39
|
+
return scope.tenant ?? NO_VECTOR_TENANT;
|
|
27
40
|
}
|
|
28
41
|
|
|
29
42
|
/**
|
|
@@ -33,12 +46,41 @@ export function tenantOf(scope: VectorScope): string {
|
|
|
33
46
|
export function narrowScope(store: string, base: VectorScope, next: VectorScope): VectorScope {
|
|
34
47
|
const tenant = narrowTenant(store, base.tenant, next.tenant);
|
|
35
48
|
const allow = narrowAllow(base.allow, next.allow);
|
|
49
|
+
const crossTenant =
|
|
50
|
+
tenant === undefined && (base.crossTenant === true || next.crossTenant === true);
|
|
36
51
|
return {
|
|
37
52
|
...(tenant === undefined ? {} : { tenant }),
|
|
38
53
|
...(allow === undefined ? {} : { allow }),
|
|
54
|
+
...(crossTenant ? { crossTenant: true as const } : {}),
|
|
39
55
|
};
|
|
40
56
|
}
|
|
41
57
|
|
|
58
|
+
/**
|
|
59
|
+
* Refuse a read that would search every tenant inside a request acting for one. Entities derive
|
|
60
|
+
* the tenant from the ambient actor; a vector store is bound by `.scoped({ tenant })`, and a
|
|
61
|
+
* forgotten call answered with every org's rows. Outside a request, or for an actor with no org,
|
|
62
|
+
* there is no tenant to leak across — a single-tenant app and a job's system actor read as before.
|
|
63
|
+
*/
|
|
64
|
+
export function assertTenantRead(store: string, scope: VectorScope): void {
|
|
65
|
+
if (scope.tenant !== undefined || scope.crossTenant === true) return;
|
|
66
|
+
if (tryUseContext()?.actor.orgId === undefined) return;
|
|
67
|
+
throw new VectorUnscopedError({ store });
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** `X_VECTOR_UNSCOPED`. Its code and title stay in `errors.ts`, beside every other `X_VECTOR_*`. */
|
|
71
|
+
export class VectorUnscopedError extends UltimateError {
|
|
72
|
+
constructor(input: { store: string }) {
|
|
73
|
+
super({
|
|
74
|
+
code: 'X_VECTOR_UNSCOPED',
|
|
75
|
+
cause:
|
|
76
|
+
`vector store "${input.store}" was read with no tenant bound, inside a request acting ` +
|
|
77
|
+
'for an org — the read would answer with every tenant’s rows. A deliberate cross-tenant ' +
|
|
78
|
+
'read (a backfill, a migration) opens the store with scope: UNSCOPED',
|
|
79
|
+
fix: 'vectorStore.scoped({ tenant: ctx.actor.orgId })',
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
42
84
|
function narrowTenant(
|
|
43
85
|
store: string,
|
|
44
86
|
base: string | undefined,
|
package/src/vector.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Vector storage and retrieval: the `VectorStore` contract and the in-memory dev store.
|
|
2
|
-
// `
|
|
2
|
+
// `PostgresVectorStore` — the production path — implements this same contract in `pg-vector.ts`.
|
|
3
3
|
//
|
|
4
4
|
// The store interface is shaped for pgvector (one table, one index, SQL you can read), with
|
|
5
5
|
// an in-memory cosine implementation as the dev default. Hybrid search is first-class rather
|
|
@@ -9,9 +9,16 @@
|
|
|
9
9
|
// be comparable — which they never are.
|
|
10
10
|
|
|
11
11
|
import { finiteCount, finiteOption } from '@ultimat3/core';
|
|
12
|
-
import { cosine,
|
|
12
|
+
import { cosine, wordTokens } from './embeddings';
|
|
13
13
|
import { VectorDimMismatchError } from './errors';
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
assertTenantRead,
|
|
16
|
+
NO_VECTOR_TENANT,
|
|
17
|
+
narrowScope,
|
|
18
|
+
scopeAdmits,
|
|
19
|
+
UNBOUND,
|
|
20
|
+
type VectorScope,
|
|
21
|
+
} from './vector-scope';
|
|
15
22
|
|
|
16
23
|
export interface VectorRecord {
|
|
17
24
|
readonly id: string;
|
|
@@ -44,7 +51,7 @@ export interface HybridSearchInput {
|
|
|
44
51
|
export interface VectorStore {
|
|
45
52
|
readonly name: string;
|
|
46
53
|
readonly dimension: number;
|
|
47
|
-
/** The tenant + policy envelope every statement carries. `
|
|
54
|
+
/** The tenant + policy envelope every statement carries. `UNBOUND` on a fresh store. */
|
|
48
55
|
readonly scope: VectorScope;
|
|
49
56
|
/** A view of the same rows through a NARROWER envelope. It can only ever tighten. */
|
|
50
57
|
scoped(scope: VectorScope): VectorStore;
|
|
@@ -56,10 +63,10 @@ export interface VectorStore {
|
|
|
56
63
|
/**
|
|
57
64
|
* Delete every row `filter` matches EXCEPT the ids in `keep`, inside this store's scope. What a
|
|
58
65
|
* re-index needs: `indexDocument` upserts a document's new chunks, then prunes the ones the
|
|
59
|
-
* shorter text no longer produced.
|
|
60
|
-
*
|
|
66
|
+
* shorter text no longer produced. Required: an optional `prune` was a re-index that silently
|
|
67
|
+
* left the old tail retrievable on any store that skipped it.
|
|
61
68
|
*/
|
|
62
|
-
prune
|
|
69
|
+
prune(filter: MetadataFilter, keep: readonly string[]): Promise<void>;
|
|
63
70
|
}
|
|
64
71
|
|
|
65
72
|
export interface MemoryVectorStoreInput {
|
|
@@ -88,6 +95,15 @@ export interface StoredRecord extends VectorRecord {
|
|
|
88
95
|
readonly tenant: string;
|
|
89
96
|
}
|
|
90
97
|
|
|
98
|
+
/** The row's primary key, `(tenant, id)` as the pg table declares it. NUL never occurs in a tenant. */
|
|
99
|
+
const storageKey = (tenant: string, id: string): string => `${tenant}\u0000${id}`;
|
|
100
|
+
|
|
101
|
+
/** A row and its score inside one ranking, before the tenant is dropped from the public hit. */
|
|
102
|
+
interface Scored {
|
|
103
|
+
readonly record: StoredRecord;
|
|
104
|
+
readonly score: number;
|
|
105
|
+
}
|
|
106
|
+
|
|
91
107
|
export class MemoryVectorStore implements VectorStore {
|
|
92
108
|
readonly name: string;
|
|
93
109
|
readonly dimension: number;
|
|
@@ -101,7 +117,7 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
101
117
|
this.input = input;
|
|
102
118
|
this.name = input.name ?? 'memory';
|
|
103
119
|
this.dimension = input.dimension;
|
|
104
|
-
this.scope = input.scope ??
|
|
120
|
+
this.scope = input.scope ?? UNBOUND;
|
|
105
121
|
this.records = input.records ?? new Map<string, StoredRecord>();
|
|
106
122
|
// Screened, not clamped: a `NaN` here makes every BM25 score `NaN`, `hit.score > 0` reads
|
|
107
123
|
// false for every document, and `searchText` answers an empty list — zero work reported as a
|
|
@@ -112,7 +128,7 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
112
128
|
}
|
|
113
129
|
|
|
114
130
|
/**
|
|
115
|
-
* Same envelope, same rule as `
|
|
131
|
+
* Same envelope, same rule as `PostgresVectorStore.scoped`. The dev store enforces it too, because
|
|
116
132
|
* a tenant leak that only reproduces against production Postgres is a leak nobody finds.
|
|
117
133
|
*/
|
|
118
134
|
scoped(scope: VectorScope): MemoryVectorStore {
|
|
@@ -124,10 +140,10 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
124
140
|
}
|
|
125
141
|
|
|
126
142
|
async upsert(records: readonly VectorRecord[]): Promise<void> {
|
|
127
|
-
const tenant = this.scope.tenant ??
|
|
143
|
+
const tenant = this.scope.tenant ?? NO_VECTOR_TENANT;
|
|
128
144
|
for (const record of records) {
|
|
129
145
|
this.assertDimension(record.vector.length);
|
|
130
|
-
this.records.set(
|
|
146
|
+
this.records.set(storageKey(tenant, record.id), { ...record, tenant });
|
|
131
147
|
}
|
|
132
148
|
}
|
|
133
149
|
|
|
@@ -136,13 +152,13 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
136
152
|
k: number,
|
|
137
153
|
filter?: MetadataFilter,
|
|
138
154
|
): Promise<readonly SearchHit[]> {
|
|
155
|
+
assertTenantRead(this.name, this.scope);
|
|
139
156
|
this.assertDimension(vector.length);
|
|
140
157
|
// `k` carries no default, so `??` never sees it and nothing else does either: `slice(0, NaN)`
|
|
141
158
|
// is `[]`, which is a search that answers "no matches" for every query and reports success.
|
|
142
|
-
return this.
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
.slice(0, finiteCount(SUBJECT, 'k', k));
|
|
159
|
+
return this.dense(vector, finiteCount(SUBJECT, 'k', k), filter).map(({ record, score }) =>
|
|
160
|
+
this.hit(record, score),
|
|
161
|
+
);
|
|
146
162
|
}
|
|
147
163
|
|
|
148
164
|
/** BM25 over the stored text. Real lexical scoring, so a rare exact term actually wins. */
|
|
@@ -151,12 +167,67 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
151
167
|
k: number,
|
|
152
168
|
filter?: MetadataFilter,
|
|
153
169
|
): Promise<readonly SearchHit[]> {
|
|
154
|
-
|
|
170
|
+
assertTenantRead(this.name, this.scope);
|
|
171
|
+
return this.lexical(query, finiteCount(SUBJECT, 'k', k), filter).map(({ record, score }) =>
|
|
172
|
+
this.hit(record, score),
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Reciprocal-rank fusion. Each ranking contributes `1 / (rrfK + rank)`, so only ORDER
|
|
178
|
+
* matters — the two score scales never have to be reconciled, and a document ranked first
|
|
179
|
+
* by exact term match beats one that merely leads a flat vector ranking.
|
|
180
|
+
*
|
|
181
|
+
* Fused on the stored `(tenant, id)`, as `hybridSql` groups: an unscoped read sees every
|
|
182
|
+
* tenant's rows, and two tenants may share an id. Fusing on the id alone summed both rows'
|
|
183
|
+
* ranks into whichever came first and dropped the other. `fuse` keys on `SearchHit.id`, so the
|
|
184
|
+
* rankings reach it under the row's storage key and leave under the caller's id again.
|
|
185
|
+
*/
|
|
186
|
+
async hybrid(input: HybridSearchInput): Promise<readonly SearchHit[]> {
|
|
187
|
+
assertTenantRead(this.name, this.scope);
|
|
188
|
+
this.assertDimension(input.vector.length);
|
|
189
|
+
// Three bounds, none of which `Math.max` screens — it PROPAGATES a `NaN`. A `NaN` `k` collapses
|
|
190
|
+
// both candidate widths and the final slice to `[]`; a `NaN` `rrfK` makes every fused score
|
|
191
|
+
// `NaN`, and the ranking this method exists to produce becomes whatever order the sort left.
|
|
192
|
+
const k = finiteCount(HYBRID, 'k', input.k);
|
|
193
|
+
const width = finiteCount(HYBRID, 'candidates', input.candidates ?? Math.max(k * 4, 20));
|
|
194
|
+
const rrfK = finiteOption(HYBRID, 'rrfK', input.rrfK ?? 60);
|
|
195
|
+
const rows = new Map<string, StoredRecord>();
|
|
196
|
+
const keyed = (ranking: readonly Scored[]): readonly SearchHit[] =>
|
|
197
|
+
ranking.map(({ record, score }) => {
|
|
198
|
+
const key = storageKey(record.tenant, record.id);
|
|
199
|
+
rows.set(key, record);
|
|
200
|
+
return { ...this.hit(record, score), id: key };
|
|
201
|
+
});
|
|
202
|
+
const fused = fuse(
|
|
203
|
+
[
|
|
204
|
+
keyed(this.dense(input.vector, width, input.filter)),
|
|
205
|
+
keyed(this.lexical(input.query, width, input.filter)),
|
|
206
|
+
],
|
|
207
|
+
rrfK,
|
|
208
|
+
);
|
|
209
|
+
// Re-sorted once the ids are the caller's again: the tie-break is `d."id" asc`, never the key.
|
|
210
|
+
return fused
|
|
211
|
+
.map((hit) => ({ ...hit, id: rows.get(hit.id)?.id ?? hit.id }))
|
|
212
|
+
.sort(byScoreDesc)
|
|
213
|
+
.slice(0, k);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** The cosine ranking `search` answers and `hybrid` fuses, rows still carrying their tenant. */
|
|
217
|
+
private dense(vector: Float32Array, k: number, filter?: MetadataFilter): readonly Scored[] {
|
|
218
|
+
return this.candidates(filter)
|
|
219
|
+
.map((record) => ({ record, score: cosine(vector, record.vector) }))
|
|
220
|
+
.sort(byRankDesc)
|
|
221
|
+
.slice(0, k);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** BM25 over the stored text, rows still carrying their tenant. Only a positive score matches. */
|
|
225
|
+
private lexical(query: string, width: number, filter?: MetadataFilter): readonly Scored[] {
|
|
155
226
|
const candidates = this.candidates(filter);
|
|
156
227
|
if (candidates.length === 0) return [];
|
|
157
|
-
const docs = candidates.map((record) => ({ record, tokens:
|
|
228
|
+
const docs = candidates.map((record) => ({ record, tokens: wordTokens(record.text) }));
|
|
158
229
|
const avgLength = docs.reduce((sum, d) => sum + d.tokens.length, 0) / docs.length;
|
|
159
|
-
const terms = [...new Set(
|
|
230
|
+
const terms = [...new Set(wordTokens(query))];
|
|
160
231
|
|
|
161
232
|
return docs
|
|
162
233
|
.map(({ record, tokens }) => {
|
|
@@ -169,32 +240,13 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
169
240
|
const norm = this.k1 * (1 - this.b + (this.b * tokens.length) / avgLength);
|
|
170
241
|
score += idf * ((tf * (this.k1 + 1)) / (tf + norm));
|
|
171
242
|
}
|
|
172
|
-
return
|
|
243
|
+
return { record, score };
|
|
173
244
|
})
|
|
174
|
-
.filter((
|
|
175
|
-
.sort(
|
|
245
|
+
.filter((scored) => scored.score > 0)
|
|
246
|
+
.sort(byRankDesc)
|
|
176
247
|
.slice(0, width);
|
|
177
248
|
}
|
|
178
249
|
|
|
179
|
-
/**
|
|
180
|
-
* Reciprocal-rank fusion. Each ranking contributes `1 / (rrfK + rank)`, so only ORDER
|
|
181
|
-
* matters — the two score scales never have to be reconciled, and a document ranked first
|
|
182
|
-
* by exact term match beats one that merely leads a flat vector ranking.
|
|
183
|
-
*/
|
|
184
|
-
async hybrid(input: HybridSearchInput): Promise<readonly SearchHit[]> {
|
|
185
|
-
// Three bounds, none of which `Math.max` screens — it PROPAGATES a `NaN`. A `NaN` `k` collapses
|
|
186
|
-
// both candidate widths and the final slice to `[]`; a `NaN` `rrfK` makes every fused score
|
|
187
|
-
// `NaN`, and the ranking this method exists to produce becomes whatever order the sort left.
|
|
188
|
-
const k = finiteCount(HYBRID, 'k', input.k);
|
|
189
|
-
const width = finiteCount(HYBRID, 'candidates', input.candidates ?? Math.max(k * 4, 20));
|
|
190
|
-
const rrfK = finiteOption(HYBRID, 'rrfK', input.rrfK ?? 60);
|
|
191
|
-
const [dense, lexical] = await Promise.all([
|
|
192
|
-
this.search(input.vector, width, input.filter),
|
|
193
|
-
this.searchText(input.query, width, input.filter),
|
|
194
|
-
]);
|
|
195
|
-
return fuse([dense, lexical], rrfK).slice(0, k);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
250
|
/** Scoped and filtered, exactly like `pruneSql`: everything `filter` matches but `keep`. */
|
|
199
251
|
async prune(filter: MetadataFilter, keep: readonly string[]): Promise<void> {
|
|
200
252
|
const kept = new Set(keep);
|
|
@@ -237,6 +289,14 @@ export class MemoryVectorStore implements VectorStore {
|
|
|
237
289
|
}
|
|
238
290
|
}
|
|
239
291
|
|
|
292
|
+
/**
|
|
293
|
+
* The one way to build the in-process store — the twin of `postgresVectorStore()`. The class is a
|
|
294
|
+
* type in the barrel only (`X_FACTORY_NAME_SPELLING`), so `new` is never a second spelling.
|
|
295
|
+
*/
|
|
296
|
+
export function memoryVectorStore(input: MemoryVectorStoreInput): MemoryVectorStore {
|
|
297
|
+
return new MemoryVectorStore(input);
|
|
298
|
+
}
|
|
299
|
+
|
|
240
300
|
/**
|
|
241
301
|
* Fuse ranked lists by reciprocal rank. Exported so a reranker can reuse it — which is why the
|
|
242
302
|
* damping is screened HERE too and not only in `hybrid`: a default parameter is a bound like any
|
|
@@ -269,4 +329,20 @@ export function fuse(
|
|
|
269
329
|
* sort then resolves them by dense-list insertion order, which no SQL engine reproduces.
|
|
270
330
|
*/
|
|
271
331
|
const byScoreDesc = (a: SearchHit, b: SearchHit): number =>
|
|
272
|
-
|
|
332
|
+
byScore(a.score, b.score) || byId(a.id, b.id);
|
|
333
|
+
|
|
334
|
+
/** The same order over a ranking's rows, which still carry their tenant. */
|
|
335
|
+
const byRankDesc = (a: Scored, b: Scored): number =>
|
|
336
|
+
byScore(a.score, b.score) || byId(a.record.id, b.record.id);
|
|
337
|
+
|
|
338
|
+
const byId = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Descending, `NaN` last. A cosine against a zero-norm vector is `NaN`, and float8 orders `NaN`
|
|
342
|
+
* above every number, so pg's `order by distance` puts that row at the end. `b - a` alone is `NaN`
|
|
343
|
+
* for any pair holding one, which reads as a tie and leaves the sort inconsistent.
|
|
344
|
+
*/
|
|
345
|
+
function byScore(a: number, b: number): number {
|
|
346
|
+
if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1;
|
|
347
|
+
return b - a;
|
|
348
|
+
}
|
package/src/wire.ts
CHANGED
|
@@ -185,9 +185,14 @@ export class MessageStream {
|
|
|
185
185
|
}
|
|
186
186
|
}
|
|
187
187
|
|
|
188
|
-
/**
|
|
188
|
+
/**
|
|
189
|
+
* True once `message_stop` arrived with every tool_use block closed. False means the connection
|
|
190
|
+
* died mid-answer — or the stream ended with a call's arguments still open, which drops the call
|
|
191
|
+
* (it is emitted whole or not at all), and for `llm()` that call IS the answer. `openai-wire.ts`
|
|
192
|
+
* refuses the same case.
|
|
193
|
+
*/
|
|
189
194
|
isComplete(): boolean {
|
|
190
|
-
return this.stopped;
|
|
195
|
+
return this.stopped && this.pending.size === 0;
|
|
191
196
|
}
|
|
192
197
|
|
|
193
198
|
state(): StreamState {
|