@ultimat3/ai 23.0.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CLAUDE.md +96 -32
  2. package/README.md +249 -75
  3. package/package.json +12 -11
  4. package/src/agent-facts.ts +9 -2
  5. package/src/agent-transcript.ts +1 -1
  6. package/src/agent.ts +43 -25
  7. package/src/budget.ts +178 -51
  8. package/src/content-blocks.ts +227 -0
  9. package/src/content-errors.ts +67 -0
  10. package/src/echo-provider.ts +102 -0
  11. package/src/embeddings.ts +40 -21
  12. package/src/error-body.ts +38 -3
  13. package/src/errors.ts +19 -11
  14. package/src/eval-baseline.ts +5 -1
  15. package/src/eval-errors.ts +3 -1
  16. package/src/evals.ts +1 -1
  17. package/src/failure-disclosure.ts +90 -0
  18. package/src/gateway.ts +73 -37
  19. package/src/hive-pool.ts +24 -16
  20. package/src/hive-result.ts +4 -1
  21. package/src/hive.ts +15 -5
  22. package/src/index.ts +66 -25
  23. package/src/llm-cache.ts +4 -3
  24. package/src/llm.ts +27 -65
  25. package/src/model-resolve.ts +79 -0
  26. package/src/models.ts +125 -111
  27. package/src/openai-body.ts +13 -2
  28. package/src/openai-messages.ts +86 -15
  29. package/src/openai-provider.ts +28 -31
  30. package/src/openai-wire.ts +11 -3
  31. package/src/pg-vector-sql.ts +11 -7
  32. package/src/pg-vector.ts +21 -10
  33. package/src/prompt-fence.ts +128 -0
  34. package/src/prompt.ts +43 -13
  35. package/src/provider.ts +82 -117
  36. package/src/rag.ts +17 -9
  37. package/src/remote-embedder.ts +15 -41
  38. package/src/respond.ts +86 -0
  39. package/src/runtime.ts +12 -14
  40. package/src/scorers.ts +14 -3
  41. package/src/tool-idempotency.ts +79 -0
  42. package/src/tools.ts +52 -39
  43. package/src/vector-scope.ts +46 -4
  44. package/src/vector.ts +117 -41
  45. package/src/wire.ts +7 -2
  46. package/src/fix-line.evals.ts +0 -35
  47. package/src/fix-line.ts +0 -27
  48. package/src/openai-models.ts +0 -84
@@ -12,11 +12,9 @@ import { isSecret, revealSecret } from '@ultimat3/core';
12
12
  import { detailOf, withoutKey } from './error-body';
13
13
  import { AiKeyMissingError, AiRequestInvalidError, AiTransportError } from './errors';
14
14
  import type { AiFetch } from './fetch-seam';
15
+ import { resolveModel } from './model-resolve';
15
16
  import type { ModelId } from './models';
16
17
  import { chatCompletionBody } from './openai-body';
17
- // Imported for its registration side effect: a provider that cannot price what it serves throws
18
- // X_AI_MODEL_UNKNOWN at the first call, and the specs belong with the format that names them.
19
- import './openai-models';
20
18
  import type { ChatAnswer } from './openai-wire';
21
19
  import { ChatCompletionStream, parseChatCompletion } from './openai-wire';
22
20
  import type {
@@ -28,6 +26,7 @@ import type {
28
26
  } from './provider';
29
27
  import { costOf, estimateInputTokens, estimateTextTokens, requiresStreaming } from './provider';
30
28
  import { readSse } from './sse';
29
+ import type { LlmToolCall } from './tools';
31
30
 
32
31
  const API_KEY_ENV = 'OPENAI_API_KEY';
33
32
  const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
@@ -75,7 +74,7 @@ export interface OpenAiProviderInput {
75
74
  *
76
75
  * ```ts
77
76
  * // OpenAI
78
- * openAiProvider({ apiKey: openAiKey, models: [...OPENAI_MODEL_IDS] })
77
+ * openAiProvider({ apiKey: openAiKey, models: [appModel] }) // ids your app registerModel-ed
79
78
  * // a company gateway, vLLM, Ollama — same class, one different field
80
79
  * openAiProvider({ apiKey: gatewayKey, baseUrl: 'https://llm.acme.internal/v1', models: ['acme-70b'] })
81
80
  * ```
@@ -87,23 +86,22 @@ export function openAiProvider(input: OpenAiProviderInput): Provider {
87
86
  class OpenAiProvider implements Provider {
88
87
  readonly name: string;
89
88
  readonly models: readonly ModelId[];
90
- private readonly defaultModel: ModelId;
91
89
  private readonly config: OpenAiProviderInput;
92
90
 
93
- constructor(config: OpenAiProviderInput) {
94
- const [first] = config.models;
95
- if (first === undefined) {
91
+ // `| undefined` for the reason `AnthropicProvider`'s is: no argument at all, from plain JS or a
92
+ // cast, is the empty-list boot mistake and gets its coded refusal, never a TypeError.
93
+ constructor(config: OpenAiProviderInput | undefined) {
94
+ if (config === undefined || (config.models ?? []).length === 0) {
96
95
  // A provider serving nothing can never be routed to, so it is a boot mistake that would
97
96
  // otherwise surface as "no provider serves <model>" — a true statement about the wrong thing.
98
97
  throw new AiRequestInvalidError({
99
98
  detail:
100
99
  'openAiProvider was given an empty models list, so the gateway can never route to it',
101
- fix: 'pass models: [...OPENAI_MODEL_IDS] to openAiProvider, or the ids your endpoint serves',
100
+ fix: "openAiProvider({ …, models: ['<id>'] }) # the ids your endpoint serves, each one your app registerModel-ed",
102
101
  });
103
102
  }
104
103
  this.name = config.name ?? 'openai';
105
104
  this.models = config.models;
106
- this.defaultModel = first;
107
105
  this.config = config;
108
106
  }
109
107
 
@@ -113,15 +111,12 @@ class OpenAiProvider implements Provider {
113
111
  * open socket past the HTTP timeout and fails after the completion was generated and billed.
114
112
  */
115
113
  async generate(request: GenerateRequest): Promise<GenerateResult> {
116
- // Resolved BEFORE the transport question is asked. `requiresStreaming` clamps `maxTokens` to
117
- // the model's `maxOutput`, and its own fallback for an absent model is the framework's
118
- // `DEFAULT_MODEL` — a Claude id no OpenAI-format endpoint serves. Sized against that, a
119
- // request naming no model was measured against somebody else's ceiling, seventy lines above
120
- // `modelOf`'s comment saying this provider never does that.
121
- const model = this.modelOf(request);
114
+ // Resolved BEFORE the socket: a direct call naming no model is refused here, never answered on
115
+ // this provider's first listed model — the app's three places are the only ones.
116
+ const model = resolveModel('provider', request.model);
122
117
  if (requiresStreaming({ ...request, model })) return this.assemble(request);
123
118
  const response = await this.send(
124
- chatCompletionBody({ request, model, stream: false }),
119
+ chatCompletionBody({ request, model, stream: false, provider: this.name }),
125
120
  false,
126
121
  request.signal,
127
122
  );
@@ -130,9 +125,9 @@ class OpenAiProvider implements Provider {
130
125
  }
131
126
 
132
127
  async *stream(request: GenerateRequest): AsyncIterable<StreamChunk> {
133
- const model = this.modelOf(request);
128
+ const model = resolveModel('provider', request.model);
134
129
  const response = await this.send(
135
- chatCompletionBody({ request, model, stream: true }),
130
+ chatCompletionBody({ request, model, stream: true, provider: this.name }),
136
131
  true,
137
132
  request.signal,
138
133
  );
@@ -171,7 +166,7 @@ class OpenAiProvider implements Provider {
171
166
 
172
167
  /** One parsed answer, priced. The only place `cost` is applied — the provider owns prices. */
173
168
  private result(request: GenerateRequest, model: ModelId, answer: ChatAnswer): GenerateResult {
174
- const usage = answer.usage ?? estimatedUsage(request, answer.text);
169
+ const usage = answer.usage ?? estimatedUsage(request, answer.text, answer.toolCalls);
175
170
  return {
176
171
  model,
177
172
  text: answer.text,
@@ -183,15 +178,6 @@ class OpenAiProvider implements Provider {
183
178
  };
184
179
  }
185
180
 
186
- /**
187
- * The model this request is for. The gateway resolves one before it routes, so the fallback is
188
- * only ever reached by a direct call — and it is this provider's first model, never the
189
- * framework's `DEFAULT_MODEL`, which names a Claude id no OpenAI-format endpoint serves.
190
- */
191
- private modelOf(request: GenerateRequest): ModelId {
192
- return request.model ?? this.defaultModel;
193
- }
194
-
195
181
  /**
196
182
  * The one place a request leaves the process. A non-2xx becomes an `AiTransportError` carrying
197
183
  * its status, because the gateway decides whether to retry from that status and a body parsed as
@@ -269,11 +255,22 @@ class OpenAiProvider implements Provider {
269
255
  * reconciling a real call against zero, which refunds the reservation in full and turns the ledger
270
256
  * into a decoration. An estimate is wrong by a few percent in the safe direction; zero is wrong by
271
257
  * all of it.
258
+ *
259
+ * The tool calls are output too, and for an `llm()` answer they are ALL of it: the answer is one
260
+ * `respond` call with no text beside it, so counting `text` alone estimated it at zero.
272
261
  */
273
- export function estimatedUsage(request: GenerateRequest, text: string): TokenUsage {
262
+ export function estimatedUsage(
263
+ request: GenerateRequest,
264
+ text: string,
265
+ toolCalls: readonly LlmToolCall[],
266
+ ): TokenUsage {
267
+ let output = estimateTextTokens(text);
268
+ for (const call of toolCalls) {
269
+ output += estimateTextTokens(call.name) + estimateTextTokens(JSON.stringify(call.input));
270
+ }
274
271
  return {
275
272
  inputTokens: estimateInputTokens(request),
276
- outputTokens: estimateTextTokens(text),
273
+ outputTokens: output,
277
274
  cacheReadTokens: 0,
278
275
  cacheWriteTokens: 0,
279
276
  };
@@ -62,9 +62,14 @@ const ERROR_STATUS: ReadonlyMap<string, number> = new Map(
62
62
  }),
63
63
  );
64
64
 
65
- /** A finish reason this format knows, or `undefined` for `null` — which means "still going". */
65
+ /**
66
+ * The finish reason, or `undefined` for a non-string (`null` on a delta means "still going").
67
+ * A string this table does not know FAILS CLOSED as `max_tokens` — `wire.ts`'s `parseStopReason`
68
+ * rule, so one unknown reason reads the same on both formats: a provider's new reason is far
69
+ * likelier to be a way of stopping early than of finishing.
70
+ */
66
71
  export function parseFinishReason(raw: unknown): StopReason | undefined {
67
- return typeof raw === 'string' ? FINISH_REASONS.get(raw) : undefined;
72
+ return typeof raw === 'string' ? (FINISH_REASONS.get(raw) ?? 'max_tokens') : undefined;
68
73
  }
69
74
 
70
75
  /**
@@ -99,7 +104,10 @@ export function parseChatCompletion(raw: unknown, provider: string): ChatAnswer
99
104
  const choice = asRecord(Array.isArray(record['choices']) ? record['choices'][0] : undefined);
100
105
  if (choice === undefined) throw malformed(provider, 'the response carried no choices');
101
106
  const message = asRecord(choice['message']) ?? {};
102
- const refusal = typeof message['refusal'] === 'string' ? message['refusal'] : undefined;
107
+ // Non-empty, as the stream decides it: a gateway that always serialises the field sends
108
+ // `"refusal": ""` beside a real answer, and that body must not refuse here and answer streamed.
109
+ const declined = message['refusal'];
110
+ const refusal = typeof declined === 'string' && declined !== '' ? declined : undefined;
103
111
  const finish = parseFinishReason(choice['finish_reason']);
104
112
  return {
105
113
  text: typeof message['content'] === 'string' ? message['content'] : '',
@@ -115,6 +115,8 @@ export interface PgSearchArgs {
115
115
  * `<=>` is cosine DISTANCE, so the score is `1 - distance` — the same scale the memory store
116
116
  * returns. The ordering stays on the raw distance, ascending, inside a subquery: HNSW answers
117
117
  * `order by embedding <=> $1` and nothing else, and `order by 1 - (...) desc` is a seq scan.
118
+ * Only the OUTER sort, over at most `k` rows, breaks a tie by `"id"` as `vector.ts` does — so
119
+ * which tied rows make the page at the `limit` boundary is still the index's call.
118
120
  */
119
121
  export function searchSql(
120
122
  target: PgVectorTable,
@@ -129,14 +131,14 @@ from (
129
131
  order by distance
130
132
  limit ${args.k}
131
133
  ) top
132
- order by distance`;
134
+ order by distance, "id" asc`;
133
135
  }
134
136
 
135
137
  export function textSql(target: PgVectorTable, query: string, args: PgSearchArgs): SqlFragment {
136
138
  return sql`select "id", "content", "metadata", ts_rank_cd("tsv", q) as score
137
139
  from ${identifier(target.table)}, websearch_to_tsquery(${target.language}::regconfig, ${query}) q
138
140
  where "tsv" @@ q and ${conditionsSql(args.scope, args.filter)}
139
- order by score desc
141
+ order by score desc, "id" asc
140
142
  limit ${args.k}`;
141
143
  }
142
144
 
@@ -149,7 +151,9 @@ export interface PgHybridArgs extends PgSearchArgs {
149
151
  * Reciprocal-rank fusion, done in SQL. Two candidate sets are ranked independently and fused by
150
152
  * `1 / (rrfK + rank)` — identical to `fuse()` in `vector.ts`, so dev and production order hits
151
153
  * the same way. Both CTEs carry the SAME scope conditions: fusing an unfiltered lexical ranking
152
- * into a filtered dense one would leak the other tenant's rows through the back door.
154
+ * into a filtered dense one would leak the other tenant's rows through the back door. A rank is a
155
+ * position, so each window breaks a tie by `"id"` as `vector.ts` does; the dense candidate scan
156
+ * keeps its bare `order by distance`, the shape HNSW answers.
153
157
  */
154
158
  export function hybridSql(
155
159
  target: PgVectorTable,
@@ -160,7 +164,7 @@ export function hybridSql(
160
164
  const table = identifier(target.table);
161
165
  const where = conditionsSql(args.scope, args.filter);
162
166
  return sql`with dense as (
163
- select "tenant", "id", row_number() over (order by distance) as rank
167
+ select "tenant", "id", row_number() over (order by distance, "id" asc) as rank
164
168
  from (
165
169
  select "tenant", "id", "embedding" <=> ${vectorLiteral(vector)}::vector as distance
166
170
  from ${table}
@@ -169,12 +173,12 @@ export function hybridSql(
169
173
  limit ${args.candidates}
170
174
  ) top
171
175
  ), lexical as (
172
- select "tenant", "id", row_number() over (order by relevance desc) as rank
176
+ select "tenant", "id", row_number() over (order by relevance desc, "id" asc) as rank
173
177
  from (
174
178
  select "tenant", "id", ts_rank_cd("tsv", q) as relevance
175
179
  from ${table}, websearch_to_tsquery(${target.language}::regconfig, ${query}) q
176
180
  where "tsv" @@ q and ${where}
177
- order by relevance desc
181
+ order by relevance desc, "id" asc
178
182
  limit ${args.candidates}
179
183
  ) top
180
184
  ), fused as (
@@ -211,7 +215,7 @@ export function deleteSql(
211
215
  ids: readonly string[],
212
216
  ): SqlFragment {
213
217
  // `in ()` is a syntax error, so an empty list takes the same constant an empty allow-list does.
214
- // `PgVectorStore.delete` returns before it gets here, but this function is exported from
218
+ // `PostgresVectorStore.delete` returns before it gets here, but this function is exported from
215
219
  // `index.ts` — an app compiling the statement itself is a caller too.
216
220
  const byId = ids.length === 0 ? NEVER : sql`"id" in (${join(ids.map((id) => sql`${id}`))})`;
217
221
  return sql`delete from ${identifier(target.table)}
package/src/pg-vector.ts CHANGED
@@ -24,9 +24,9 @@ import type {
24
24
  VectorRecord,
25
25
  VectorStore,
26
26
  } from './vector';
27
- import { narrowScope, tenantOf, UNSCOPED, type VectorScope } from './vector-scope';
27
+ import { assertTenantRead, narrowScope, tenantOf, UNBOUND, type VectorScope } from './vector-scope';
28
28
 
29
- export interface PgVectorStoreInput {
29
+ export interface PostgresVectorStoreInput {
30
30
  /** The table. Also the store's name in errors, and the stem of every index name. */
31
31
  readonly name: string;
32
32
  readonly dimension: number;
@@ -49,18 +49,18 @@ interface HitRow {
49
49
  readonly score: unknown;
50
50
  }
51
51
 
52
- export class PgVectorStore implements VectorStore {
52
+ export class PostgresVectorStore implements VectorStore {
53
53
  readonly name: string;
54
54
  readonly dimension: number;
55
55
  readonly scope: VectorScope;
56
- private readonly input: PgVectorStoreInput;
56
+ private readonly input: PostgresVectorStoreInput;
57
57
  private readonly target: PgVectorTable;
58
58
 
59
- constructor(input: PgVectorStoreInput) {
59
+ constructor(input: PostgresVectorStoreInput) {
60
60
  this.input = input;
61
61
  this.name = input.name;
62
62
  this.dimension = input.dimension;
63
- this.scope = input.scope ?? UNSCOPED;
63
+ this.scope = input.scope ?? UNBOUND;
64
64
  this.target = {
65
65
  table: input.name,
66
66
  dimension: input.dimension,
@@ -75,11 +75,11 @@ export class PgVectorStore implements VectorStore {
75
75
  }
76
76
 
77
77
  /**
78
- * A view of the same table through a narrower envelope. The unscoped store is the backfill
79
- * path; a request handler derives from it and can never widen back out.
78
+ * A view of the same table through a narrower envelope. A store opened with `scope: UNSCOPED`
79
+ * is the backfill path; a request handler derives from it and can never widen back out.
80
80
  */
81
- scoped(scope: VectorScope): PgVectorStore {
82
- return new PgVectorStore({
81
+ scoped(scope: VectorScope): PostgresVectorStore {
82
+ return new PostgresVectorStore({
83
83
  ...this.input,
84
84
  scope: narrowScope(this.name, this.scope, scope),
85
85
  });
@@ -111,6 +111,7 @@ export class PgVectorStore implements VectorStore {
111
111
  k: number,
112
112
  filter?: MetadataFilter,
113
113
  ): Promise<readonly SearchHit[]> {
114
+ assertTenantRead(this.name, this.scope);
114
115
  this.assertDimension(vector.length);
115
116
  // `k` is the statement's `limit` and reaches Postgres as a bound parameter, so a `NaN` is the
116
117
  // database's error to report rather than this store's — and a fractional one is an error there
@@ -125,12 +126,14 @@ export class PgVectorStore implements VectorStore {
125
126
  k: number,
126
127
  filter?: MetadataFilter,
127
128
  ): Promise<readonly SearchHit[]> {
129
+ assertTenantRead(this.name, this.scope);
128
130
  return this.run(
129
131
  textSql(this.target, query, { scope: this.scope, filter, k: finiteCount(HYBRID, 'k', k) }),
130
132
  );
131
133
  }
132
134
 
133
135
  async hybrid(input: HybridSearchInput): Promise<readonly SearchHit[]> {
136
+ assertTenantRead(this.name, this.scope);
134
137
  this.assertDimension(input.vector.length);
135
138
  // The same three bounds `MemoryVectorStore.hybrid` screens, and they have to be screened in
136
139
  // BOTH stores: `rrfK` lands in `1.0 / (rrfK + rank)`, Postgres has a float8 `NaN`, and it
@@ -176,6 +179,14 @@ export class PgVectorStore implements VectorStore {
176
179
  }
177
180
  }
178
181
 
182
+ /**
183
+ * The one way to build the pgvector-backed store — the twin of `memoryVectorStore()`. The class is
184
+ * a type in the barrel only (`X_FACTORY_NAME_SPELLING`), so `new` is never a second spelling.
185
+ */
186
+ export function postgresVectorStore(input: PostgresVectorStoreInput): PostgresVectorStore {
187
+ return new PostgresVectorStore(input);
188
+ }
189
+
179
190
  function toHit(row: HitRow): SearchHit {
180
191
  return {
181
192
  id: row.id,
@@ -0,0 +1,128 @@
1
+ // The data fences of a prompt template: which XML-style tag pairs enclose each `{{slot}}`, and the
2
+ // neutraliser `render` applies to the prompt those values were assembled into. A fence is the template's declaration
3
+ // that a slot is DATA; a value that writes the fence's own closer would end it early and speak as
4
+ // the prompt (#689), so that closer is broken — never deleted, the model still reads every word.
5
+
6
+ /** `<name>` or `<name attr="…">` — a fence opener. Self-closing and closing tags are not openers. */
7
+ const OPEN = /<([A-Za-z_][\w.-]*)(?:\s[^<>]*)?(?<!\/)>/g;
8
+ /** `</name>`, as the template author writes it. */
9
+ const CLOSE = /<\/([A-Za-z_][\w.-]*)\s*>/g;
10
+ /**
11
+ * A `{{slot}}`. The ONE pattern: `render` substitutes by it and `fenceMap` keys offsets by it, so
12
+ * the two can never disagree about where a slot is.
13
+ */
14
+ export const SLOT = /\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g;
15
+
16
+ interface TagMark {
17
+ readonly name: string;
18
+ readonly at: number;
19
+ }
20
+
21
+ const marks = (template: string, pattern: RegExp): readonly TagMark[] =>
22
+ [...template.matchAll(pattern)].map((match) => ({ name: match[1] ?? '', at: match.index }));
23
+
24
+ /**
25
+ * The fences enclosing the offset `at`, outermost first. A tag encloses it when its LAST opener
26
+ * before `at` is not closed before `at`, and a closer follows — read per name, never as a stack, so
27
+ * a stray `<note>` in prose fences nothing and cannot unbalance the real fences around it.
28
+ */
29
+ function enclosing(
30
+ opens: readonly TagMark[],
31
+ closes: readonly TagMark[],
32
+ at: number,
33
+ ): readonly string[] {
34
+ const found: TagMark[] = [];
35
+ for (const name of new Set(opens.map((mark) => mark.name))) {
36
+ const opener = opens.findLast((mark) => mark.name === name && mark.at < at);
37
+ if (opener === undefined) continue;
38
+ const named = closes.filter((mark) => mark.name === name);
39
+ const closedBefore = named.some((mark) => mark.at > opener.at && mark.at < at);
40
+ if (!closedBefore && named.some((mark) => mark.at > at)) found.push(opener);
41
+ }
42
+ return found.sort((a, b) => a.at - b.at).map((mark) => mark.name);
43
+ }
44
+
45
+ /**
46
+ * Each FENCED slot occurrence's offset, mapped to the closers its value must not keep: every fence
47
+ * the template draws, not only the ones around it. A title is no safer for writing `</post_body>`
48
+ * than `</post_title>` — a model reads tags in order, and a forged closer of a later fence is one
49
+ * more place the data seems to end.
50
+ */
51
+ export function fenceMap(template: string): ReadonlyMap<number, readonly string[]> {
52
+ const opens = marks(template, OPEN);
53
+ const closes = marks(template, CLOSE);
54
+ const drawn = [...new Set(opens.map((mark) => mark.name))].filter((name) =>
55
+ closes.some(
56
+ (close) => close.name === name && opens.some((o) => o.name === name && o.at < close.at),
57
+ ),
58
+ );
59
+ const map = new Map<number, readonly string[]>();
60
+ for (const slot of template.matchAll(SLOT)) {
61
+ if (enclosing(opens, closes, slot.index).length > 0) map.set(slot.index, drawn);
62
+ }
63
+ return map;
64
+ }
65
+
66
+ /**
67
+ * Per slot name, the fences enclosing EVERY occurrence of it — so one bare occurrence reports the
68
+ * slot as unfenced, which is what an app's own test asserts against ("is `body` data?").
69
+ */
70
+ export function promptFences(template: string): Readonly<Record<string, readonly string[]>> {
71
+ const opens = marks(template, OPEN);
72
+ const closes = marks(template, CLOSE);
73
+ const out: Record<string, readonly string[]> = {};
74
+ for (const slot of template.matchAll(SLOT)) {
75
+ const name = slot[1] ?? '';
76
+ const here = enclosing(opens, closes, slot.index);
77
+ // Own keys only: a slot named `constructor` must not read Object.prototype's.
78
+ const before = Object.hasOwn(out, name) ? out[name] : undefined;
79
+ out[name] = before === undefined ? here : before.filter((tag) => here.includes(tag));
80
+ }
81
+ return out;
82
+ }
83
+
84
+ const escapeName = (name: string): string => name.replace(/[.-]/g, '\\$&');
85
+
86
+ /** Where a fenced slot's value landed in the rendered prompt: `[start, end)`. */
87
+ export interface FencedSpan {
88
+ readonly start: number;
89
+ readonly end: number;
90
+ }
91
+
92
+ /**
93
+ * Break every closer of `fences` in the ASSEMBLED prompt that a fenced value wrote any character
94
+ * of. Per value was not enough: `{{a}}{{b}}` with `</post_` and `body>` rebuilt a whole closer from
95
+ * two halves, and a value ending `</post_` beside template text `body>` completed one. So the scan
96
+ * runs over the rendered text, and a closer the template wrote alone — no value inside it — stands.
97
+ *
98
+ * A closer is `<`, `/` and the name, any case, whitespace either side of the slash, and ANYTHING
99
+ * after the name that does not continue it: `</post_body foo=1>`, `</post_body/>` and a bare
100
+ * `</post_body` read as one to a lenient model. `<\/name` is the shape `assembleContext` uses for
101
+ * `</document>`: the marker is broken, the text survives.
102
+ */
103
+ export function neutraliseFenced(
104
+ rendered: string,
105
+ spans: readonly FencedSpan[],
106
+ fences: readonly string[],
107
+ ): string {
108
+ if (fences.length === 0 || spans.length === 0) return rendered;
109
+ const closer = new RegExp(`<\\s*\\/\\s*(?:${fences.map(escapeName).join('|')})(?![\\w.-])`, 'gi');
110
+ let out = '';
111
+ let from = 0;
112
+ for (const match of rendered.matchAll(closer)) {
113
+ const start = match.index;
114
+ const end = start + match[0].length;
115
+ if (!spans.some((span) => span.start < end && span.end > start)) continue;
116
+ const slash = rendered.indexOf('/', start);
117
+ out += `${rendered.slice(from, slash)}\\`;
118
+ from = slash;
119
+ }
120
+ return out + rendered.slice(from);
121
+ }
122
+
123
+ /**
124
+ * A value that is data from its first character to its last — a retrieved document, not a slot in
125
+ * a template. The same pass as `neutraliseFenced`, one span wide: one neutraliser in this package.
126
+ */
127
+ export const neutraliseFences = (value: string, fences: readonly string[]): string =>
128
+ neutraliseFenced(value, [{ start: 0, end: value.length }], fences);
package/src/prompt.ts CHANGED
@@ -12,6 +12,7 @@
12
12
  import { canonicalJson } from '@ultimat3/core';
13
13
  import { AiPromptRenderError, AiPromptVersionError } from './errors';
14
14
  import type { Effort, ModelId, ThinkingMode } from './models';
15
+ import { type FencedSpan, fenceMap, neutraliseFenced, SLOT } from './prompt-fence';
15
16
  import type { JsonSchema } from './tools';
16
17
 
17
18
  /** Template variables. Values are stringified at render time with no formatting magic. */
@@ -21,7 +22,11 @@ export interface DefinePromptInput<V extends PromptVars> {
21
22
  readonly id: string;
22
23
  /** Semver-ish, author-assigned. Must change whenever `template` changes. */
23
24
  readonly version: string;
24
- /** `{{name}}` placeholders. Every key in `V` must appear; unfilled ones throw. */
25
+ /**
26
+ * `{{name}}` placeholders. Every key in `V` must appear; unfilled ones throw. A slot inside an
27
+ * XML-style tag pair (`<post_body>\n{{body}}\n</post_body>`) is fenced DATA: render breaks every
28
+ * fence closer in the value, so user text cannot end the fence and pose as instructions.
29
+ */
25
30
  readonly template: string;
26
31
  /** Optional system prompt. Part of the hash — it changes behaviour. */
27
32
  readonly system?: string;
@@ -58,7 +63,10 @@ export interface Prompt<V extends PromptVars = PromptVars> {
58
63
  readonly model: ModelId | undefined;
59
64
  readonly effort: Effort | undefined;
60
65
  readonly thinking: ThinkingMode | undefined;
61
- /** Substitute variables. Throws on an unfilled placeholder. */
66
+ /**
67
+ * Substitute variables. In a fenced slot every fence closer the template draws is broken, never
68
+ * deleted (`prompt-fence.ts`). Throws on an unfilled placeholder.
69
+ */
62
70
  render(vars: V): string;
63
71
  /** `id@version` — the identity an eval result is filed under. */
64
72
  readonly ref: string;
@@ -67,7 +75,7 @@ export interface Prompt<V extends PromptVars = PromptVars> {
67
75
  const registry = new Map<string, Prompt>();
68
76
 
69
77
  export function definePrompt<V extends PromptVars>(input: DefinePromptInput<V>): Prompt<V> {
70
- const hash = contentHash(input);
78
+ const hash = promptHash(input);
71
79
  const key = `${input.id}@${input.version}`;
72
80
  const existing = registry.get(key);
73
81
  if (existing !== undefined && existing.hash !== hash) {
@@ -78,6 +86,8 @@ export function definePrompt<V extends PromptVars>(input: DefinePromptInput<V>):
78
86
  });
79
87
  }
80
88
 
89
+ // Read once, at declaration: the fences are the template's, and a render is a hot path.
90
+ const fences = fenceMap(input.template);
81
91
  const prompt: Prompt<V> = {
82
92
  id: input.id,
83
93
  version: input.version,
@@ -90,7 +100,7 @@ export function definePrompt<V extends PromptVars>(input: DefinePromptInput<V>):
90
100
  effort: input.effort,
91
101
  thinking: input.thinking,
92
102
  ref: key,
93
- render: (vars) => render(input.template, vars, key),
103
+ render: (vars) => render(input.template, vars, key, fences),
94
104
  };
95
105
  registry.set(key, prompt as Prompt);
96
106
  return prompt;
@@ -126,11 +136,23 @@ export function resetPrompts(): void {
126
136
  registry.clear();
127
137
  }
128
138
 
129
- const PLACEHOLDER = /\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g;
130
-
131
- function render(template: string, vars: PromptVars, ref: string): string {
139
+ function render(
140
+ template: string,
141
+ vars: PromptVars,
142
+ ref: string,
143
+ fences: ReadonlyMap<number, readonly string[]>,
144
+ ): string {
132
145
  const missing: string[] = [];
133
- const out = template.replace(PLACEHOLDER, (_match, name: string) => {
146
+ const spans: FencedSpan[] = [];
147
+ let names: readonly string[] = [];
148
+ let out = '';
149
+ let from = 0;
150
+ // Assembled by hand rather than `replace`: the fence pass below needs where each fenced value
151
+ // LANDED, because a closer can be built across two values or a value and the template beside it.
152
+ for (const match of template.matchAll(SLOT)) {
153
+ out += template.slice(from, match.index);
154
+ from = match.index + match[0].length;
155
+ const name = match[1] ?? '';
134
156
  // `Object.hasOwn`, never `vars[name] === undefined`: a plain object inherits `constructor`,
135
157
  // `toString` and `valueOf`, so `{{constructor}}` in a template rendered JS SOURCE into the
136
158
  // prompt instead of raising the unfilled-slot error this file promises — and that source was
@@ -139,16 +161,24 @@ function render(template: string, vars: PromptVars, ref: string): string {
139
161
  const value = Object.hasOwn(vars, name) ? vars[name] : undefined;
140
162
  if (value === undefined) {
141
163
  missing.push(name);
142
- return '';
164
+ continue;
165
+ }
166
+ const text = String(value);
167
+ const fenced = fences.get(match.index);
168
+ if (fenced !== undefined) {
169
+ spans.push({ start: out.length, end: out.length + text.length });
170
+ names = fenced;
143
171
  }
144
- return String(value);
145
- });
172
+ out += text;
173
+ }
174
+ out += template.slice(from);
146
175
  if (missing.length > 0) {
147
176
  // Loud, like an i18n miss: a silently blank variable is a prompt that reads fine and
148
177
  // means something else.
149
178
  throw new AiPromptRenderError({ ref, missing });
150
179
  }
151
- return out;
180
+ // A fenced slot is data; no closer a value wrote any part of survives.
181
+ return neutraliseFenced(out, spans, names);
152
182
  }
153
183
 
154
184
  /**
@@ -161,7 +191,7 @@ function render(template: string, vars: PromptVars, ref: string): string {
161
191
  * An absent schema stays the empty string rather than becoming `canonicalJson(undefined)`'s
162
192
  * `null` — that is what keeps every hash already recorded in a baseline the same value.
163
193
  */
164
- export function contentHash<V extends PromptVars>(input: DefinePromptInput<V>): string {
194
+ export function promptHash<V extends PromptVars>(input: DefinePromptInput<V>): string {
165
195
  const fields = [
166
196
  `id:${input.id}`,
167
197
  `version:${input.version}`,