@panaversity/ksor 0.0.52 → 0.0.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +162 -0
- package/README.md +30 -0
- package/dist/cli.mjs +266 -46
- package/dist/{gateway-api-Ck1l_Sgs-B-_wMYLB.mjs → gateway-api-uhx2l1kC-C2BAxISt.mjs} +197 -17
- package/dist/gateway.mjs +1 -1
- package/docs/deploying.md +41 -8
- package/docs/ingesting.md +25 -8
- package/package.json +3 -3
- package/templates/scaffold/AGENTS.md +34 -5
- package/templates/scaffold/README.md +78 -7
- package/templates/scaffold/mcp.json +13 -0
package/dist/cli.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { n as resolveCommand, r as verbs, t as exitCodes } from "./src-dqpI-p1a.mjs";
|
|
3
|
-
import { A as parseViewer, B as
|
|
3
|
+
import { A as parseViewer, B as tallyHandlers, C as contentPoolMin, D as outlineHandler, E as keyRingFromEnv, F as recordIsUndescribed, G as z$1, H as validateViewer, I as runProbe$1, L as searchHandler, M as prewarmPool, N as providerKeyEnv$1, O as parseInstanceText$1, P as readHandler, R as servingPolicy$1, S as contentPool$1, T as instancePathOf$1, U as withPgRetry$1, V as tlsPosture, W as withProbeDeadline$1, _ as assertGovernanceServable$1, a as GovernanceGateError$1, b as checkEmbeddingSpace$1, c as McpServer$1, d as READ_ONLY, f as READ_OUTPUT, g as TextSearchConfigMismatch, h as TRUST_TIERS$1, i as FLOOR, j as pooledEndpointFor, k as parseTrustFloor, l as MissingProviderKeyError$1, m as SchemaVersionError, n as ContentStoreError$1, o as MAX_OUTLINE_LIMIT, p as SEARCH_OUTPUT, r as EmbeddingSpaceMismatch$1, t as AudienceError$1, u as OUTLINE_OUTPUT, v as assertSchemaCompatible, w as embedQueryVlit, x as composeInstructions, y as buildShippedProvider$1, z as storedTextSearchConfig } from "./gateway-api-uhx2l1kC-C2BAxISt.mjs";
|
|
4
4
|
import { appendFileSync, chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
6
6
|
import { InMemoryTransport, LATEST_PROTOCOL_VERSION, createMcpHandler } from "@modelcontextprotocol/server";
|
|
@@ -17,7 +17,7 @@ import { Document, YAMLParseError, isCollection, isMap, isPair, isSeq, parseAllD
|
|
|
17
17
|
import { parseArgs } from "node:util";
|
|
18
18
|
import { readFile } from "node:fs/promises";
|
|
19
19
|
import { execFileSync, spawnSync } from "node:child_process";
|
|
20
|
-
//#region ../content-gateway/dist/main-
|
|
20
|
+
//#region ../content-gateway/dist/main-BHQDvsKA.mjs
|
|
21
21
|
/**
|
|
22
22
|
* The default registration — and the ORIGINAL of the file `ksor init` emits.
|
|
23
23
|
*
|
|
@@ -1008,7 +1008,7 @@ async function compose(rawInstancePath, version) {
|
|
|
1008
1008
|
let provider;
|
|
1009
1009
|
try {
|
|
1010
1010
|
provider = buildShippedProvider$1(instance.embeddingProvider, {
|
|
1011
|
-
apiKey: process.env["
|
|
1011
|
+
apiKey: process.env[providerKeyEnv$1(instance.embeddingProvider) ?? ""] ?? null,
|
|
1012
1012
|
modelId: instance.embeddingModel,
|
|
1013
1013
|
dim: instance.embeddingDim
|
|
1014
1014
|
});
|
|
@@ -4709,7 +4709,7 @@ async function withPgRetry(op, options = {}) {
|
|
|
4709
4709
|
throw lastError;
|
|
4710
4710
|
}
|
|
4711
4711
|
//#endregion
|
|
4712
|
-
//#region ../content/dist/commands-
|
|
4712
|
+
//#region ../content/dist/commands-DM5TajmH.mjs
|
|
4713
4713
|
/**
|
|
4714
4714
|
* EVAL-LOCKED constants, quarried verbatim from the oracle
|
|
4715
4715
|
* (sor-agentfactory @ b554f91, config.py) — changing any of these is a
|
|
@@ -5823,13 +5823,6 @@ const INGEST_RETRY = {
|
|
|
5823
5823
|
initialS: 1,
|
|
5824
5824
|
maxS: 30
|
|
5825
5825
|
};
|
|
5826
|
-
/** Fail-fast for the query path: 2 attempts, sub-second backoff, no 429 retry — worst case a
|
|
5827
|
-
* query embed costs ~1 s before the caller degrades, never a 15 s stall on the read. */
|
|
5828
|
-
const QUERY_RETRY = {
|
|
5829
|
-
attempts: 2,
|
|
5830
|
-
initialS: .2,
|
|
5831
|
-
maxS: .5
|
|
5832
|
-
};
|
|
5833
5826
|
function sleepS(seconds) {
|
|
5834
5827
|
return new Promise((resolve) => setTimeout(resolve, seconds * 1e3));
|
|
5835
5828
|
}
|
|
@@ -5855,17 +5848,6 @@ async function embedIntent(texts, opts) {
|
|
|
5855
5848
|
return contract(texts, await withRetry(INGEST_RETRY, (exc) => provider.isRetryable(exc), () => provider.reset(), () => provider.embed(texts, { intent })), provider.dim);
|
|
5856
5849
|
}
|
|
5857
5850
|
/**
|
|
5858
|
-
* The READ plane's door (query-embed, intent "query"). Same contract;
|
|
5859
|
-
* FAIL-FAST retry (no 429, 2 attempts): a rate limit degrades the search to
|
|
5860
|
-
* keyword-only in under a second instead of stalling the read ~15 s. Async
|
|
5861
|
-
* like everything here — the oracle's `a` prefix marked Python's async
|
|
5862
|
-
* flavor and is kept only for greppability against the Python.
|
|
5863
|
-
*/
|
|
5864
|
-
async function aembedIntent(texts, opts) {
|
|
5865
|
-
const { provider, intent } = opts;
|
|
5866
|
-
return contract(texts, await withRetry(QUERY_RETRY, (exc) => provider.isRetryableQuery(exc), () => provider.reset(), () => provider.embed(texts, { intent })), provider.dim);
|
|
5867
|
-
}
|
|
5868
|
-
/**
|
|
5869
5851
|
* The deterministic, key-free fake provider — a KSOR ADDITION (no oracle
|
|
5870
5852
|
* counterpart; decision 6 conversion note). It exists so the DB tier and CI
|
|
5871
5853
|
* can exercise ingest + retrieval end to end without a vendor key: same text
|
|
@@ -5937,7 +5919,7 @@ var FakeEmbeddingProvider = class {
|
|
|
5937
5919
|
}
|
|
5938
5920
|
reset() {}
|
|
5939
5921
|
};
|
|
5940
|
-
const DEFAULT_BASE = "https://generativelanguage.googleapis.com/v1beta";
|
|
5922
|
+
const DEFAULT_BASE$1 = "https://generativelanguage.googleapis.com/v1beta";
|
|
5941
5923
|
/**
|
|
5942
5924
|
* An HTTP-shaped failure carrying the status the retry classifier reads.
|
|
5943
5925
|
*
|
|
@@ -5955,7 +5937,7 @@ var GeminiHttpError = class extends Error {
|
|
|
5955
5937
|
};
|
|
5956
5938
|
/** One POST, with the key in a HEADER — never the query string, which is logged. */
|
|
5957
5939
|
async function post(opts, apiKey, path, body, timeoutMs) {
|
|
5958
|
-
const res = await (opts.fetchImpl ?? fetch)(`${opts.baseUrl ?? DEFAULT_BASE}${path}`, {
|
|
5940
|
+
const res = await (opts.fetchImpl ?? fetch)(`${opts.baseUrl ?? DEFAULT_BASE$1}${path}`, {
|
|
5959
5941
|
method: "POST",
|
|
5960
5942
|
headers: {
|
|
5961
5943
|
"x-goog-api-key": apiKey,
|
|
@@ -6025,33 +6007,33 @@ function geminiRestTextClient(apiKey, opts = {}) {
|
|
|
6025
6007
|
* Python-SDK failure mode with no JS equivalent; `reset()`
|
|
6026
6008
|
* keeps its drop-never-close contract regardless.
|
|
6027
6009
|
*/
|
|
6028
|
-
function httpStatusOf(exc) {
|
|
6010
|
+
function httpStatusOf$1(exc) {
|
|
6029
6011
|
if (!(exc instanceof Error)) return void 0;
|
|
6030
6012
|
const status = exc.status;
|
|
6031
6013
|
return typeof status === "number" ? status : void 0;
|
|
6032
6014
|
}
|
|
6033
|
-
function isTransportBlip(exc, depth = 0) {
|
|
6015
|
+
function isTransportBlip$1(exc, depth = 0) {
|
|
6034
6016
|
if (depth > 5 || !(exc instanceof Error)) return false;
|
|
6035
6017
|
if (exc.name === "AbortError" || exc.name === "TimeoutError") return true;
|
|
6036
6018
|
const code = exc.code;
|
|
6037
6019
|
if (typeof code === "string" && /^E[A-Z0-9_]+$/.test(code)) return true;
|
|
6038
6020
|
if (exc instanceof TypeError && exc.message.includes("fetch failed")) return true;
|
|
6039
|
-
return isTransportBlip(exc.cause, depth + 1);
|
|
6021
|
+
return isTransportBlip$1(exc.cause, depth + 1);
|
|
6040
6022
|
}
|
|
6041
6023
|
/** The Gemini taxonomy for the INGEST plane: transport blips, 5xx, AND 429 —
|
|
6042
6024
|
* batch work is resumable and has no user waiting. */
|
|
6043
|
-
function isRetryable(exc) {
|
|
6044
|
-
if (isTransportBlip(exc)) return true;
|
|
6045
|
-
const status = httpStatusOf(exc);
|
|
6025
|
+
function isRetryable$1(exc) {
|
|
6026
|
+
if (isTransportBlip$1(exc)) return true;
|
|
6027
|
+
const status = httpStatusOf$1(exc);
|
|
6046
6028
|
if (status === void 0) return false;
|
|
6047
6029
|
return status >= 500 && status <= 599 || status === 429;
|
|
6048
6030
|
}
|
|
6049
6031
|
/** The READ path's predicate: transport blips + 5xx only — NEVER 429. A
|
|
6050
6032
|
* rate-limited project stays rate-limited on the next second; the correct
|
|
6051
6033
|
* move is to degrade to keyword-only NOW, not stall the read behind backoff. */
|
|
6052
|
-
function isRetryableQuery(exc) {
|
|
6053
|
-
if (isTransportBlip(exc)) return true;
|
|
6054
|
-
const status = httpStatusOf(exc);
|
|
6034
|
+
function isRetryableQuery$1(exc) {
|
|
6035
|
+
if (isTransportBlip$1(exc)) return true;
|
|
6036
|
+
const status = httpStatusOf$1(exc);
|
|
6055
6037
|
return status !== void 0 && status >= 500 && status <= 599;
|
|
6056
6038
|
}
|
|
6057
6039
|
/**
|
|
@@ -6102,10 +6084,10 @@ var GeminiEmbeddingProvider = class {
|
|
|
6102
6084
|
})).embeddings ?? []).map((e) => [...e.values ?? []]);
|
|
6103
6085
|
}
|
|
6104
6086
|
isRetryable(exc) {
|
|
6105
|
-
return isRetryable(exc);
|
|
6087
|
+
return isRetryable$1(exc);
|
|
6106
6088
|
}
|
|
6107
6089
|
isRetryableQuery(exc) {
|
|
6108
|
-
return isRetryableQuery(exc);
|
|
6090
|
+
return isRetryableQuery$1(exc);
|
|
6109
6091
|
}
|
|
6110
6092
|
};
|
|
6111
6093
|
/**
|
|
@@ -6118,7 +6100,7 @@ var GeminiTextGenerator = class {
|
|
|
6118
6100
|
clientFactory;
|
|
6119
6101
|
client = null;
|
|
6120
6102
|
constructor(opts) {
|
|
6121
|
-
this.model = opts.model ?? "gemini-
|
|
6103
|
+
this.model = opts.model ?? "gemini-3.7-flash";
|
|
6122
6104
|
this.clientFactory = opts.clientFactory ?? (() => geminiRestTextClient(opts.apiKey));
|
|
6123
6105
|
}
|
|
6124
6106
|
getClient() {
|
|
@@ -6139,9 +6121,163 @@ var GeminiTextGenerator = class {
|
|
|
6139
6121
|
}
|
|
6140
6122
|
})).text ?? "";
|
|
6141
6123
|
}
|
|
6124
|
+
isRetryable(exc) {
|
|
6125
|
+
return isRetryable$1(exc);
|
|
6126
|
+
}
|
|
6127
|
+
};
|
|
6128
|
+
/**
|
|
6129
|
+
* OpenAI's embedding endpoint, over `fetch` — no SDK.
|
|
6130
|
+
*
|
|
6131
|
+
* The same call decision 12's 2026-08-22 revision made for Gemini, for the same
|
|
6132
|
+
* reason: this is ONE HTTP call behind a structurally-typed slice, and a vendor
|
|
6133
|
+
* SDK would put megabytes and a transitive dependency tree into every
|
|
6134
|
+
* `ksor init` that never embeds anything. If a provider ever needs an SDK, the
|
|
6135
|
+
* seam takes one through `clientFactory`; nothing here forecloses that.
|
|
6136
|
+
*
|
|
6137
|
+
* WHAT DIFFERS FROM GEMINI, and it is worth knowing before choosing:
|
|
6138
|
+
*
|
|
6139
|
+
* Gemini embeds ASYMMETRICALLY — `RETRIEVAL_DOCUMENT` and `RETRIEVAL_QUERY`
|
|
6140
|
+
* produce different vectors for the same text, and the plane's intent picks
|
|
6141
|
+
* which. OpenAI has no task type: a query and a document are embedded
|
|
6142
|
+
* identically. The seam anticipates exactly this — "a provider whose two
|
|
6143
|
+
* vendor labels are equal can never mis-route a plane"
|
|
6144
|
+
* (`lib/embedding.ts`) — so both labels are the empty string and the intent
|
|
6145
|
+
* reaches this transport and is deliberately ignored.
|
|
6146
|
+
*
|
|
6147
|
+
* `dimensions` is supported on `text-embedding-3-*` only. Asking an older
|
|
6148
|
+
* model for a dimension is an error there, not a silent full-width vector,
|
|
6149
|
+
* which is the failure the framework's own width check would catch anyway.
|
|
6150
|
+
*/
|
|
6151
|
+
/**
|
|
6152
|
+
* An HTTP-shaped failure carrying the status the retry classifier reads — and
|
|
6153
|
+
* the vendor's own error `type`, which the status alone does not distinguish.
|
|
6154
|
+
*
|
|
6155
|
+
* OpenAI answers a spent balance with **429**, the same status as a rate limit:
|
|
6156
|
+
* `{"error":{"type":"insufficient_quota","code":"credit_balance_exhausted"}}`
|
|
6157
|
+
* (observed live, 2026-09-01). One clears by waiting and one never will, so a
|
|
6158
|
+
* classifier reading only the status retries a billing problem five times with
|
|
6159
|
+
* exponential backoff and then reports it — slowly, and as if it had been
|
|
6160
|
+
* transient.
|
|
6161
|
+
*/
|
|
6162
|
+
var OpenAiHttpError = class extends Error {
|
|
6163
|
+
status;
|
|
6164
|
+
/** The vendor's `error.type`, when it sends one. */
|
|
6165
|
+
kind;
|
|
6166
|
+
constructor(status, detail, kind = null) {
|
|
6167
|
+
super(`OpenAI API error ${status}: ${detail}`);
|
|
6168
|
+
this.name = "OpenAiHttpError";
|
|
6169
|
+
this.status = status;
|
|
6170
|
+
this.kind = kind;
|
|
6171
|
+
}
|
|
6172
|
+
};
|
|
6173
|
+
const DEFAULT_BASE = "https://api.openai.com/v1";
|
|
6174
|
+
function openAiRestEmbedClient(apiKey, opts = {}) {
|
|
6175
|
+
return { async embed(params) {
|
|
6176
|
+
const res = await (opts.fetchImpl ?? fetch)(`${opts.baseUrl ?? DEFAULT_BASE}/embeddings`, {
|
|
6177
|
+
method: "POST",
|
|
6178
|
+
headers: {
|
|
6179
|
+
authorization: `Bearer ${apiKey}`,
|
|
6180
|
+
"content-type": "application/json"
|
|
6181
|
+
},
|
|
6182
|
+
body: JSON.stringify({
|
|
6183
|
+
model: params.model,
|
|
6184
|
+
input: [...params.input],
|
|
6185
|
+
dimensions: params.dimensions
|
|
6186
|
+
}),
|
|
6187
|
+
signal: AbortSignal.timeout(params.timeoutMs)
|
|
6188
|
+
});
|
|
6189
|
+
const text = await res.text();
|
|
6190
|
+
if (!res.ok) {
|
|
6191
|
+
let detail = text.slice(0, 300);
|
|
6192
|
+
let kind = null;
|
|
6193
|
+
try {
|
|
6194
|
+
const err = JSON.parse(text).error;
|
|
6195
|
+
if (typeof err?.message === "string") detail = err.message;
|
|
6196
|
+
if (typeof err?.type === "string") kind = err.type;
|
|
6197
|
+
} catch {}
|
|
6198
|
+
throw new OpenAiHttpError(res.status, detail, kind);
|
|
6199
|
+
}
|
|
6200
|
+
const data = [...JSON.parse(text).data ?? []];
|
|
6201
|
+
data.sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
|
|
6202
|
+
return { embeddings: data.map((d) => ({ values: d.embedding })) };
|
|
6203
|
+
} };
|
|
6204
|
+
}
|
|
6205
|
+
/** True for a transport blip with no HTTP status of its own. */
|
|
6206
|
+
function isTransportBlip(exc) {
|
|
6207
|
+
if (exc instanceof OpenAiHttpError) return false;
|
|
6208
|
+
const name = exc?.name;
|
|
6209
|
+
return name === "AbortError" || name === "TimeoutError" || name === "TypeError";
|
|
6210
|
+
}
|
|
6211
|
+
function httpStatusOf(exc) {
|
|
6212
|
+
return exc instanceof OpenAiHttpError ? exc.status : void 0;
|
|
6213
|
+
}
|
|
6214
|
+
/**
|
|
6215
|
+
* The INGEST plane's taxonomy: transport blips, 5xx, AND 429 — batch work is
|
|
6216
|
+
* resumable and has nobody waiting. Deliberately the same shape as the Gemini
|
|
6217
|
+
* adapter's, because the two planes are a property of ksor, not of a vendor.
|
|
6218
|
+
*/
|
|
6219
|
+
function isRetryable(exc) {
|
|
6220
|
+
if (isTransportBlip(exc)) return true;
|
|
6221
|
+
if (exc instanceof OpenAiHttpError && exc.kind === "insufficient_quota") return false;
|
|
6222
|
+
const status = httpStatusOf(exc);
|
|
6223
|
+
if (status === void 0) return false;
|
|
6224
|
+
return status >= 500 && status <= 599 || status === 429;
|
|
6225
|
+
}
|
|
6226
|
+
/**
|
|
6227
|
+
* The READ plane's: transport blips + 5xx only, NEVER 429. A rate-limited
|
|
6228
|
+
* project stays rate-limited on the next second, so a search degrades to
|
|
6229
|
+
* keyword-only now rather than stalling a reader behind backoff.
|
|
6230
|
+
*/
|
|
6231
|
+
function isRetryableQuery(exc) {
|
|
6232
|
+
if (isTransportBlip(exc)) return true;
|
|
6233
|
+
const status = httpStatusOf(exc);
|
|
6234
|
+
return status !== void 0 && status >= 500 && status <= 599;
|
|
6235
|
+
}
|
|
6236
|
+
var OpenAiEmbeddingProvider = class {
|
|
6237
|
+
providerId = "openai";
|
|
6238
|
+
modelId;
|
|
6239
|
+
dim;
|
|
6240
|
+
documentTaskLabel;
|
|
6241
|
+
queryTaskLabel;
|
|
6242
|
+
documentTimeoutMs;
|
|
6243
|
+
queryTimeoutMs;
|
|
6244
|
+
clientFactory;
|
|
6245
|
+
client = null;
|
|
6246
|
+
constructor(opts) {
|
|
6247
|
+
this.modelId = opts.modelId;
|
|
6248
|
+
this.dim = opts.dim;
|
|
6249
|
+
this.documentTaskLabel = opts.documentTaskLabel;
|
|
6250
|
+
this.queryTaskLabel = opts.queryTaskLabel;
|
|
6251
|
+
this.documentTimeoutMs = Math.trunc(opts.documentTimeoutS * 1e3);
|
|
6252
|
+
this.queryTimeoutMs = Math.trunc(opts.queryTimeoutS * 1e3);
|
|
6253
|
+
this.clientFactory = opts.clientFactory ?? (() => openAiRestEmbedClient(opts.apiKey));
|
|
6254
|
+
}
|
|
6255
|
+
get recipe() {
|
|
6256
|
+
return `${this.modelId}/d${this.dim}/${this.documentTaskLabel}`;
|
|
6257
|
+
}
|
|
6258
|
+
getClient() {
|
|
6259
|
+
this.client ??= this.clientFactory();
|
|
6260
|
+
return this.client;
|
|
6261
|
+
}
|
|
6262
|
+
/** DROP the client reference, never close it: in-flight calls finish on their
|
|
6263
|
+
* own reference and the next call rebuilds lazily. Idempotent. */
|
|
6264
|
+
reset() {
|
|
6265
|
+
this.client = null;
|
|
6266
|
+
}
|
|
6267
|
+
async embed(texts, opts) {
|
|
6268
|
+
return (await this.getClient().embed({
|
|
6269
|
+
model: this.modelId,
|
|
6270
|
+
input: texts,
|
|
6271
|
+
dimensions: this.dim,
|
|
6272
|
+
timeoutMs: opts.intent === "document" ? this.documentTimeoutMs : this.queryTimeoutMs
|
|
6273
|
+
})).embeddings.map((e) => [...e.values ?? []]);
|
|
6274
|
+
}
|
|
6142
6275
|
isRetryable(exc) {
|
|
6143
6276
|
return isRetryable(exc);
|
|
6144
6277
|
}
|
|
6278
|
+
isRetryableQuery(exc) {
|
|
6279
|
+
return isRetryableQuery(exc);
|
|
6280
|
+
}
|
|
6145
6281
|
};
|
|
6146
6282
|
/**
|
|
6147
6283
|
* The embedding-provider registry — a plain object, NOT any discovery
|
|
@@ -6177,11 +6313,30 @@ var MissingProviderKeyError = class extends Error {
|
|
|
6177
6313
|
const PROVIDERS = {
|
|
6178
6314
|
gemini: {
|
|
6179
6315
|
build: (opts) => new GeminiEmbeddingProvider(opts),
|
|
6180
|
-
needsApiKey: true
|
|
6316
|
+
needsApiKey: true,
|
|
6317
|
+
keyEnv: "GEMINI_API_KEY",
|
|
6318
|
+
taskLabels: {
|
|
6319
|
+
document: EMBED_TASK_DOCUMENT,
|
|
6320
|
+
query: EMBED_TASK_QUERY
|
|
6321
|
+
}
|
|
6322
|
+
},
|
|
6323
|
+
openai: {
|
|
6324
|
+
build: (opts) => new OpenAiEmbeddingProvider(opts),
|
|
6325
|
+
needsApiKey: true,
|
|
6326
|
+
keyEnv: "OPENAI_API_KEY",
|
|
6327
|
+
taskLabels: {
|
|
6328
|
+
document: "",
|
|
6329
|
+
query: ""
|
|
6330
|
+
}
|
|
6181
6331
|
},
|
|
6182
6332
|
fake: {
|
|
6183
6333
|
build: (opts) => new FakeEmbeddingProvider(opts),
|
|
6184
|
-
needsApiKey: false
|
|
6334
|
+
needsApiKey: false,
|
|
6335
|
+
keyEnv: null,
|
|
6336
|
+
taskLabels: {
|
|
6337
|
+
document: EMBED_TASK_DOCUMENT,
|
|
6338
|
+
query: EMBED_TASK_QUERY
|
|
6339
|
+
}
|
|
6185
6340
|
}
|
|
6186
6341
|
};
|
|
6187
6342
|
function entryFor(name) {
|
|
@@ -6196,6 +6351,13 @@ function providerNeedsApiKey(name) {
|
|
|
6196
6351
|
return entryFor(name).needsApiKey;
|
|
6197
6352
|
}
|
|
6198
6353
|
/**
|
|
6354
|
+
* The environment variable this provider's key comes from, or null when it
|
|
6355
|
+
* needs none. Unknown name → the same loud error as building it.
|
|
6356
|
+
*/
|
|
6357
|
+
function providerKeyEnv(name) {
|
|
6358
|
+
return entryFor(name).keyEnv;
|
|
6359
|
+
}
|
|
6360
|
+
/**
|
|
6199
6361
|
* The port door: the named provider bound to the DECLARED embedding space and
|
|
6200
6362
|
* the framework's timeout knobs. `modelId`/`dim` omitted = the shipped
|
|
6201
6363
|
* config space (EMBED_MODEL / EMBED_DIM, eval-locked); an instance may
|
|
@@ -6212,8 +6374,8 @@ function buildShippedProvider(name, opts) {
|
|
|
6212
6374
|
apiKey: opts.apiKey ?? "",
|
|
6213
6375
|
modelId: opts.modelId ?? "gemini-embedding-001",
|
|
6214
6376
|
dim: opts.dim ?? 1536,
|
|
6215
|
-
documentTaskLabel:
|
|
6216
|
-
queryTaskLabel:
|
|
6377
|
+
documentTaskLabel: entry.taskLabels.document,
|
|
6378
|
+
queryTaskLabel: entry.taskLabels.query,
|
|
6217
6379
|
documentTimeoutS: EMBED_TIMEOUT_S(),
|
|
6218
6380
|
queryTimeoutS: QUERY_EMBED_TIMEOUT_S()
|
|
6219
6381
|
});
|
|
@@ -6711,7 +6873,7 @@ function calibrationScope(viewer) {
|
|
|
6711
6873
|
async function scoreQueries(pool, scope, gucs, provider, queries, inCorpus) {
|
|
6712
6874
|
const out = [];
|
|
6713
6875
|
for (const query of queries) {
|
|
6714
|
-
const [vector] = await
|
|
6876
|
+
const [vector] = await embedIntent([query], {
|
|
6715
6877
|
provider,
|
|
6716
6878
|
intent: "query"
|
|
6717
6879
|
});
|
|
@@ -9743,6 +9905,43 @@ function pct(fraction) {
|
|
|
9743
9905
|
return `${(fraction * 100).toFixed(0)}%`;
|
|
9744
9906
|
}
|
|
9745
9907
|
/**
|
|
9908
|
+
* What to do when calibration is refused for quota — named, not left as the
|
|
9909
|
+
* vendor's sentence.
|
|
9910
|
+
*
|
|
9911
|
+
* Both of these were hit walking a real free-tier key, and they are DIFFERENT
|
|
9912
|
+
* failures with different remedies, which is why the raw error is not enough:
|
|
9913
|
+
*
|
|
9914
|
+
* generate_content …/gemini-3.7-flash, limit 5 the SYNTHESIZED door writes
|
|
9915
|
+
* one probe question per
|
|
9916
|
+
* sampled passage, and a free
|
|
9917
|
+
* key allows a handful of
|
|
9918
|
+
* generations a minute. No
|
|
9919
|
+
* amount of waiting fixes a
|
|
9920
|
+
* corpus of any size; the
|
|
9921
|
+
* answer is the zero-LLM door.
|
|
9922
|
+
* global_embed_content_requests_per_minute the EMBEDDING endpoint, which
|
|
9923
|
+
* both doors use. Transient:
|
|
9924
|
+
* an ingest immediately before
|
|
9925
|
+
* a calibration spends the same
|
|
9926
|
+
* per-minute budget.
|
|
9927
|
+
*
|
|
9928
|
+
* Product principle 4: a failure states what is wrong, why the rule exists, and
|
|
9929
|
+
* how to fix it. The vendor's message states only the first.
|
|
9930
|
+
*/
|
|
9931
|
+
/** The generation quota, which no wait resolves on a free key. */
|
|
9932
|
+
const GENERATION = /generate_content|generativelanguage\.googleapis\.com\/generate/i;
|
|
9933
|
+
/** The embedding quota, which clears on its own. */
|
|
9934
|
+
const EMBEDDING = /embed_content|global_embed/i;
|
|
9935
|
+
/**
|
|
9936
|
+
* The remedy for a quota refusal, or null when the failure is not one this
|
|
9937
|
+
* knows — in which case the vendor's own message stands, unembellished.
|
|
9938
|
+
*/
|
|
9939
|
+
function quotaRemedy(message) {
|
|
9940
|
+
if (GENERATION.test(message)) return "the SYNTHESIZED door writes one probe question per sampled passage with an LLM, and a free-tier key allows only a few generations a minute — a bigger corpus makes this worse, not better.\n fix: calibrate with zero LLM — write your in-corpus questions one per line and pass\n --queries-file PATH. The floor is measured the same way; only the questions differ,\n and the door is recorded beside the number so the two are never compared.";
|
|
9941
|
+
if (EMBEDDING.test(message)) return "the EMBEDDING endpoint is limited per minute, and both doors use it — an ingest immediately before this spends the same budget.\n fix: wait about a minute and run it again. Nothing was written; calibration only reads.";
|
|
9942
|
+
return null;
|
|
9943
|
+
}
|
|
9944
|
+
/**
|
|
9746
9945
|
* How close to the floor an answer has to be to count as marginal.
|
|
9747
9946
|
*
|
|
9748
9947
|
* 0.01 is not a tuned threshold — it is the size of the decision in this
|
|
@@ -10175,8 +10374,9 @@ function composeProvider(instance) {
|
|
|
10175
10374
|
try {
|
|
10176
10375
|
let apiKey = null;
|
|
10177
10376
|
if (providerNeedsApiKey(instance.embeddingProvider)) {
|
|
10178
|
-
|
|
10179
|
-
|
|
10377
|
+
const keyEnv = providerKeyEnv(instance.embeddingProvider) ?? "";
|
|
10378
|
+
apiKey = process.env[keyEnv] || null;
|
|
10379
|
+
if (apiKey === null) return fail$1(ENVIRONMENT, `${keyEnv} is required (the instance's embedding provider ${JSON.stringify(instance.embeddingProvider)} needs a key)\n fix: export ${keyEnv}=... and rerun`);
|
|
10180
10380
|
}
|
|
10181
10381
|
return buildShippedProvider(instance.embeddingProvider, {
|
|
10182
10382
|
apiKey,
|
|
@@ -10437,7 +10637,7 @@ async function calibrateCommand(args) {
|
|
|
10437
10637
|
textGenerator = new GeminiTextGenerator({ apiKey });
|
|
10438
10638
|
}
|
|
10439
10639
|
const ooc = values["ooc-file"] === void 0 ? null : parseQueriesFile(readFileSync(values["ooc-file"], "utf8"));
|
|
10440
|
-
const report = await withPool(dsn, async (pool) => runCalibration(pool, {
|
|
10640
|
+
const report = await withQuotaRemedy(async () => withPool(dsn, async (pool) => runCalibration(pool, {
|
|
10441
10641
|
tenantId: instance.tenantId,
|
|
10442
10642
|
corpusId: instance.corpusId,
|
|
10443
10643
|
viewer: await widestViewer(pool, instance),
|
|
@@ -10448,12 +10648,31 @@ async function calibrateCommand(args) {
|
|
|
10448
10648
|
oocProbes: ooc,
|
|
10449
10649
|
perNode: values["per-node"] === void 0 ? void 0 : intFlag("--per-node", values["per-node"]),
|
|
10450
10650
|
minChars: values["min-chars"] === void 0 ? void 0 : intFlag("--min-chars", values["min-chars"])
|
|
10451
|
-
}));
|
|
10651
|
+
})));
|
|
10452
10652
|
process.stdout.write(renderReport(report, GATE_PREDICATE_DIGEST) + "\n");
|
|
10453
10653
|
const advice = overlapAdvice(report);
|
|
10454
10654
|
if (advice !== null) process.stdout.write(advice);
|
|
10455
10655
|
return 0;
|
|
10456
10656
|
}
|
|
10657
|
+
/**
|
|
10658
|
+
* Run a calibration, turning a quota refusal into the remedy for THAT quota.
|
|
10659
|
+
*
|
|
10660
|
+
* Both failures reach here as the vendor's own sentence, which states what is
|
|
10661
|
+
* wrong and neither why nor how to fix it — and the two need opposite answers
|
|
10662
|
+
* (change door vs wait a minute). Anything `quotaRemedy` does not recognise is
|
|
10663
|
+
* re-thrown untouched: inventing advice for an error nobody has read is worse
|
|
10664
|
+
* than passing the vendor's through.
|
|
10665
|
+
*/
|
|
10666
|
+
async function withQuotaRemedy(op) {
|
|
10667
|
+
try {
|
|
10668
|
+
return await op();
|
|
10669
|
+
} catch (exc) {
|
|
10670
|
+
const message = exc instanceof Error ? exc.message : String(exc);
|
|
10671
|
+
const remedy = quotaRemedy(message);
|
|
10672
|
+
if (remedy === null) throw exc;
|
|
10673
|
+
throw Object.assign(/* @__PURE__ */ new Error(`${message}\n why: ${remedy}`), { slug: "ksor-calibrate-quota" });
|
|
10674
|
+
}
|
|
10675
|
+
}
|
|
10457
10676
|
/** How many days of traffic one --check reads. Bounded so a busy record cannot make it expensive. */
|
|
10458
10677
|
const DRIFT_DEFAULT_DAYS = 30;
|
|
10459
10678
|
/**
|
|
@@ -13350,7 +13569,8 @@ function isEnvironmentError(value) {
|
|
|
13350
13569
|
const EMITTED_NAMES = /* @__PURE__ */ new Map([
|
|
13351
13570
|
["gitignore", ".gitignore"],
|
|
13352
13571
|
["env.example", ".env.example"],
|
|
13353
|
-
["dockerignore", ".dockerignore"]
|
|
13572
|
+
["dockerignore", ".dockerignore"],
|
|
13573
|
+
["mcp.json", ".mcp.json"]
|
|
13354
13574
|
]);
|
|
13355
13575
|
const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
13356
13576
|
".md",
|