@nacre.work/core 0.17.2 → 0.17.4

package/dist/ingest-failure.js

@@ -0,0 +1,135 @@
+/**
+ * What a caller is told about a document that failed to index.
+ *
+ * ## Why this is not just the error string
+ *
+ * The worker stores whatever went wrong verbatim, which is right for the
+ * operator reading a log and wrong for the caller reading an API. That string
+ * carries the embedding endpoint's URL, the parser's, and whatever a sidecar
+ * put in its message — infrastructure names, on an internal network, written
+ * by somebody who was not thinking about who reads them.
+ *
+ * The caller who most needs this is the one who should see it least: an agent
+ * over MCP, possibly acting through a delegation somebody granted to a third
+ * party. It needs to know its document failed and whether re-sending would
+ * help. It does not need `http://embedder.internal:8080/embeddings`.
+ *
+ * So a failure becomes a **reason** — a stable string a program can branch on —
+ * beside the message itself with anything shaped like a host or a URL taken
+ * out. Relaying the real wording matters: a scanned PDF fails with a message
+ * naming the scan and pointing at OCR, and a sentence invented here would
+ * send an operator looking for a corrupt file. The unredacted text stays in
+ * Postgres and in the worker's log, where the operator is.
+ *
+ * ## The reasons
+ *
+ * Deliberately few. Each one answers "what should the caller do now", because
+ * that is the only question the answer is for; two failures with the same
+ * answer do not need two reasons.
+ */
+/**
+ * Classify a stored failure.
+ *
+ * Matched on wording, which is unavoidable: what arrives here is a string built
+ * from somebody else's error. The mapping is deliberately loose — an unmatched
+ * failure is `internal`, which is honest, rather than a guess that sends the
+ * caller to re-send a document that will never index.
+ */
+export function classifyIngestFailure(raw) {
+    const text = raw.toLowerCase();
+    const reason = /must have less than \d+ tokens|too long|maximum (?:input )?length|context length/.test(text)
+        ? 'too_long'
+        : /quota|max_documents|limit reached|too many documents/.test(text)
+            ? 'quota'
+            : /parse|unsupported|not a pdf|%pdf|decode|extract/.test(text)
+                ? 'unreadable'
+                : /econnrefused|enotfound|etimedout|timeout|fetch failed|socket|answered 5\d\d|unavailable|network/.test(text)
+                    ? 'unavailable'
+                    : 'internal';
+    // The message is the real one with hosts removed, not a sentence written
+    // here. Replacing it looked safer and threw away the only thing that made
+    // some failures actionable: a scanned PDF fails with a message naming the
+    // scan and pointing at OCR, and "This document could not be read" sends an
+    // operator hunting for a corrupt file instead. The compose smoke asserts
+    // that specific wording survives, and caught exactly this.
+    //
+    // What the caller must not receive is the *infrastructure* in it, which is
+    // what `withoutHosts` takes out. The advice that used to be canned here —
+    // whether re-sending helps — is what `reason` is for, and it is documented
+    // against the reason rather than repeated into every message.
+    const message = withoutHosts(raw).slice(0, 300);
+    return { reason, message };
+}
+/**
+ * Whatever is safe to repeat from a stored failure.
+ *
+ * ## Redacted by context, not by the shape of a token
+ *
+ * The first version matched anything with a dot and a two-letter tail, and that
+ * is wrong in both directions.
+ *
+ * It **missed** every hostname this product actually deploys with. Service
+ * names in `docker-compose.yml` and in the chart are single-label — `embedder`,
+ * `qdrant`, `parser`, `minio` — so a rule needing a dot held for the example in
+ * the paragraph above (`embedder.internal`) and for nothing that ships. The way
+ * the name survived is worth keeping: the URL was removed, and then undici
+ * appended its cause verbatim, so `getaddrinfo ENOTFOUND embedder` sat at the
+ * end of an otherwise redacted sentence. IPv6 was untouched entirely.
+ *
+ * So a host is now recognised where a host can *be*: after a scheme, inside
+ * brackets, as `name:port`, after a DNS or socket error code, and after the
+ * handful of phrases this repository writes itself. Those are the positions a
+ * machine puts one in, and unlike a token's shape they are not shared with
+ * anything a person wrote.
+ *
+ * ## What it still over-redacts, and why that is the direction to fail in
+ *
+ * The dotted-name rule stays, last, and it cannot tell `contract.pdf` from
+ * `example.com` — nothing can, they are the same string with a different tail,
+ * and a list of file extensions is a list that goes stale. So a filename in a
+ * parser's message becomes `[host]`, which costs the sender the name of their
+ * own document.
+ *
+ * That is the trade taken deliberately: over-redacting a filename is a
+ * usability cost on one message, and under-redacting a hostname is the thing
+ * this function exists to prevent, on a surface a third party reaches through
+ * a delegation. It is stated here and in `docs/openapi.yaml` rather than left
+ * for somebody to discover.
+ *
+ * Not a general redactor and does not pretend to be one: a credential that is
+ * not inside a URL passes through, because there is no shape to match it on.
+ */
+export function withoutHosts(raw) {
+    return (raw
+        // A URL, whatever the scheme. Everything after `://` goes, which is what
+        // takes the userinfo with it: `postgres://user:hunter2@db:5432/x`.
+        .replace(/[a-z][a-z0-9+.-]*:\/\/[^\s"']+/gi, '[endpoint]')
+        // IPv6 in brackets, with or without a port: `[fd00:ec2::23]:8080`.
+        .replace(/\[[0-9a-f]{0,4}(?::[0-9a-f]{0,4}){2,}\](?::\d{1,5})?/gi, '[host]')
+        // Bare IPv6. Three colons at least, so a clock (`12:30:45`) is not one
+        // and `::1:8080` is — the lookbehind rather than `\b` because a leading
+        // `::` has no word boundary in front of it, which is how that one
+        // survived the first attempt at this rule.
+        .replace(/(?<![\w:.])(?:[0-9a-f]{0,4}:){3,}[0-9a-f]{0,4}/gi, '[host]')
+        .replace(/\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b/g, '[host]')
+        // `host:port`, which is the shape that makes a bare label a host rather
+        // than a word. This is what catches `embedder:8080` — every service name
+        // this product ships with is single-label.
+        .replace(/\b[a-z][a-z0-9-]*(?:\.[a-z0-9-]+)*:\d{2,5}\b/gi, '[host]')
+        // Node names the host straight after the code, and undici appends the
+        // cause verbatim to a message whose URL has already been taken out — so
+        // `getaddrinfo ENOTFOUND embedder` is where the name actually survived.
+        //
+        // Case-sensitive, and the lookahead skips a second code, because the
+        // real string is `getaddrinfo ENOTFOUND embedder` and the naive rule
+        // redacts `ENOTFOUND` and keeps the host.
+        .replace(/\b(ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH)\s+(?![A-Z_]{3,}\b)[A-Za-z0-9_.[\]-]*[A-Za-z0-9\]]/g, '$1 [host]')
+        // And our own wording says it in prose, where there is no port and no
+        // code to anchor on.
+        .replace(/\b(could not reach|could not connect to|connection to|connecting to|connect to|no route to)\s+[A-Za-z0-9_.[\]-]*[A-Za-z0-9\]]/gi, '$1 [host]')
+        // A dotted name. Last, and deliberately kept even though it is the rule
+        // that cannot tell `contract.pdf` from `example.com` — see the note in
+        // this function's header about which way that trade goes.
+        .replace(/\b(?:[a-z0-9-]+\.)+[a-z]{2,}(?::\d+)?\b/gi, '[host]'));
+}
+//# sourceMappingURL=ingest-failure.js.map

package/dist/ingest-failure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest-failure.js","sourceRoot":"","sources":["../ingest-failure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAoBH;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAW;IAC/C,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,EAAE,CAAA;IAE9B,MAAM,MAAM,GACV,kFAAkF,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3F,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC;YACjE,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5D,CAAC,CAAC,YAAY;gBACd,CAAC,CAAC,iGAAiG,CAAC,IAAI,CAClG,IAAI,CACL;oBACH,CAAC,CAAC,aAAa;oBACf,CAAC,CAAC,UAAU,CAAA;IAEtB,yEAAyE;IACzE,0EAA0E;IAC1E,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,2DAA2D;IAC3D,EAAE;IACF,2EAA2E;IAC3E,0EAA0E;IAC1E,2EAA2E;IAC3E,8DAA8D;IAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IAC/C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;AAC5B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAO,CACL,GAAG;QACD,yEAAyE;QACzE,mEAAmE;SAClE,OAAO,CAAC,kCAAkC,EAAE,YAAY,CAAC;QAC1D,mEAAmE;SAClE,OAAO,CAAC,wDAAwD,EAAE,QAAQ,CAAC;QAC5E,uEAAuE;QACvE,wEAAwE;QACxE,kEAAkE;QAClE,2CAA2C;SAC1C,OAAO,CAAC,kDAAkD,EAAE,QAAQ,CAAC;SACrE,OAAO,CAAC,uCAAuC,EAAE,QAAQ,CAAC;QAC3D,wEAAwE;QACxE,yEAAyE;QACzE,2CAA2C;SAC1C,OAAO,CAAC,gDAAgD,EAAE,QAAQ,CAAC;QACpE,sEAAsE;QACtE,wEAAwE;QACxE,wEAAwE;QACxE,EAAE;QACF,qEAAqE;QACrE,qEAAqE;QACrE,0CAA0C;SACzC,OAAO,CACN,uIAAuI,EACvI,WAAW,CACZ;QACD,sEAAsE;QACtE,qBAAqB;SACpB,OAAO,CACN,iIAAiI,EACjI,WAAW,CACZ;QACD,wEAAwE;QACxE,uEAAuE;QACvE,0DAA0D;SACzD,OAAO,CAAC,2CAA2C,EAAE,QAAQ,CAAC,CAClE,CAAA;AACH,CAAC"}

package/dist/provision.d.ts

@@ -41,6 +41,23 @@ import type { VectorStore } from './vector/search.js';
  * So the rule is here, `provisionOrganization` applies it before anything else,
  * and no caller can be the one that forgot.
  */
+/**
+ * An email address, checked only enough to refuse what cannot be one.
+ *
+ * Here rather than beside the endpoints because `provisionOrganization` writes
+ * the first `users` row of every organization, and it was the one writer that
+ * did not ask. `init` asked for an `@` and nothing else, so
+ * `--email admin@localhost` created an organization whose administrator holds
+ * an address `POST /v1/users` refuses for anybody else — and the comment above
+ * that line already stated the rule it was breaking: the CLI asks in order to
+ * produce a usage message, and the function refuses regardless.
+ *
+ * Deliberately not RFC 5322. A stricter test rejects addresses that work; what
+ * matters is that the value is a single token with an `@` and a dot after it,
+ * so a name or a uuid typed into the wrong field is refused where the operator
+ * can see it rather than becoming a user nobody can sign in as.
+ */
+export declare function looksLikeEmail(value: string): boolean;
 export declare function organizationSlugError(slug: string): string | undefined;
 export interface ProvisionOptions {
     readonly slug: string;

package/dist/provision.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provision.d.ts","sourceRoot":"","sources":["../provision.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAG9B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAErD;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAWtE;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B;;;;;;;;OAQG;IACH,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAA;IAC/B,8EAA8E;IAC9E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,gFAAgF;IAChF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB;;;;;;;;OAQG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAA;CAC9B;AAED,KAAK,GAAG,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;AAEjE;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,YAAY,EACtB,UAAU,EAAE,MAAM,EAClB,GAAG,GAAE,GAAqB,GACzB,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC,CAkK9C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,EAC9C,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,YAAY,EACtB,GAAG,GAAE,GAAqB,GACzB,OAAO,CAAC,eAAe,CAAC,CAkB1B"}
+{"version":3,"file":"provision.d.ts","sourceRoot":"","sources":["../provision.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAG9B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAErD;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAWtE;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B;;;;;;;;OAQG;IACH,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAA;IAC/B,8EAA8E;IAC9E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,gFAAgF;IAChF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB;;;;;;;;OAQG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAA;CAC9B;AAED,KAAK,GAAG,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;AAEjE;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,YAAY,EACtB,UAAU,EAAE,MAAM,EAClB,GAAG,GAAE,GAAqB,GACzB,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC,CAkK9C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,EAC9C,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,YAAY,EACtB,GAAG,GAAE,GAAqB,GACzB,OAAO,CAAC,eAAe,CAAC,CAqB1B"}

package/dist/provision.js

@@ -40,6 +40,25 @@ import { collectionName, vectorName } from './vector/query.js';
  * So the rule is here, `provisionOrganization` applies it before anything else,
  * and no caller can be the one that forgot.
  */
+/**
+ * An email address, checked only enough to refuse what cannot be one.
+ *
+ * Here rather than beside the endpoints because `provisionOrganization` writes
+ * the first `users` row of every organization, and it was the one writer that
+ * did not ask. `init` asked for an `@` and nothing else, so
+ * `--email admin@localhost` created an organization whose administrator holds
+ * an address `POST /v1/users` refuses for anybody else — and the comment above
+ * that line already stated the rule it was breaking: the CLI asks in order to
+ * produce a usage message, and the function refuses regardless.
+ *
+ * Deliberately not RFC 5322. A stricter test rejects addresses that work; what
+ * matters is that the value is a single token with an `@` and a dot after it,
+ * so a name or a uuid typed into the wrong field is refused where the operator
+ * can see it rather than becoming a user nobody can sign in as.
+ */
+export function looksLikeEmail(value) {
+    return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value);
+}
 export function organizationSlugError(slug) {
     // The slug becomes the Qdrant collection name, so the characters it may
     // contain are not a matter of taste: one with a slash or a quote in it
@@ -227,6 +246,9 @@ export async function provisionOrganization(pool, vectors, options, provider, lo
     const wrong = organizationSlugError(options.slug);
     if (wrong !== undefined)
         throw new Error(`organization slug ${wrong}`);
+    if (!looksLikeEmail(options.email)) {
+        throw new Error(`administrator email does not look like an address: ${options.email}`);
+    }
     const collection = collectionName(options.slug);
     const ids = await provisionInPostgres(pool, options, provider, collection, log);
     // `ensureCollection` is already idempotent and already creates the payload

package/dist/provision.js.map

@@ -1 +1 @@
-{"version":3,"file":"provision.js","sourceRoot":"","sources":["../provision.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAG9D;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,wEAAwE;IACxE,uEAAuE;IACvE,sCAAsC;IACtC,IAAI,CAAC,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpD,OAAO,CACL,oFAAoF;YACpF,oBAAoB,IAAI,EAAE,CAC3B,CAAA;IACH,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAwDD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAU,EACV,OAAyB,EACzB,QAAsB,EACtB,UAAkB,EAClB,MAAW,GAAG,EAAE,CAAC,SAAS;IAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAE3B,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC;;;;;;;;eAQS,EACT,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CACzC,CAAA;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACnB,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;QACjF,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,sBAAsB,EAAE;YACzE,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAA;QAEF,2EAA2E;QAC3E,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,iBAAiB,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElF,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK;QACxC,uEAAuE;QACvE,4DAA4D;QAC5D,wEAAwE;QACxE,EAAE;QACF,wEAAwE;QACxE,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,sEAAsE;QACtE,uCAAuC;QACvC;;;;gGAI0F,EAC1F,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,CACtD,CAAA;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QAC3B,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QAC5E,GAAG,CAAC,kBAAkB,EAAE;YACtB,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,IAAI;SAC9C,CAAC,CAAA;QAEF,2EAA2E;QAC3E,mEAAmE;QACnE,wDAAwD;QACxD,EAAE;QACF,wEAAwE;QACxE,uEAAuE;QACvE,2EAA2E;QAC3E,2EAA2E;QAC3E,kEAAkE;QAClE,sEAAsE;QACtE,mEAAmE;QACnE,2EAA2E;QAC3E,+DAA+D;QAC/D,EAAE;QACF,uEAAuE;QACvE,4EAA4E;QAC5E,2EAA2E;QAC3E,yEAAyE;QACzE,2EAA2E;QAC3E,0EAA0E;QAC1E,2EAA2E;QAC3E,4EAA4E;QAC5E,6DAA6D;QAC7D,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QACjD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK;QAM5C,uEAAuE;QACvE,uEAAuE;QACvE,uEAAuE;QACvE,sEAAsE;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,yEAAyE;QACzE,EAAE;QACF,wEAAwE;QACxE,oEAAoE;QACpE,qCAAqC;QACrC;;;;;;;;;;;;;eAaS,EACT,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CACzD,CAAA;QACD,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QAChC,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,QAAQ,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;QAC5F,IAAI,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC7F,GAAG,CAAC,wEAAwE,EAAE;gBAC5E,SAAS,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;gBAC7D,oBAAoB,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;aACzE,CAAC,CAAA;QACJ,CAAC;QACD,GAAG,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;QAEpF,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC7C;;;;;;;;;;;eAWS,EACT,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAC/C,CAAA;QACD,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QACrC,IAAI,WAAW,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACtF,GAAG,CAAC,iBAAiB,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;QAE9E,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAC5B,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,MAAM;YACN,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,QAAQ,EAAE;gBACR,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;aACxC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,OAAO;YACrB,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,IAAI;SAC7C,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC9C,MAAM,KAAK,CAAA;IACb,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAA;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAU,EACV,OAA8C,EAC9C,OAAyB,EACzB,QAAsB,EACtB,MAAW,GAAG,EAAE,CAAC,SAAS;IAE1B,4EAA4E;IAC5E,8EAA8E;IAC9E,6CAA6C;IAC7C,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;IAEtE,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,CAAA;IAE/E,2EAA2E;IAC3E,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IACpE,MAAM,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAC3E,GAAG,CAAC,kBAAkB,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1F,OAAO,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/B,CAAC"}
+{"version":3,"file":"provision.js","sourceRoot":"","sources":["../provision.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAG9D;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,wEAAwE;IACxE,uEAAuE;IACvE,sCAAsC;IACtC,IAAI,CAAC,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpD,OAAO,CACL,oFAAoF;YACpF,oBAAoB,IAAI,EAAE,CAC3B,CAAA;IACH,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAwDD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAU,EACV,OAAyB,EACzB,QAAsB,EACtB,UAAkB,EAClB,MAAW,GAAG,EAAE,CAAC,SAAS;IAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAE3B,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC;;;;;;;;eAQS,EACT,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CACzC,CAAA;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACnB,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;QACjF,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,sBAAsB,EAAE;YACzE,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAA;QAEF,2EAA2E;QAC3E,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,iBAAiB,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElF,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK;QACxC,uEAAuE;QACvE,4DAA4D;QAC5D,wEAAwE;QACxE,EAAE;QACF,wEAAwE;QACxE,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,sEAAsE;QACtE,uCAAuC;QACvC;;;;gGAI0F,EAC1F,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,CACtD,CAAA;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QAC3B,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QAC5E,GAAG,CAAC,kBAAkB,EAAE;YACtB,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,IAAI;SAC9C,CAAC,CAAA;QAEF,2EAA2E;QAC3E,mEAAmE;QACnE,wDAAwD;QACxD,EAAE;QACF,wEAAwE;QACxE,uEAAuE;QACvE,2EAA2E;QAC3E,2EAA2E;QAC3E,kEAAkE;QAClE,sEAAsE;QACtE,mEAAmE;QACnE,2EAA2E;QAC3E,+DAA+D;QAC/D,EAAE;QACF,uEAAuE;QACvE,4EAA4E;QAC5E,2EAA2E;QAC3E,yEAAyE;QACzE,2EAA2E;QAC3E,0EAA0E;QAC1E,2EAA2E;QAC3E,4EAA4E;QAC5E,6DAA6D;QAC7D,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QACjD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK;QAM5C,uEAAuE;QACvE,uEAAuE;QACvE,uEAAuE;QACvE,sEAAsE;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,yEAAyE;QACzE,EAAE;QACF,wEAAwE;QACxE,oEAAoE;QACpE,qCAAqC;QACrC;;;;;;;;;;;;;eAaS,EACT,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CACzD,CAAA;QACD,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QAChC,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,QAAQ,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;QAC5F,IAAI,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC7F,GAAG,CAAC,wEAAwE,EAAE;gBAC5E,SAAS,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;gBAC7D,oBAAoB,EAAE,GAAG,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;aACzE,CAAC,CAAA;QACJ,CAAC;QACD,GAAG,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;QAEpF,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC7C;;;;;;;;;;;eAWS,EACT,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAC/C,CAAA;QACD,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QACrC,IAAI,WAAW,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACtF,GAAG,CAAC,iBAAiB,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;QAE9E,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAC5B,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,MAAM;YACN,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,QAAQ,EAAE;gBACR,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;aACxC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,OAAO;YACrB,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,IAAI;SAC7C,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC9C,MAAM,KAAK,CAAA;IACb,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAA;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAU,EACV,OAA8C,EAC9C,OAAyB,EACzB,QAAsB,EACtB,MAAW,GAAG,EAAE,CAAC,SAAS;IAE1B,4EAA4E;IAC5E,8EAA8E;IAC9E,6CAA6C;IAC7C,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;IACtE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,sDAAsD,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACxF,CAAC;IAED,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,CAAA;IAE/E,2EAA2E;IAC3E,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IACpE,MAAM,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAC3E,GAAG,CAAC,kBAAkB,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1F,OAAO,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/B,CAAC"}

package/dist/text/tokens.d.ts

@@ -0,0 +1,99 @@
+/**
+ * What a piece of text will cost an embedding model, in tokens.
+ *
+ * ## Why this exists
+ *
+ * A chunk size in **characters** says nothing about whether the embedder will
+ * accept it. Every embedding model has a hard ceiling on input length measured
+ * in tokens — 512 for the BGE and E5 families, which is what these Compose
+ * profiles start and what most self-hosted deployments run — and a character
+ * is not a token. Under an English tokenizer a Cyrillic or Korean character
+ * costs several.
+ *
+ * Measured against a real `BAAI/bge-small-en-v1.5` through Text Embeddings
+ * Inference's `/tokenize`, at the 800-character chunk this repository shipped:
+ *
+ * | script  | chars | tokens | over 512? |
+ * |---------|-------|--------|-----------|
+ * | English |   800 |    149 | no        |
+ * | Hindi   |   800 |    471 | no        |
+ * | Greek   |   800 |    642 | **yes**   |
+ * | Russian |   800 |    655 | **yes**   |
+ * | Hebrew  |   800 |    664 | **yes**   |
+ * | Arabic  |   800 |    676 | **yes**   |
+ * | Chinese |   800 |    802 | **yes**   |
+ * | Japanese|   800 |    802 | **yes**   |
+ * | Korean  |   800 |   1094 | **yes**   |
+ *
+ * Seven scripts out of eleven tried. The endpoint answers `413` and the worker
+ * marks the document `failed`, which nothing retries — so a Russian corpus
+ * indexed nothing at all while the API answered `queued`. Exactly the shape of
+ * the batch-size defect, arriving through the other bound on the same request.
+ *
+ * ## The cost model, and why it is not the provable one
+ *
+ * The provable bound is the UTF-8 **byte** length: a byte-fallback tokenizer
+ * emits at most one token per byte, and the worst case measured was 0.97
+ * tokens per byte. Using it would be correct and would also cut English chunks
+ * from 800 characters to about 510, for text that was never near the ceiling —
+ * a 57% increase in vectors to fix a problem English does not have.
+ *
+ * So ASCII is charged at **half a token per character** and everything else at
+ * its byte length. That is above ASCII prose (0.18 measured) and identifiers
+ * (0.42), and below the one ASCII case that beats it — a run of pure
+ * punctuation, at 0.97. This estimate is therefore *not* an upper bound, and
+ * that is a deliberate trade rather than an oversight: `ingest.ts` re-chunks
+ * and retries when the endpoint refuses, so an underestimate costs one extra
+ * request and never a document. Being provably safe here would cost every
+ * English deployment half its context, permanently, to avoid a retry.
+ */
+/**
+ * The ceiling to assume when a deployment has not said and the endpoint will
+ * not answer.
+ *
+ * **512, because that is what the BGE and E5 families accept** — the models
+ * this project's own profiles start. The same argument as `DEFAULT_EMBED_BATCH`
+ * being 32: a bound the ecosystem's most common self-hosted server actually
+ * enforces, rather than a guess at a good number.
+ */
+export declare const DEFAULT_EMBED_MAX_TOKENS = 512;
+/**
+ * Reserved out of the ceiling for whatever the model adds around the input.
+ *
+ * BERT-family tokenizers wrap every input in `[CLS]` and `[SEP]`, and those
+ * count against the same 512. Four rather than two: a tokenizer that adds a
+ * language or task prefix — E5's `query:` and `passage:`, which some
+ * deployments prepend — costs a little more, and the difference between four
+ * tokens and two is not worth a refusal.
+ */
+export declare const TOKEN_RESERVE = 4;
+/**
+ * An estimate of what `text` costs the embedder, in tokens.
+ *
+ * Deliberately cheap: one pass, no table, no allocation per character. It runs
+ * on every chunk boundary of every document, so it has to cost about what
+ * measuring a length costs.
+ */
+export declare function estimateTokens(text: string): number;
+/**
+ * How much of a model's window a chunk may fill.
+ *
+ * Separate from the ceiling itself so the reserve is applied in one place
+ * rather than at each call site — the defect this file exists for is a bound
+ * that two pieces of code disagreed about.
+ */
+export declare function tokenBudget(maxTokens?: number): number;
+/**
+ * Whether an error from an embedding endpoint says the input was too long.
+ *
+ * Matched on the message rather than the status, and both are checked, because
+ * the status alone is ambiguous: Text Embeddings Inference answers **413** for
+ * a batch with too many texts *and* for one text with too many tokens, and the
+ * two want opposite repairs — send fewer texts, or send shorter ones. The
+ * wording is what tells them apart.
+ *
+ * Checked against a real TEI: the OpenAI-shaped route answers
+ * `Input validation error: \`inputs\` must have less than 512 tokens. Given: 1482`.
+ */
+export declare function refusedForLength(message: string): boolean;
+//# sourceMappingURL=tokens.d.ts.map

package/dist/text/tokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../text/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,MAAM,CAAA;AAE3C;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa,IAAI,CAAA;AAE9B;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAkBnD;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,SAAS,GAAE,MAAiC,GAAG,MAAM,CAShF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAIzD"}

package/dist/text/tokens.js

@@ -0,0 +1,129 @@
+/**
+ * What a piece of text will cost an embedding model, in tokens.
+ *
+ * ## Why this exists
+ *
+ * A chunk size in **characters** says nothing about whether the embedder will
+ * accept it. Every embedding model has a hard ceiling on input length measured
+ * in tokens — 512 for the BGE and E5 families, which is what these Compose
+ * profiles start and what most self-hosted deployments run — and a character
+ * is not a token. Under an English tokenizer a Cyrillic or Korean character
+ * costs several.
+ *
+ * Measured against a real `BAAI/bge-small-en-v1.5` through Text Embeddings
+ * Inference's `/tokenize`, at the 800-character chunk this repository shipped:
+ *
+ * | script  | chars | tokens | over 512? |
+ * |---------|-------|--------|-----------|
+ * | English |   800 |    149 | no        |
+ * | Hindi   |   800 |    471 | no        |
+ * | Greek   |   800 |    642 | **yes**   |
+ * | Russian |   800 |    655 | **yes**   |
+ * | Hebrew  |   800 |    664 | **yes**   |
+ * | Arabic  |   800 |    676 | **yes**   |
+ * | Chinese |   800 |    802 | **yes**   |
+ * | Japanese|   800 |    802 | **yes**   |
+ * | Korean  |   800 |   1094 | **yes**   |
+ *
+ * Seven scripts out of eleven tried. The endpoint answers `413` and the worker
+ * marks the document `failed`, which nothing retries — so a Russian corpus
+ * indexed nothing at all while the API answered `queued`. Exactly the shape of
+ * the batch-size defect, arriving through the other bound on the same request.
+ *
+ * ## The cost model, and why it is not the provable one
+ *
+ * The provable bound is the UTF-8 **byte** length: a byte-fallback tokenizer
+ * emits at most one token per byte, and the worst case measured was 0.97
+ * tokens per byte. Using it would be correct and would also cut English chunks
+ * from 800 characters to about 510, for text that was never near the ceiling —
+ * a 57% increase in vectors to fix a problem English does not have.
+ *
+ * So ASCII is charged at **half a token per character** and everything else at
+ * its byte length. That is above ASCII prose (0.18 measured) and identifiers
+ * (0.42), and below the one ASCII case that beats it — a run of pure
+ * punctuation, at 0.97. This estimate is therefore *not* an upper bound, and
+ * that is a deliberate trade rather than an oversight: `ingest.ts` re-chunks
+ * and retries when the endpoint refuses, so an underestimate costs one extra
+ * request and never a document. Being provably safe here would cost every
+ * English deployment half its context, permanently, to avoid a retry.
+ */
+/**
+ * The ceiling to assume when a deployment has not said and the endpoint will
+ * not answer.
+ *
+ * **512, because that is what the BGE and E5 families accept** — the models
+ * this project's own profiles start. The same argument as `DEFAULT_EMBED_BATCH`
+ * being 32: a bound the ecosystem's most common self-hosted server actually
+ * enforces, rather than a guess at a good number.
+ */
+export const DEFAULT_EMBED_MAX_TOKENS = 512;
+/**
+ * Reserved out of the ceiling for whatever the model adds around the input.
+ *
+ * BERT-family tokenizers wrap every input in `[CLS]` and `[SEP]`, and those
+ * count against the same 512. Four rather than two: a tokenizer that adds a
+ * language or task prefix — E5's `query:` and `passage:`, which some
+ * deployments prepend — costs a little more, and the difference between four
+ * tokens and two is not worth a refusal.
+ */
+export const TOKEN_RESERVE = 4;
+/**
+ * An estimate of what `text` costs the embedder, in tokens.
+ *
+ * Deliberately cheap: one pass, no table, no allocation per character. It runs
+ * on every chunk boundary of every document, so it has to cost about what
+ * measuring a length costs.
+ */
+export function estimateTokens(text) {
+    let cost = 0;
+    for (const character of text) {
+        const code = character.codePointAt(0);
+        if (code < 0x80) {
+            // ASCII: half a token each. See the note above — over prose and
+            // identifiers, under a run of pure punctuation.
+            cost += 0.5;
+        }
+        else if (code < 0x800) {
+            cost += 2;
+        }
+        else if (code < 0x10000) {
+            cost += 3;
+        }
+        else {
+            // Astral: emoji, and the rarer CJK planes. Four bytes.
+            cost += 4;
+        }
+    }
+    return Math.ceil(cost);
+}
+/**
+ * How much of a model's window a chunk may fill.
+ *
+ * Separate from the ceiling itself so the reserve is applied in one place
+ * rather than at each call site — the defect this file exists for is a bound
+ * that two pieces of code disagreed about.
+ */
+export function tokenBudget(maxTokens = DEFAULT_EMBED_MAX_TOKENS) {
+    const budget = maxTokens - TOKEN_RESERVE;
+    if (budget < 1) {
+        throw new Error(`an embedding token ceiling of ${String(maxTokens)} leaves no room for input once ` +
+            `${String(TOKEN_RESERVE)} tokens are reserved for what the model wraps around it`);
+    }
+    return budget;
+}
+/**
+ * Whether an error from an embedding endpoint says the input was too long.
+ *
+ * Matched on the message rather than the status, and both are checked, because
+ * the status alone is ambiguous: Text Embeddings Inference answers **413** for
+ * a batch with too many texts *and* for one text with too many tokens, and the
+ * two want opposite repairs — send fewer texts, or send shorter ones. The
+ * wording is what tells them apart.
+ *
+ * Checked against a real TEI: the OpenAI-shaped route answers
+ * `Input validation error: \`inputs\` must have less than 512 tokens. Given: 1482`.
+ */
+export function refusedForLength(message) {
+    return /must have less than \d+ tokens|too long|exceeds? (?:the )?maximum (?:input )?length|context length/i.test(message);
+}
+//# sourceMappingURL=tokens.js.map

package/dist/text/tokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../text/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAA;AAE3C;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAA;AAE9B;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAW,CAAA;QAC/C,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;YAChB,gEAAgE;YAChE,gDAAgD;YAChD,IAAI,IAAI,GAAG,CAAA;QACb,CAAC;aAAM,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,CAAA;QACX,CAAC;aAAM,IAAI,IAAI,GAAG,OAAO,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,CAAA;QACX,CAAC;aAAM,CAAC;YACN,uDAAuD;YACvD,IAAI,IAAI,CAAC,CAAA;QACX,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,YAAoB,wBAAwB;IACtE,MAAM,MAAM,GAAG,SAAS,GAAG,aAAa,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,iCAAiC;YACjF,GAAG,MAAM,CAAC,aAAa,CAAC,yDAAyD,CACpF,CAAA;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,OAAO,qGAAqG,CAAC,IAAI,CAC/G,OAAO,CACR,CAAA;AACH,CAAC"}