@stabrise/scaledp 0.1.0

package/dist/ner-SsZLZ6ed.js

@@ -0,0 +1,1028 @@
+import { h as getConfig, i as Stage, l as NerError } from "./pipeline-DACqGkpN.js";
+import { n as ensureModelFiles } from "./model-cache-BEaqqRZ9.js";
+import { i as resolveParams, t as BASE_STAGE_DEFAULTS } from "./params-DapwK9Ns.js";
+import { t as createNerOutput } from "./entity-D6Hxaugj.js";
+import { t as createSession } from "./ort-CXDoPrtw.js";
+//#region src/ner/chunking.ts
+/** Python's `split_text` default, and what the cloud /ner/text endpoint uses. */
+const DEFAULT_CHUNK_LENGTH = 500;
+/** 500 - 480 leaves a 20-character overlap so entities on a seam survive. */
+const DEFAULT_CHUNK_STRIDE = 480;
+function chunkText(text, maxLength = 500, stride = 480) {
+	if (stride <= 0) throw new RangeError(`stride must be positive, received ${stride}`);
+	if (maxLength <= 0) throw new RangeError(`maxLength must be positive, received ${maxLength}`);
+	if (text.length === 0) return [];
+	if (text.length <= maxLength) return [{
+		text,
+		offset: 0
+	}];
+	const chunks = [];
+	for (let offset = 0; offset < text.length; offset += stride) {
+		const slice = text.slice(offset, offset + maxLength);
+		chunks.push({
+			text: slice,
+			offset
+		});
+		if (slice.length < maxLength) break;
+	}
+	return chunks;
+}
+/** Shift a chunk-local span onto the original text's coordinates. */
+function rebaseSpan(span, offset) {
+	return {
+		...span,
+		start: span.start + offset,
+		end: span.end + offset
+	};
+}
+/**
+* Drop duplicates produced by the chunk overlap, keeping the highest score for
+* each distinct (start, end, label).
+*/
+function dedupeSpans(spans) {
+	const best = /* @__PURE__ */ new Map();
+	for (const span of spans) {
+		const key = `${span.start}:${span.end}:${span.label}`;
+		const existing = best.get(key);
+		if (!existing || span.score > existing.score) best.set(key, span);
+	}
+	return [...best.values()].sort((a, b) => a.start - b.start);
+}
+/** Ratio of uppercase to cased letters. */
+function uppercaseRatio(text) {
+	let upper = 0;
+	let lower = 0;
+	for (const char of text) if (char >= "A" && char <= "Z") upper++;
+	else if (char >= "a" && char <= "z") lower++;
+	else if (char !== char.toLowerCase()) upper++;
+	else if (char !== char.toUpperCase()) lower++;
+	return upper + lower === 0 ? 0 : upper / (upper + lower);
+}
+const ALL_CAPS_RATIO = .6;
+function isMostlyUppercase(text) {
+	return uppercaseRatio(text) > ALL_CAPS_RATIO;
+}
+/**
+* Title-case runs of capitals, preserving length.
+*
+* GLiNER1 models are cased and scanned documents are frequently set in all
+* caps, which reads to the model as unlike anything in training. Length
+* preservation is essential: every character offset the decoder returns is used
+* to index the original text.
+*/
+function titleCaseAllCapsWords(text) {
+	return text.replace(/\p{Lu}[\p{Lu}\p{N}'’-]*\p{Lu}/gu, (word) => {
+		const titled = word[0] + word.slice(1).toLowerCase();
+		return titled.length === word.length ? titled : word;
+	});
+}
+/** Apply the casing fix only when the text is predominantly uppercase. */
+function normaliseCasing(text) {
+	return isMostlyUppercase(text) ? titleCaseAllCapsWords(text) : text;
+}
+//#endregion
+//#region src/ner/vendor/math.ts
+/** Numeric helpers for the GLiNER runtimes. Adapted from @lmoe/gliner-onnx (MIT). */
+/** Numerically stable sigmoid: exp(-x) overflows for large negative x. */
+function sigmoid(x) {
+	if (x >= 0) return 1 / (1 + Math.exp(-x));
+	const expX = Math.exp(x);
+	return expX / (1 + expX);
+}
+function softmax(values) {
+	let max = Number.NEGATIVE_INFINITY;
+	for (let i = 0; i < values.length; i++) {
+		const v = values[i];
+		if (v > max) max = v;
+	}
+	const out = new Float32Array(values.length);
+	let sum = 0;
+	for (let i = 0; i < values.length; i++) {
+		const e = Math.exp(values[i] - max);
+		out[i] = e;
+		sum += e;
+	}
+	for (let i = 0; i < out.length; i++) out[i] = out[i] / sum;
+	return out;
+}
+/** Gather rows out of a flat [n, hiddenSize] matrix. */
+function gatherRows(source, positions, hiddenSize) {
+	const out = new Float32Array(positions.length * hiddenSize);
+	for (let i = 0; i < positions.length; i++) {
+		const from = positions[i] * hiddenSize;
+		out.set(source.subarray(from, from + hiddenSize), i * hiddenSize);
+	}
+	return out;
+}
+/** Contiguous row slice of a flat [n, hiddenSize] matrix. */
+function sliceRows(source, startRow, rowCount, hiddenSize) {
+	const from = startRow * hiddenSize;
+	return source.slice(from, from + rowCount * hiddenSize);
+}
+/** Token ids from a transformers.js `tolist()` result, flattened and de-bigint'd. */
+function extractTokenIds(tolistResult) {
+	return (Array.isArray(tolistResult[0]) ? tolistResult.flat() : tolistResult).map((v) => typeof v === "bigint" ? Number(v) : v);
+}
+//#endregion
+//#region src/ner/tokenizer-types.ts
+/** Shared tokenizer adaptation, kept separate so backends need not import the loader. */
+/**
+* Adapt a transformers.js tokenizer to the span processor's interface.
+*
+* CLS/SEP ids are derived empirically -- encode a throwaway token, read the
+* first and last id -- rather than read from `cls_token_id`. Not every GLiNER
+* repo populates those fields, and a wrong id corrupts every sequence silently
+* instead of failing loudly.
+*/
+function toSpanTokenizer(tokenizer) {
+	const encode = (text) => extractTokenIds(tokenizer(text, { add_special_tokens: true }).input_ids.tolist());
+	const probe = encode("x");
+	if (probe.length < 2) throw new NerError("Tokenizer produced no special tokens; cannot derive CLS/SEP ids", "toSpanTokenizer");
+	return {
+		encode,
+		clsTokenId: probe[0],
+		sepTokenId: probe[probe.length - 1]
+	};
+}
+//#endregion
+//#region src/ner/vendor/span-decoder.ts
+/**
+* GLiNER1 span decoder. Adapted from @lmoe/gliner-onnx (MIT).
+*
+* The model emits logits shaped [batch, seqLen, maxWidth, entityCount]. Every
+* (start word, width, label) triple above threshold becomes a candidate span;
+* greedy non-maximum suppression then keeps the highest-scoring
+* non-overlapping set.
+*/
+/** Model class ids are 1-based; index 0 is reserved. */
+const ENTITY_ID_OFFSET = 1;
+function spansOverlap(aStart, aEnd, bStart, bEnd, allowNested, allowMultiLabel) {
+	if (aStart === bStart && aEnd === bEnd) return !allowMultiLabel;
+	if (aStart > bEnd || bStart > aEnd) return false;
+	if (allowNested) {
+		if (aStart <= bStart && aEnd >= bEnd || bStart <= aStart && bEnd >= aEnd) return false;
+	}
+	return true;
+}
+/** Keep the highest-scoring spans that do not collide. */
+function greedySearch(spans, flatNer, multiLabel) {
+	const byScore = [...spans].sort((a, b) => b.score - a.score);
+	const kept = [];
+	for (const span of byScore) if (!kept.some((other) => spansOverlap(span.start, span.end, other.start, other.end, !flatNer, multiLabel))) kept.push(span);
+	return kept.sort((a, b) => a.start - b.start);
+}
+function decodeSpans(logits, params, options = {}) {
+	const threshold = options.threshold ?? .5;
+	const flatNer = options.flatNer ?? true;
+	const multiLabel = options.multiLabel ?? false;
+	const { batchSize, inputLength, maxWidth, entityCount } = params;
+	const batchStride = inputLength * maxWidth * entityCount;
+	const tokenStride = maxWidth * entityCount;
+	const spans = Array.from({ length: batchSize }, () => []);
+	for (let index = 0; index < logits.length; index++) {
+		const score = sigmoid(logits[index]);
+		if (score < threshold) continue;
+		const batchIdx = Math.floor(index / batchStride);
+		const startWord = Math.floor(index / tokenStride) % inputLength;
+		const endWord = startWord + Math.floor(index / entityCount) % maxWidth;
+		const entityIdx = index % entityCount;
+		const words = params.batchWords[batchIdx];
+		if (!words || startWord >= words.length || endWord >= words.length) continue;
+		const start = words[startWord][1];
+		const end = words[endWord][2];
+		const text = params.texts[batchIdx] ?? "";
+		spans[batchIdx].push({
+			text: text.slice(start, end),
+			label: params.idToClass[entityIdx + ENTITY_ID_OFFSET] ?? "",
+			start,
+			end,
+			score
+		});
+	}
+	return spans.map((batch) => greedySearch(batch, flatNer, multiLabel));
+}
+//#endregion
+//#region src/ner/vendor/splitter.ts
+/** Word splitting for GLiNER. Adapted from @lmoe/gliner-onnx (MIT). */
+/**
+* Unicode word classes (`\p{L}\p{N}` with the `u` flag), deliberately not `\w`.
+*
+* JavaScript's `\w` is ASCII-only even under `/u`, so it would shatter accented
+* names -- "Müller", "García" -- into single-character tokens and wreck
+* multi-word span detection in German, Polish and Spanish text. Python's `\w`
+* is Unicode-aware, so this restores parity with the reference tokenizer.
+*/
+const WORD_PATTERN = /[\p{L}\p{N}_]+(?:[-_][\p{L}\p{N}_]+)*|\S/gu;
+/**
+* As above, plus leading branches for URLs, emails and @mentions so they stay
+* whole. Those branches are ASCII on purpose, mirroring the Python pattern.
+*/
+const RICH_WORD_PATTERN = /(?:https?:\/\/[^\s]+|www\.[^\s]+)|[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}|@[a-z0-9_]+|[\p{L}\p{N}_]+(?:[-_][\p{L}\p{N}_]+)*|\S/giu;
+/**
+* Split text into words with their character offsets.
+*
+* A fresh RegExp per call: the `g` flag makes `lastIndex` stateful, so sharing
+* one instance across calls silently skips matches.
+*/
+function splitWords(text, pattern = WORD_PATTERN) {
+	const regex = new RegExp(pattern.source, pattern.flags);
+	const out = [];
+	for (;;) {
+		const match = regex.exec(text);
+		if (match === null) break;
+		out.push([
+			match[0],
+			match.index,
+			regex.lastIndex
+		]);
+	}
+	return out;
+}
+//#endregion
+//#region src/ner/vendor/span-processor.ts
+/**
+* GLiNER1 span-enumeration input builder. Adapted from @lmoe/gliner-onnx (MIT).
+*
+* Builds the prompt `<<ENT>> label1 <<ENT>> label2 <<SEP>> text` and enumerates
+* every candidate span up to `maxWidth` words, which is what the model scores.
+*/
+const PAD = 0;
+function pad(arrays, value) {
+	const max = Math.max(...arrays.map((a) => a.length));
+	return arrays.map((a) => [...a, ...new Array(max - a.length).fill(value)]);
+}
+var SpanProcessor = class {
+	config;
+	tokenizer;
+	constructor(config, tokenizer) {
+		this.config = config;
+		this.tokenizer = tokenizer;
+	}
+	prepare(texts, labels) {
+		const batchWords = texts.map((text) => splitWords(text));
+		const idToClass = {};
+		for (const [i, label] of labels.entries()) idToClass[i + 1] = label;
+		const inputIds = [];
+		const attentionMasks = [];
+		const wordsMasks = [];
+		const textLengths = [];
+		const spanIdxs = [];
+		const spanMasks = [];
+		for (const words of batchWords) {
+			textLengths.push(words.length);
+			const prompt = [];
+			for (const label of labels) prompt.push(this.config.entToken, label);
+			prompt.push(this.config.sepToken);
+			const sequence = [...prompt, ...words.map(([w]) => w)];
+			const ids = [this.tokenizer.clsTokenId];
+			const attention = [1];
+			const wordsMask = [PAD];
+			let wordCounter = 1;
+			for (const [wordIdx, word] of sequence.entries()) {
+				const subTokens = this.tokenizer.encode(word).slice(1, -1);
+				for (const [tokenIdx, id] of subTokens.entries()) {
+					ids.push(id);
+					attention.push(1);
+					if (wordIdx < prompt.length) wordsMask.push(PAD);
+					else if (tokenIdx === 0) wordsMask.push(wordCounter++);
+					else wordsMask.push(PAD);
+				}
+			}
+			ids.push(this.tokenizer.sepTokenId);
+			attention.push(1);
+			wordsMask.push(PAD);
+			inputIds.push(ids);
+			attentionMasks.push(attention);
+			wordsMasks.push(wordsMask);
+			const spanIdx = [];
+			const spanMask = [];
+			for (let start = 0; start < words.length; start++) for (let width = 0; width < this.config.maxWidth; width++) {
+				const end = Math.min(start + width, words.length - 1);
+				spanIdx.push([start, end]);
+				spanMask.push(end < words.length);
+			}
+			spanIdxs.push(spanIdx);
+			spanMasks.push(spanMask);
+		}
+		const maxSpans = Math.max(...spanIdxs.map((s) => s.length));
+		return {
+			inputIds: pad(inputIds, PAD),
+			attentionMasks: pad(attentionMasks, PAD),
+			wordsMasks: pad(wordsMasks, PAD),
+			textLengths,
+			spanIdxs: spanIdxs.map((s) => [...s, ...Array.from({ length: maxSpans - s.length }, () => [PAD, PAD])]),
+			spanMasks: pad(spanMasks, false),
+			idToClass,
+			batchWords
+		};
+	}
+};
+//#endregion
+//#region src/ner/gliner1-backend.ts
+/**
+* GLiNER1 span-enumeration runtime on onnxruntime-web.
+*
+* One ONNX graph. Inputs:
+*   input_ids       int64  [batch, tokens]
+*   attention_mask  int64  [batch, tokens]
+*   words_mask      int64  [batch, tokens]   1-based word index on first sub-token
+*   text_lengths    int64  [batch, 1]
+*   span_idx        int64  [batch, spans, 2]
+*   span_mask       bool   [batch, spans]
+* Output:
+*   logits          float32 [batch, words, maxWidth, entityCount]
+*/
+/** Fallback when a repo's config omits max_width. */
+const DEFAULT_MAX_WIDTH = 12;
+const DEFAULT_ENT_TOKEN = "<<ENT>>";
+const DEFAULT_SEP_TOKEN = "<<SEP>>";
+const CONFIG_CANDIDATES = ["gliner_config.json", "config.json"];
+/** Some repos put the graph at the root rather than under onnx/. */
+const MODEL_EXTENSIONS = [".onnx", ".model"];
+const OUTPUT_LOGITS = "logits";
+function readConfig(files) {
+	for (const name of CONFIG_CANDIDATES) {
+		const raw = files[name];
+		if (!raw) continue;
+		try {
+			const parsed = JSON.parse(new TextDecoder().decode(raw));
+			return {
+				maxWidth: Number(parsed.max_width) || DEFAULT_MAX_WIDTH,
+				entToken: String(parsed.ent_token ?? DEFAULT_ENT_TOKEN),
+				sepToken: String(parsed.sep_token ?? DEFAULT_SEP_TOKEN)
+			};
+		} catch {}
+	}
+	return {
+		maxWidth: DEFAULT_MAX_WIDTH,
+		entToken: DEFAULT_ENT_TOKEN,
+		sepToken: DEFAULT_SEP_TOKEN
+	};
+}
+function findModelBytes(files) {
+	for (const [path, bytes] of Object.entries(files)) if (MODEL_EXTENSIONS.some((ext) => path.endsWith(ext))) return bytes;
+	throw new NerError("No .onnx or .model file in the downloaded model", "Gliner1Backend");
+}
+var Gliner1Backend = class {
+	arch = "gliner1";
+	session = null;
+	processor = null;
+	config = null;
+	async load(files, options) {
+		this.config = readConfig(files);
+		this.processor = new SpanProcessor(this.config, toSpanTokenizer(options.tokenizer));
+		this.session = await createSession(findModelBytes(files), { executionProviders: options.executionProviders });
+	}
+	async extract(text, labels, threshold) {
+		const session = this.session;
+		const processor = this.processor;
+		const config = this.config;
+		if (!session || !processor || !config) throw new NerError("Backend used before load()", "Gliner1Backend");
+		if (labels.length === 0) return [];
+		const batch = processor.prepare([text], labels);
+		const words = batch.batchWords[0] ?? [];
+		if (words.length === 0) return [];
+		const { Tensor } = await import("onnxruntime-web");
+		const tokenCount = batch.inputIds[0].length;
+		const spanCount = batch.spanIdxs[0].length;
+		const big = (values) => BigInt64Array.from(values, BigInt);
+		const feeds = {
+			input_ids: new Tensor("int64", big(batch.inputIds.flat()), [1, tokenCount]),
+			attention_mask: new Tensor("int64", big(batch.attentionMasks.flat()), [1, tokenCount]),
+			words_mask: new Tensor("int64", big(batch.wordsMasks.flat()), [1, tokenCount]),
+			text_lengths: new Tensor("int64", big(batch.textLengths), [1, 1]),
+			span_idx: new Tensor("int64", big(batch.spanIdxs.flat(2)), [
+				1,
+				spanCount,
+				2
+			]),
+			span_mask: new Tensor("bool", Uint8Array.from(batch.spanMasks.flat(), (v) => v ? 1 : 0), [1, spanCount])
+		};
+		const inputs = {};
+		for (const name of session.inputNames) {
+			const tensor = feeds[name];
+			if (!tensor) throw new NerError(`Model expects unknown input "${name}"`, "Gliner1Backend");
+			inputs[name] = tensor;
+		}
+		const outputs = await session.run(inputs);
+		const logits = (outputs[OUTPUT_LOGITS] ?? outputs[session.outputNames[0] ?? ""])?.data;
+		if (!logits) throw new NerError("Model produced no logits", "Gliner1Backend");
+		const [spans = []] = decodeSpans(logits, {
+			batchSize: 1,
+			inputLength: words.length,
+			maxWidth: config.maxWidth,
+			entityCount: labels.length,
+			texts: [text],
+			batchWords: batch.batchWords,
+			idToClass: batch.idToClass
+		}, {
+			threshold,
+			flatNer: true,
+			multiLabel: false
+		});
+		return spans;
+	}
+	async dispose() {
+		await this.session?.release();
+		this.session = null;
+		this.processor = null;
+		this.config = null;
+	}
+};
+//#endregion
+//#region src/ner/vendor/gliner2-decoder.ts
+/** GLiNER2 span generation and decoding. Adapted from @lmoe/gliner-onnx (MIT). */
+/**
+* Every span up to `maxWidth` words.
+*
+* Out-of-range slots are filled with (0, 0) rather than dropped, because the
+* span axis must stay a fixed `seqLen * maxWidth` for the ONNX graph. The
+* decoder skips them by re-checking the word bounds.
+*/
+function generateSpans(seqLen, maxWidth) {
+	const spanStart = [];
+	const spanEnd = [];
+	for (let i = 0; i < seqLen; i++) for (let j = 0; j < maxWidth; j++) {
+		const inRange = i + j < seqLen;
+		spanStart.push(inRange ? i : 0);
+		spanEnd.push(inRange ? i + j : 0);
+	}
+	return {
+		spanStart,
+		spanEnd,
+		spanCount: spanStart.length
+	};
+}
+/** Sigmoid of the dot product between each span and each label embedding. */
+function computeDotProductScores(spanRep, labelRep, spanCount, labelCount, hiddenSize) {
+	const scores = new Float32Array(spanCount * labelCount);
+	for (let s = 0; s < spanCount; s++) {
+		const spanOffset = s * hiddenSize;
+		for (let l = 0; l < labelCount; l++) {
+			const labelOffset = l * hiddenSize;
+			let dot = 0;
+			for (let h = 0; h < hiddenSize; h++) dot += spanRep[spanOffset + h] * labelRep[labelOffset + h];
+			scores[s * labelCount + l] = sigmoid(dot);
+		}
+	}
+	return scores;
+}
+/** Score matrix -> entities, then drop same-label overlaps keeping the best. */
+function decodeEntities(scoreData, wordCount, labels, wordOffsets, text, threshold) {
+	const { scores, wordSpanStart, wordSpanEnd, spanCount } = scoreData;
+	const labelCount = labels.length;
+	const entities = [];
+	for (let s = 0; s < spanCount; s++) {
+		const startWord = wordSpanStart[s];
+		const endWord = wordSpanEnd[s];
+		if (startWord >= wordCount || endWord >= wordCount) continue;
+		for (let l = 0; l < labelCount; l++) {
+			const score = scores[s * labelCount + l];
+			if (score < threshold) continue;
+			const start = wordOffsets[startWord][0];
+			const end = wordOffsets[endWord][1];
+			entities.push({
+				text: text.slice(start, end),
+				label: labels[l],
+				start,
+				end,
+				score
+			});
+		}
+	}
+	return deduplicateEntities(entities);
+}
+/**
+* Keep the highest-scoring entity among overlapping ones *of the same label*.
+* Different labels may overlap: a person name inside an organization is a
+* legitimate reading, not a conflict.
+*/
+function deduplicateEntities(entities) {
+	const byScore = [...entities].sort((a, b) => b.score - a.score);
+	const kept = [];
+	for (const entity of byScore) if (!kept.some((other) => entity.label === other.label && entity.start < other.end && entity.end > other.start)) kept.push(entity);
+	return kept.sort((a, b) => a.start - b.start);
+}
+//#endregion
+//#region src/ner/gliner2-backend.ts
+/**
+* GLiNER2 runtime on onnxruntime-web.
+*
+* Three graphs, orchestrated in JS:
+*   encoder      input_ids, attention_mask [1, seq] int64 -> hidden states
+*   span_rep     hidden_states [1, seq, hidden] + span_start_idx/span_end_idx
+*                [1, spans] int64 -> span representations
+*   count_embed  label_embeddings [labels, hidden] -> transformed embeddings
+*
+* The `classifier` graph is for text classification only and is not downloaded.
+* Scoring (dot product + sigmoid) happens in JS between span_rep and
+* count_embed.
+*
+* Unlike GLiNER1's single prompt string, GLiNER2 builds a *schema*:
+*   ( [P] entities ( [E] label1 [E] label2 ) ) [SEP_TEXT] <text>
+* with each label's position recorded so its embedding can be gathered from
+* the encoder output.
+*/
+const CONFIG_FILE = "config.json";
+const GLINER2_CONFIG_FILE = "gliner2_config.json";
+const TOKEN_P = "[P]";
+const TOKEN_E = "[E]";
+const TOKEN_SEP_TEXT = "[SEP_TEXT]";
+const SCHEMA_OPEN = "(";
+const SCHEMA_CLOSE = ")";
+const NER_TASK_NAME = "entities";
+function decodeJson(buffer, name) {
+	if (!buffer) throw new NerError(`Model is missing ${name}`, "Gliner2Backend");
+	return JSON.parse(new TextDecoder().decode(buffer));
+}
+var Gliner2Backend = class {
+	arch = "gliner2";
+	config = null;
+	encoder = null;
+	spanRep = null;
+	countEmbed = null;
+	tokenize = null;
+	async load(files, options) {
+		const transformer = decodeJson(files[CONFIG_FILE], CONFIG_FILE);
+		const gliner2 = decodeJson(files[GLINER2_CONFIG_FILE], GLINER2_CONFIG_FILE);
+		if (typeof transformer.hidden_size !== "number") throw new NerError(`${CONFIG_FILE} is missing hidden_size`, "Gliner2Backend");
+		if (typeof gliner2.max_width !== "number") throw new NerError(`${GLINER2_CONFIG_FILE} is missing max_width`, "Gliner2Backend");
+		this.config = {
+			hiddenSize: transformer.hidden_size,
+			maxWidth: gliner2.max_width,
+			specialTokens: gliner2.special_tokens
+		};
+		const tokenizer = options.tokenizer;
+		this.tokenize = (text) => extractTokenIds(tokenizer(text, { add_special_tokens: false }).input_ids.tolist());
+		const graph = (path) => {
+			const bytes = files[path];
+			if (!bytes) throw new NerError(`Model is missing ${path}`, "Gliner2Backend");
+			return bytes;
+		};
+		const providers = options.executionProviders ?? ["wasm"];
+		const [encoder, spanRep, countEmbed] = await Promise.all([
+			createSession(graph("onnx/encoder.onnx"), { executionProviders: providers }),
+			createSession(graph("onnx/span_rep.onnx"), { executionProviders: providers }),
+			createSession(graph("onnx/count_embed.onnx"), { executionProviders: providers })
+		]);
+		this.encoder = encoder;
+		this.spanRep = spanRep;
+		this.countEmbed = countEmbed;
+	}
+	async extract(text, labels, threshold) {
+		const { config, tokenize } = this;
+		if (!config || !tokenize || !this.encoder || !this.spanRep || !this.countEmbed) throw new NerError("Backend used before load()", "Gliner2Backend");
+		if (text.trim().length === 0 || labels.length === 0) return [];
+		const schema = this.buildSchema(labels);
+		const words = this.tokenizeWords(text);
+		if (words.wordOffsets.length === 0) return [];
+		const allTokens = [...schema.tokens, ...words.tokens];
+		const hidden = await this.encode(allTokens);
+		const labelEmbeddings = gatherRows(hidden, schema.labelPositions, config.hiddenSize);
+		const textTokenCount = allTokens.length - schema.tokens.length;
+		if (textTokenCount === 0) return [];
+		const textHidden = sliceRows(hidden, schema.tokens.length, textTokenCount, config.hiddenSize);
+		const { spanStart, spanEnd, spanCount } = generateSpans(words.wordOffsets.length, config.maxWidth);
+		const toToken = (wordIdx) => words.firstTokenPositions[wordIdx] ?? 0;
+		return decodeEntities({
+			scores: computeDotProductScores(await this.runSpanRep(textHidden, textTokenCount, spanStart.map(toToken), spanEnd.map(toToken), spanCount), await this.runCountEmbed(labelEmbeddings, labels.length), spanCount, labels.length, config.hiddenSize),
+			wordSpanStart: spanStart,
+			wordSpanEnd: spanEnd,
+			spanCount
+		}, words.wordOffsets.length, labels, words.wordOffsets, text, threshold);
+	}
+	/** `( [P] entities ( [E] label1 [E] label2 ) ) [SEP_TEXT]` */
+	buildSchema(labels) {
+		const config = this.config;
+		const tokenize = this.tokenize;
+		const open = tokenize(SCHEMA_OPEN);
+		const close = tokenize(SCHEMA_CLOSE);
+		const tokens = [...open, config.specialTokens[TOKEN_P]];
+		tokens.push(...tokenize(NER_TASK_NAME), ...open);
+		const labelPositions = [];
+		for (const label of labels) {
+			labelPositions.push(tokens.length);
+			tokens.push(config.specialTokens[TOKEN_E], ...tokenize(label));
+		}
+		tokens.push(...close, ...close, config.specialTokens[TOKEN_SEP_TEXT]);
+		return {
+			tokens,
+			labelPositions
+		};
+	}
+	/**
+	* Word-split and tokenize the text.
+	*
+	* Lower-cased before splitting, matching the reference implementation --
+	* offsets stay valid because `toLowerCase` is length-preserving for the
+	* scripts these models cover.
+	*/
+	tokenizeWords(text) {
+		const tokenize = this.tokenize;
+		const tokens = [];
+		const wordOffsets = [];
+		const firstTokenPositions = [];
+		for (const [word, start, end] of splitWords(text.toLowerCase(), RICH_WORD_PATTERN)) {
+			wordOffsets.push([start, end]);
+			firstTokenPositions.push(tokens.length);
+			tokens.push(...tokenize(word));
+		}
+		return {
+			tokens,
+			wordOffsets,
+			firstTokenPositions
+		};
+	}
+	async encode(tokens) {
+		const { Tensor } = await import("onnxruntime-web");
+		const session = this.encoder;
+		const seqLen = tokens.length;
+		return firstOutput(await session.run({
+			input_ids: new Tensor("int64", BigInt64Array.from(tokens, BigInt), [1, seqLen]),
+			attention_mask: new Tensor("int64", new BigInt64Array(seqLen).fill(1n), [1, seqLen])
+		}), "encoder");
+	}
+	async runSpanRep(hidden, seqLen, spanStart, spanEnd, spanCount) {
+		const { Tensor } = await import("onnxruntime-web");
+		const session = this.spanRep;
+		const hiddenSize = this.config.hiddenSize;
+		return firstOutput(await session.run({
+			hidden_states: new Tensor("float32", hidden, [
+				1,
+				seqLen,
+				hiddenSize
+			]),
+			span_start_idx: new Tensor("int64", BigInt64Array.from(spanStart, BigInt), [1, spanCount]),
+			span_end_idx: new Tensor("int64", BigInt64Array.from(spanEnd, BigInt), [1, spanCount])
+		}), "span_rep");
+	}
+	async runCountEmbed(labelEmbeddings, labelCount) {
+		const { Tensor } = await import("onnxruntime-web");
+		const session = this.countEmbed;
+		const hiddenSize = this.config.hiddenSize;
+		return firstOutput(await session.run({ label_embeddings: new Tensor("float32", labelEmbeddings, [labelCount, hiddenSize]) }), "count_embed");
+	}
+	async dispose() {
+		await Promise.all([
+			this.encoder?.release(),
+			this.spanRep?.release(),
+			this.countEmbed?.release()
+		]);
+		this.encoder = null;
+		this.spanRep = null;
+		this.countEmbed = null;
+		this.config = null;
+		this.tokenize = null;
+	}
+};
+function firstOutput(outputs, graph) {
+	const [name] = Object.keys(outputs);
+	if (!name) throw new NerError(`Graph ${graph} produced no output`, "Gliner2Backend");
+	return outputs[name].data;
+}
+//#endregion
+//#region src/ner/registry.ts
+/** Generic PII labels, matching the pdftools prototype's default set. */
+const DEFAULT_PII_LABELS = Object.freeze([
+	"person",
+	"organization",
+	"location",
+	"email",
+	"phone_number",
+	"url",
+	"id",
+	"account_number",
+	"zip_code",
+	"address",
+	"ip_address",
+	"date",
+	"ssn",
+	"driver_license",
+	"passport",
+	"age",
+	"credit_card",
+	"medical_condition"
+]);
+/**
+* Label prompts the StabRise GLiNER2 PII model was fine-tuned on.
+*
+* These must match the cloud endpoint's tag list (scaledp-api
+* deidentify/views.py) or scores drop: GLiNER scores a label by its prompt
+* text, so a renamed label is a different label.
+*/
+const GLINER2_PII_LABELS = Object.freeze([
+	"date",
+	"person_name",
+	"person_title",
+	"organization",
+	"location",
+	"email",
+	"phone",
+	"id",
+	"account",
+	"zip_code",
+	"address",
+	"ip",
+	"url",
+	"ssn",
+	"driver_license",
+	"passport",
+	"age",
+	"credit_card",
+	"medical_condition",
+	"technology"
+]);
+const NER_MODELS = Object.freeze([
+	{
+		id: "gliner-multi-pii",
+		name: "GLiNER multilingual PII, int8 (~333 MB)",
+		arch: "gliner1",
+		repo: "onnx-community/gliner_multi_pii-v1",
+		files: [{
+			path: "gliner_config.json",
+			approxBytes: 800
+		}, {
+			path: "onnx/model_int8.onnx",
+			approxBytes: 349e6
+		}],
+		labels: DEFAULT_PII_LABELS,
+		languages: ["multi"]
+	},
+	{
+		id: "gliner-small",
+		name: "GLiNER small English, int8 (~183 MB)",
+		arch: "gliner1",
+		repo: "onnx-community/gliner_small-v2.1",
+		files: [{
+			path: "gliner_config.json",
+			approxBytes: 731
+		}, {
+			path: "onnx/model_quantized.onnx",
+			approxBytes: 183403734
+		}],
+		labels: DEFAULT_PII_LABELS,
+		languages: ["en"]
+	},
+	{
+		id: "stabrise-pii-multi",
+		name: "StabRise PII multilingual, int8 (~404 MB)",
+		arch: "gliner1",
+		repo: "StabRise/pii-detection-en-fr-ge-it-es",
+		files: [{
+			path: "config.json",
+			approxBytes: 3417
+		}, {
+			path: "model_int8.model",
+			approxBytes: 403923207
+		}],
+		labels: DEFAULT_PII_LABELS,
+		languages: [
+			"en",
+			"fr",
+			"de",
+			"it",
+			"es"
+		],
+		private: true
+	},
+	{
+		id: "stabrise-pii-multi-g2",
+		name: "StabRise PII multilingual GLiNER2, fp32 (~1.2 GB)",
+		arch: "gliner2",
+		repo: "StabRise/pii-multi-g2-v1-onnx",
+		files: [
+			{
+				path: "config.json",
+				approxBytes: 48
+			},
+			{
+				path: "gliner2_config.json",
+				approxBytes: 691
+			},
+			{
+				path: "onnx/encoder.onnx",
+				approxBytes: 1111055946
+			},
+			{
+				path: "onnx/span_rep.onnx",
+				approxBytes: 66111424
+			},
+			{
+				path: "onnx/count_embed.onnx",
+				approxBytes: 42506885
+			}
+		],
+		labels: GLINER2_PII_LABELS,
+		languages: [
+			"en",
+			"de",
+			"pl",
+			"es"
+		],
+		private: true,
+		executionProviders: ["wasm"]
+	}
+]);
+/** Public, zero-configuration default. */
+const DEFAULT_NER_MODEL_ID = "gliner-multi-pii";
+function getNerModel(id) {
+	return NER_MODELS.find((m) => m.id === id);
+}
+/** Total download size in bytes, for a progress estimate before fetching. */
+function modelSizeBytes(model) {
+	return model.files.reduce((sum, f) => sum + (f.approxBytes ?? 0), 0);
+}
+//#endregion
+//#region src/ner/tokenizer.ts
+/**
+* Tokenizer loading and adaptation for the GLiNER runtimes.
+*
+* `@huggingface/transformers` provides the tokenizer only; no model runs
+* through it. Its remote host is configurable so gated repos can be proxied
+* through the consuming application's own origin.
+*/
+let modulePromise = null;
+async function loadTransformers() {
+	if (modulePromise) return modulePromise;
+	modulePromise = (async () => {
+		let mod;
+		try {
+			mod = await import("@huggingface/transformers");
+		} catch (cause) {
+			throw new Error("@huggingface/transformers is required for NER tokenization. Install it: npm i @huggingface/transformers", { cause });
+		}
+		const { hf } = getConfig();
+		mod.env.allowLocalModels = false;
+		if (hf.remoteHost) mod.env.remoteHost = hf.remoteHost;
+		if (hf.remotePathTemplate) mod.env.remotePathTemplate = hf.remotePathTemplate;
+		return mod;
+	})();
+	return modulePromise;
+}
+/** Reset the cached module. Tests only. */
+function resetTransformers() {
+	modulePromise = null;
+}
+const tokenizers = /* @__PURE__ */ new Map();
+async function loadTokenizer(repo) {
+	const existing = tokenizers.get(repo);
+	if (existing) return existing;
+	const promise = (async () => {
+		const { AutoTokenizer } = await loadTransformers();
+		return AutoTokenizer.from_pretrained(repo);
+	})();
+	promise.catch(() => tokenizers.delete(repo));
+	tokenizers.set(repo, promise);
+	return promise;
+}
+/** Adapt to the callable form the GLiNER2 runtime expects. */
+function toCallableTokenizer(tokenizer) {
+	return (text) => {
+		return extractTokenIds(tokenizer(text, { add_special_tokens: false }).input_ids.tolist());
+	};
+}
+//#endregion
+//#region src/ner/gliner-ner.ts
+/**
+* Named-entity recognition over a Document, mirroring ScaleDP's `Ner` stage.
+*
+* Chunks long text, runs a GLiNER backend, and maps the resulting character
+* offsets back onto the OCR boxes so every entity carries its position on the
+* page.
+*/
+const GLINER_NER_DEFAULTS = Object.freeze({
+	...BASE_STAGE_DEFAULTS,
+	inputCol: "text",
+	outputCol: "ner",
+	keepInputData: true,
+	model: DEFAULT_NER_MODEL_ID,
+	labels: DEFAULT_PII_LABELS,
+	threshold: .5,
+	whiteList: [],
+	chunkLength: 500,
+	chunkStride: 480,
+	normaliseCasing: true
+});
+/**
+* Map each character of the joined document text to the box it came from.
+*
+* Built from the *actual* text the OCR stage produced rather than assuming one
+* separator per box. Python derives the mapping from `len(box.text) + 1`, which
+* silently drifts whenever `keepFormatting` inserted several spaces or a
+* newline, shifting every entity's boxes after the first wide gap.
+*/
+function buildCharToBoxMap(text, boxes) {
+	const mapping = new Int32Array(text.length).fill(-1);
+	let cursor = 0;
+	for (const [index, box] of boxes.entries()) {
+		if (box.text.length === 0) continue;
+		const found = text.indexOf(box.text, cursor);
+		if (found === -1) continue;
+		mapping.fill(index, found, found + box.text.length);
+		cursor = found + box.text.length;
+	}
+	return mapping;
+}
+/** Boxes a character range touches, in document order and without repeats. */
+function boxesForRange(mapping, boxes, start, end) {
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (let i = Math.max(0, start); i < Math.min(end, mapping.length); i++) {
+		const index = mapping[i];
+		if (index < 0 || seen.has(index)) continue;
+		seen.add(index);
+		const box = boxes[index];
+		if (box) out.push(box);
+	}
+	return out;
+}
+var GlinerNer = class extends Stage {
+	name = "GlinerNer";
+	backend = null;
+	loading = null;
+	constructor(options = {}) {
+		super(resolveParams(GLINER_NER_DEFAULTS, options, {
+			threshold: (value) => {
+				if (!(value >= 0 && value <= 1)) throw new RangeError(`threshold must be between 0 and 1, received ${value}`);
+			},
+			model: (value) => {
+				if (!getNerModel(value)) throw new RangeError(`Unknown NER model "${value}". See NER_MODELS for valid ids.`);
+			}
+		}));
+	}
+	async init() {
+		await this.getBackend();
+	}
+	getBackend() {
+		if (this.backend) return Promise.resolve(this.backend);
+		if (this.loading) return this.loading;
+		this.loading = (async () => {
+			const model = getNerModel(this.params.model);
+			if (!model) throw new NerError(`Unknown model ${this.params.model}`, this.name);
+			if (model.private && !getConfig().auth) throw new NerError(`Model ${model.id} lives in a private repo. Supply a token via configure({ auth }).`, this.name);
+			const [files, tokenizer] = await Promise.all([ensureModelFiles({
+				repo: model.repo,
+				files: model.files
+			}), loadTokenizer(model.repo)]);
+			const backend = model.arch === "gliner2" ? new Gliner2Backend() : new Gliner1Backend();
+			await backend.load(files, {
+				tokenizer,
+				executionProviders: model.executionProviders ?? getConfig().executionProviders
+			});
+			this.backend = backend;
+			return backend;
+		})();
+		this.loading.catch(() => {
+			this.loading = null;
+		});
+		return this.loading;
+	}
+	async apply(input, row) {
+		const document = input;
+		if (!document || typeof document.text !== "string") throw new NerError("Expected a Document with text", this.name);
+		if (document.exception) throw new NerError(`Upstream stage failed: ${document.exception}`, this.name);
+		const entities = await this.extract(document);
+		return createNerOutput({
+			path: String(row[this.params.pathCol] ?? document.path ?? "memory"),
+			entities,
+			json: JSON.stringify(entities)
+		});
+	}
+	/** Run NER over a document and attach boxes to every entity found. */
+	async extract(document) {
+		const { labels, threshold, whiteList, chunkLength, chunkStride } = this.params;
+		if (document.text.trim().length === 0 || labels.length === 0) return [];
+		const backend = await this.getBackend();
+		const source = this.params.normaliseCasing ? normaliseCasing(document.text) : document.text;
+		const spans = [];
+		for (const chunk of chunkText(source, chunkLength, chunkStride)) {
+			const found = await backend.extract(chunk.text, labels, threshold);
+			for (const span of found) spans.push(rebaseSpan(span, chunk.offset));
+		}
+		const mapping = buildCharToBoxMap(document.text, document.bboxes);
+		const allowed = new Set(whiteList);
+		return dedupeSpans(spans).filter((span) => allowed.size === 0 || allowed.has(span.label)).map((span) => ({
+			entity_group: span.label,
+			score: span.score,
+			word: document.text.slice(span.start, span.end),
+			start: span.start,
+			end: span.end,
+			boxes: boxesForRange(mapping, document.bboxes, span.start, span.end)
+		}));
+	}
+	onError(message, row) {
+		return createNerOutput({
+			path: String(row[this.params.pathCol] ?? "memory"),
+			exception: message
+		});
+	}
+	async dispose() {
+		await this.backend?.dispose();
+		this.backend = null;
+		this.loading = null;
+	}
+};
+//#endregion
+export { isMostlyUppercase as A, toSpanTokenizer as C, DEFAULT_CHUNK_STRIDE as D, DEFAULT_CHUNK_LENGTH as E, rebaseSpan as M, titleCaseAllCapsWords as N, chunkText as O, decodeSpans as S, softmax as T, generateSpans as _, loadTokenizer as a, WORD_PATTERN as b, DEFAULT_NER_MODEL_ID as c, NER_MODELS as d, getNerModel as f, decodeEntities as g, computeDotProductScores as h, buildCharToBoxMap as i, normaliseCasing as j, dedupeSpans as k, DEFAULT_PII_LABELS as l, Gliner2Backend as m, GlinerNer as n, resetTransformers as o, modelSizeBytes as p, boxesForRange as r, toCallableTokenizer as s, GLINER_NER_DEFAULTS as t, GLINER2_PII_LABELS as u, Gliner1Backend as v, sigmoid as w, splitWords as x, RICH_WORD_PATTERN as y };
+
+//# sourceMappingURL=ner-SsZLZ6ed.js.map