@stabrise/scaledp 0.1.0

package/dist/ner/index.d.ts

@@ -0,0 +1,293 @@
+import { l as Stage, m as BaseStageParams, s as Row } from "../pipeline-DeLO-OCE.js";
+import { n as Box } from "../box-DAfzwfhA.js";
+import { t as Document } from "../document-B8I61TiY.js";
+import { n as NerOutput, t as Entity } from "../entity-CedtRhU1.js";
+import { n as ModelFiles, t as ModelFile } from "../model-cache-BhFYpfZz.js";
+//#region src/ner/vendor/splitter.d.ts
+/** Word splitting for GLiNER. Adapted from @lmoe/gliner-onnx (MIT). */
+/**
+ * Unicode word classes (`\p{L}\p{N}` with the `u` flag), deliberately not `\w`.
+ *
+ * JavaScript's `\w` is ASCII-only even under `/u`, so it would shatter accented
+ * names -- "Müller", "García" -- into single-character tokens and wreck
+ * multi-word span detection in German, Polish and Spanish text. Python's `\w`
+ * is Unicode-aware, so this restores parity with the reference tokenizer.
+ */
+declare const WORD_PATTERN: RegExp;
+/**
+ * As above, plus leading branches for URLs, emails and @mentions so they stay
+ * whole. Those branches are ASCII on purpose, mirroring the Python pattern.
+ */
+declare const RICH_WORD_PATTERN: RegExp;
+type SplitWord = [text: string, start: number, end: number];
+/**
+ * Split text into words with their character offsets.
+ *
+ * A fresh RegExp per call: the `g` flag makes `lastIndex` stateful, so sharing
+ * one instance across calls silently skips matches.
+ */
+declare function splitWords(text: string, pattern?: RegExp): SplitWord[];
+//#endregion
+//#region src/ner/vendor/span-decoder.d.ts
+interface DecodedSpan {
+  text: string;
+  label: string;
+  start: number;
+  end: number;
+  score: number;
+}
+interface DecodeOptions {
+  threshold?: number;
+  /** Flat NER forbids nesting; set false to allow a span inside another. */
+  flatNer?: boolean;
+  /** Allow the same span to carry more than one label. */
+  multiLabel?: boolean;
+}
+declare function decodeSpans(logits: ArrayLike<number>, params: {
+  batchSize: number;
+  /** Words per sequence, i.e. the model's span-start axis. */
+  inputLength: number;
+  maxWidth: number;
+  entityCount: number;
+  texts: readonly string[];
+  batchWords: readonly SplitWord[][];
+  idToClass: Record<number, string>;
+}, options?: DecodeOptions): DecodedSpan[][];
+//#endregion
+//#region src/ner/backend.d.ts
+interface NerBackendLoadOptions {
+  /** A `@huggingface/transformers` tokenizer instance. */
+  tokenizer: unknown;
+  executionProviders?: readonly string[];
+}
+interface NerBackend {
+  readonly arch: 'gliner1' | 'gliner2';
+  load(files: ModelFiles, options: NerBackendLoadOptions): Promise<void>;
+  /** Entities with character offsets into `text`. */
+  extract(text: string, labels: readonly string[], threshold: number): Promise<DecodedSpan[]>;
+  dispose(): Promise<void>;
+}
+//#endregion
+//#region src/ner/chunking.d.ts
+/** Python's `split_text` default, and what the cloud /ner/text endpoint uses. */
+declare const DEFAULT_CHUNK_LENGTH = 500;
+/** 500 - 480 leaves a 20-character overlap so entities on a seam survive. */
+declare const DEFAULT_CHUNK_STRIDE = 480;
+interface Chunk {
+  text: string;
+  /** Character offset of this chunk within the original text. */
+  offset: number;
+}
+declare function chunkText(text: string, maxLength?: number, stride?: number): Chunk[];
+/** Shift a chunk-local span onto the original text's coordinates. */
+declare function rebaseSpan(span: DecodedSpan, offset: number): DecodedSpan;
+/**
+ * Drop duplicates produced by the chunk overlap, keeping the highest score for
+ * each distinct (start, end, label).
+ */
+declare function dedupeSpans(spans: readonly DecodedSpan[]): DecodedSpan[];
+declare function isMostlyUppercase(text: string): boolean;
+/**
+ * Title-case runs of capitals, preserving length.
+ *
+ * GLiNER1 models are cased and scanned documents are frequently set in all
+ * caps, which reads to the model as unlike anything in training. Length
+ * preservation is essential: every character offset the decoder returns is used
+ * to index the original text.
+ */
+declare function titleCaseAllCapsWords(text: string): string;
+/** Apply the casing fix only when the text is predominantly uppercase. */
+declare function normaliseCasing(text: string): string;
+//#endregion
+//#region src/ner/gliner-ner.d.ts
+interface GlinerNerParams extends BaseStageParams {
+  /** Registry id, e.g. 'gliner-multi-pii'. */
+  model: string;
+  /** Entity types to look for. GLiNER scores a label by its prompt text. */
+  labels: readonly string[];
+  /** Minimum score an entity must reach (0-1). */
+  threshold: number;
+  /** Only keep these entity groups; empty keeps everything. */
+  whiteList: readonly string[];
+  chunkLength: number;
+  chunkStride: number;
+  /**
+   * Title-case runs of capitals before inference. GLiNER1 models are cased
+   * and scanned documents are often set in all caps.
+   */
+  normaliseCasing: boolean;
+}
+declare const GLINER_NER_DEFAULTS: GlinerNerParams;
+/**
+ * Map each character of the joined document text to the box it came from.
+ *
+ * Built from the *actual* text the OCR stage produced rather than assuming one
+ * separator per box. Python derives the mapping from `len(box.text) + 1`, which
+ * silently drifts whenever `keepFormatting` inserted several spaces or a
+ * newline, shifting every entity's boxes after the first wide gap.
+ */
+declare function buildCharToBoxMap(text: string, boxes: readonly Box[]): Int32Array;
+/** Boxes a character range touches, in document order and without repeats. */
+declare function boxesForRange(mapping: Int32Array, boxes: readonly Box[], start: number, end: number): Box[];
+declare class GlinerNer extends Stage<GlinerNerParams> {
+  readonly name = "GlinerNer";
+  private backend;
+  private loading;
+  constructor(options?: Partial<GlinerNerParams>);
+  init(): Promise<void>;
+  private getBackend;
+  protected apply(input: unknown, row: Row): Promise<NerOutput>;
+  /** Run NER over a document and attach boxes to every entity found. */
+  extract(document: Document): Promise<Entity[]>;
+  protected onError(message: string, row: Row): NerOutput;
+  dispose(): Promise<void>;
+}
+//#endregion
+//#region src/ner/gliner1-backend.d.ts
+declare class Gliner1Backend implements NerBackend {
+  readonly arch: "gliner1";
+  private session;
+  private processor;
+  private config;
+  load(files: ModelFiles, options: NerBackendLoadOptions): Promise<void>;
+  extract(text: string, labels: readonly string[], threshold: number): Promise<DecodedSpan[]>;
+  dispose(): Promise<void>;
+}
+//#endregion
+//#region src/ner/gliner2-backend.d.ts
+declare class Gliner2Backend implements NerBackend {
+  readonly arch: "gliner2";
+  private config;
+  private encoder;
+  private spanRep;
+  private countEmbed;
+  private tokenize;
+  load(files: ModelFiles, options: NerBackendLoadOptions): Promise<void>;
+  extract(text: string, labels: readonly string[], threshold: number): Promise<DecodedSpan[]>;
+  /** `( [P] entities ( [E] label1 [E] label2 ) ) [SEP_TEXT]` */
+  private buildSchema;
+  /**
+   * Word-split and tokenize the text.
+   *
+   * Lower-cased before splitting, matching the reference implementation --
+   * offsets stay valid because `toLowerCase` is length-preserving for the
+   * scripts these models cover.
+   */
+  private tokenizeWords;
+  private encode;
+  private runSpanRep;
+  private runCountEmbed;
+  dispose(): Promise<void>;
+}
+//#endregion
+//#region src/ner/registry.d.ts
+type NerArchitecture = 'gliner1' | 'gliner2';
+interface NerModel {
+  /** Short id callers pass to GlinerNer. */
+  id: string;
+  /** Human-readable name including the download size. */
+  name: string;
+  arch: NerArchitecture;
+  /** Hugging Face repo id. */
+  repo: string;
+  files: ModelFile[];
+  /** Labels this model was tuned for. Using others still works, less well. */
+  labels: readonly string[];
+  languages: readonly string[];
+  /** Private repos need `configure({ auth })` to supply a bearer token. */
+  private?: boolean;
+  /** Execution providers this model requires; overrides the global config. */
+  executionProviders?: readonly string[];
+}
+/** Generic PII labels, matching the pdftools prototype's default set. */
+declare const DEFAULT_PII_LABELS: readonly string[];
+/**
+ * Label prompts the StabRise GLiNER2 PII model was fine-tuned on.
+ *
+ * These must match the cloud endpoint's tag list (scaledp-api
+ * deidentify/views.py) or scores drop: GLiNER scores a label by its prompt
+ * text, so a renamed label is a different label.
+ */
+declare const GLINER2_PII_LABELS: readonly string[];
+declare const NER_MODELS: readonly NerModel[];
+/** Public, zero-configuration default. */
+declare const DEFAULT_NER_MODEL_ID = "gliner-multi-pii";
+declare function getNerModel(id: string): NerModel | undefined;
+/** Total download size in bytes, for a progress estimate before fetching. */
+declare function modelSizeBytes(model: NerModel): number;
+//#endregion
+//#region src/ner/tokenizer.d.ts
+/**
+ * Tokenizer loading and adaptation for the GLiNER runtimes.
+ *
+ * `@huggingface/transformers` provides the tokenizer only; no model runs
+ * through it. Its remote host is configurable so gated repos can be proxied
+ * through the consuming application's own origin.
+ */
+type Transformers = typeof import('@huggingface/transformers');
+type PretrainedTokenizer = Awaited<ReturnType<Transformers['AutoTokenizer']['from_pretrained']>>;
+/** Reset the cached module. Tests only. */
+declare function resetTransformers(): void;
+declare function loadTokenizer(repo: string): Promise<PretrainedTokenizer>;
+/** Adapt to the callable form the GLiNER2 runtime expects. */
+declare function toCallableTokenizer(tokenizer: PretrainedTokenizer): (text: string) => number[];
+//#endregion
+//#region src/ner/vendor/span-processor.d.ts
+/** Minimal tokenizer surface the processor needs. */
+interface SpanTokenizer {
+  encode(text: string): number[];
+  clsTokenId: number;
+  sepTokenId: number;
+}
+//#endregion
+//#region src/ner/tokenizer-types.d.ts
+type PretrainedTokenizerLike = (text: string, options?: {
+  add_special_tokens?: boolean;
+}) => {
+  input_ids: {
+    tolist(): (bigint | number)[][] | (bigint | number)[];
+  };
+};
+/**
+ * Adapt a transformers.js tokenizer to the span processor's interface.
+ *
+ * CLS/SEP ids are derived empirically -- encode a throwaway token, read the
+ * first and last id -- rather than read from `cls_token_id`. Not every GLiNER
+ * repo populates those fields, and a wrong id corrupts every sequence silently
+ * instead of failing loudly.
+ */
+declare function toSpanTokenizer(tokenizer: PretrainedTokenizerLike): SpanTokenizer;
+//#endregion
+//#region src/ner/vendor/gliner2-decoder.d.ts
+type WordOffset = [start: number, end: number];
+/**
+ * Every span up to `maxWidth` words.
+ *
+ * Out-of-range slots are filled with (0, 0) rather than dropped, because the
+ * span axis must stay a fixed `seqLen * maxWidth` for the ONNX graph. The
+ * decoder skips them by re-checking the word bounds.
+ */
+declare function generateSpans(seqLen: number, maxWidth: number): {
+  spanStart: number[];
+  spanEnd: number[];
+  spanCount: number;
+};
+/** Sigmoid of the dot product between each span and each label embedding. */
+declare function computeDotProductScores(spanRep: Float32Array, labelRep: Float32Array, spanCount: number, labelCount: number, hiddenSize: number): Float32Array;
+interface NerScoreData {
+  scores: Float32Array;
+  wordSpanStart: number[];
+  wordSpanEnd: number[];
+  spanCount: number;
+}
+/** Score matrix -> entities, then drop same-label overlaps keeping the best. */
+declare function decodeEntities(scoreData: NerScoreData, wordCount: number, labels: readonly string[], wordOffsets: readonly WordOffset[], text: string, threshold: number): DecodedSpan[];
+//#endregion
+//#region src/ner/vendor/math.d.ts
+/** Numeric helpers for the GLiNER runtimes. Adapted from @lmoe/gliner-onnx (MIT). */
+/** Numerically stable sigmoid: exp(-x) overflows for large negative x. */
+declare function sigmoid(x: number): number;
+declare function softmax(values: ArrayLike<number>): Float32Array;
+//#endregion
+export { type Chunk, DEFAULT_CHUNK_LENGTH, DEFAULT_CHUNK_STRIDE, DEFAULT_NER_MODEL_ID, DEFAULT_PII_LABELS, type DecodedSpan, GLINER2_PII_LABELS, GLINER_NER_DEFAULTS, Gliner1Backend, Gliner2Backend, GlinerNer, type GlinerNerParams, NER_MODELS, type NerArchitecture, type NerBackend, type NerBackendLoadOptions, type NerModel, RICH_WORD_PATTERN, type SplitWord, WORD_PATTERN, boxesForRange, buildCharToBoxMap, chunkText, computeDotProductScores, decodeEntities, decodeSpans, dedupeSpans, generateSpans, getNerModel, isMostlyUppercase, loadTokenizer, modelSizeBytes, normaliseCasing, rebaseSpan, resetTransformers, sigmoid, softmax, splitWords, titleCaseAllCapsWords, toCallableTokenizer, toSpanTokenizer };
+//# sourceMappingURL=index.d.ts.map

package/dist/ner/index.js

@@ -0,0 +1,2 @@
+import { A as isMostlyUppercase, C as toSpanTokenizer, D as DEFAULT_CHUNK_STRIDE, E as DEFAULT_CHUNK_LENGTH, M as rebaseSpan, N as titleCaseAllCapsWords, O as chunkText, S as decodeSpans, T as softmax, _ as generateSpans, a as loadTokenizer, b as WORD_PATTERN, c as DEFAULT_NER_MODEL_ID, d as NER_MODELS, f as getNerModel, g as decodeEntities, h as computeDotProductScores, i as buildCharToBoxMap, j as normaliseCasing, k as dedupeSpans, l as DEFAULT_PII_LABELS, m as Gliner2Backend, n as GlinerNer, o as resetTransformers, p as modelSizeBytes, r as boxesForRange, s as toCallableTokenizer, t as GLINER_NER_DEFAULTS, u as GLINER2_PII_LABELS, v as Gliner1Backend, w as sigmoid, x as splitWords, y as RICH_WORD_PATTERN } from "../ner-SsZLZ6ed.js";
+export { DEFAULT_CHUNK_LENGTH, DEFAULT_CHUNK_STRIDE, DEFAULT_NER_MODEL_ID, DEFAULT_PII_LABELS, GLINER2_PII_LABELS, GLINER_NER_DEFAULTS, Gliner1Backend, Gliner2Backend, GlinerNer, NER_MODELS, RICH_WORD_PATTERN, WORD_PATTERN, boxesForRange, buildCharToBoxMap, chunkText, computeDotProductScores, decodeEntities, decodeSpans, dedupeSpans, generateSpans, getNerModel, isMostlyUppercase, loadTokenizer, modelSizeBytes, normaliseCasing, rebaseSpan, resetTransformers, sigmoid, softmax, splitWords, titleCaseAllCapsWords, toCallableTokenizer, toSpanTokenizer };