@stabrise/scaledp 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,269 @@
+import { _ as assertPositiveInt, a as PipelineOptions, c as RowTime, d as StageDescriptor, f as toRows, g as assertInRange, h as Validator, i as PipelineInput, l as Stage, m as BaseStageParams, n as ExecutionTime, o as ROW_TIME_COL, p as BASE_STAGE_DEFAULTS, r as Pipeline, s as Row, t as EXECUTION_TIME_COL, u as StageContext, v as resolveParams } from "./pipeline-DeLO-OCE.js";
+import { S as polygonPerimeter, _ as RotatedRect, a as bbox, b as minAreaRect, c as boxIou, d as isRotated, f as mergeBoxes, g as Point, h as shape, i as ROTATION_EPSILON_DEGREES, l as createBox, m as scaleBox, n as Box, o as boxFromBBox, p as mergeOverlappingBoxes, r as BoxShape, s as boxFromPolygon, t as BBox, u as isOnSameLine, v as boxPoints, x as polygonArea, y as convexHull } from "./box-DAfzwfhA.js";
+import { n as createDetectorOutput, t as DetectorOutput } from "./detector-output-C0Qt-jEq.js";
+import { n as createDocument, r as mergeDocuments, t as Document } from "./document-B8I61TiY.js";
+import { n as NerOutput, r as createNerOutput, t as Entity } from "./entity-CedtRhU1.js";
+import { n as ScaleDpImage, r as createImage, t as ImageFormat } from "./image-Dc5TSg46.js";
+import { a as getConfig, i as defaultNumThreads, n as ScaleDpConfig, o as resetConfig, r as configure, s as resolveNumThreads, t as ModelProgress } from "./config-g6IrKlDC.js";
+import { a as ensureModelFiles, c as isCached, i as cacheKey, n as ModelFiles, o as evict, r as ModelSpec, s as fileUrl, t as ModelFile } from "./model-cache-BhFYpfZz.js";
+//#region src/core/errors.d.ts
+/**
+ * ScaleDP stages never throw by default: a failure is captured into the output
+ * schema's `exception` field so a pipeline always completes and partial results
+ * survive. `propagateError: true` opts a stage into throwing instead.
+ */
+declare class ScaleDpError extends Error {
+  readonly stage: string;
+  readonly cause?: unknown | undefined;
+  constructor(message: string, stage: string, cause?: unknown | undefined);
+}
+declare class ImageError extends ScaleDpError {
+  readonly name = "ImageError";
+}
+declare class OcrError extends ScaleDpError {
+  readonly name = "OcrError";
+}
+declare class DetectionError extends ScaleDpError {
+  readonly name = "DetectionError";
+}
+declare class NerError extends ScaleDpError {
+  readonly name = "NerError";
+}
+declare class ConfigError extends ScaleDpError {
+  readonly name = "ConfigError";
+}
+/** Render a caught value the way Python writes a traceback into `exception`. */
+declare function formatException(stage: string, error: unknown): string;
+//#endregion
+//#region src/core/image.d.ts
+interface Size {
+  width: number;
+  height: number;
+}
+declare function createCanvas(width: number, height: number): OffscreenCanvas;
+declare function context2d(canvas: OffscreenCanvas): OffscreenCanvasRenderingContext2D;
+declare function decodeImage(data: Uint8Array | Blob): Promise<ImageBitmap>;
+declare function toImageData(source: ImageBitmap | OffscreenCanvas): ImageData;
+declare function imageDataToCanvas(image: ImageData): OffscreenCanvas;
+declare function encodeImage(source: ImageData | OffscreenCanvas, type?: 'image/png' | 'image/webp' | 'image/jpeg', quality?: number): Promise<Uint8Array>;
+/** Read the intrinsic size of encoded image bytes without keeping the bitmap. */
+declare function probeImageSize(data: Uint8Array | Blob): Promise<Size>;
+interface LetterboxResult {
+  canvas: OffscreenCanvas;
+  /** Uniform scale applied to the source. */
+  scale: number;
+  /** Size the source occupies inside the target canvas. */
+  resized: Size;
+  /** Original source size. */
+  source: Size;
+}
+/**
+ * Fit an image into `target` preserving aspect ratio.
+ *
+ * `padding: 'end'` pads bottom and right only, matching PaddleOCR's detection
+ * preprocessing -- coordinates then restore by dividing by `scale`, with no
+ * offset to subtract. `padding: 'center'` centres the image, matching the YOLO
+ * preprocessing, where the pad offsets must be subtracted before unscaling.
+ */
+declare function letterbox(source: ImageBitmap | OffscreenCanvas, target: Size, opts?: {
+  padding?: 'end' | 'center';
+  fill?: string;
+}): LetterboxResult;
+/** Uniform resize by a scale factor. */
+declare function resize(source: ImageBitmap | OffscreenCanvas, factor: number): OffscreenCanvas;
+/**
+ * Crop a box out of an image, straightening it if it is rotated.
+ *
+ * Port of `TesseractRecognizer._prepare_box_for_ocr`. For upright boxes this is
+ * a plain crop; for rotated ones it applies the affine transform that maps the
+ * box's own corners onto the destination rectangle, which is what makes a
+ * recognizer see level text.
+ */
+/**
+ * The transform `cropBox` applies, as a function from crop pixel to source
+ * pixel.
+ *
+ * Kept separate so anything that needs to map *back* -- word boxes recognised
+ * inside a crop, say -- uses the same geometry the crop was made with, rather
+ * than a second copy of it that can drift.
+ */
+interface CropGeometry {
+  /** The box after scaling and padding, in source coordinates. */
+  scaled: Box;
+  width: number;
+  height: number;
+  /** Crop pixel -> source pixel. */
+  map: (x: number, y: number) => Point;
+}
+declare function cropGeometry(box: Box, opts?: {
+  scaleFactor?: number;
+  padding?: number;
+}): CropGeometry;
+declare function cropBox(source: ImageBitmap | OffscreenCanvas, box: Box, opts?: {
+  scaleFactor?: number;
+  padding?: number;
+}): OffscreenCanvas;
+/**
+ * Build an NCHW float32 tensor from an image.
+ *
+ * `bgr` exists because ScaleDP's DBNet path converts RGB->BGR and never swaps
+ * back, so the model is fed BGR channels against RGB ImageNet statistics.
+ * Replicating that quirk is required for box parity.
+ */
+declare function toNchwFloat32(image: ImageData, opts?: {
+  mean?: readonly number[];
+  std?: readonly number[];
+  scale?: number;
+  bgr?: boolean;
+}): Float32Array;
+declare const IMAGENET_MEAN: readonly [0.485, 0.456, 0.406];
+declare const IMAGENET_STD: readonly [0.229, 0.224, 0.225];
+//#endregion
+//#region src/core/text.d.ts
+/**
+ * Arrange items into groups whose successive elements differ by no more than
+ * `maxGap`. Unlike Python's `cluster`, this does not sort the caller's array
+ * in place.
+ */
+declare function cluster<T>(items: readonly T[], maxGap: number, key?: (item: T) => number): T[][];
+/**
+ * A robust central value: drop the first and last quartile, then take the mode.
+ *
+ * Trimming the tails is what makes this survive OCR outliers -- a single giant
+ * heading would otherwise skew a plain mean. Always returns a truncated
+ * integer, matching Python's `int(...)` wrapper.
+ */
+declare function getSize<T>(items: readonly T[], key?: (item: T) => number): number;
+/**
+ * Median-ish width of a single character, used to size inter-word gaps.
+ * Takes grouped lines, matching Python's signature.
+ */
+declare function getCharacterWidth(lines: readonly (readonly Box[])[]): number;
+/**
+ * Group boxes into text lines, each sorted left to right.
+ *
+ * `lineTolerance` of 0 means "derive it": a third of the typical character
+ * height -- tight enough to keep adjacent lines apart, loose enough to tolerate
+ * baseline jitter.
+ */
+declare function groupBoxesIntoLines(boxes: readonly Box[], lineTolerance?: number): Box[][];
+/**
+ * Render grouped lines back to text, preserving horizontal spacing and blank
+ * lines.
+ *
+ * Each line is indented from x = 0, so the left margin is preserved rather than
+ * every line being flushed left. Vertical gaps wider than two character heights
+ * become blank lines. The running `y` starts at 0, which means the first line is
+ * measured against the top of the page -- that is Python's behaviour and it is
+ * what makes a top margin survive.
+ */
+declare function linesToFormattedText(lines: readonly (readonly Box[])[], characterHeight: number): string;
+/** Rebuild layout-preserving text straight from a flat box list. */
+declare function boxesToFormattedText(boxes: readonly Box[], lineTolerance?: number): string;
+/** Plain reading-order join, the default when `keepFormatting` is off. */
+declare function boxesToText(boxes: readonly Box[], separator?: string): string;
+//#endregion
+//#region src/stages/data-to-image.d.ts
+interface DataToImageParams extends BaseStageParams {
+  /** Encoding recorded on the output; the bytes are passed through unchanged. */
+  imageType: ImageFormat;
+  /** DPI to record when the row carries no `resolution` field. */
+  resolution: number;
+}
+declare const DATA_TO_IMAGE_DEFAULTS: DataToImageParams;
+declare class DataToImage extends Stage<DataToImageParams> {
+  readonly name = "DataToImage";
+  constructor(options?: Partial<DataToImageParams>);
+  protected apply(input: unknown, row: Row): Promise<ScaleDpImage>;
+  protected onError(message: string, row: Row): ScaleDpImage;
+}
+/** Coerce any accepted binary representation into bytes. */
+declare function toBytes(input: unknown): Uint8Array;
+//#endregion
+//#region src/stages/image-crop-boxes.d.ts
+interface ImageCropBoxesParams extends BaseStageParams {
+  /** [imageColumn, boxColumn]. */
+  inputCols: string[];
+  imageType: ImageFormat;
+  /** Grow each box by this many pixels before cropping. */
+  padding: number;
+  /** Maximum crops per page; 0 means all of them. */
+  limit: number;
+  /** Rotate portrait crops a quarter turn, so text reads horizontally. */
+  autoRotate: boolean;
+  /** Emit the whole page when nothing was detected, instead of failing. */
+  returnEmpty: boolean;
+  /** Column to write the source box alongside each crop. */
+  boxCol: string;
+}
+declare const IMAGE_CROP_BOXES_DEFAULTS: ImageCropBoxesParams;
+declare class ImageCropBoxes extends Stage<ImageCropBoxesParams> {
+  readonly name = "ImageCropBoxes";
+  constructor(options?: Partial<ImageCropBoxesParams>);
+  protected expand(_input: unknown, row: Row, ctx: StageContext): Promise<Row[]>;
+  protected apply(): Promise<never>;
+  protected onError(message: string, row: Row): ScaleDpImage;
+}
+//#endregion
+//#region src/stages/image-draw-boxes.d.ts
+interface ImageDrawBoxesParams extends BaseStageParams {
+  /**
+   * First entry is the image; the rest are box or entity sources. `inputCol`
+   * is inherited and unused here -- multi-input stages address their columns
+   * through this list, as they do in Python.
+   */
+  inputCols: string[];
+  imageType: ImageFormat;
+  /** Fill boxes as well as outlining them. */
+  filled: boolean;
+  /** Fixed colour for every box. Unset colours by group instead. */
+  color: string | null;
+  lineWidth: number;
+  textSize: number;
+  /**
+   * Box or entity fields to render as a label above each box, joined by ':'.
+   * e.g. ['entity_group'] or ['text', 'score'].
+   */
+  displayDataList: string[];
+  /** Grow each box by this many pixels before drawing. */
+  padding: number;
+  /** Only draw these entity groups; empty draws all. */
+  whiteList: string[];
+  /** Never draw these entity groups. */
+  blackList: string[];
+}
+declare const IMAGE_DRAW_BOXES_DEFAULTS: ImageDrawBoxesParams;
+/**
+ * A stable colour per group name.
+ *
+ * Python picks a random colour per group, which changes on every run and makes
+ * two renders of the same document impossible to compare. Hashing the name
+ * instead keeps 'PERSON' the same colour everywhere, and the fixed saturation
+ * and lightness keep every colour legible on a white page.
+ */
+declare function colorForGroup(name: string): string;
+declare class ImageDrawBoxes extends Stage<ImageDrawBoxesParams> {
+  readonly name = "ImageDrawBoxes";
+  constructor(options?: Partial<ImageDrawBoxesParams>);
+  /** The base class drives `apply` off inputCol; this stage reads several. */
+  protected apply(_input: unknown, row: Row): Promise<ScaleDpImage>;
+  private drawBoxes;
+  private drawEntities;
+  private drawBox;
+  /**
+   * The label chip, anchored to the box's own top edge.
+   *
+   * Python places it at `box.x, box.y`, which is the corner of the *axis-
+   * aligned envelope*. For an upright box that is the top-left corner and the
+   * label sits where you expect; for a rotated one the envelope corner can be
+   * a long way off the box -- a vertical line's label lands beside its middle,
+   * detached from the box it names. Here the chip is drawn in the box's own
+   * frame instead, so it rides the top edge at the box's angle whatever that
+   * angle is. Upright boxes come out exactly where Python puts them.
+   */
+  private drawLabel;
+  protected onError(message: string, row: Row): ScaleDpImage;
+}
+//#endregion
+export { BASE_STAGE_DEFAULTS, BBox, type BaseStageParams, Box, BoxShape, ConfigError, CropGeometry, DATA_TO_IMAGE_DEFAULTS, DataToImage, type DataToImageParams, DetectionError, DetectorOutput, Document, EXECUTION_TIME_COL, Entity, type ExecutionTime, IMAGENET_MEAN, IMAGENET_STD, IMAGE_CROP_BOXES_DEFAULTS, IMAGE_DRAW_BOXES_DEFAULTS, ImageCropBoxes, type ImageCropBoxesParams, ImageDrawBoxes, type ImageDrawBoxesParams, ImageError, ImageFormat, LetterboxResult, type ModelFile, type ModelFiles, type ModelProgress, type ModelSpec, NerError, NerOutput, OcrError, Pipeline, type PipelineInput, type PipelineOptions, Point, ROTATION_EPSILON_DEGREES, ROW_TIME_COL, RotatedRect, type Row, type RowTime, type ScaleDpConfig, ScaleDpError, ScaleDpImage, Size, Stage, type StageContext, type StageDescriptor, type Validator, assertInRange, assertPositiveInt, bbox, boxFromBBox, boxFromPolygon, boxIou, boxPoints, boxesToFormattedText, boxesToText, cacheKey, cluster, colorForGroup, configure, context2d, convexHull, createBox, createCanvas, createDetectorOutput, createDocument, createImage, createNerOutput, cropBox, cropGeometry, decodeImage, defaultNumThreads, encodeImage, ensureModelFiles, evict, fileUrl, formatException, getCharacterWidth, getConfig, getSize, groupBoxesIntoLines, imageDataToCanvas, isCached, isOnSameLine, isRotated, letterbox, linesToFormattedText, mergeBoxes, mergeDocuments, mergeOverlappingBoxes, minAreaRect, polygonArea, polygonPerimeter, probeImageSize, resetConfig, resize, resolveNumThreads, resolveParams, scaleBox, shape, toBytes, toImageData, toNchwFloat32, toRows };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,11 @@
+import { _ as resolveNumThreads, a as toRows, c as ImageError, d as ScaleDpError, f as formatException, g as resetConfig, h as getConfig, i as Stage, l as NerError, m as defaultNumThreads, n as Pipeline, o as ConfigError, p as configure, r as ROW_TIME_COL, s as DetectionError, t as EXECUTION_TIME_COL, u as OcrError } from "./pipeline-DACqGkpN.js";
+import { A as polygonArea, C as mergeBoxes, D as boxPoints, E as shape, O as convexHull, S as isRotated, T as scaleBox, _ as boxFromBBox, a as cropBox, b as createBox, c as encodeImage, d as probeImageSize, f as resize, g as bbox, h as ROTATION_EPSILON_DEGREES, i as createCanvas, j as polygonPerimeter, k as minAreaRect, l as imageDataToCanvas, m as toNchwFloat32, n as IMAGENET_STD, o as cropGeometry, p as toImageData, r as context2d, s as decodeImage, t as IMAGENET_MEAN, u as letterbox, v as boxFromPolygon, w as mergeOverlappingBoxes, x as isOnSameLine, y as boxIou } from "./image-CAH2rLv9.js";
+import { a as isCached, i as fileUrl, n as ensureModelFiles, r as evict, t as cacheKey } from "./model-cache-BEaqqRZ9.js";
+import { i as resolveParams, n as assertInRange, r as assertPositiveInt, t as BASE_STAGE_DEFAULTS } from "./params-DapwK9Ns.js";
+import { a as getSize, i as getCharacterWidth, n as boxesToText, o as groupBoxesIntoLines, r as cluster, s as linesToFormattedText, t as boxesToFormattedText } from "./text-ahMLpxN9.js";
+import { t as createDetectorOutput } from "./detector-output-lyF1Mqb8.js";
+import { n as createDocument, r as mergeDocuments, t as createImage } from "./image-DoZDJkcR.js";
+import { t as createNerOutput } from "./entity-D6Hxaugj.js";
+import { n as DataToImage, r as toBytes, t as DATA_TO_IMAGE_DEFAULTS } from "./data-to-image-DoZ4jQ3R.js";
+import { a as ImageCropBoxes, i as IMAGE_CROP_BOXES_DEFAULTS, n as ImageDrawBoxes, r as colorForGroup, t as IMAGE_DRAW_BOXES_DEFAULTS } from "./image-draw-boxes-De0QbFv9.js";
+export { BASE_STAGE_DEFAULTS, ConfigError, DATA_TO_IMAGE_DEFAULTS, DataToImage, DetectionError, EXECUTION_TIME_COL, IMAGENET_MEAN, IMAGENET_STD, IMAGE_CROP_BOXES_DEFAULTS, IMAGE_DRAW_BOXES_DEFAULTS, ImageCropBoxes, ImageDrawBoxes, ImageError, NerError, OcrError, Pipeline, ROTATION_EPSILON_DEGREES, ROW_TIME_COL, ScaleDpError, Stage, assertInRange, assertPositiveInt, bbox, boxFromBBox, boxFromPolygon, boxIou, boxPoints, boxesToFormattedText, boxesToText, cacheKey, cluster, colorForGroup, configure, context2d, convexHull, createBox, createCanvas, createDetectorOutput, createDocument, createImage, createNerOutput, cropBox, cropGeometry, decodeImage, defaultNumThreads, encodeImage, ensureModelFiles, evict, fileUrl, formatException, getCharacterWidth, getConfig, getSize, groupBoxesIntoLines, imageDataToCanvas, isCached, isOnSameLine, isRotated, letterbox, linesToFormattedText, mergeBoxes, mergeDocuments, mergeOverlappingBoxes, minAreaRect, polygonArea, polygonPerimeter, probeImageSize, resetConfig, resize, resolveNumThreads, resolveParams, scaleBox, shape, toBytes, toImageData, toNchwFloat32, toRows };

package/dist/model-cache-BEaqqRZ9.js

@@ -0,0 +1,182 @@
+import { h as getConfig } from "./pipeline-DACqGkpN.js";
+//#region src/core/model-cache.ts
+/**
+* Model file fetching with IndexedDB persistence and streaming progress.
+*
+* The pdftools prototype implements this twice -- once for GLiNER and once for
+* PaddleOCR -- differing only in where sizes come from and whether an auth
+* token is attached. Both behaviours are folded in here: sizes come from a HEAD
+* request (real percentages, rather than the hardcoded byte counts the GLiNER
+* copy used) with a declared `approxBytes` as fallback.
+*/
+const STORE = "files";
+const DB_VERSION = 1;
+function openDb(name) {
+	return new Promise((resolve, reject) => {
+		const request = indexedDB.open(name, DB_VERSION);
+		request.onupgradeneeded = () => {
+			const db = request.result;
+			if (!db.objectStoreNames.contains(STORE)) db.createObjectStore(STORE);
+		};
+		request.onsuccess = () => resolve(request.result);
+		request.onerror = () => reject(request.error);
+	});
+}
+/**
+* Run one store operation.
+*
+* Writes resolve on transaction *commit*, not on request success -- a request
+* succeeding only means it was queued, so resolving there would report a
+* durable write that a later abort could still roll back.
+*/
+async function withStore(mode, fn) {
+	const db = await openDb(getConfig().cacheDbName);
+	try {
+		return await new Promise((resolve, reject) => {
+			const tx = db.transaction(STORE, mode);
+			const request = fn(tx.objectStore(STORE));
+			request.onerror = () => reject(request.error);
+			if (mode === "readonly") request.onsuccess = () => resolve(request.result);
+			else {
+				tx.oncomplete = () => resolve(request.result);
+				tx.onabort = () => reject(tx.error);
+				tx.onerror = () => reject(tx.error);
+			}
+		});
+	} finally {
+		db.close();
+	}
+}
+const cacheAvailable = () => getConfig().cache === "indexeddb" && typeof indexedDB !== "undefined";
+function cacheKey(spec, path) {
+	return `${spec.repo}@${spec.revision ?? "main"}/${path}`;
+}
+/**
+* Resolve a repo-relative path to a URL under the configured model host.
+*
+* A path that is already absolute is returned untouched -- some catalogues
+* (ppu-paddle-ocr's, for one) publish full URLs against their own host, and
+* rewriting those against `modelHost` would point at files that do not exist.
+*/
+function fileUrl(spec, path) {
+	if (/^https?:\/\//.test(path)) return path;
+	const { modelHost } = getConfig();
+	const host = modelHost.replace(/\/+$/, "");
+	if (!/^https?:\/\/(?:[^/]*\.)?huggingface\.co/.test(host)) return `${host}/${spec.repo}/${path}`;
+	return `${host}/${spec.repo}/resolve/${spec.revision ?? "main"}/${path}`;
+}
+async function authHeaders(repo) {
+	const auth = getConfig().auth;
+	if (!auth) return void 0;
+	const token = await auth(repo);
+	return token ? { Authorization: `Bearer ${token}` } : void 0;
+}
+/** Fetch a URL as an ArrayBuffer, reporting each chunk's byte length. */
+async function fetchWithProgress(url, headers, onChunk, signal) {
+	const response = await fetch(url, {
+		headers,
+		signal
+	});
+	if (!response.ok) throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
+	if (!response.body) {
+		const buffer = await response.arrayBuffer();
+		onChunk(buffer.byteLength);
+		return buffer;
+	}
+	const reader = response.body.getReader();
+	const chunks = [];
+	let total = 0;
+	for (;;) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		if (!value) continue;
+		chunks.push(value);
+		total += value.byteLength;
+		onChunk(value.byteLength);
+	}
+	const out = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		out.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return out.buffer;
+}
+/** Byte size from a HEAD request, falling back to the declared estimate. */
+async function contentLength(url, headers, fallback) {
+	try {
+		const response = await fetch(url, {
+			method: "HEAD",
+			headers
+		});
+		return Number(response.headers.get("content-length") ?? 0) || fallback;
+	} catch {
+		return fallback;
+	}
+}
+async function isCached(spec) {
+	if (!cacheAvailable()) return false;
+	for (const file of spec.files) if (!await withStore("readonly", (s) => s.get(cacheKey(spec, file.path)))) return false;
+	return true;
+}
+async function evict(spec) {
+	if (!cacheAvailable()) return;
+	for (const file of spec.files) await withStore("readwrite", (s) => s.delete(cacheKey(spec, file.path)));
+}
+/**
+* Ensure every file in `spec` is available, downloading and caching what is
+* missing. Returns buffers keyed by repo-relative path.
+*
+* Progress is aggregate across the files still to fetch; already-cached files
+* count as instantly complete.
+*/
+async function ensureModelFiles(spec, signal) {
+	const { onProgress } = getConfig();
+	const usable = cacheAvailable();
+	const result = {};
+	const missing = [];
+	for (const file of spec.files) {
+		const hit = usable ? await withStore("readonly", (s) => s.get(cacheKey(spec, file.path))) : void 0;
+		if (hit) result[file.path] = hit;
+		else missing.push(file);
+	}
+	if (missing.length === 0) {
+		onProgress?.({
+			repo: spec.repo,
+			file: "",
+			loaded: 0,
+			total: 0,
+			phase: "ready"
+		});
+		return result;
+	}
+	const headers = await authHeaders(spec.repo);
+	const total = (await Promise.all(missing.map((file) => file.approxBytes ? Promise.resolve(file.approxBytes) : contentLength(fileUrl(spec, file.path), headers, 0)))).reduce((sum, size) => sum + size, 0);
+	let loaded = 0;
+	for (const file of missing) {
+		const buffer = await fetchWithProgress(fileUrl(spec, file.path), headers, (bytes) => {
+			loaded += bytes;
+			onProgress?.({
+				repo: spec.repo,
+				file: file.path,
+				loaded,
+				total,
+				phase: "downloading"
+			});
+		}, signal);
+		if (usable) await withStore("readwrite", (s) => s.put(buffer, cacheKey(spec, file.path)));
+		result[file.path] = buffer;
+	}
+	onProgress?.({
+		repo: spec.repo,
+		file: "",
+		loaded: total,
+		total,
+		phase: "initializing"
+	});
+	return result;
+}
+//#endregion
+export { isCached as a, fileUrl as i, ensureModelFiles as n, evict as r, cacheKey as t };
+
+//# sourceMappingURL=model-cache-BEaqqRZ9.js.map

package/dist/model-cache-BEaqqRZ9.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-cache-BEaqqRZ9.js","names":[],"sources":["../src/core/model-cache.ts"],"sourcesContent":["/**\n * Model file fetching with IndexedDB persistence and streaming progress.\n *\n * The pdftools prototype implements this twice -- once for GLiNER and once for\n * PaddleOCR -- differing only in where sizes come from and whether an auth\n * token is attached. Both behaviours are folded in here: sizes come from a HEAD\n * request (real percentages, rather than the hardcoded byte counts the GLiNER\n * copy used) with a declared `approxBytes` as fallback.\n */\n\nimport { getConfig, type ModelProgress } from './config.js'\n\nconst STORE = 'files'\nconst DB_VERSION = 1\n\nexport interface ModelFile {\n    /** Repo-relative path, e.g. 'onnx/model_quantized.onnx'. */\n    path: string\n    /** Fallback size used for progress when the server sends no content-length. */\n    approxBytes?: number\n}\n\nexport interface ModelSpec {\n    /** Hugging Face repo id, e.g. 'onnx-community/gliner_multi_pii-v1'. */\n    repo: string\n    revision?: string\n    files: ModelFile[]\n}\n\nexport type ModelFiles = Record<string, ArrayBuffer>\n\nfunction openDb(name: string): Promise<IDBDatabase> {\n    return new Promise((resolve, reject) => {\n        const request = indexedDB.open(name, DB_VERSION)\n        request.onupgradeneeded = () => {\n            const db = request.result\n            if (!db.objectStoreNames.contains(STORE)) db.createObjectStore(STORE)\n        }\n        request.onsuccess = () => resolve(request.result)\n        request.onerror = () => reject(request.error)\n    })\n}\n\n/**\n * Run one store operation.\n *\n * Writes resolve on transaction *commit*, not on request success -- a request\n * succeeding only means it was queued, so resolving there would report a\n * durable write that a later abort could still roll back.\n */\nasync function withStore<T>(\n    mode: IDBTransactionMode,\n    fn: (store: IDBObjectStore) => IDBRequest<T>\n): Promise<T> {\n    const db = await openDb(getConfig().cacheDbName)\n    try {\n        return await new Promise<T>((resolve, reject) => {\n            const tx = db.transaction(STORE, mode)\n            const request = fn(tx.objectStore(STORE))\n            request.onerror = () => reject(request.error)\n            if (mode === 'readonly') {\n                request.onsuccess = () => resolve(request.result)\n            } else {\n                tx.oncomplete = () => resolve(request.result)\n                tx.onabort = () => reject(tx.error)\n                tx.onerror = () => reject(tx.error)\n            }\n        })\n    } finally {\n        db.close()\n    }\n}\n\nconst cacheAvailable = (): boolean => getConfig().cache === 'indexeddb' && typeof indexedDB !== 'undefined'\n\nexport function cacheKey(spec: ModelSpec, path: string): string {\n    return `${spec.repo}@${spec.revision ?? 'main'}/${path}`\n}\n\n/**\n * Resolve a repo-relative path to a URL under the configured model host.\n *\n * A path that is already absolute is returned untouched -- some catalogues\n * (ppu-paddle-ocr's, for one) publish full URLs against their own host, and\n * rewriting those against `modelHost` would point at files that do not exist.\n */\nexport function fileUrl(spec: ModelSpec, path: string): string {\n    if (/^https?:\\/\\//.test(path)) return path\n\n    const { modelHost } = getConfig()\n    const host = modelHost.replace(/\\/+$/, '')\n    // A bare directory host (self-hosting) has no HF resolve/<revision> segment.\n    if (!/^https?:\\/\\/(?:[^/]*\\.)?huggingface\\.co/.test(host)) {\n        return `${host}/${spec.repo}/${path}`\n    }\n    return `${host}/${spec.repo}/resolve/${spec.revision ?? 'main'}/${path}`\n}\n\nasync function authHeaders(repo: string): Promise<HeadersInit | undefined> {\n    const auth = getConfig().auth\n    if (!auth) return undefined\n    const token = await auth(repo)\n    return token ? { Authorization: `Bearer ${token}` } : undefined\n}\n\n/** Fetch a URL as an ArrayBuffer, reporting each chunk's byte length. */\nasync function fetchWithProgress(\n    url: string,\n    headers: HeadersInit | undefined,\n    onChunk: (bytes: number) => void,\n    signal?: AbortSignal\n): Promise<ArrayBuffer> {\n    const response = await fetch(url, { headers, signal })\n    if (!response.ok) {\n        throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`)\n    }\n    if (!response.body) {\n        const buffer = await response.arrayBuffer()\n        onChunk(buffer.byteLength)\n        return buffer\n    }\n\n    const reader = response.body.getReader()\n    const chunks: Uint8Array[] = []\n    let total = 0\n    for (;;) {\n        const { done, value } = await reader.read()\n        if (done) break\n        if (!value) continue\n        chunks.push(value)\n        total += value.byteLength\n        onChunk(value.byteLength)\n    }\n\n    const out = new Uint8Array(total)\n    let offset = 0\n    for (const chunk of chunks) {\n        out.set(chunk, offset)\n        offset += chunk.byteLength\n    }\n    return out.buffer\n}\n\n/** Byte size from a HEAD request, falling back to the declared estimate. */\nasync function contentLength(\n    url: string,\n    headers: HeadersInit | undefined,\n    fallback: number\n): Promise<number> {\n    try {\n        const response = await fetch(url, { method: 'HEAD', headers })\n        return Number(response.headers.get('content-length') ?? 0) || fallback\n    } catch {\n        return fallback\n    }\n}\n\nexport async function isCached(spec: ModelSpec): Promise<boolean> {\n    if (!cacheAvailable()) return false\n    for (const file of spec.files) {\n        const hit = await withStore('readonly', (s) => s.get(cacheKey(spec, file.path)))\n        if (!hit) return false\n    }\n    return true\n}\n\nexport async function evict(spec: ModelSpec): Promise<void> {\n    if (!cacheAvailable()) return\n    for (const file of spec.files) {\n        await withStore('readwrite', (s) => s.delete(cacheKey(spec, file.path)))\n    }\n}\n\n/**\n * Ensure every file in `spec` is available, downloading and caching what is\n * missing. Returns buffers keyed by repo-relative path.\n *\n * Progress is aggregate across the files still to fetch; already-cached files\n * count as instantly complete.\n */\nexport async function ensureModelFiles(spec: ModelSpec, signal?: AbortSignal): Promise<ModelFiles> {\n    const { onProgress } = getConfig()\n    const usable = cacheAvailable()\n    const result: ModelFiles = {}\n    const missing: ModelFile[] = []\n\n    for (const file of spec.files) {\n        const hit = usable\n            ? await withStore<ArrayBuffer | undefined>('readonly', (s) => s.get(cacheKey(spec, file.path)))\n            : undefined\n        if (hit) result[file.path] = hit\n        else missing.push(file)\n    }\n\n    if (missing.length === 0) {\n        onProgress?.({ repo: spec.repo, file: '', loaded: 0, total: 0, phase: 'ready' })\n        return result\n    }\n\n    const headers = await authHeaders(spec.repo)\n\n    // Size everything up front so progress is a real percentage rather than a\n    // byte count climbing toward an unknown ceiling.\n    //\n    // A declared `approxBytes` is used as-is: a HEAD costs a full round-trip per\n    // file (and on Hugging Face it follows a redirect to the CDN), which doubles\n    // the request count for a download that is already the slow part. Only\n    // files with no estimate are probed, and those probes run together.\n    const sizes = await Promise.all(\n        missing.map((file) =>\n            file.approxBytes\n                ? Promise.resolve(file.approxBytes)\n                : contentLength(fileUrl(spec, file.path), headers, 0)\n        )\n    )\n    const total = sizes.reduce((sum, size) => sum + size, 0)\n\n    let loaded = 0\n    for (const file of missing) {\n        const buffer = await fetchWithProgress(\n            fileUrl(spec, file.path),\n            headers,\n            (bytes) => {\n                loaded += bytes\n                onProgress?.({\n                    repo: spec.repo,\n                    file: file.path,\n                    loaded,\n                    total,\n                    phase: 'downloading',\n                })\n            },\n            signal\n        )\n        if (usable) {\n            await withStore('readwrite', (s) => s.put(buffer, cacheKey(spec, file.path)))\n        }\n        result[file.path] = buffer\n    }\n\n    onProgress?.({ repo: spec.repo, file: '', loaded: total, total, phase: 'initializing' })\n    return result\n}\n\nexport type { ModelProgress }\n"],"mappings":";;;;;;;;;;;AAYA,MAAM,QAAQ;AACd,MAAM,aAAa;AAkBnB,SAAS,OAAO,MAAoC;CAChD,OAAO,IAAI,SAAS,SAAS,WAAW;EACpC,MAAM,UAAU,UAAU,KAAK,MAAM,UAAU;EAC/C,QAAQ,wBAAwB;GAC5B,MAAM,KAAK,QAAQ;GACnB,IAAI,CAAC,GAAG,iBAAiB,SAAS,KAAK,GAAG,GAAG,kBAAkB,KAAK;EACxE;EACA,QAAQ,kBAAkB,QAAQ,QAAQ,MAAM;EAChD,QAAQ,gBAAgB,OAAO,QAAQ,KAAK;CAChD,CAAC;AACL;;;;;;;;AASA,eAAe,UACX,MACA,IACU;CACV,MAAM,KAAK,MAAM,OAAO,UAAU,CAAC,CAAC,WAAW;CAC/C,IAAI;EACA,OAAO,MAAM,IAAI,SAAY,SAAS,WAAW;GAC7C,MAAM,KAAK,GAAG,YAAY,OAAO,IAAI;GACrC,MAAM,UAAU,GAAG,GAAG,YAAY,KAAK,CAAC;GACxC,QAAQ,gBAAgB,OAAO,QAAQ,KAAK;GAC5C,IAAI,SAAS,YACT,QAAQ,kBAAkB,QAAQ,QAAQ,MAAM;QAC7C;IACH,GAAG,mBAAmB,QAAQ,QAAQ,MAAM;IAC5C,GAAG,gBAAgB,OAAO,GAAG,KAAK;IAClC,GAAG,gBAAgB,OAAO,GAAG,KAAK;GACtC;EACJ,CAAC;CACL,UAAU;EACN,GAAG,MAAM;CACb;AACJ;AAEA,MAAM,uBAAgC,UAAU,CAAC,CAAC,UAAU,eAAe,OAAO,cAAc;AAEhG,SAAgB,SAAS,MAAiB,MAAsB;CAC5D,OAAO,GAAG,KAAK,KAAK,GAAG,KAAK,YAAY,OAAO,GAAG;AACtD;;;;;;;;AASA,SAAgB,QAAQ,MAAiB,MAAsB;CAC3D,IAAI,eAAe,KAAK,IAAI,GAAG,OAAO;CAEtC,MAAM,EAAE,cAAc,UAAU;CAChC,MAAM,OAAO,UAAU,QAAQ,QAAQ,EAAE;CAEzC,IAAI,CAAC,0CAA0C,KAAK,IAAI,GACpD,OAAO,GAAG,KAAK,GAAG,KAAK,KAAK,GAAG;CAEnC,OAAO,GAAG,KAAK,GAAG,KAAK,KAAK,WAAW,KAAK,YAAY,OAAO,GAAG;AACtE;AAEA,eAAe,YAAY,MAAgD;CACvE,MAAM,OAAO,UAAU,CAAC,CAAC;CACzB,IAAI,CAAC,MAAM,OAAO,KAAA;CAClB,MAAM,QAAQ,MAAM,KAAK,IAAI;CAC7B,OAAO,QAAQ,EAAE,eAAe,UAAU,QAAQ,IAAI,KAAA;AAC1D;;AAGA,eAAe,kBACX,KACA,SACA,SACA,QACoB;CACpB,MAAM,WAAW,MAAM,MAAM,KAAK;EAAE;EAAS;CAAO,CAAC;CACrD,IAAI,CAAC,SAAS,IACV,MAAM,IAAI,MAAM,sBAAsB,IAAI,IAAI,SAAS,OAAO,GAAG,SAAS,YAAY;CAE1F,IAAI,CAAC,SAAS,MAAM;EAChB,MAAM,SAAS,MAAM,SAAS,YAAY;EAC1C,QAAQ,OAAO,UAAU;EACzB,OAAO;CACX;CAEA,MAAM,SAAS,SAAS,KAAK,UAAU;CACvC,MAAM,SAAuB,CAAC;CAC9B,IAAI,QAAQ;CACZ,SAAS;EACL,MAAM,EAAE,MAAM,UAAU,MAAM,OAAO,KAAK;EAC1C,IAAI,MAAM;EACV,IAAI,CAAC,OAAO;EACZ,OAAO,KAAK,KAAK;EACjB,SAAS,MAAM;EACf,QAAQ,MAAM,UAAU;CAC5B;CAEA,MAAM,MAAM,IAAI,WAAW,KAAK;CAChC,IAAI,SAAS;CACb,KAAK,MAAM,SAAS,QAAQ;EACxB,IAAI,IAAI,OAAO,MAAM;EACrB,UAAU,MAAM;CACpB;CACA,OAAO,IAAI;AACf;;AAGA,eAAe,cACX,KACA,SACA,UACe;CACf,IAAI;EACA,MAAM,WAAW,MAAM,MAAM,KAAK;GAAE,QAAQ;GAAQ;EAAQ,CAAC;EAC7D,OAAO,OAAO,SAAS,QAAQ,IAAI,gBAAgB,KAAK,CAAC,KAAK;CAClE,QAAQ;EACJ,OAAO;CACX;AACJ;AAEA,eAAsB,SAAS,MAAmC;CAC9D,IAAI,CAAC,eAAe,GAAG,OAAO;CAC9B,KAAK,MAAM,QAAQ,KAAK,OAEpB,IAAI,CAAC,MADa,UAAU,aAAa,MAAM,EAAE,IAAI,SAAS,MAAM,KAAK,IAAI,CAAC,CAAC,GACrE,OAAO;CAErB,OAAO;AACX;AAEA,eAAsB,MAAM,MAAgC;CACxD,IAAI,CAAC,eAAe,GAAG;CACvB,KAAK,MAAM,QAAQ,KAAK,OACpB,MAAM,UAAU,cAAc,MAAM,EAAE,OAAO,SAAS,MAAM,KAAK,IAAI,CAAC,CAAC;AAE/E;;;;;;;;AASA,eAAsB,iBAAiB,MAAiB,QAA2C;CAC/F,MAAM,EAAE,eAAe,UAAU;CACjC,MAAM,SAAS,eAAe;CAC9B,MAAM,SAAqB,CAAC;CAC5B,MAAM,UAAuB,CAAC;CAE9B,KAAK,MAAM,QAAQ,KAAK,OAAO;EAC3B,MAAM,MAAM,SACN,MAAM,UAAmC,aAAa,MAAM,EAAE,IAAI,SAAS,MAAM,KAAK,IAAI,CAAC,CAAC,IAC5F,KAAA;EACN,IAAI,KAAK,OAAO,KAAK,QAAQ;OACxB,QAAQ,KAAK,IAAI;CAC1B;CAEA,IAAI,QAAQ,WAAW,GAAG;EACtB,aAAa;GAAE,MAAM,KAAK;GAAM,MAAM;GAAI,QAAQ;GAAG,OAAO;GAAG,OAAO;EAAQ,CAAC;EAC/E,OAAO;CACX;CAEA,MAAM,UAAU,MAAM,YAAY,KAAK,IAAI;CAgB3C,MAAM,SAAQ,MAPM,QAAQ,IACxB,QAAQ,KAAK,SACT,KAAK,cACC,QAAQ,QAAQ,KAAK,WAAW,IAChC,cAAc,QAAQ,MAAM,KAAK,IAAI,GAAG,SAAS,CAAC,CAC5D,CACJ,EAAA,CACoB,QAAQ,KAAK,SAAS,MAAM,MAAM,CAAC;CAEvD,IAAI,SAAS;CACb,KAAK,MAAM,QAAQ,SAAS;EACxB,MAAM,SAAS,MAAM,kBACjB,QAAQ,MAAM,KAAK,IAAI,GACvB,UACC,UAAU;GACP,UAAU;GACV,aAAa;IACT,MAAM,KAAK;IACX,MAAM,KAAK;IACX;IACA;IACA,OAAO;GACX,CAAC;EACL,GACA,MACJ;EACA,IAAI,QACA,MAAM,UAAU,cAAc,MAAM,EAAE,IAAI,QAAQ,SAAS,MAAM,KAAK,IAAI,CAAC,CAAC;EAEhF,OAAO,KAAK,QAAQ;CACxB;CAEA,aAAa;EAAE,MAAM,KAAK;EAAM,MAAM;EAAI,QAAQ;EAAO;EAAO,OAAO;CAAe,CAAC;CACvF,OAAO;AACX"}

package/dist/model-cache-BhFYpfZz.d.ts

@@ -0,0 +1,36 @@
+//#region src/core/model-cache.d.ts
+interface ModelFile {
+  /** Repo-relative path, e.g. 'onnx/model_quantized.onnx'. */
+  path: string;
+  /** Fallback size used for progress when the server sends no content-length. */
+  approxBytes?: number;
+}
+interface ModelSpec {
+  /** Hugging Face repo id, e.g. 'onnx-community/gliner_multi_pii-v1'. */
+  repo: string;
+  revision?: string;
+  files: ModelFile[];
+}
+type ModelFiles = Record<string, ArrayBuffer>;
+declare function cacheKey(spec: ModelSpec, path: string): string;
+/**
+ * Resolve a repo-relative path to a URL under the configured model host.
+ *
+ * A path that is already absolute is returned untouched -- some catalogues
+ * (ppu-paddle-ocr's, for one) publish full URLs against their own host, and
+ * rewriting those against `modelHost` would point at files that do not exist.
+ */
+declare function fileUrl(spec: ModelSpec, path: string): string;
+declare function isCached(spec: ModelSpec): Promise<boolean>;
+declare function evict(spec: ModelSpec): Promise<void>;
+/**
+ * Ensure every file in `spec` is available, downloading and caching what is
+ * missing. Returns buffers keyed by repo-relative path.
+ *
+ * Progress is aggregate across the files still to fetch; already-cached files
+ * count as instantly complete.
+ */
+declare function ensureModelFiles(spec: ModelSpec, signal?: AbortSignal): Promise<ModelFiles>;
+//#endregion
+export { ensureModelFiles as a, isCached as c, cacheKey as i, ModelFiles as n, evict as o, ModelSpec as r, fileUrl as s, ModelFile as t };
+//# sourceMappingURL=model-cache-BhFYpfZz.d.ts.map