@texion/core 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Texion contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,27 @@
-# Temporary Holding Version
+# @texion/core
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Document parsing and file state for text and attachment inputs, without React.
+
+```sh
+npm install @texion/core
+```
+
+```ts
+import { processDocument } from '@texion/core';
+
+const document = await processDocument('# Hello from Texion');
+
+if (document.status === 'ready') {
+  console.log(document.raw, document.getPreview());
+}
+```
+
+Pass a `File`, `Blob`, or text source. Markdown, TXT, JSON, CSV, PDF, DOCX, and PNG/JPEG/GIF/WebP attachments use the same document shape. PDF and Word extraction load their parsers on demand. Images keep their original bytes and metadata; they do not automatically provide extracted text.
+
+Use `createController` to manage multiple files, cancellation, duplicates, and pending/processing/ready/error states. Limits bound file sizes, document pages, extracted characters, and DOCX decompression. Safe previews escape content and apply a URL policy; Texion is not a malware scanner.
+
+Parsing runs locally. Your app owns uploads, storage, and anything sent to an AI service. Default PDF extraction reads native text; optional OCR packages support scanned PDFs. Complex PDF reading order and table reconstruction are not guaranteed.
+
+Experimental release. Works in the browser and Node. Node 20 or newer is required.
+
+[Documentation](https://texion.pages.dev/docs) · [Source](https://github.com/Sirily23/texion/tree/main/packages/core) · [MIT license](LICENSE)

package/dist/index.d.ts

@@ -0,0 +1,420 @@
+interface OcrScheduler {
+    run<T>(work: () => Promise<T>, signal: AbortSignal): Promise<T>;
+}
+/** Share one scheduler across adapters/controllers to bound active OCR sessions. */
+declare function createOcrScheduler(concurrency?: number): OcrScheduler;
+
+type OcrErrorCode = 'ocr-unavailable' | 'ocr-consent-denied' | 'ocr-timeout' | 'ocr-limit-exceeded' | 'ocr-page-failed';
+type ErrorCode = 'unsupported' | 'too-large' | 'too-many' | 'empty' | 'binary' | 'read-failed' | 'invalid-image' | 'invalid-document' | OcrErrorCode;
+interface DocumentError {
+    readonly code: ErrorCode;
+    readonly message: string;
+}
+interface DocumentWarning {
+    readonly code: 'no-text' | 'invalid-utf8' | 'parse-failed' | 'truncated' | 'invalid-json' | 'invalid-csv' | 'too-complex' | OcrErrorCode;
+    readonly message: string;
+}
+declare class IngestionError extends Error {
+    readonly code: ErrorCode;
+    constructor(code: ErrorCode, message: string);
+}
+
+interface DocumentBlock {
+    readonly text: string;
+    readonly bbox?: {
+        readonly x: number;
+        readonly y: number;
+        readonly w: number;
+        readonly h: number;
+    };
+    readonly confidence?: number;
+    readonly kind?: 'paragraph' | 'line' | 'word' | 'table';
+}
+interface DocumentPage {
+    readonly pageNumber: number;
+    readonly text: string;
+    readonly source: string;
+    readonly classification: 'native' | 'ocr-candidate';
+    readonly confidence?: number;
+    readonly nativeText?: string;
+    readonly image?: {
+        readonly width: number;
+        readonly height: number;
+        readonly dpi: number;
+        readonly rotation?: number;
+    };
+    readonly blocks?: readonly DocumentBlock[];
+    readonly error?: DocumentError;
+}
+interface OcrSummary {
+    readonly engineId: string;
+    readonly state: 'off' | 'denied' | 'done' | 'partial' | 'failed';
+    readonly pagesAttempted: number;
+    readonly pagesProcessed: number;
+    readonly pagesSkipped: number;
+    readonly pagesFailed: number;
+}
+interface DocumentProgress {
+    readonly phase: 'extracting' | 'pending-consent' | 'initializing-ocr' | 'ocr';
+    readonly done: number;
+    readonly total: number;
+    readonly pageNumber?: number;
+}
+interface OcrImage {
+    readonly rotation?: number;
+    readonly encoding: 'png';
+    readonly bytes: Uint8Array;
+    readonly width: number;
+    readonly height: number;
+    readonly dpi: number;
+}
+interface OcrPageInput extends OcrImage {
+    readonly pageNumber: number;
+    readonly signal: AbortSignal;
+}
+interface OcrPageResult {
+    readonly text: string;
+    /** Normalized 0..1, only when provided by the engine. */
+    readonly confidence?: number;
+    readonly blocks?: readonly DocumentBlock[];
+}
+/** A session belongs to exactly one recognition operation. terminate must release owned work. */
+interface OcrSession {
+    initialize?(signal: AbortSignal): Promise<void>;
+    recognize(input: OcrPageInput): Promise<OcrPageResult>;
+    terminate(): void | Promise<void>;
+}
+interface OcrEngine {
+    readonly id: string;
+    readonly kind: 'local' | 'server' | 'hosted';
+    /** Trusted application declaration, not a network sandbox. Non-local engines always require approval. */
+    readonly uploadsData: boolean;
+    /** Create a cancellable session synchronously; defer model initialization to recognize. */
+    createSession(): OcrSession;
+}
+interface OcrRenderRequest {
+    readonly pageNumber: number;
+    readonly dpi: number;
+    readonly maxPixels: number;
+    readonly maxBytes: number;
+    readonly signal: AbortSignal;
+}
+type OcrRenderer = (request: OcrRenderRequest) => Promise<OcrImage>;
+interface OcrApproval {
+    readonly document?: {
+        readonly id: string;
+        readonly name: string;
+    };
+    readonly engineId: string;
+    readonly uploadsData: boolean;
+    readonly pageNumbers: readonly number[];
+    readonly signal: AbortSignal;
+}
+interface OcrOptions {
+    readonly engine: OcrEngine;
+    readonly scheduler?: OcrScheduler;
+    readonly trigger?: 'auto' | 'ask' | 'off';
+    readonly requestApproval?: (request: OcrApproval) => boolean | Promise<boolean>;
+    readonly maxPages?: number;
+    readonly pageTimeoutMs?: number;
+    readonly totalTimeoutMs?: number;
+    readonly approvalTimeoutMs?: number;
+    readonly concurrency?: number;
+    readonly dpi?: number;
+    readonly minCharsPerPage?: number;
+    readonly maxImagePixels?: number;
+    readonly maxImageBytes?: number;
+    readonly maxBlocksPerPage?: number;
+    readonly onPageError?: 'skip' | 'fail';
+}
+interface OcrOutput {
+    readonly raw: string;
+    readonly pages: readonly DocumentPage[];
+    readonly ocr: OcrSummary;
+    readonly warnings: readonly DocumentWarning[];
+}
+
+interface PreviewNode {
+    readonly type: string;
+    readonly value?: string;
+    readonly language?: string;
+    readonly children?: readonly PreviewNode[];
+    readonly url?: string;
+    readonly depth?: number;
+    readonly ordered?: boolean;
+    readonly start?: number;
+    readonly checked?: boolean;
+}
+declare function safeUrl(url: string): string | undefined;
+
+type DocumentStatus = 'pending' | 'processing' | 'ready' | 'error';
+type DocumentSource = 'file' | 'paste' | 'text';
+interface TextInput {
+    readonly text: string;
+    readonly name?: string;
+    readonly source?: 'paste' | 'text';
+    readonly format?: string;
+}
+type DocumentInput = string | Blob | TextInput;
+interface ImageInfo {
+    readonly mimeType: 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp';
+    readonly width: number;
+    readonly height: number;
+}
+interface DocumentPreview {
+    readonly image?: ImageInfo;
+    readonly nodes: readonly PreviewNode[];
+    readonly truncated: boolean;
+    readonly warnings: readonly DocumentWarning[];
+}
+interface TexionDocument {
+    readonly id: string;
+    readonly name: string;
+    readonly format: string;
+    readonly source: DocumentSource;
+    readonly status: DocumentStatus;
+    readonly size: number;
+    /** Decoded text source, or extracted PDF/DOCX text. Absent for image attachments. */
+    readonly raw?: string;
+    readonly pages?: readonly DocumentPage[];
+    readonly ocr?: OcrSummary;
+    readonly progress?: DocumentProgress;
+    readonly file?: File;
+    readonly blob?: Blob;
+    readonly metadata: Readonly<Record<string, unknown>>;
+    readonly warnings: readonly DocumentWarning[];
+    readonly error?: DocumentError;
+    readonly getPreview?: () => DocumentPreview;
+}
+interface DocumentSnapshot {
+    readonly documents: readonly TexionDocument[];
+}
+
+interface Limits {
+    readonly maxExtractedCharacters: number;
+    readonly maxDocumentPages: number;
+    readonly maxArchiveBytes: number;
+    readonly maxImagePixels: number;
+    readonly maxFiles: number;
+    readonly maxFileSize: number;
+    readonly maxPreviewCharacters: number;
+    readonly maxPreviewNodes: number;
+    readonly maxPreviewDepth: number;
+}
+declare const defaultLimits: Limits;
+
+interface AdapterInspection {
+    readonly metadata?: Readonly<Record<string, unknown>>;
+    readonly warnings?: readonly DocumentWarning[];
+}
+interface BinaryExtractionResult extends AdapterInspection {
+    readonly raw: string;
+    readonly pages?: readonly DocumentPage[];
+    readonly ocr?: OcrSummary;
+}
+interface ExtractionContext {
+    readonly document?: {
+        readonly id: string;
+        readonly name: string;
+    };
+    readonly reportProgress: (progress: DocumentProgress) => void;
+}
+interface AdapterIdentity {
+    readonly format: string;
+    readonly extensions: readonly string[];
+    readonly mimeTypes: readonly string[];
+}
+interface TextFormatAdapter extends AdapterIdentity {
+    readonly kind?: 'text';
+    readonly detect: (text: string) => boolean;
+    readonly metadata?: (text: string) => Readonly<Record<string, unknown>>;
+    readonly inspect?: (text: string, limits: Limits) => AdapterInspection;
+    readonly parse: (text: string, limits: Limits, metadata?: Readonly<Record<string, unknown>>) => DocumentPreview;
+}
+interface BinaryFormatAdapter extends AdapterIdentity {
+    readonly kind: 'binary';
+    /** Extract document text during processing; heavy parsers can load on demand. */
+    readonly extract?: (bytes: Uint8Array, limits: Limits, signal: AbortSignal, context?: ExtractionContext) => Promise<BinaryExtractionResult>;
+    readonly detectBytes: (bytes: Uint8Array) => boolean;
+    readonly inspect: (bytes: Uint8Array, limits: Limits, source: {
+        readonly name: string;
+        readonly mime: string;
+    }) => AdapterInspection;
+    readonly parse: (bytes: Uint8Array, limits: Limits, metadata: Readonly<Record<string, unknown>>) => DocumentPreview;
+}
+/** Third-party adapters are trusted code. Preview generation stays synchronous and lazy; binary text extraction can be async. */
+type FormatAdapter = TextFormatAdapter | BinaryFormatAdapter;
+
+interface ControllerOptions extends Partial<Limits> {
+    readonly adapters?: readonly FormatAdapter[];
+    readonly acceptedFormats?: readonly string[];
+}
+declare function createController(options?: ControllerOptions): {
+    getSnapshot: () => DocumentSnapshot;
+    subscribe: (listener: () => void) => () => void;
+    add: (input: DocumentInput) => string;
+    remove: (id: string) => void;
+    cancel: (id: string) => void;
+    clear: () => void;
+    setText(text: string, format?: string): void;
+    whenSettled(id: string): Promise<TexionDocument | undefined>;
+    waitForIdle(): Promise<void>;
+};
+type DocumentController = ReturnType<typeof createController>;
+
+declare function processDocument(input: DocumentInput, options?: ControllerOptions & {
+    readonly signal?: AbortSignal;
+}): Promise<TexionDocument>;
+
+declare function createRegistry(adapters?: readonly FormatAdapter[], acceptedFormats?: readonly string[]): readonly FormatAdapter[];
+
+declare function detectMarkdown(text: string): boolean;
+declare const markdownAdapter: TextFormatAdapter;
+
+interface DecodeResult {
+    readonly text: string;
+    readonly warnings: readonly DocumentWarning[];
+}
+declare function decode(bytes: Uint8Array): DecodeResult;
+
+type HighlightKind = 'heading' | 'bold' | 'italic' | 'code' | 'link' | 'strikethrough' | 'list-marker' | 'blockquote' | 'frontmatter' | 'thematic-break' | 'property' | 'string' | 'number' | 'boolean' | 'null' | 'punctuation' | 'keyword' | 'comment';
+/** Half-open offsets measured in JavaScript UTF-16 code units, including Markdown markers. */
+interface HighlightToken {
+    readonly start: number;
+    readonly end: number;
+    readonly kind: HighlightKind;
+}
+interface HighlightLimits {
+    readonly maxCharacters?: number;
+    readonly maxTokens?: number;
+    readonly maxDepth?: number;
+}
+declare const defaultHighlightLimits: Readonly<{
+    maxCharacters: 20000;
+    maxTokens: 2000;
+    maxDepth: 64;
+}>;
+declare function tokenizeMarkdownSource(text: string, options?: HighlightLimits): {
+    readonly tokens: readonly HighlightToken[];
+    readonly limited: boolean;
+};
+
+type TextFormat = 'markdown' | 'json' | 'csv' | 'text';
+type ContentFormat = TextFormat | (string & {});
+type BuiltinFormat = TextFormat | 'image' | 'pdf' | 'docx';
+/** Framework-independent, JSON-serializable content exchanged by editors and viewers. */
+interface TexionContent {
+    readonly version: 1;
+    readonly format: ContentFormat;
+    readonly source: string;
+    /** Original imported Markdown, when available. Rich edits may normalize source. */
+    readonly originalSource?: string;
+}
+/** Persistable attachment metadata. reference is an opaque key owned by the host. */
+interface TexionAttachmentReference {
+    readonly id: string;
+    readonly name: string;
+    readonly format: string;
+    readonly size: number;
+    readonly reference: string;
+    readonly mimeType?: string;
+}
+/** A saved message contains no File/Blob objects, functions, or temporary object URLs. */
+interface TexionMessage {
+    readonly version: 1;
+    readonly content: TexionContent;
+    readonly attachments: readonly TexionAttachmentReference[];
+}
+
+interface HighlightSegment {
+    readonly start: number;
+    readonly end: number;
+    readonly kind?: HighlightKind;
+}
+interface HighlightModel {
+    readonly text: string;
+    readonly enabled: boolean;
+    readonly reason?: 'disabled' | 'too-large' | 'too-complex' | 'parse-failed';
+    readonly tokens: readonly HighlightToken[];
+    readonly segments: readonly HighlightSegment[];
+}
+interface HighlightOptions extends HighlightLimits {
+    readonly enabled?: boolean;
+    readonly format?: TextFormat;
+}
+declare function createHighlightModel(text: string, options?: HighlightOptions): HighlightModel;
+
+declare function isTexionContent(value: unknown): value is TexionContent;
+declare function createContent(source: string, options?: {
+    readonly originalSource?: string;
+    readonly format?: ContentFormat;
+}): TexionContent;
+
+declare function serializeContent(content: TexionContent): string;
+declare function deserializeContent(serialized: string): TexionContent;
+declare function exportMarkdown(content: TexionContent): string;
+declare function exportSource(content: TexionContent): string;
+
+/** Inspect bounded Markdown syntax without rendering HTML or changing the source. */
+declare function inspectMarkdownSource(source: string, maxCharacters?: number): {
+    readonly nodeTypes: readonly string[];
+    readonly limited: boolean;
+};
+
+declare const jsonAdapter: TextFormatAdapter;
+
+declare const csvAdapter: TextFormatAdapter;
+
+declare const imageAdapter: BinaryFormatAdapter;
+
+declare const textAdapter: TextFormatAdapter;
+
+/** A host-supplied canvas module; interpreted by the pinned unpdf runtime. */
+type PdfCanvasImport = () => Promise<unknown>;
+/** Original PDF bytes plus bounded page rendering instructions. Copy before transferring bytes to a worker. */
+type PdfOcrRenderer = (bytes: Uint8Array, request: OcrRenderRequest) => Promise<OcrImage>;
+
+interface PdfAdapterOptions {
+    readonly ocr?: OcrOptions;
+    readonly renderer?: PdfOcrRenderer;
+    readonly canvasImport?: PdfCanvasImport;
+}
+declare function createPdfAdapter(options?: PdfAdapterOptions): BinaryFormatAdapter;
+declare const pdfAdapter: BinaryFormatAdapter;
+
+declare const docxAdapter: BinaryFormatAdapter;
+
+declare function isTextFormat(value: string): value is TextFormat;
+declare function detectTextFormat(source: string, allowed?: readonly TextFormat[]): TextFormat;
+declare function detectTextFormat(source: string, allowed?: readonly string[], adapters?: readonly FormatAdapter[]): ContentFormat;
+declare function sourceWarnings(source: string, format: ContentFormat, maxCharacters?: number): readonly DocumentWarning[];
+
+declare function tokenizeDataSource(source: string, format: 'json' | 'csv', options?: HighlightLimits): {
+    tokens: readonly HighlightToken[];
+    limited: boolean;
+};
+
+declare function isAttachmentReference(value: unknown): value is TexionAttachmentReference;
+declare function isTexionMessage(value: unknown): value is TexionMessage;
+declare function createMessage(content: TexionContent, options?: {
+    readonly attachments?: readonly TexionAttachmentReference[];
+}): TexionMessage;
+/** The callback stores each attachment in the host's storage and returns its stable key. */
+declare function createMessageFromDocuments(content: TexionContent, documents: readonly TexionDocument[], referenceFor: (document: TexionDocument) => string | Promise<string>): Promise<TexionMessage>;
+declare function serializeMessage(message: TexionMessage): string;
+declare function deserializeMessage(serialized: string): TexionMessage;
+
+/** Bounded lexical highlighting; SQL is highlighted, never executed or syntax-validated. */
+declare function inspectCodeBlock(source: string, language?: string, options?: HighlightLimits): Readonly<{
+    language: string;
+    tokens: readonly HighlightToken[];
+    warnings: readonly DocumentWarning[];
+    limited: boolean;
+}>;
+
+/** A routing heuristic; short text does not prove that a page is scanned or empty. */
+declare function classifyPdfPage(text: string, minCharsPerPage?: number): 'native' | 'ocr-candidate';
+
+declare function recognizePdfPages(nativePages: readonly DocumentPage[], options: OcrOptions, render: OcrRenderer, limits: Limits, signal: AbortSignal, report?: (progress: DocumentProgress) => void, document?: OcrApproval['document']): Promise<OcrOutput>;
+
+export { type AdapterInspection, type BinaryExtractionResult, type BinaryFormatAdapter, type BuiltinFormat, type ContentFormat, type ControllerOptions, type DocumentBlock, type DocumentController, type DocumentError, type DocumentInput, type DocumentPage, type DocumentPreview, type DocumentProgress, type DocumentSnapshot, type DocumentSource, type DocumentStatus, type DocumentWarning, type ErrorCode, type ExtractionContext, type FormatAdapter, type HighlightKind, type HighlightLimits, type HighlightModel, type HighlightOptions, type HighlightSegment, type HighlightToken, type ImageInfo, IngestionError, type Limits, type OcrApproval, type OcrEngine, type OcrImage, type OcrOptions, type OcrOutput, type OcrPageInput, type OcrPageResult, type OcrRenderRequest, type OcrRenderer, type OcrScheduler, type OcrSession, type OcrSummary, type PdfAdapterOptions, type PdfCanvasImport, type PdfOcrRenderer, type PreviewNode, type TexionAttachmentReference, type TexionContent, type TexionDocument, type TexionMessage, type TextFormat, type TextFormatAdapter, type TextInput, classifyPdfPage, createContent, createController, createHighlightModel, createMessage, createMessageFromDocuments, createOcrScheduler, createPdfAdapter, createRegistry, csvAdapter, decode, defaultHighlightLimits, defaultLimits, deserializeContent, deserializeMessage, detectMarkdown, detectTextFormat, docxAdapter, exportMarkdown, exportSource, imageAdapter, inspectCodeBlock, inspectMarkdownSource, isAttachmentReference, isTexionContent, isTexionMessage, isTextFormat, jsonAdapter, markdownAdapter, pdfAdapter, processDocument, recognizePdfPages, safeUrl, serializeContent, serializeMessage, sourceWarnings, textAdapter, tokenizeDataSource, tokenizeMarkdownSource };