@yolk-sdk/extractors 0.1.0-canary.98

package/dist/knowledge.mjs

@@ -0,0 +1,77 @@
+import { FileExtractor } from "./service.mjs";
+import { Effect, Layer, Match, Predicate } from "effect";
+import { KnowledgeExtractionError } from "@yolk-sdk/knowledge/errors";
+import { KnowledgeExtractor } from "@yolk-sdk/knowledge/extraction";
+//#region src/knowledge.ts
+/** Percent-decode a path segment; malformed escapes (`%zz`, invalid UTF-8) stay encoded. */
+const decodeSegment = (segment) => {
+	try {
+		return decodeURIComponent(segment);
+	} catch {
+		return segment;
+	}
+};
+const urlFilename = (url) => {
+	if (!URL.canParse(url)) return url;
+	const last = new URL(url).pathname.split("/").filter((segment) => segment.length > 0).at(-1);
+	return last === void 0 ? url : decodeSegment(last);
+};
+/** The filename used for format detection: file name or ref, URL path, or text label. */
+const filenameFor = (source) => Match.value(source).pipe(Match.tagsExhaustive({
+	File: (file) => file.name ?? file.ref,
+	Url: (url) => urlFilename(url.url),
+	Text: (text) => text.label ?? "text"
+}));
+/** Loaded media type first, then the file source's, then `text/plain` for text sources. */
+const mediaTypeFor = (loaded) => {
+	if (loaded.mediaType !== void 0) return loaded.mediaType;
+	if (Predicate.isTagged(loaded.source, "File") && loaded.source.mediaType !== void 0) return loaded.source.mediaType;
+	return Predicate.isTagged(loaded.source, "Text") ? "text/plain" : "";
+};
+const documentFrom = (loaded, extracted) => {
+	const { title, ...fileMetadata } = extracted.metadata;
+	const metadata = {
+		...fileMetadata,
+		...loaded.metadata
+	};
+	return title === void 0 || title.trim().length === 0 ? {
+		content: extracted.content,
+		metadata
+	} : {
+		content: extracted.content,
+		title: title.trim(),
+		metadata
+	};
+};
+/**
+* A `KnowledgeExtractor` backed by a `FileExtractor`. String content is already text and passes
+* through unchanged (it must not be blank); bytes are extracted with the format chosen from the
+* source name and media type. File metadata (`format`, `pageCount`, `sheetNames`) is merged
+* under the loaded source's own metadata, and the extracted title becomes the document title.
+*/
+const makeFileKnowledgeExtractor = (extractor) => ({ extract: (loaded) => {
+	if (Predicate.isString(loaded.content)) {
+		const content = loaded.content;
+		return content.trim().length === 0 ? Effect.fail(new KnowledgeExtractionError({ message: "Knowledge source text is empty" })) : Effect.succeed(loaded.metadata === void 0 ? { content } : {
+			content,
+			metadata: loaded.metadata
+		});
+	}
+	return extractor.extract({
+		filename: filenameFor(loaded.source),
+		mediaType: mediaTypeFor(loaded),
+		bytes: loaded.content
+	}).pipe(Effect.map((extracted) => documentFrom(loaded, extracted)), Effect.mapError((error) => new KnowledgeExtractionError({
+		message: error.message,
+		cause: error
+	})));
+} });
+/** Provide `KnowledgeExtractor` from the `FileExtractor` in context (for example the Node layer). */
+const FileKnowledgeExtractorLayer = Layer.effect(KnowledgeExtractor, Effect.gen(function* () {
+	const extractor = yield* FileExtractor;
+	return KnowledgeExtractor.of(makeFileKnowledgeExtractor(extractor));
+}));
+//#endregion
+export { FileKnowledgeExtractorLayer, makeFileKnowledgeExtractor };
+
+//# sourceMappingURL=knowledge.mjs.map

package/dist/knowledge.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledge.mjs","names":[],"sources":["../src/knowledge.ts"],"sourcesContent":["import { Effect, Layer, Match, Predicate } from 'effect'\nimport type { ExtractedKnowledgeDocument, KnowledgeSource } from '@yolk-sdk/knowledge/documents'\nimport { KnowledgeExtractionError } from '@yolk-sdk/knowledge/errors'\nimport { KnowledgeExtractor } from '@yolk-sdk/knowledge/extraction'\nimport type { KnowledgeExtractorApi, LoadedKnowledgeSource } from '@yolk-sdk/knowledge/extraction'\nimport type { ExtractedFile } from './format.ts'\nimport { FileExtractor } from './service.ts'\nimport type { FileExtractorApi } from './service.ts'\n\n/** Percent-decode a path segment; malformed escapes (`%zz`, invalid UTF-8) stay encoded. */\nconst decodeSegment = (segment: string) => {\n  try {\n    return decodeURIComponent(segment)\n  } catch {\n    return segment\n  }\n}\n\nconst urlFilename = (url: string) => {\n  if (!URL.canParse(url)) return url\n\n  const segments = new URL(url).pathname.split('/').filter(segment => segment.length > 0)\n  const last = segments.at(-1)\n\n  return last === undefined ? url : decodeSegment(last)\n}\n\n/** The filename used for format detection: file name or ref, URL path, or text label. */\nconst filenameFor = (source: KnowledgeSource) =>\n  Match.value(source).pipe(\n    Match.tagsExhaustive({\n      File: file => file.name ?? file.ref,\n      Url: url => urlFilename(url.url),\n      Text: text => text.label ?? 'text'\n    })\n  )\n\n/** Loaded media type first, then the file source's, then `text/plain` for text sources. */\nconst mediaTypeFor = (loaded: LoadedKnowledgeSource) => {\n  if (loaded.mediaType !== undefined) return loaded.mediaType\n\n  if (Predicate.isTagged(loaded.source, 'File') && loaded.source.mediaType !== undefined)\n    return loaded.source.mediaType\n\n  return Predicate.isTagged(loaded.source, 'Text') ? 'text/plain' : ''\n}\n\nconst documentFrom = (\n  loaded: LoadedKnowledgeSource,\n  extracted: ExtractedFile\n): ExtractedKnowledgeDocument => {\n  const { title, ...fileMetadata } = extracted.metadata\n  const metadata = { ...fileMetadata, ...loaded.metadata }\n\n  return title === undefined || title.trim().length === 0\n    ? { content: extracted.content, metadata }\n    : { content: extracted.content, title: title.trim(), metadata }\n}\n\n/**\n * A `KnowledgeExtractor` backed by a `FileExtractor`. String content is already text and passes\n * through unchanged (it must not be blank); bytes are extracted with the format chosen from the\n * source name and media type. File metadata (`format`, `pageCount`, `sheetNames`) is merged\n * under the loaded source's own metadata, and the extracted title becomes the document title.\n */\nexport const makeFileKnowledgeExtractor = (extractor: FileExtractorApi): KnowledgeExtractorApi => ({\n  extract: loaded => {\n    if (Predicate.isString(loaded.content)) {\n      const content = loaded.content\n\n      return content.trim().length === 0\n        ? Effect.fail(new KnowledgeExtractionError({ message: 'Knowledge source text is empty' }))\n        : Effect.succeed(\n            loaded.metadata === undefined ? { content } : { content, metadata: loaded.metadata }\n          )\n    }\n\n    return extractor\n      .extract({\n        filename: filenameFor(loaded.source),\n        mediaType: mediaTypeFor(loaded),\n        bytes: loaded.content\n      })\n      .pipe(\n        Effect.map(extracted => documentFrom(loaded, extracted)),\n        Effect.mapError(\n          error => new KnowledgeExtractionError({ message: error.message, cause: error })\n        )\n      )\n  }\n})\n\n/** Provide `KnowledgeExtractor` from the `FileExtractor` in context (for example the Node layer). */\nexport const FileKnowledgeExtractorLayer = Layer.effect(\n  KnowledgeExtractor,\n  Effect.gen(function* () {\n    const extractor = yield* FileExtractor\n\n    return KnowledgeExtractor.of(makeFileKnowledgeExtractor(extractor))\n  })\n)\n"],"mappings":";;;;;;AAUA,MAAM,iBAAiB,YAAoB;CACzC,IAAI;EACF,OAAO,mBAAmB,OAAO;CACnC,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,eAAe,QAAgB;CACnC,IAAI,CAAC,IAAI,SAAS,GAAG,GAAG,OAAO;CAG/B,MAAM,OADW,IAAI,IAAI,GAAG,EAAE,SAAS,MAAM,GAAG,EAAE,QAAO,YAAW,QAAQ,SAAS,CACjE,EAAE,GAAG,EAAE;CAE3B,OAAO,SAAS,KAAA,IAAY,MAAM,cAAc,IAAI;AACtD;;AAGA,MAAM,eAAe,WACnB,MAAM,MAAM,MAAM,EAAE,KAClB,MAAM,eAAe;CACnB,OAAM,SAAQ,KAAK,QAAQ,KAAK;CAChC,MAAK,QAAO,YAAY,IAAI,GAAG;CAC/B,OAAM,SAAQ,KAAK,SAAS;AAC9B,CAAC,CACH;;AAGF,MAAM,gBAAgB,WAAkC;CACtD,IAAI,OAAO,cAAc,KAAA,GAAW,OAAO,OAAO;CAElD,IAAI,UAAU,SAAS,OAAO,QAAQ,MAAM,KAAK,OAAO,OAAO,cAAc,KAAA,GAC3E,OAAO,OAAO,OAAO;CAEvB,OAAO,UAAU,SAAS,OAAO,QAAQ,MAAM,IAAI,eAAe;AACpE;AAEA,MAAM,gBACJ,QACA,cAC+B;CAC/B,MAAM,EAAE,OAAO,GAAG,iBAAiB,UAAU;CAC7C,MAAM,WAAW;EAAE,GAAG;EAAc,GAAG,OAAO;CAAS;CAEvD,OAAO,UAAU,KAAA,KAAa,MAAM,KAAK,EAAE,WAAW,IAClD;EAAE,SAAS,UAAU;EAAS;CAAS,IACvC;EAAE,SAAS,UAAU;EAAS,OAAO,MAAM,KAAK;EAAG;CAAS;AAClE;;;;;;;AAQA,MAAa,8BAA8B,eAAwD,EACjG,UAAS,WAAU;CACjB,IAAI,UAAU,SAAS,OAAO,OAAO,GAAG;EACtC,MAAM,UAAU,OAAO;EAEvB,OAAO,QAAQ,KAAK,EAAE,WAAW,IAC7B,OAAO,KAAK,IAAI,yBAAyB,EAAE,SAAS,iCAAiC,CAAC,CAAC,IACvF,OAAO,QACL,OAAO,aAAa,KAAA,IAAY,EAAE,QAAQ,IAAI;GAAE;GAAS,UAAU,OAAO;EAAS,CACrF;CACN;CAEA,OAAO,UACJ,QAAQ;EACP,UAAU,YAAY,OAAO,MAAM;EACnC,WAAW,aAAa,MAAM;EAC9B,OAAO,OAAO;CAChB,CAAC,EACA,KACC,OAAO,KAAI,cAAa,aAAa,QAAQ,SAAS,CAAC,GACvD,OAAO,UACL,UAAS,IAAI,yBAAyB;EAAE,SAAS,MAAM;EAAS,OAAO;CAAM,CAAC,CAChF,CACF;AACJ,EACF;;AAGA,MAAa,8BAA8B,MAAM,OAC/C,oBACA,OAAO,IAAI,aAAa;CACtB,MAAM,YAAY,OAAO;CAEzB,OAAO,mBAAmB,GAAG,2BAA2B,SAAS,CAAC;AACpE,CAAC,CACH"}

package/dist/limits.d.mts

@@ -0,0 +1,28 @@
+import * as Schema from "effect/Schema";
+
+//#region src/limits.d.ts
+/**
+ * Smallest `maxXlsxTextCharacters`: one more than the space always reserved for the
+ * `[Some hyperlinks omitted: output limit and hyperlink limit]` marker (61 characters with its
+ * leading blank line), so a workbook with links can still produce text.
+ */
+declare const minimumXlsxTextCharacters = 62;
+/**
+ * Work and output bounds for one `extract` call. Every value is a positive integer, and
+ * `maxXlsxTextCharacters` is at least `minimumXlsxTextCharacters`.
+ */
+declare const FileExtractorLimits: Schema.Struct<{
+  /** Input bytes accepted for any format. */readonly maxInputBytes: Schema.Int; /** Entries in a DOCX, XLSX, or PPTX ZIP archive. */
+  readonly maxArchiveEntries: Schema.Int; /** Total inflated bytes of a DOCX, XLSX, or PPTX archive, counted while inflating. */
+  readonly maxExpandedBytes: Schema.Int; /** Worksheets in a workbook. */
+  readonly maxXlsxSheets: Schema.Int; /** Cells visited across all worksheet ranges (absent cells count too). */
+  readonly maxXlsxCellVisits: Schema.Int; /** Characters of XLSX text, including hyperlink annotations and the omitted-links marker. */
+  readonly maxXlsxTextCharacters: Schema.Int; /** Hyperlinks read per workbook; later ones are ignored (still removed before SheetJS). */
+  readonly maxXlsxHyperlinks: Schema.Int;
+}>;
+type FileExtractorLimits = typeof FileExtractorLimits.Type;
+/** Default limits; override any of them through `makeFileExtractorLayer({ limits })`. */
+declare const defaultFileExtractorLimits: FileExtractorLimits;
+//#endregion
+export { FileExtractorLimits, defaultFileExtractorLimits, minimumXlsxTextCharacters };
+//# sourceMappingURL=limits.d.mts.map

package/dist/limits.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"limits.d.mts","names":[],"sources":["../src/limits.ts"],"mappings":";;;;;AAYA;;;cAAa,yBAAA;AAAyB;AAMtC;;;AANsC,cAMzB,mBAAA,EAAmB,MAAA,CAAA,MAAA;;;yCAAA;EAAA,oCAAA;EAAA,wCAAA;EAAA;;;KAmBpB,mBAAA,UAA6B,mBAAA,CAAoB,IAAI;;cAGpD,0BAAA,EAA4B,mBAQxC"}

package/dist/limits.mjs

@@ -0,0 +1,43 @@
+import * as Schema from "effect/Schema";
+//#region src/limits.ts
+const PositiveSafeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)), Schema.check(Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER)));
+/**
+* Smallest `maxXlsxTextCharacters`: one more than the space always reserved for the
+* `[Some hyperlinks omitted: output limit and hyperlink limit]` marker (61 characters with its
+* leading blank line), so a workbook with links can still produce text.
+*/
+const minimumXlsxTextCharacters = 62;
+/**
+* Work and output bounds for one `extract` call. Every value is a positive integer, and
+* `maxXlsxTextCharacters` is at least `minimumXlsxTextCharacters`.
+*/
+const FileExtractorLimits = Schema.Struct({
+	/** Input bytes accepted for any format. */
+	maxInputBytes: PositiveSafeInteger,
+	/** Entries in a DOCX, XLSX, or PPTX ZIP archive. */
+	maxArchiveEntries: PositiveSafeInteger,
+	/** Total inflated bytes of a DOCX, XLSX, or PPTX archive, counted while inflating. */
+	maxExpandedBytes: PositiveSafeInteger,
+	/** Worksheets in a workbook. */
+	maxXlsxSheets: PositiveSafeInteger,
+	/** Cells visited across all worksheet ranges (absent cells count too). */
+	maxXlsxCellVisits: PositiveSafeInteger,
+	/** Characters of XLSX text, including hyperlink annotations and the omitted-links marker. */
+	maxXlsxTextCharacters: PositiveSafeInteger.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(62))),
+	/** Hyperlinks read per workbook; later ones are ignored (still removed before SheetJS). */
+	maxXlsxHyperlinks: PositiveSafeInteger
+});
+/** Default limits; override any of them through `makeFileExtractorLayer({ limits })`. */
+const defaultFileExtractorLimits = {
+	maxInputBytes: 50 * 1024 * 1024,
+	maxArchiveEntries: 1e4,
+	maxExpandedBytes: 50 * 1024 * 1024,
+	maxXlsxSheets: 100,
+	maxXlsxCellVisits: 1e5,
+	maxXlsxTextCharacters: 512 * 1024,
+	maxXlsxHyperlinks: 1e4
+};
+//#endregion
+export { FileExtractorLimits, defaultFileExtractorLimits, minimumXlsxTextCharacters };
+
+//# sourceMappingURL=limits.mjs.map

package/dist/limits.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"limits.mjs","names":[],"sources":["../src/limits.ts"],"sourcesContent":["import * as Schema from 'effect/Schema'\n\nconst PositiveSafeInteger = Schema.Int.pipe(\n  Schema.check(Schema.isGreaterThan(0)),\n  Schema.check(Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER))\n)\n\n/**\n * Smallest `maxXlsxTextCharacters`: one more than the space always reserved for the\n * `[Some hyperlinks omitted: output limit and hyperlink limit]` marker (61 characters with its\n * leading blank line), so a workbook with links can still produce text.\n */\nexport const minimumXlsxTextCharacters = 62\n\n/**\n * Work and output bounds for one `extract` call. Every value is a positive integer, and\n * `maxXlsxTextCharacters` is at least `minimumXlsxTextCharacters`.\n */\nexport const FileExtractorLimits = Schema.Struct({\n  /** Input bytes accepted for any format. */\n  maxInputBytes: PositiveSafeInteger,\n  /** Entries in a DOCX, XLSX, or PPTX ZIP archive. */\n  maxArchiveEntries: PositiveSafeInteger,\n  /** Total inflated bytes of a DOCX, XLSX, or PPTX archive, counted while inflating. */\n  maxExpandedBytes: PositiveSafeInteger,\n  /** Worksheets in a workbook. */\n  maxXlsxSheets: PositiveSafeInteger,\n  /** Cells visited across all worksheet ranges (absent cells count too). */\n  maxXlsxCellVisits: PositiveSafeInteger,\n  /** Characters of XLSX text, including hyperlink annotations and the omitted-links marker. */\n  maxXlsxTextCharacters: PositiveSafeInteger.pipe(\n    Schema.check(Schema.isGreaterThanOrEqualTo(minimumXlsxTextCharacters))\n  ),\n  /** Hyperlinks read per workbook; later ones are ignored (still removed before SheetJS). */\n  maxXlsxHyperlinks: PositiveSafeInteger\n})\n\nexport type FileExtractorLimits = typeof FileExtractorLimits.Type\n\n/** Default limits; override any of them through `makeFileExtractorLayer({ limits })`. */\nexport const defaultFileExtractorLimits: FileExtractorLimits = {\n  maxInputBytes: 50 * 1024 * 1024,\n  maxArchiveEntries: 10_000,\n  maxExpandedBytes: 50 * 1024 * 1024,\n  maxXlsxSheets: 100,\n  maxXlsxCellVisits: 100_000,\n  maxXlsxTextCharacters: 512 * 1024,\n  maxXlsxHyperlinks: 10_000\n}\n"],"mappings":";;AAEA,MAAM,sBAAsB,OAAO,IAAI,KACrC,OAAO,MAAM,OAAO,cAAc,CAAC,CAAC,GACpC,OAAO,MAAM,OAAO,oBAAoB,OAAO,gBAAgB,CAAC,CAClE;;;;;;AAOA,MAAa,4BAA4B;;;;;AAMzC,MAAa,sBAAsB,OAAO,OAAO;;CAE/C,eAAe;;CAEf,mBAAmB;;CAEnB,kBAAkB;;CAElB,eAAe;;CAEf,mBAAmB;;CAEnB,uBAAuB,oBAAoB,KACzC,OAAO,MAAM,OAAO,uBAAA,EAAgD,CAAC,CACvE;;CAEA,mBAAmB;AACrB,CAAC;;AAKD,MAAa,6BAAkD;CAC7D,eAAe,KAAK,OAAO;CAC3B,mBAAmB;CACnB,kBAAkB,KAAK,OAAO;CAC9B,eAAe;CACf,mBAAmB;CACnB,uBAAuB,MAAM;CAC7B,mBAAmB;AACrB"}

package/dist/node/extract-file.d.mts

@@ -0,0 +1,31 @@
+import { FileExtractionError, FileExtractorError } from "../errors.mjs";
+import { ExtractedFile, ExtractedFileFormat, ExtractedFileMetadata, FileInput, OfficeFileFormat } from "../format.mjs";
+import { FileExtractorLimits } from "../limits.mjs";
+import { SheetJsLoader } from "./sheetjs.mjs";
+import { Effect } from "effect";
+
+//#region src/node/extract-file.d.ts
+/** Formats whose bytes go through a parser (and, by default, an isolated worker). */
+type ParsedFileFormat = 'pdf' | OfficeFileFormat;
+declare const isParsedFileFormat: (format: ExtractedFileFormat) => format is ParsedFileFormat;
+/** Sanitize extracted text; empty results fail. */
+declare const makeExtractedFile: (content: string, metadata: ExtractedFileMetadata) => Effect.Effect<never, FileExtractionError, never> | Effect.Effect<ExtractedFile, never, never>;
+/**
+ * PDF.js 6 releases a document through its loading task (`PDFDocumentProxy` no longer has
+ * `destroy()`); unpdf releases the documents it opens the same way.
+ */
+type PdfDocumentResource = {
+  readonly loadingTask: {
+    readonly destroy: () => Promise<void>;
+  };
+};
+/** Run `use` with an opened PDF document and always release the parser afterwards. */
+declare const withAcquiredPdfDocument: <D extends PdfDocumentResource, A, E, R>(open: Effect.Effect<D, E, R>, use: (document: D) => Effect.Effect<A, E, R>, format: ExtractedFileFormat) => Effect.Effect<A, E, Exclude<R, import("effect/Scope").Scope>>;
+/**
+ * Extract a PDF, DOCX, XLSX, or PPTX file in the current thread. The Node layer runs this inside
+ * an isolated worker unless `isolation: 'none'` is configured.
+ */
+declare const extractParsedFile: (input: FileInput, format: ParsedFileFormat, limits: FileExtractorLimits, loader: SheetJsLoader) => Effect.Effect<ExtractedFile, FileExtractorError>;
+//#endregion
+export { ParsedFileFormat, extractParsedFile, isParsedFileFormat, makeExtractedFile, withAcquiredPdfDocument };
+//# sourceMappingURL=extract-file.d.mts.map

package/dist/node/extract-file.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-file.d.mts","names":[],"sources":["../../src/node/extract-file.ts"],"mappings":";;;;;;;;KAuBY,gBAAA,WAA2B,gBAAgB;AAAA,cAE1C,kBAAA,GAAsB,MAAA,EAAQ,mBAAA,KAAsB,MAAA,IAAU,gBACM;;cAGpE,iBAAA,GAAqB,OAAA,UAAiB,QAAA,EAAU,qBAAA,KAAqB,MAAA,CAAA,MAAA,QAAA,mBAAA,WAAA,MAAA,CAAA,MAAA,CAAA,aAAA;AAN3B;AAEvD;;;AAFuD,KAwBlD,mBAAA;EAAA,SACM,WAAA;IAAA,SAAwB,OAAA,QAAe,OAAO;EAAA;AAAA;;cAI5C,uBAAA,aAAqC,mBAAA,WAChD,IAAA,EAAM,MAAA,CAAO,MAAA,CAAO,CAAA,EAAG,CAAA,EAAG,CAAA,GAC1B,GAAA,GAAM,QAAA,EAAU,CAAA,KAAM,MAAA,CAAO,MAAA,CAAO,CAAA,EAAG,CAAA,EAAG,CAAA,GAC1C,MAAA,EAAQ,mBAAA,KAAmB,MAAA,CAAA,MAAA,CAAA,CAAA,EAAA,CAAA,EAAA,OAAA,CAAA,CAAA,yBAAA,KAAA;AA1B7B;;;;AAAA,cAkOa,iBAAA,GACX,KAAA,EAAO,SAAA,EACP,MAAA,EAAQ,gBAAA,EACR,MAAA,EAAQ,mBAAA,EACR,MAAA,EAAQ,aAAA,KACP,MAAA,CAAO,MAAA,CAAO,aAAA,EAAe,kBAAA"}

package/dist/node/extract-file.mjs

@@ -0,0 +1,183 @@
+import { FileExtractionError } from "../errors.mjs";
+import { sanitizeExtractedText } from "../sanitize.mjs";
+import { readOfficeArchive, storedArchive } from "./office-archive.mjs";
+import { extractPptxText } from "./pptx-text.mjs";
+import { asXlsxWorkbook, loadSheetJs } from "./sheetjs.mjs";
+import { resolveXlsxHyperlinks } from "./xlsx-hyperlinks.mjs";
+import { buildSheetJsInput } from "./xlsx-sheetjs-input.mjs";
+import { extractBoundedXlsxText } from "./xlsx-text.mjs";
+import { Effect, Option, Predicate } from "effect";
+import { Buffer } from "node:buffer";
+//#region src/node/extract-file.ts
+const isParsedFileFormat = (format) => format === "pdf" || format === "docx" || format === "pptx" || format === "xlsx";
+/** Sanitize extracted text; empty results fail. */
+const makeExtractedFile = (content, metadata) => {
+	const sanitized = sanitizeExtractedText(content);
+	if (sanitized.length === 0) return Effect.fail(new FileExtractionError({
+		message: "Extracted file content is empty",
+		format: metadata.format
+	}));
+	return Effect.succeed({
+		content: sanitized,
+		metadata
+	});
+};
+/** Run `use` with an opened PDF document and always release the parser afterwards. */
+const withAcquiredPdfDocument = (open, use, format) => Effect.scoped(Effect.gen(function* () {
+	return yield* use(yield* Effect.acquireRelease(open, (document) => Effect.tryPromise({
+		try: () => document.loadingTask.destroy(),
+		catch: () => new FileExtractionError({
+			message: "Could not release PDF parser",
+			format
+		})
+	}).pipe(Effect.ignore)));
+}));
+const extractPdf = (input) => Effect.gen(function* () {
+	const { extractText, getDocumentProxy, getMeta } = yield* Effect.tryPromise({
+		try: () => import("unpdf"),
+		catch: (cause) => new FileExtractionError({
+			message: "Could not read PDF",
+			format: "pdf",
+			cause
+		})
+	});
+	return yield* withAcquiredPdfDocument(Effect.tryPromise({
+		try: () => getDocumentProxy(new Uint8Array(input.bytes)),
+		catch: (cause) => new FileExtractionError({
+			message: "Could not read PDF",
+			format: "pdf",
+			cause
+		})
+	}), (document) => Effect.gen(function* () {
+		const extracted = yield* Effect.tryPromise({
+			try: () => extractText(document, { mergePages: true }),
+			catch: (cause) => new FileExtractionError({
+				message: "Could not extract PDF text",
+				format: "pdf",
+				cause
+			})
+		});
+		const meta = yield* Effect.tryPromise({
+			try: () => getMeta(document),
+			catch: (cause) => new FileExtractionError({
+				message: "Could not read PDF metadata",
+				format: "pdf",
+				cause
+			})
+		}).pipe(Effect.option);
+		const rawTitle = Option.isSome(meta) ? meta.value.info.Title : void 0;
+		const title = Predicate.isString(rawTitle) && rawTitle.length > 0 ? rawTitle : void 0;
+		const metadata = title === void 0 ? {
+			format: "pdf",
+			pageCount: extracted.totalPages
+		} : {
+			format: "pdf",
+			title,
+			pageCount: extracted.totalPages
+		};
+		return yield* makeExtractedFile(extracted.text, metadata);
+	}), "pdf");
+});
+const validatedArchive = (input, format, limits) => readOfficeArchive(input.bytes, format, limits, format === "xlsx" ? { maxHyperlinkTags: limits.maxXlsxHyperlinks + 1 } : {}).pipe(Effect.mapError((error) => new FileExtractionError({
+	message: error.message,
+	format,
+	cause: error
+})));
+const extractDocx = (input, limits) => Effect.gen(function* () {
+	const { parts } = yield* validatedArchive(input, "docx", limits);
+	return yield* makeExtractedFile((yield* Effect.tryPromise({
+		try: async () => {
+			const { default: mammoth } = await import("mammoth");
+			return await mammoth.extractRawText({ buffer: Buffer.from(storedArchive(parts)) });
+		},
+		catch: (cause) => new FileExtractionError({
+			message: "Could not extract DOCX text",
+			format: "docx",
+			cause
+		})
+	})).value, {
+		format: "docx",
+		title: input.filename
+	});
+});
+/** Keep the first `max` tags in archive order. */
+const capHyperlinkTags = (tagsByPart, max) => {
+	const capped = /* @__PURE__ */ new Map();
+	let remaining = max;
+	for (const [part, tags] of tagsByPart) {
+		if (remaining <= 0) break;
+		const kept = tags.slice(0, remaining);
+		remaining -= kept.length;
+		capped.set(part, kept);
+	}
+	return capped;
+};
+const extractXlsx = (input, limits, loader) => Effect.gen(function* () {
+	const normalized = yield* validatedArchive(input, "xlsx", limits);
+	const sheetJsInput = yield* Effect.try({
+		try: () => buildSheetJsInput(normalized.parts, limits.maxXlsxSheets),
+		catch: (cause) => cause instanceof FileExtractionError ? cause : new FileExtractionError({
+			message: "Could not read XLSX",
+			format: "xlsx",
+			cause
+		})
+	});
+	const sheetJs = yield* loadSheetJs(loader);
+	const workbook = asXlsxWorkbook(yield* Effect.try({
+		try: () => sheetJs.read(sheetJsInput.archive),
+		catch: (cause) => new FileExtractionError({
+			message: "Could not read XLSX",
+			format: "xlsx",
+			cause
+		})
+	}));
+	if (workbook === void 0) return yield* Effect.fail(new FileExtractionError({
+		message: "Could not read XLSX",
+		format: "xlsx"
+	}));
+	const hyperlinksTruncated = [...normalized.hyperlinkTags.values()].reduce((total, tags) => total + tags.length, 0) > limits.maxXlsxHyperlinks;
+	const hyperlinks = resolveXlsxHyperlinks(normalized.parts, hyperlinksTruncated ? capHyperlinkTags(normalized.hyperlinkTags, limits.maxXlsxHyperlinks) : normalized.hyperlinkTags, sheetJsInput.sheets);
+	return yield* makeExtractedFile(yield* Effect.try({
+		try: () => extractBoundedXlsxText(workbook, limits, {
+			hyperlinks,
+			hyperlinksTruncated
+		}),
+		catch: (cause) => cause instanceof FileExtractionError ? cause : new FileExtractionError({
+			message: "Could not extract XLSX text",
+			format: "xlsx",
+			cause
+		})
+	}), {
+		format: "xlsx",
+		title: sheetJsInput.title ?? input.filename,
+		sheetNames: workbook.SheetNames
+	});
+});
+const extractPptx = (input, limits) => Effect.gen(function* () {
+	const { parts } = yield* validatedArchive(input, "pptx", limits);
+	return yield* makeExtractedFile(yield* Effect.try({
+		try: () => extractPptxText(parts),
+		catch: (cause) => new FileExtractionError({
+			message: "Could not extract PPTX text",
+			format: "pptx",
+			cause
+		})
+	}), {
+		format: "pptx",
+		title: input.filename
+	});
+});
+/**
+* Extract a PDF, DOCX, XLSX, or PPTX file in the current thread. The Node layer runs this inside
+* an isolated worker unless `isolation: 'none'` is configured.
+*/
+const extractParsedFile = (input, format, limits, loader) => {
+	if (format === "pdf") return extractPdf(input);
+	if (format === "docx") return extractDocx(input, limits);
+	if (format === "xlsx") return extractXlsx(input, limits, loader);
+	return extractPptx(input, limits);
+};
+//#endregion
+export { extractParsedFile, isParsedFileFormat, makeExtractedFile, withAcquiredPdfDocument };
+
+//# sourceMappingURL=extract-file.mjs.map

package/dist/node/extract-file.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-file.mjs","names":[],"sources":["../../src/node/extract-file.ts"],"sourcesContent":["import { Buffer } from 'node:buffer'\nimport { Effect, Option, Predicate } from 'effect'\nimport { FileExtractionError } from '../errors.ts'\nimport type { FileExtractorError } from '../errors.ts'\nimport type {\n  ExtractedFile,\n  ExtractedFileFormat,\n  ExtractedFileMetadata,\n  FileInput,\n  OfficeFileFormat\n} from '../format.ts'\nimport type { FileExtractorLimits } from '../limits.ts'\nimport { sanitizeExtractedText } from '../sanitize.ts'\nimport { readOfficeArchive, storedArchive } from './office-archive.ts'\nimport type { NormalizedOfficeArchive } from './office-archive.ts'\nimport { extractPptxText } from './pptx-text.ts'\nimport { asXlsxWorkbook, loadSheetJs } from './sheetjs.ts'\nimport type { SheetJsLoader } from './sheetjs.ts'\nimport { resolveXlsxHyperlinks } from './xlsx-hyperlinks.ts'\nimport { buildSheetJsInput } from './xlsx-sheetjs-input.ts'\nimport { extractBoundedXlsxText } from './xlsx-text.ts'\n\n/** Formats whose bytes go through a parser (and, by default, an isolated worker). */\nexport type ParsedFileFormat = 'pdf' | OfficeFileFormat\n\nexport const isParsedFileFormat = (format: ExtractedFileFormat): format is ParsedFileFormat =>\n  format === 'pdf' || format === 'docx' || format === 'pptx' || format === 'xlsx'\n\n/** Sanitize extracted text; empty results fail. */\nexport const makeExtractedFile = (content: string, metadata: ExtractedFileMetadata) => {\n  const sanitized = sanitizeExtractedText(content)\n\n  if (sanitized.length === 0)\n    return Effect.fail(\n      new FileExtractionError({\n        message: 'Extracted file content is empty',\n        format: metadata.format\n      })\n    )\n\n  return Effect.succeed<ExtractedFile>({ content: sanitized, metadata })\n}\n\n/**\n * PDF.js 6 releases a document through its loading task (`PDFDocumentProxy` no longer has\n * `destroy()`); unpdf releases the documents it opens the same way.\n */\ntype PdfDocumentResource = {\n  readonly loadingTask: { readonly destroy: () => Promise<void> }\n}\n\n/** Run `use` with an opened PDF document and always release the parser afterwards. */\nexport const withAcquiredPdfDocument = <D extends PdfDocumentResource, A, E, R>(\n  open: Effect.Effect<D, E, R>,\n  use: (document: D) => Effect.Effect<A, E, R>,\n  format: ExtractedFileFormat\n) =>\n  Effect.scoped(\n    Effect.gen(function* () {\n      const document = yield* Effect.acquireRelease(open, document =>\n        Effect.tryPromise({\n          try: () => document.loadingTask.destroy(),\n          catch: () => new FileExtractionError({ message: 'Could not release PDF parser', format })\n        }).pipe(Effect.ignore)\n      )\n\n      return yield* use(document)\n    })\n  )\n\nconst extractPdf = (input: FileInput) =>\n  Effect.gen(function* () {\n    // Loaded on first use, so a worker extracting another format never loads PDF.js.\n    const { extractText, getDocumentProxy, getMeta } = yield* Effect.tryPromise({\n      try: () => import('unpdf'),\n      catch: cause =>\n        new FileExtractionError({ message: 'Could not read PDF', format: 'pdf', cause })\n    })\n\n    return yield* withAcquiredPdfDocument(\n      Effect.tryPromise({\n        // PDF.js may detach the buffer it is given; never hand it the caller's bytes.\n        try: () => getDocumentProxy(new Uint8Array(input.bytes)),\n        catch: cause =>\n          new FileExtractionError({ message: 'Could not read PDF', format: 'pdf', cause })\n      }),\n      document =>\n        Effect.gen(function* () {\n          const extracted = yield* Effect.tryPromise({\n            try: () => extractText(document, { mergePages: true }),\n            catch: cause =>\n              new FileExtractionError({\n                message: 'Could not extract PDF text',\n                format: 'pdf',\n                cause\n              })\n          })\n\n          const meta = yield* Effect.tryPromise({\n            try: () => getMeta(document),\n            catch: cause =>\n              new FileExtractionError({\n                message: 'Could not read PDF metadata',\n                format: 'pdf',\n                cause\n              })\n          }).pipe(Effect.option)\n\n          const rawTitle = Option.isSome(meta) ? meta.value.info.Title : undefined\n          const title = Predicate.isString(rawTitle) && rawTitle.length > 0 ? rawTitle : undefined\n\n          const metadata: ExtractedFileMetadata =\n            title === undefined\n              ? { format: 'pdf', pageCount: extracted.totalPages }\n              : { format: 'pdf', title, pageCount: extracted.totalPages }\n\n          return yield* makeExtractedFile(extracted.text, metadata)\n        }),\n      'pdf'\n    )\n  })\n\nconst validatedArchive = (\n  input: FileInput,\n  format: OfficeFileFormat,\n  limits: FileExtractorLimits\n): Effect.Effect<NormalizedOfficeArchive, FileExtractionError> =>\n  readOfficeArchive(\n    input.bytes,\n    format,\n    limits,\n    // One extra tag detects a workbook over the cap.\n    format === 'xlsx' ? { maxHyperlinkTags: limits.maxXlsxHyperlinks + 1 } : {}\n  ).pipe(\n    Effect.mapError(\n      error => new FileExtractionError({ message: error.message, format, cause: error })\n    )\n  )\n\nconst extractDocx = (input: FileInput, limits: FileExtractorLimits) =>\n  Effect.gen(function* () {\n    const { parts } = yield* validatedArchive(input, 'docx', limits)\n\n    const result = yield* Effect.tryPromise({\n      try: async () => {\n        const { default: mammoth } = await import('mammoth')\n\n        return await mammoth.extractRawText({ buffer: Buffer.from(storedArchive(parts)) })\n      },\n      catch: cause =>\n        new FileExtractionError({ message: 'Could not extract DOCX text', format: 'docx', cause })\n    })\n\n    return yield* makeExtractedFile(result.value, { format: 'docx', title: input.filename })\n  })\n\n/** Keep the first `max` tags in archive order. */\nconst capHyperlinkTags = (\n  tagsByPart: ReadonlyMap<string, ReadonlyArray<string>>,\n  max: number\n): ReadonlyMap<string, ReadonlyArray<string>> => {\n  const capped = new Map<string, ReadonlyArray<string>>()\n  let remaining = max\n\n  for (const [part, tags] of tagsByPart) {\n    if (remaining <= 0) break\n\n    const kept = tags.slice(0, remaining)\n    remaining -= kept.length\n    capped.set(part, kept)\n  }\n\n  return capped\n}\n\nconst extractXlsx = (input: FileInput, limits: FileExtractorLimits, loader: SheetJsLoader) =>\n  Effect.gen(function* () {\n    const normalized = yield* validatedArchive(input, 'xlsx', limits)\n\n    // SheetJS never sees the uploaded archive: only this allowlisted rebuild of its parts.\n    const sheetJsInput = yield* Effect.try({\n      try: () => buildSheetJsInput(normalized.parts, limits.maxXlsxSheets),\n      catch: cause =>\n        cause instanceof FileExtractionError\n          ? cause\n          : new FileExtractionError({ message: 'Could not read XLSX', format: 'xlsx', cause })\n    })\n\n    const sheetJs = yield* loadSheetJs(loader)\n\n    const parsed = yield* Effect.try({\n      try: () => sheetJs.read(sheetJsInput.archive),\n      catch: cause =>\n        new FileExtractionError({ message: 'Could not read XLSX', format: 'xlsx', cause })\n    })\n\n    const workbook = asXlsxWorkbook(parsed)\n\n    if (workbook === undefined)\n      return yield* Effect.fail(\n        new FileExtractionError({ message: 'Could not read XLSX', format: 'xlsx' })\n      )\n\n    // One extra tag was captured to detect that the workbook exceeds the hyperlink cap.\n    const capturedTags = [...normalized.hyperlinkTags.values()].reduce(\n      (total, tags) => total + tags.length,\n      0\n    )\n\n    const hyperlinksTruncated = capturedTags > limits.maxXlsxHyperlinks\n\n    const hyperlinks = resolveXlsxHyperlinks(\n      normalized.parts,\n      hyperlinksTruncated\n        ? capHyperlinkTags(normalized.hyperlinkTags, limits.maxXlsxHyperlinks)\n        : normalized.hyperlinkTags,\n      sheetJsInput.sheets\n    )\n\n    const content = yield* Effect.try({\n      try: () => extractBoundedXlsxText(workbook, limits, { hyperlinks, hyperlinksTruncated }),\n      catch: cause =>\n        cause instanceof FileExtractionError\n          ? cause\n          : new FileExtractionError({\n              message: 'Could not extract XLSX text',\n              format: 'xlsx',\n              cause\n            })\n    })\n\n    return yield* makeExtractedFile(content, {\n      format: 'xlsx',\n      title: sheetJsInput.title ?? input.filename,\n      sheetNames: workbook.SheetNames\n    })\n  })\n\nconst extractPptx = (input: FileInput, limits: FileExtractorLimits) =>\n  Effect.gen(function* () {\n    const { parts } = yield* validatedArchive(input, 'pptx', limits)\n\n    const text = yield* Effect.try({\n      try: () => extractPptxText(parts),\n      catch: cause =>\n        new FileExtractionError({ message: 'Could not extract PPTX text', format: 'pptx', cause })\n    })\n\n    return yield* makeExtractedFile(text, { format: 'pptx', title: input.filename })\n  })\n\n/**\n * Extract a PDF, DOCX, XLSX, or PPTX file in the current thread. The Node layer runs this inside\n * an isolated worker unless `isolation: 'none'` is configured.\n */\nexport const extractParsedFile = (\n  input: FileInput,\n  format: ParsedFileFormat,\n  limits: FileExtractorLimits,\n  loader: SheetJsLoader\n): Effect.Effect<ExtractedFile, FileExtractorError> => {\n  if (format === 'pdf') return extractPdf(input)\n\n  if (format === 'docx') return extractDocx(input, limits)\n\n  if (format === 'xlsx') return extractXlsx(input, limits, loader)\n\n  return extractPptx(input, limits)\n}\n"],"mappings":";;;;;;;;;;;AAyBA,MAAa,sBAAsB,WACjC,WAAW,SAAS,WAAW,UAAU,WAAW,UAAU,WAAW;;AAG3E,MAAa,qBAAqB,SAAiB,aAAoC;CACrF,MAAM,YAAY,sBAAsB,OAAO;CAE/C,IAAI,UAAU,WAAW,GACvB,OAAO,OAAO,KACZ,IAAI,oBAAoB;EACtB,SAAS;EACT,QAAQ,SAAS;CACnB,CAAC,CACH;CAEF,OAAO,OAAO,QAAuB;EAAE,SAAS;EAAW;CAAS,CAAC;AACvE;;AAWA,MAAa,2BACX,MACA,KACA,WAEA,OAAO,OACL,OAAO,IAAI,aAAa;CAQtB,OAAO,OAAO,IAAI,OAPM,OAAO,eAAe,OAAM,aAClD,OAAO,WAAW;EAChB,WAAW,SAAS,YAAY,QAAQ;EACxC,aAAa,IAAI,oBAAoB;GAAE,SAAS;GAAgC;EAAO,CAAC;CAC1F,CAAC,EAAE,KAAK,OAAO,MAAM,CACvB,CAE0B;AAC5B,CAAC,CACH;AAEF,MAAM,cAAc,UAClB,OAAO,IAAI,aAAa;CAEtB,MAAM,EAAE,aAAa,kBAAkB,YAAY,OAAO,OAAO,WAAW;EAC1E,WAAW,OAAO;EAClB,QAAO,UACL,IAAI,oBAAoB;GAAE,SAAS;GAAsB,QAAQ;GAAO;EAAM,CAAC;CACnF,CAAC;CAED,OAAO,OAAO,wBACZ,OAAO,WAAW;EAEhB,WAAW,iBAAiB,IAAI,WAAW,MAAM,KAAK,CAAC;EACvD,QAAO,UACL,IAAI,oBAAoB;GAAE,SAAS;GAAsB,QAAQ;GAAO;EAAM,CAAC;CACnF,CAAC,IACD,aACE,OAAO,IAAI,aAAa;EACtB,MAAM,YAAY,OAAO,OAAO,WAAW;GACzC,WAAW,YAAY,UAAU,EAAE,YAAY,KAAK,CAAC;GACrD,QAAO,UACL,IAAI,oBAAoB;IACtB,SAAS;IACT,QAAQ;IACR;GACF,CAAC;EACL,CAAC;EAED,MAAM,OAAO,OAAO,OAAO,WAAW;GACpC,WAAW,QAAQ,QAAQ;GAC3B,QAAO,UACL,IAAI,oBAAoB;IACtB,SAAS;IACT,QAAQ;IACR;GACF,CAAC;EACL,CAAC,EAAE,KAAK,OAAO,MAAM;EAErB,MAAM,WAAW,OAAO,OAAO,IAAI,IAAI,KAAK,MAAM,KAAK,QAAQ,KAAA;EAC/D,MAAM,QAAQ,UAAU,SAAS,QAAQ,KAAK,SAAS,SAAS,IAAI,WAAW,KAAA;EAE/E,MAAM,WACJ,UAAU,KAAA,IACN;GAAE,QAAQ;GAAO,WAAW,UAAU;EAAW,IACjD;GAAE,QAAQ;GAAO;GAAO,WAAW,UAAU;EAAW;EAE9D,OAAO,OAAO,kBAAkB,UAAU,MAAM,QAAQ;CAC1D,CAAC,GACH,KACF;AACF,CAAC;AAEH,MAAM,oBACJ,OACA,QACA,WAEA,kBACE,MAAM,OACN,QACA,QAEA,WAAW,SAAS,EAAE,kBAAkB,OAAO,oBAAoB,EAAE,IAAI,CAAC,CAC5E,EAAE,KACA,OAAO,UACL,UAAS,IAAI,oBAAoB;CAAE,SAAS,MAAM;CAAS;CAAQ,OAAO;AAAM,CAAC,CACnF,CACF;AAEF,MAAM,eAAe,OAAkB,WACrC,OAAO,IAAI,aAAa;CACtB,MAAM,EAAE,UAAU,OAAO,iBAAiB,OAAO,QAAQ,MAAM;CAY/D,OAAO,OAAO,mBAAkB,OAVV,OAAO,WAAW;EACtC,KAAK,YAAY;GACf,MAAM,EAAE,SAAS,YAAY,MAAM,OAAO;GAE1C,OAAO,MAAM,QAAQ,eAAe,EAAE,QAAQ,OAAO,KAAK,cAAc,KAAK,CAAC,EAAE,CAAC;EACnF;EACA,QAAO,UACL,IAAI,oBAAoB;GAAE,SAAS;GAA+B,QAAQ;GAAQ;EAAM,CAAC;CAC7F,CAAC,GAEsC,OAAO;EAAE,QAAQ;EAAQ,OAAO,MAAM;CAAS,CAAC;AACzF,CAAC;;AAGH,MAAM,oBACJ,YACA,QAC+C;CAC/C,MAAM,yBAAS,IAAI,IAAmC;CACtD,IAAI,YAAY;CAEhB,KAAK,MAAM,CAAC,MAAM,SAAS,YAAY;EACrC,IAAI,aAAa,GAAG;EAEpB,MAAM,OAAO,KAAK,MAAM,GAAG,SAAS;EACpC,aAAa,KAAK;EAClB,OAAO,IAAI,MAAM,IAAI;CACvB;CAEA,OAAO;AACT;AAEA,MAAM,eAAe,OAAkB,QAA6B,WAClE,OAAO,IAAI,aAAa;CACtB,MAAM,aAAa,OAAO,iBAAiB,OAAO,QAAQ,MAAM;CAGhE,MAAM,eAAe,OAAO,OAAO,IAAI;EACrC,WAAW,kBAAkB,WAAW,OAAO,OAAO,aAAa;EACnE,QAAO,UACL,iBAAiB,sBACb,QACA,IAAI,oBAAoB;GAAE,SAAS;GAAuB,QAAQ;GAAQ;EAAM,CAAC;CACzF,CAAC;CAED,MAAM,UAAU,OAAO,YAAY,MAAM;CAQzC,MAAM,WAAW,eAAe,OANV,OAAO,IAAI;EAC/B,WAAW,QAAQ,KAAK,aAAa,OAAO;EAC5C,QAAO,UACL,IAAI,oBAAoB;GAAE,SAAS;GAAuB,QAAQ;GAAQ;EAAM,CAAC;CACrF,CAAC,CAEqC;CAEtC,IAAI,aAAa,KAAA,GACf,OAAO,OAAO,OAAO,KACnB,IAAI,oBAAoB;EAAE,SAAS;EAAuB,QAAQ;CAAO,CAAC,CAC5E;CAQF,MAAM,sBALe,CAAC,GAAG,WAAW,cAAc,OAAO,CAAC,EAAE,QACzD,OAAO,SAAS,QAAQ,KAAK,QAC9B,CAGqC,IAAI,OAAO;CAElD,MAAM,aAAa,sBACjB,WAAW,OACX,sBACI,iBAAiB,WAAW,eAAe,OAAO,iBAAiB,IACnE,WAAW,eACf,aAAa,MACf;CAcA,OAAO,OAAO,kBAAkB,OAZT,OAAO,IAAI;EAChC,WAAW,uBAAuB,UAAU,QAAQ;GAAE;GAAY;EAAoB,CAAC;EACvF,QAAO,UACL,iBAAiB,sBACb,QACA,IAAI,oBAAoB;GACtB,SAAS;GACT,QAAQ;GACR;EACF,CAAC;CACT,CAAC,GAEwC;EACvC,QAAQ;EACR,OAAO,aAAa,SAAS,MAAM;EACnC,YAAY,SAAS;CACvB,CAAC;AACH,CAAC;AAEH,MAAM,eAAe,OAAkB,WACrC,OAAO,IAAI,aAAa;CACtB,MAAM,EAAE,UAAU,OAAO,iBAAiB,OAAO,QAAQ,MAAM;CAQ/D,OAAO,OAAO,kBAAkB,OANZ,OAAO,IAAI;EAC7B,WAAW,gBAAgB,KAAK;EAChC,QAAO,UACL,IAAI,oBAAoB;GAAE,SAAS;GAA+B,QAAQ;GAAQ;EAAM,CAAC;CAC7F,CAAC,GAEqC;EAAE,QAAQ;EAAQ,OAAO,MAAM;CAAS,CAAC;AACjF,CAAC;;;;;AAMH,MAAa,qBACX,OACA,QACA,QACA,WACqD;CACrD,IAAI,WAAW,OAAO,OAAO,WAAW,KAAK;CAE7C,IAAI,WAAW,QAAQ,OAAO,YAAY,OAAO,MAAM;CAEvD,IAAI,WAAW,QAAQ,OAAO,YAAY,OAAO,QAAQ,MAAM;CAE/D,OAAO,YAAY,OAAO,MAAM;AAClC"}

package/dist/node/extraction-isolation.d.mts

@@ -0,0 +1,94 @@
+import { FileExtractorError } from "../errors.mjs";
+import { ExtractedFile, FileInput } from "../format.mjs";
+import { FileExtractorLimits } from "../limits.mjs";
+import { ParsedFileFormat } from "./extract-file.mjs";
+import { Effect } from "effect";
+import * as Schema from "effect/Schema";
+
+//#region src/node/extraction-isolation.d.ts
+/** Limits for the worker each PDF, DOCX, XLSX, or PPTX extraction runs in. */
+type WorkerIsolationOptions = {
+  /** V8 old-generation heap of the worker, in MB. Default 256. */readonly maxOldGenerationSizeMb?: number; /** V8 young-generation heap of the worker, in MB. Default 32. */
+  readonly maxYoungGenerationSizeMb?: number; /** Stack of the worker's main thread, in MB. Default 4 (Node's own default). */
+  readonly stackSizeMb?: number;
+  /**
+   * Wall-clock time a worker may run before it is terminated, in ms. Default 30,000. At most
+   * 2³¹−1 (Node's largest timer delay).
+   */
+  readonly timeoutMs?: number;
+  /**
+   * This layer's share of the realm-wide worker pool: at most this many of its extractions run at
+   * once. Every layer in a JavaScript realm (the main thread, or each worker thread that builds
+   * the layer) shares one pool of 4 workers, however often the layer is built, so this can only
+   * lower the layer's share. 1 to 4; default 4.
+   */
+  readonly maxConcurrentWorkers?: number;
+  /**
+   * How long an extraction may wait for a worker slot, in ms, before it fails with
+   * `reason: 'busy'` without starting a worker. Slots are handed out first come, first served, and
+   * a slot freed after the deadline never admits the extraction. Default: the layer's `timeoutMs`.
+   * At most 2³¹−1 (Node's largest timer delay).
+   */
+  readonly maxQueueWaitMs?: number;
+  /**
+   * The worker entry. Default: the package's self-contained `dist/node/extraction-worker.mjs`,
+   * found from this module's own location (in `dist`, or in `src` when a workspace or a bundler
+   * that keeps module locations, such as Next.js Turbopack, runs the source). When this module has
+   * been bundled into a file of another name, there is no default and extraction fails with
+   * `reason: 'worker-unavailable'` without starting a worker; point this at a copy of
+   * `@yolk-sdk/extractors/node/extraction-worker` instead.
+   */
+  readonly workerUrl?: string | URL;
+};
+/**
+ * Where parsers run. `'worker'` (the default) and an options object run each PDF, DOCX, XLSX, and
+ * PPTX extraction in a fresh `worker_threads` worker with V8 heap, stack, and time limits.
+ * `'none'` runs parsers in the calling thread: only for environments without worker threads, and
+ * unsafe for untrusted input (a crafted file can exhaust the process heap or block the event loop).
+ */
+type FileExtractorIsolation = 'worker' | 'none' | WorkerIsolationOptions;
+/** Resolved worker settings; every value is a positive integer, and timers fit `setTimeout`. */
+declare const WorkerIsolationSettings: Schema.Struct<{
+  readonly maxOldGenerationSizeMb: Schema.Int;
+  readonly maxYoungGenerationSizeMb: Schema.Int;
+  readonly stackSizeMb: Schema.Int;
+  readonly timeoutMs: Schema.Int;
+  readonly maxConcurrentWorkers: Schema.Int;
+  readonly maxQueueWaitMs: Schema.Int;
+}>;
+type WorkerIsolationSettings = typeof WorkerIsolationSettings.Type;
+/**
+ * Defaults. 256 MB of old generation holds SheetJS's cell objects for the default limits (100,000
+ * visited cells, 50 MiB expanded) several times over and PDF.js's working set for ordinary PDFs,
+ * and the realm-wide pool of four workers keeps their V8 heaps near 1 GB together. That bounds
+ * heap, not the process: Buffers and PDF.js's decoded data live outside it (see the README).
+ * 32 MB of young generation is twice V8's usual 64-bit default, enough for short-lived parser
+ * strings. 30 s is far above legitimate parse times at the default limits (well under a second
+ * for the research workbooks) and below common serverless request budgets; a queued extraction
+ * waits at most as long again for a slot.
+ */
+declare const defaultWorkerIsolation: WorkerIsolationSettings;
+/**
+ * The built worker for an isolation module at `moduleUrl`: `dist/node/extraction-worker.mjs` of
+ * the same package, from `dist/node/extraction-isolation.mjs` (installed) or
+ * `src/node/extraction-isolation.ts` (workspace source, which must be built first). Any other
+ * location (this module bundled into a chunk) has no default: `undefined`, and nothing is started.
+ * The URL is derived from the module URL at runtime, never written as
+ * `new URL('./…', import.meta.url)`: bundlers rewrite that pattern into a copied asset.
+ */
+declare const extractionWorkerUrlFor: (moduleUrl: string) => URL | undefined;
+/** The default worker entry for this module; see `extractionWorkerUrlFor`. */
+declare const defaultExtractionWorkerUrl: () => URL | undefined;
+type WorkerExtractor = (input: FileInput, format: ParsedFileFormat, limits: FileExtractorLimits) => Effect.Effect<ExtractedFile, FileExtractorError>;
+/**
+ * Run each extraction in a fresh worker: the input is copied into a transferred buffer, the
+ * worker returns only text and metadata, and the worker is terminated when the result arrives,
+ * the timeout fires, or the caller is interrupted. Admission goes through this layer's share and
+ * the realm-wide pool (`worker-admission.ts`); a slot is freed only once its worker has
+ * terminated. A worker that cannot start, or a missing `workerUrl`, fails closed with
+ * `reason: 'worker-unavailable'`; there is no in-process fallback.
+ */
+declare const makeWorkerExtractor: (settings: WorkerIsolationSettings, workerUrl: string | URL | undefined) => Effect.Effect<WorkerExtractor>;
+//#endregion
+export { FileExtractorIsolation, WorkerExtractor, WorkerIsolationOptions, WorkerIsolationSettings, defaultExtractionWorkerUrl, defaultWorkerIsolation, extractionWorkerUrlFor, makeWorkerExtractor };
+//# sourceMappingURL=extraction-isolation.d.mts.map

package/dist/node/extraction-isolation.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-isolation.d.mts","names":[],"sources":["../../src/node/extraction-isolation.ts"],"mappings":";;;;;;;;;KAaY,sBAAA;EAAA,yEAED,sBAAA;WAEA,wBAAA,WAFA;EAAA,SAIA,WAAA;EAAA;;;;EAAA,SAKA,SAAA;EAuBqB;;AAAG;AASnC;;;EATgC,SAhBrB,oBAAA;EAyBoE;AAW/E;;;;;EAX+E,SAlBpE,cAAA;;;;;;;;;WASA,SAAA,YAAqB,GAAG;AAAA;;;;;;;KASvB,sBAAA,uBAA6C,sBAAsB;;cAWlE,uBAAA,EAAuB,MAAA,CAAA,MAAA;EAAA;;;;;;;KAWxB,uBAAA,UAAiC,uBAAA,CAAwB,IAAI;;AAAzE;;;;AAAyE;AAYzE;;;;cAAa,sBAAA,EAAwB,uBAOpC;AAaD;;;;AAA8D;AAM9D;;;AANA,cAAa,sBAAA,GAA0B,SAAA,aAAoB,GAAG;AAMvB;AAAA,cAA1B,0BAAA,QAA0B,GAAA;AAAA,KAiG3B,eAAA,IACV,KAAA,EAAO,SAAA,EACP,MAAA,EAAQ,gBAAA,EACR,MAAA,EAAQ,mBAAA,KACL,MAAA,CAAO,MAAA,CAAO,aAAA,EAAe,kBAAA;;;;;;;;;cAcrB,mBAAA,GACX,QAAA,EAAU,uBAAA,EACV,SAAA,WAAoB,GAAA,iBACnB,MAAA,CAAO,MAAA,CAAO,eAAA"}