@yolk-sdk/extractors 0.1.0-canary.98

package/src/node/xlsx-text.ts

@@ -0,0 +1,288 @@
+import { Predicate } from 'effect'
+import { FileExtractionError } from '../errors.ts'
+import type { FileExtractorLimits } from '../limits.ts'
+import { makeHyperlinkLookup } from './xlsx-hyperlinks.ts'
+import type { HyperlinkLookup, XlsxHyperlinks } from './xlsx-hyperlinks.ts'
+import type { CellRange } from './xlsx-range.ts'
+import { cellCount, encodeCellAddress, parseCellRange } from './xlsx-range.ts'
+
+export type XlsxTextLimits = Pick<
+  FileExtractorLimits,
+  'maxXlsxCellVisits' | 'maxXlsxSheets' | 'maxXlsxTextCharacters'
+>
+
+/** The parsed workbook as SheetJS returns it; sheets and cells are read as own properties. */
+export type XlsxWorkbook = {
+  readonly SheetNames: ReadonlyArray<string>
+  readonly Sheets: object
+}
+
+export type XlsxTextOptions = {
+  readonly hyperlinks?: XlsxHyperlinks
+  /** Some hyperlinks were never read because the workbook exceeded `maxXlsxHyperlinks`. */
+  readonly hyperlinksTruncated?: boolean
+}
+
+const outputLimitReason = 'output limit'
+
+const hyperlinkLimitReason = 'hyperlink limit'
+
+const omittedHyperlinksMarker = (reasons: ReadonlyArray<string>) =>
+  `\n\n[Some hyperlinks omitted: ${reasons.join(' and ')}]`
+
+/**
+ * Space reserved for the marker whenever a workbook has hyperlinks, so it always fits.
+ * `maxXlsxTextCharacters` must exceed it (`minimumXlsxTextCharacters`).
+ */
+export const omittedHyperlinksMarkerReserve = omittedHyperlinksMarker([
+  outputLimitReason,
+  hyperlinkLimitReason
+]).length
+
+const invalidRange = () =>
+  new FileExtractionError({ format: 'xlsx', message: 'Invalid XLSX worksheet range.' })
+
+const tooLarge = () =>
+  new FileExtractionError({
+    format: 'xlsx',
+    message: 'XLSX exceeds the worksheet or cell-visit limit.'
+  })
+
+const outputTooLarge = () =>
+  new FileExtractionError({
+    format: 'xlsx',
+    message: 'XLSX extracted text exceeds the output limit.'
+  })
+
+/** Read an own property without walking the prototype chain (or trusting a polluted one). */
+const ownProperty = (target: object, key: string): unknown => {
+  const descriptor = Object.getOwnPropertyDescriptor(target, key)
+
+  if (descriptor === undefined) return undefined
+
+  return 'value' in descriptor ? descriptor.value : descriptor.get?.call(target)
+}
+
+const cellText = (cell: object): string => {
+  if (ownProperty(cell, 't') === 'z') return ''
+
+  const value = ownProperty(cell, 'v')
+
+  // SheetJS runs with `cellFormula: false`: only cached values exist, formula-only cells are empty.
+  if (value === undefined || value === null) return ''
+
+  // Prefer parser-provided display text. Do not run an untrusted format template here.
+  const display = ownProperty(cell, 'w')
+
+  if (Predicate.isString(display)) return display
+
+  if (Predicate.isString(value)) return value
+
+  if (Predicate.isBoolean(value)) return value ? 'TRUE' : 'FALSE'
+
+  if (value instanceof Date && Number.isFinite(value.getTime())) return value.toISOString()
+
+  if (Predicate.isNumber(value) && Number.isFinite(value)) return String(value)
+
+  throw new FileExtractionError({ format: 'xlsx', message: 'Invalid XLSX cell value.' })
+}
+
+/**
+ * CSV length of `text` (quotes doubled, wrapped when needed). The scan stops once the length
+ * exceeds `limit`; the returned length is then only known to be above it.
+ */
+const csvField = (text: string, limit = Number.POSITIVE_INFINITY) => {
+  let quoteCount = 0
+  let quote = text === 'ID'
+
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text.charCodeAt(index)
+
+    // '"', ',', '\n', '\r'
+    if (character === 34) quoteCount += 1
+
+    if (character === 34 || character === 44 || character === 10 || character === 13) {
+      quote = true
+
+      if (text.length + quoteCount + 2 > limit) break
+    }
+  }
+
+  return { quote, length: text.length + quoteCount + (quote ? 2 : 0) }
+}
+
+type VisitedSheet = {
+  readonly name: string
+  readonly sheet: object | undefined
+  readonly range: CellRange | undefined
+}
+
+/** Annotated text for a cell, or `undefined` to write it plain. */
+type CellDecorator = (input: {
+  readonly sheet: VisitedSheet
+  readonly row: number
+  readonly column: number
+  readonly text: string
+}) => string | undefined
+
+/** Write bounded CSV for every sheet; throws once `budget` characters would be exceeded. */
+const renderSheets = (
+  sheets: ReadonlyArray<VisitedSheet>,
+  budget: number,
+  decorate?: CellDecorator
+) => {
+  let characters = 0
+  const output: Array<string> = []
+
+  const append = (text: string) => {
+    if (text.length > budget - characters) throw outputTooLarge()
+
+    characters += text.length
+    output.push(text)
+  }
+
+  const appendCell = (text: string) => {
+    if (text.length > budget - characters) throw outputTooLarge()
+
+    const field = csvField(text, budget - characters)
+
+    if (field.length > budget - characters) throw outputTooLarge()
+
+    append(field.quote ? `"${text.replaceAll('"', '""')}"` : text)
+  }
+
+  for (const visited of sheets) {
+    const { name, sheet, range } = visited
+
+    if (sheet === undefined) continue
+
+    if (output.length > 0) append('\n\n')
+
+    append('# ')
+    append(name)
+    append('\n')
+
+    if (range === undefined) continue
+
+    for (let row = range.start.r; row <= range.end.r; row += 1) {
+      if (row > range.start.r) append('\n')
+
+      for (let column = range.start.c; column <= range.end.c; column += 1) {
+        if (column > range.start.c) append(',')
+
+        const cell = ownProperty(sheet, encodeCellAddress({ r: row, c: column }))
+
+        if (!Predicate.isObject(cell)) {
+          appendCell('')
+          continue
+        }
+
+        const text = cellText(cell)
+
+        appendCell(decorate?.({ sheet: visited, row, column, text }) ?? text)
+      }
+    }
+  }
+
+  return output.join('')
+}
+
+/**
+ * Validate every sheet before touching any cell, then generate bounded CSV incrementally, never
+ * with `sheet_to_csv` (which can walk billions of absent cells or allocate an unbounded quoted
+ * string).
+ *
+ * Existing cells inside an external hyperlink are written as `text <url>`. Plain text is
+ * rendered first, so links only use budget the plain text leaves over: in document order, an
+ * annotation that no longer fits leaves its cell plain and one marker ends the output.
+ */
+export const extractBoundedXlsxText = (
+  workbook: XlsxWorkbook,
+  limits: XlsxTextLimits,
+  options: XlsxTextOptions = {}
+) => {
+  if (workbook.SheetNames.length > limits.maxXlsxSheets) throw tooLarge()
+
+  let visits = 0
+
+  const sheets = workbook.SheetNames.map((name): VisitedSheet => {
+    const candidate = ownProperty(workbook.Sheets, name)
+    const sheet = Predicate.isObject(candidate) ? candidate : undefined
+    const ref = sheet === undefined ? undefined : ownProperty(sheet, '!ref')
+
+    if (ref !== undefined && !Predicate.isString(ref)) throw invalidRange()
+
+    const range = ref === undefined ? undefined : parseCellRange(ref)
+
+    if (ref !== undefined && range === undefined) throw invalidRange()
+
+    visits += range === undefined ? 0 : cellCount(range)
+
+    if (!Number.isSafeInteger(visits) || visits > limits.maxXlsxCellVisits) throw tooLarge()
+
+    return { name, sheet, range }
+  })
+
+  const links = options.hyperlinks ?? new Map()
+  const truncated = options.hyperlinksTruncated === true
+
+  if (links.size === 0 && !truncated) return renderSheets(sheets, limits.maxXlsxTextCharacters)
+
+  // Reserve the marker's space so it always fits inside the character limit.
+  const budget = limits.maxXlsxTextCharacters - omittedHyperlinksMarkerReserve
+  let spare = budget - renderSheets(sheets, budget).length
+  let dropped = false
+  const lookups = new Map<string, HyperlinkLookup>()
+
+  const lookupFor = (sheet: VisitedSheet) => {
+    const sheetLinks = links.get(sheet.name)
+
+    if (sheetLinks === undefined || sheet.range === undefined) return undefined
+
+    const existing = lookups.get(sheet.name)
+
+    if (existing !== undefined) return existing
+
+    const lookup = makeHyperlinkLookup(sheetLinks, sheet.range)
+    lookups.set(sheet.name, lookup)
+
+    return lookup
+  }
+
+  const output = renderSheets(sheets, budget, ({ sheet, row, column, text }) => {
+    const link = lookupFor(sheet)?.at(row, column)
+
+    if (link === undefined || link.target === text) return undefined
+
+    const label = text.length > 0 ? text : (link.display ?? '')
+    const plainLength = csvField(text).length
+    const annotatedLength = label.length + (label.length > 0 ? 3 : 2) + link.target.length
+
+    // Constant-time bound first (CSV escaping only adds), then a scan capped at the budget.
+    if (annotatedLength - plainLength > spare) {
+      dropped = true
+
+      return undefined
+    }
+
+    const annotated = label.length > 0 ? `${label} <${link.target}>` : `<${link.target}>`
+    const extra = csvField(annotated, plainLength + spare).length - plainLength
+
+    if (extra > spare) {
+      dropped = true
+
+      return undefined
+    }
+
+    spare -= extra
+
+    return annotated
+  })
+
+  const reasons = [
+    ...(dropped ? [outputLimitReason] : []),
+    ...(truncated ? [hyperlinkLimitReason] : [])
+  ]
+
+  return reasons.length === 0 ? output : output + omittedHyperlinksMarker(reasons)
+}

package/src/node/xlsx-workbook.ts

@@ -0,0 +1,133 @@
+import { Buffer } from 'node:buffer'
+import { FileExtractionError } from '../errors.ts'
+import {
+  sheetJsAttributeEscape,
+  sheetJsAttributeText,
+  sheetJsTags,
+  stripSheetJsNamespace,
+  sheetJsXmlHeader,
+  spreadsheetMainNamespace,
+  officeDocumentRelationshipsNamespace
+} from './sheetjs-xml.ts'
+import type { SheetJsTag } from './sheetjs-xml.ts'
+import { decodeXmlEntities, prefixedAttribute, rawXmlAttributes } from './xml-text.ts'
+
+/**
+ * `xl/workbook.xml` is never handed to SheetJS as uploaded. SheetJS's workbook parser slices and
+ * decodes the whole prefix of the part at every `</definedName>` (quadratic), and walks every
+ * `<sheet>` its own tag pattern finds, resolving each through its `r:id`, so one worksheet can be
+ * parsed once per declaration. The extractor reads the sheet list itself, checks that its strict
+ * scan and SheetJS's tag grammar see the same sheets, and writes a minimal workbook: the sheets in
+ * order (name, `sheetId`, hidden state, a fresh `r:id`) and the 1904 date system.
+ */
+
+export type WorkbookSheetDeclaration = {
+  /** The sheet name exactly as SheetJS reads it (`unescapexml(utf8read(name))`). */
+  readonly name: string
+  readonly sheetId: string
+  readonly state: 'hidden' | 'veryHidden' | undefined
+  /** The `r:id` of the sheet's relationship in the uploaded workbook, entity-decoded. */
+  readonly relationshipId: string | undefined
+}
+
+export type WorkbookModel = {
+  readonly sheets: ReadonlyArray<WorkbookSheetDeclaration>
+  readonly date1904: boolean
+}
+
+export const malformedWorkbookMessage = 'XLSX workbook is malformed.'
+
+const malformed = () =>
+  new FileExtractionError({ format: 'xlsx', message: malformedWorkbookMessage })
+
+const tooManySheets = () =>
+  new FileExtractionError({
+    format: 'xlsx',
+    message: 'XLSX exceeds the worksheet or cell-visit limit.'
+  })
+
+// `[^<>]` keeps each candidate inside one tag, so the strict scan stays linear.
+const strictSheetTag = /<(?:[\w.-]+:)?sheet(?=[\s/>])[^<>]*>/g
+
+const plainSheetId = /^[1-9]\d{0,8}$/
+
+/**
+ * The sheets of `xl/workbook.xml` and its date system. Throws `FileExtractionError` when the
+ * workbook declares more than `maxSheets` sheets (counted by either scan), when the strict scan
+ * and SheetJS's grammar disagree on the number or names of sheets, when a name is missing, holds
+ * CDATA, or repeats another ignoring case, or when there are no sheets.
+ */
+export const readWorkbookModel = (bytes: Uint8Array, maxSheets: number): WorkbookModel => {
+  // SheetJS parses the Latin-1 ("binary") view and decodes attribute text as UTF-8 afterwards.
+  const text = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('latin1')
+  const sheetJsSheets: Array<SheetJsTag> = []
+  let date1904 = false
+
+  for (const tag of sheetJsTags(text)) {
+    const head = stripSheetJsNamespace(tag.head)
+
+    if (head === '<sheet') {
+      sheetJsSheets.push(tag)
+
+      if (sheetJsSheets.length > maxSheets) throw tooManySheets()
+    } else if (head === '<workbookPr' || head === '<workbookPr/>') {
+      // Each `workbookPr` that carries the attribute overrides the last (SheetJS `parsexmlbool`).
+      const value = tag.attributes.get('date1904')
+
+      if (value !== undefined) date1904 = value === '1' || value === 'true'
+    }
+  }
+
+  const strictSheets: Array<ReadonlyMap<string, string>> = []
+
+  for (const [tag] of text.matchAll(strictSheetTag)) {
+    strictSheets.push(rawXmlAttributes(tag))
+
+    if (strictSheets.length > maxSheets) throw tooManySheets()
+  }
+
+  if (sheetJsSheets.length === 0 || sheetJsSheets.length !== strictSheets.length) throw malformed()
+
+  const seen = new Set<string>()
+
+  const sheets = sheetJsSheets.map((tag, index): WorkbookSheetDeclaration => {
+    const strict = strictSheets[index]
+    const rawName = tag.attributes.get('name')
+
+    if (strict === undefined || rawName === undefined || strict.get('name') !== rawName)
+      throw malformed()
+
+    const name = sheetJsAttributeText(rawName)
+
+    // Excel compares sheet names ignoring case.
+    if (name === undefined || seen.has(name.toLowerCase())) throw malformed()
+
+    seen.add(name.toLowerCase())
+
+    const state = tag.attributes.get('state')
+    const sheetId = tag.attributes.get('sheetId')
+    const relationshipId = prefixedAttribute(strict, 'id')
+
+    return {
+      name,
+      sheetId: sheetId !== undefined && plainSheetId.test(sheetId) ? sheetId : `${index + 1}`,
+      state: state === 'hidden' || state === 'veryHidden' ? state : undefined,
+      relationshipId: relationshipId === undefined ? undefined : decodeXmlEntities(relationshipId)
+    }
+  })
+
+  return { sheets, date1904 }
+}
+
+/** The generated workbook: sheet `n` has `r:id="rId<n>"`. */
+export const workbookXml = (model: WorkbookModel) =>
+  `${sheetJsXmlHeader}<workbook xmlns="${spreadsheetMainNamespace}" xmlns:r="${officeDocumentRelationshipsNamespace}">${
+    model.date1904 ? '<workbookPr date1904="1"/>' : ''
+  }<sheets>${model.sheets
+    .map(
+      (sheet, index) =>
+        `<sheet name="${sheetJsAttributeEscape(sheet.name)}" sheetId="${sheet.sheetId}"${
+          sheet.state === undefined ? '' : ` state="${sheet.state}"`
+        } r:id="rId${index + 1}"/>`
+    )
+    .join('')}</sheets></workbook>`

package/src/node/xml-text.ts

@@ -0,0 +1,77 @@
+const xmlEntity = /&([^;&\s]{1,16});/g
+
+const hexEntity = /^#x([0-9a-fA-F]+)$/
+
+const decimalEntity = /^#(\d+)$/
+
+const namedEntities: ReadonlyMap<string, string> = new Map([
+  ['amp', '&'],
+  ['lt', '<'],
+  ['gt', '>'],
+  ['quot', '"'],
+  ['apos', "'"]
+])
+
+const decodeCodePoint = (raw: string, codePointText: string, radix: number) => {
+  const codePoint = Number.parseInt(codePointText, radix)
+
+  if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) return raw
+
+  return String.fromCodePoint(codePoint)
+}
+
+const decodeXmlEntity = (raw: string, entity: string) => {
+  const named = namedEntities.get(entity)
+
+  if (named !== undefined) return named
+
+  const hex = hexEntity.exec(entity)?.[1]
+
+  if (hex !== undefined) return decodeCodePoint(raw, hex, 16)
+
+  const decimal = decimalEntity.exec(entity)?.[1]
+
+  if (decimal !== undefined) return decodeCodePoint(raw, decimal, 10)
+
+  return raw
+}
+
+/** Decode the predefined XML entities and numeric character references; keep unknown ones. */
+export const decodeXmlEntities = (text: string) =>
+  text.replace(xmlEntity, (raw, entity: string) => decodeXmlEntity(raw, entity))
+
+// A name may only start after a non-name character, so a long run of name characters is scanned
+// once rather than once per starting position.
+const attributePattern = /(?<![\w.:-])([\w.:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g
+
+/** Attributes of one start tag as written (not decoded), keyed by qualified name; first wins. */
+export const rawXmlAttributes = (tag: string): ReadonlyMap<string, string> => {
+  const attributes = new Map<string, string>()
+
+  for (const match of tag.matchAll(attributePattern)) {
+    const name = match[1]
+    const value = match[2] ?? match[3]
+
+    if (name !== undefined && value !== undefined && !attributes.has(name))
+      attributes.set(name, value)
+  }
+
+  return attributes
+}
+
+/** Attributes of one start tag, entity-decoded, keyed by their qualified name. */
+export const xmlAttributes = (tag: string): ReadonlyMap<string, string> =>
+  new Map(
+    Array.from(rawXmlAttributes(tag), ([name, value]) => [name, decodeXmlEntities(value)] as const)
+  )
+
+/** The value of a namespace-prefixed attribute such as `r:id`, whatever the prefix. */
+export const prefixedAttribute = (attributes: ReadonlyMap<string, string>, localName: string) => {
+  for (const [name, value] of attributes) {
+    const separator = name.indexOf(':')
+
+    if (separator > 0 && name.slice(separator + 1) === localName) return value
+  }
+
+  return undefined
+}

package/src/sanitize.ts

@@ -0,0 +1,18 @@
+const nonPrintableCharacters = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g
+
+const longDotRuns = /\.{4,}/g
+
+const horizontalWhitespaceRuns = /[\t ]{2,}/g
+
+const blankLineRuns = /\n{3,}/g
+
+/** Normalize line endings, drop control characters, and collapse layout noise. */
+export const sanitizeExtractedText = (text: string) =>
+  text
+    .replaceAll('\r\n', '\n')
+    .replaceAll('\r', '\n')
+    .replace(nonPrintableCharacters, '')
+    .replace(longDotRuns, '…')
+    .replace(horizontalWhitespaceRuns, ' ')
+    .replace(blankLineRuns, '\n\n')
+    .trim()

package/src/service.ts

@@ -0,0 +1,21 @@
+import { Context } from 'effect'
+import type { Effect } from 'effect'
+import type { FileExtractorError } from './errors.ts'
+import type { ExtractedFile, FileInput } from './format.ts'
+
+export type FileExtractorApi = {
+  /**
+   * Extract sanitized text from one file. Fails with `UnsupportedFileFormatError` for unknown
+   * formats, `FileExtractionError` for unreadable, oversized, or empty files, and
+   * `SheetJsUnavailableError` when an XLSX file arrives but SheetJS 0.20.3+ is not installed.
+   */
+  readonly extract: (input: FileInput) => Effect.Effect<ExtractedFile, FileExtractorError>
+}
+
+/**
+ * The file extractor service. The tag is runtime-portable; the Node implementation lives in
+ * `@yolk-sdk/extractors/node`.
+ */
+export class FileExtractor extends Context.Service<FileExtractor, FileExtractorApi>()(
+  '@yolk-sdk/extractors/FileExtractor'
+) {}