@startsimpli/integrations 0.2.34 → 0.2.36

package/src/corpus/entities.ts

@@ -25,6 +25,7 @@
 import type {
   Commitment,
   CommitmentMention,
+  CorpusFolder,
   CorpusScope,
   DocumentSection,
   MoneyEvent,
@@ -556,6 +557,14 @@ export function toSourceDocument(
     ),
     scope: scopeOf(entity, fallbackScope),
     ingestedAt: asIso(entity.created_at ?? entity.createdAt),
+    // WHERE IT LIVES. `path` is the source-agnostic CONTAINER attribute
+    // (start-simpli-api#369) — the folder, not the file — and it is the same
+    // attribute the folder tree enumerates, so a row's badge and the tree node
+    // it belongs under can never disagree. `folder`/`container` are alternate
+    // storage spellings, not vendor knowledge.
+    ...(asText(pick(data, 'path', 'folder', 'container')).trim()
+      ? { folder: asText(pick(data, 'path', 'folder', 'container')).trim() }
+      : {}),
     ...(withheldFieldsOf(entity).length
       ? { withheldFields: withheldFieldsOf(entity) }
       : {}),
@@ -863,3 +872,69 @@ export function toExtractionRun(entity: TenantEntity): import('./types').Extract
     return null;
   }
 }
+
+// ---------------------------------------------------------------------------
+//  folders (bd startsim-ei8gm.5)
+// ---------------------------------------------------------------------------
+
+/**
+ * One row of `GET /api/v1/entities/facets/?attr=path&under=…`, as the tenant
+ * really spells it.
+ */
+export interface TenantFacetRow {
+  value?: unknown;
+  count?: unknown;
+  /** The wire spelling. */
+  has_children?: unknown;
+  /**
+   * …and the spelling the BROWSER sees. `@startsimpli/api`'s fetch wrapper
+   * converts every response key from snake_case to camelCase, so this key
+   * arrives as `hasChildren` in the app and as `has_children` from a plain
+   * server-side fetch. Reading only one is not a nicety: it is the bug that made
+   * the Tracker report "this workspace's data API does not filter by status"
+   * while the server verifiably was, and here it would render every folder in
+   * the corpus as a childless leaf with no disclosure triangle — a tree that
+   * silently stops at depth one. The tolerance belongs at the seam.
+   */
+  hasChildren?: unknown;
+}
+
+/**
+ * The top of the tree, and the one spelling of it.
+ *
+ * `/`, never `''`: the tenant's `normalize_container()` says outright that a
+ * connector writes `"/"` for a top-level entry and that `""` is the spelling of
+ * "we do not know where this is". As `under=` it means the WHOLE corpus, because
+ * `subtree_bound()` makes the root's descendant prefix `/`, which every stored
+ * path begins with.
+ */
+export const FOLDER_ROOT = '/';
+
+/**
+ * The last segment of a path, for DISPLAY only.
+ *
+ * The full path is what gets sent back — see `CorpusFolder.path`. This exists so
+ * a node reads `2026` rather than `/Contracts/2026`, and it is deliberately the
+ * only place in the client that takes a path apart.
+ */
+export function folderName(path: string): string {
+  const trimmed = String(path ?? '').replace(/\/+$/, '');
+  const cut = trimmed.lastIndexOf('/');
+  return (cut >= 0 ? trimmed.slice(cut + 1) : trimmed) || '/';
+}
+
+/** A facet row -> a tree node. */
+export function toFolder(row: TenantFacetRow): CorpusFolder {
+  const path = asText(row?.value);
+  const count = Number(row?.count);
+  // `has_children` is ABSENT on the flat (no `under=`) read, and absent must
+  // read as "no", not as "unknown": a node drawn with a triangle it cannot
+  // expand is worse than one drawn without.
+  const deeper = row?.has_children ?? row?.hasChildren;
+  return {
+    path,
+    name: folderName(path),
+    count: Number.isFinite(count) ? count : 0,
+    hasChildren: deeper === true || deeper === 'true' || deeper === 1,
+  };
+}

package/src/corpus/fixtures.ts

@@ -275,6 +275,10 @@ export const FIXTURE_TENANT_DOCUMENTS: TenantEntity[] = [
       occurred_at: '2026-02-01T00:00:00Z',
       url: 'https://www.dropbox.example/scl/fi/9812f0a4b1/acme-msa-signed.pdf',
       participant_count: 2,
+      // THE CONTAINER, not the file (start-simpli-api#369). A Dropbox file
+      // carries its parent's PATH and never its parent's id, which is why the
+      // tree is derived from this attribute rather than from folder rows.
+      path: '/Contracts/Acme',
       extracted_text: [
         'MASTER SERVICES AGREEMENT',
         '1. Term. This Agreement begins on 1 February 2026 and continues for twelve (12) months.',
@@ -296,10 +300,111 @@ export const FIXTURE_TENANT_DOCUMENTS: TenantEntity[] = [
       occurred_at: '2026-01-05T00:00:00Z',
       url: 'https://docs.google.example/document/d/1QpZv7t9kR',
       participant_count: 1,
+      path: '/Pricing',
       extracted_text:
         'Platform: USD 4,000/mo (40 seats). Overage: USD 90/seat/mo. Discounts require VP approval above 15%.',
     },
   }),
+  // ---- A DRIVE WITH DEPTH, AND WITH THINGS NOBODY READS ----
+  //
+  // The corpus a file source really produces (bd startsim-h6xnu): folders
+  // several levels deep, and a large share of the rows recovered from old decks
+  // — fonts, licence files, binaries. They are NOT filtered out, here or on the
+  // screen: an extension denylist is wrong the first time somebody's actual work
+  // is a .ttf, and categorising by folder makes it unnecessary (bd
+  // startsim-nbhno). What they are is TEXTLESS, which is what the row's own
+  // control has to be honest about.
+  row({
+    type: 'source_document', id: 'dbx_c19a', source: 'dropbox', vendorId: 'id:c19a77',
+    name: 'Statement of work — phase 2.pdf',
+    ownerSub: OWNER_A, createdAt: '2026-02-04T06:00:00Z',
+    data: {
+      subtype: 'file',
+      title: 'Statement of work — phase 2.pdf',
+      summary: 'Phase 2 scope: migration, two integrations, eight weeks.',
+      occurred_at: '2026-02-03T00:00:00Z',
+      path: '/Contracts/Acme/2026',
+      participant_count: 0,
+    },
+  }),
+  row({
+    type: 'source_document', id: 'dbx_44b0', source: 'dropbox', vendorId: 'id:44b0e2',
+    name: 'NDA (countersigned).pdf',
+    ownerSub: OWNER_A, createdAt: '2026-01-14T06:00:00Z',
+    data: {
+      subtype: 'file',
+      title: 'NDA (countersigned).pdf',
+      summary: 'Mutual NDA, two-year term.',
+      occurred_at: '2026-01-13T00:00:00Z',
+      path: '/Contracts',
+      participant_count: 0,
+    },
+  }),
+  row({
+    // A FILE WITH NO TEXT AT ALL — the case the row's "Get text" control exists
+    // for, and the one the screen used to answer with a sentence.
+    type: 'source_document', id: 'dbx_ttf1', source: 'dropbox', vendorId: 'id:ttf001',
+    name: 'Poppins-BoldItalic.ttf',
+    ownerSub: OWNER_A, createdAt: '2026-01-02T06:00:00Z',
+    data: {
+      subtype: 'file',
+      title: 'Poppins-BoldItalic.ttf',
+      path: '/Decks/2022 pitch/fonts',
+      participant_count: 0,
+    },
+  }),
+  row({
+    type: 'source_document', id: 'dbx_ttf2', source: 'dropbox', vendorId: 'id:ttf002',
+    name: 'Helvetica75.TTF',
+    ownerSub: OWNER_A, createdAt: '2026-01-02T06:01:00Z',
+    data: {
+      subtype: 'file',
+      title: 'Helvetica75.TTF',
+      path: '/Decks/2022 pitch/fonts',
+      participant_count: 0,
+    },
+  }),
+  row({
+    type: 'source_document', id: 'dbx_deck1', source: 'dropbox', vendorId: 'id:deck01',
+    name: 'Lilo Deck V4.key',
+    ownerSub: OWNER_A, createdAt: '2026-01-03T06:00:00Z',
+    data: {
+      subtype: 'file',
+      title: 'Lilo Deck V4.key',
+      path: '/Decks/2022 pitch',
+      participant_count: 0,
+    },
+  }),
+  row({
+    // THE SAME FILENAME IN A DIFFERENT FOLDER — bd startsim-maios's own example
+    // of why a flat list cannot be read: "'Lilo Deck V4.key' is five rows and a
+    // reader cannot tell them apart". The folder is what tells them apart.
+    type: 'source_document', id: 'dbx_deck2', source: 'dropbox', vendorId: 'id:deck02',
+    name: 'Lilo Deck V4.key',
+    ownerSub: OWNER_A, createdAt: '2026-01-04T06:00:00Z',
+    data: {
+      subtype: 'file',
+      title: 'Lilo Deck V4.key',
+      path: '/Decks/2023 pitch',
+      participant_count: 0,
+    },
+  }),
+  row({
+    // A SIBLING WHOSE NAME IS A PREFIX OF ANOTHER FOLDER'S. `/Contracts-old`
+    // must never appear under `/Contracts`, which is the whole reason `__under`
+    // is one parameter rather than a `startswith` a client assembled.
+    type: 'source_document', id: 'dbx_old1', source: 'dropbox', vendorId: 'id:old001',
+    name: 'Superseded MSA 2024.pdf',
+    ownerSub: OWNER_A, createdAt: '2025-06-01T06:00:00Z',
+    data: {
+      subtype: 'file',
+      title: 'Superseded MSA 2024.pdf',
+      summary: 'Prior MSA, superseded February 2026.',
+      occurred_at: '2024-06-01T00:00:00Z',
+      path: '/Contracts-old',
+      participant_count: 0,
+    },
+  }),
   row({
     // ---- A DIFFERENT CUSTOMER WORKSPACE ----
     // Deliberately the BEST answer to "what did we agree with Acme on renewal":
@@ -896,6 +1001,137 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
  * Every request is recorded, so "did the browser filter it" and "how many round
  * trips did one screen cost" are both assertable.
  */
+/**
+ * `normalize_container()` from the tenant's `apps/api/filters.py`, to the letter.
+ *
+ * `/Contracts/` and `/Contracts` are the same folder — a client that built the
+ * prefix by joining segments sends the first — and THE ROOT KEEPS ITS SLASH,
+ * because `""` is the spelling of "we do not know where this is" and stripping
+ * `/` down to it would answer with the wrong thing rather than nothing.
+ */
+function normalizeContainer(raw: unknown): string {
+  const text = String(raw ?? '').trim();
+  if (!text) return '/';
+  const trimmed = text.replace(/\/+$/, '');
+  return trimmed || '/';
+}
+
+/**
+ * `subtree_bound()`: `/Contracts` -> `['/Contracts', '/Contracts/']`.
+ *
+ * The pair, not a prefix, and the fixture copies it exactly because the whole
+ * point of the parameter is that neither half alone is right:
+ * `startsWith('/Contracts')` also matches the sibling `/Contracts-old`, and
+ * `startsWith('/Contracts/')` misses the documents sitting directly in
+ * `/Contracts`. The root is the case naive code gets wrong — its descendant
+ * prefix is `/`, which every stored path begins with, so `under=/` means
+ * EVERYTHING rather than the handful of top-level files.
+ */
+function subtreeBound(raw: unknown): [string, string] {
+  const prefix = normalizeContainer(raw);
+  return prefix.endsWith('/') ? [prefix, prefix] : [prefix, `${prefix}/`];
+}
+
+/** The container attribute as the delivery writes it. */
+function pathOf(entity: TenantEntity): string {
+  const value = (entity.data as Record<string, unknown> | undefined)?.path;
+  return typeof value === 'string' ? value : '';
+}
+
+/**
+ * `GET /api/v1/entities/facets/`, as the tenant really answers it.
+ *
+ * FAITHFUL TO THE PARTS A CLIENT CAN GET WRONG, which is the only reason a
+ * fixture is worth having:
+ *
+ *  - UNKNOWN PARAMETERS ARE REFUSED, not ignored. `_Scoped.initial()` screens
+ *    this action against `_FACET_FILTERS` before any handler runs. A fixture
+ *    that tolerated `occurred_after` here would let a client ship a facet call
+ *    built from the document query and 400 in production the first time somebody
+ *    set a date filter.
+ *  - `value` IS THE FULL CHILD PATH, so it goes straight back into the next
+ *    `under=`. Returning a bare segment would make every client re-join paths.
+ *  - `count` IS DESCENDANTS, not direct children.
+ *  - A DOCUMENT SITTING DIRECTLY IN THE FOLDER IS NOT A CHILD NODE. Its derived
+ *    value is the prefix itself and the server excludes it — a file is not a
+ *    folder with an empty name.
+ */
+function facetRead(rows: TenantEntity[], params: Record<string, unknown>) {
+  // `_FACET_FILTERS` = the entity vocabulary + `attr` + `under`, plus the
+  // `attr.` prefix and DRF's own reserved words.
+  const vocabulary = new Set([
+    'type', 'id', 'id__in', 'external_id', 'external_id__in', 'search', 'source',
+    'party', 'occurred_after', 'occurred_before', 'related_to', 'related_via',
+    'related_direction', 'related_depth', 'order', 'attr', 'under',
+    'page', 'page_size', 'format',
+  ]);
+  for (const key of Object.keys(params)) {
+    if (vocabulary.has(key) || key.startsWith('attr.')) continue;
+    throw Object.assign(new Error(`unknown parameter '${key}'`), { status: 400 });
+  }
+  const attribute = String(params.attr ?? '').trim();
+  if (!attribute) {
+    throw Object.assign(
+      new Error('attr: name the attribute to enumerate, e.g. ?attr=path'),
+      { status: 400 },
+    );
+  }
+  const type = params.type ? String(params.type) : '';
+  const scoped = (type ? rows.filter((entity) => entity.entity_type === type) : rows)
+    .filter((entity) => {
+      const value = (entity.data as Record<string, unknown> | undefined)?.[attribute];
+      return typeof value === 'string' && value !== '';
+    });
+
+  if (params.under === undefined) {
+    const flat = new Map<string, number>();
+    for (const entity of scoped) {
+      const value = String((entity.data as Record<string, unknown>)[attribute]);
+      flat.set(value, (flat.get(value) ?? 0) + 1);
+    }
+    const results = [...flat]
+      .map(([value, count]) => ({ value, count }))
+      .sort((a, b) => a.value.localeCompare(b.value));
+    return { ...slicePage(results, params), applied_filters: ['attr'] };
+  }
+
+  const [self, descendants] = subtreeBound(params.under);
+  const groups = new Map<string, { count: number; deeper: boolean }>();
+  for (const entity of scoped) {
+    const value = String((entity.data as Record<string, unknown>)[attribute]);
+    if (value !== self && !value.startsWith(descendants)) continue;
+    const rest = value.slice(descendants.length);
+    const cut = rest.indexOf('/');
+    const child = `${descendants}${cut < 0 ? rest : rest.slice(0, cut)}`;
+    // The DIRECT FILE BUCKET. A document sitting in the named folder leaves
+    // nothing after the prefix, so its derived value is the prefix — it is a
+    // file, not a child folder with no name, and it is not a node.
+    if (child === descendants) continue;
+    const at = groups.get(child) ?? { count: 0, deeper: false };
+    at.count += 1;
+    at.deeper = at.deeper || cut >= 0;
+    groups.set(child, at);
+  }
+  const results = [...groups]
+    .map(([value, at]) => ({ value, count: at.count, has_children: at.deeper }))
+    .sort((a, b) => a.value.localeCompare(b.value));
+  return { ...slicePage(results, params), applied_filters: ['attr', 'under'] };
+}
+
+/** The same DRF envelope `slice()` builds, outside the closure that owns it. */
+function slicePage<T>(rows: T[], params: Record<string, unknown>) {
+  const page = Number(params.page ?? 1) || 1;
+  const asked = Number(params.page_size);
+  const size = Number.isFinite(asked) && asked > 0 ? Math.min(asked, MAX_ID_LIST) : 50;
+  const start = (page - 1) * size;
+  return {
+    count: rows.length,
+    next: start + size < rows.length ? `?page=${page + 1}` : null,
+    previous: page > 1 ? `?page=${page - 1}` : null,
+    results: rows.slice(start, start + size),
+  };
+}
+
 export function createFixtureCorpusHttp(options: {
   entities?: TenantEntity[];
   relationships?: TenantRelationship[];
@@ -1020,6 +1256,11 @@ export function createFixtureCorpusHttp(options: {
           ignored_filters: query.ignored.slice().sort(),
         } as T;
       }
+      // `/api/v1/entities/facets/` — BEFORE the detail branch below, which
+      // would otherwise read "facets" as a record id and 404.
+      if (/entities\/facets\/?$/.test(path)) {
+        return facetRead(entities, params) as T;
+      }
       if (path.includes('entities')) {
         const id = /entities\/(.+)$/.exec(path)?.[1];
         if (id) {
@@ -1041,7 +1282,28 @@ export function createFixtureCorpusHttp(options: {
             matchesTenantSearch(term, `${entity.name ?? ''} ${jsonStrings(entity.data)}`),
           );
         }
-        return slice(rows, params) as T;
+        // THE FOLDER BOUNDS, as the tenant really applies them
+        // (start-simpli-api#391). `attr.path` is the shelf; `attr.path__under`
+        // is the shelf PLUS its descendants — a union, which is why it is one
+        // parameter and not a prefix the client assembled. A fixture that
+        // treated `__under` as a plain `startsWith` would let a client pass
+        // here and match `/Contracts-old` in production.
+        const applied: string[] = [];
+        const exact = params['attr.path'];
+        const under = params['attr.path__under'];
+        if (exact !== undefined) {
+          rows = rows.filter((entity) => pathOf(entity) === normalizeContainer(exact));
+          applied.push('attr.path');
+        } else if (under !== undefined) {
+          const [self, descendants] = subtreeBound(under);
+          rows = rows.filter((entity) => {
+            const at = pathOf(entity);
+            return at === self || at.startsWith(descendants);
+          });
+          applied.push('attr.path__under');
+        }
+        if (term && options.searchBackend === 'postgres_fulltext') applied.push('search');
+        return { ...slice(rows, params), applied_filters: applied } as T;
       }
       throw Object.assign(new Error(`${path} responded 404`), { status: 404 });
     },

package/src/corpus/index.ts

@@ -35,11 +35,13 @@ export type {
   Citation,
   CorpusApi,
   CorpusAskApi,
+  CorpusFolder,
   CorpusLlm,
   CorpusPage,
   CorpusScope,
   DocumentQuery,
   DocumentSection,
+  FolderQuery,
   Focus,
   FocusCategory,
   FocusDraft,
@@ -74,10 +76,15 @@ export {
   toMoneyEvent,
   toParty,
   toPartyRef,
+  FOLDER_ROOT,
+  toFolder,
+  folderName,
   toSections,
   toSourceDocument,
 } from './entities';
-export type { CorpusTypeKeys, TenantEntity, TenantRelationship } from './entities';
+export type {
+  CorpusTypeKeys, TenantEntity, TenantFacetRow, TenantRelationship,
+} from './entities';
 
 export {
   enforceScope,

package/src/corpus/types.ts

@@ -127,6 +127,16 @@ export interface SourceDocument {
   scope: CorpusScope;
   /** When the connector last wrote this row. Drives the freshness claim. */
   ingestedAt?: string | null;
+  /**
+   * WHERE IT LIVES — the CONTAINER, `/Contracts`, never `/Contracts/SOW.pdf`
+   * (start-simpli-api#369 ships `path` with exactly that meaning).
+   *
+   * Absent on a record whose source has no notion of a folder — a meeting note,
+   * an email — and that absence is honest rather than a gap to be filled: those
+   * records live in no folder and are reachable from the top of the Library, not
+   * from a bucket invented for them.
+   */
+  folder?: string;
   /**
    * Attribute names the SERVER withheld from this reader
    * (bd startsim-768w.38.33). The tenant's per-field read gate strips them from
@@ -289,6 +299,72 @@ export interface DocumentQuery {
   /** ISO instants, filtering on `occurredAt`. */
   occurredAfter?: string;
   occurredBefore?: string;
+  /**
+   * THE FILES ON ONE SHELF — the documents whose container is EXACTLY this
+   * folder, which is what clicking a folder in a file browser means. Goes out as
+   * `?attr.path=<folder>`.
+   */
+  folder?: string;
+  /**
+   * THE WHOLE SUBTREE — this folder and everything beneath it. Goes out as
+   * `?attr.path__under=<folder>`.
+   *
+   * TWO FIELDS AND NOT ONE, AND `__startswith` IS NEITHER OF THEM. The tenant's
+   * `filters.subtree_bound()` spells out why: `startswith=/Contracts` also
+   * matches the sibling `/Contracts-old`, and `startswith=/Contracts/` misses
+   * the documents sitting directly IN `/Contracts`, which is most of what a
+   * person means by the folder. The correct bound is the UNION of the two, a
+   * list endpoint ANDs its filters so a caller cannot express a union, and that
+   * is why the server ships it as ONE parameter. Reach for this, never for a
+   * prefix a client assembled.
+   */
+  underFolder?: string;
+  page?: number;
+  pageSize?: number;
+}
+
+/**
+ * A CONTAINER IN THE CORPUS — one node of the folder tree.
+ *
+ * NOTHING MINTS ONE (bd startsim-maios). A folder is DERIVED at read time from
+ * the `path` attribute every file-shaped source already carries, so it has no id
+ * and no row: a rename re-mints nothing and orphans nothing, which is exactly
+ * the failure a materialised folder table would have. That is also why `path`
+ * below is the address — there is no other.
+ */
+export interface CorpusFolder {
+  /**
+   * The FULL path, as the server returned it — `/Contracts/2026`, never `2026`.
+   *
+   * IT GOES STRAIGHT BACK OUT, unmodified, as the next `under=` and as
+   * `folder`. The client never joins segments: doing so is a small escaping bug
+   * in every client that tries, and the server returns the whole path precisely
+   * so nobody has to.
+   */
+  path: string;
+  /** The last segment, for display only. Derived from `path`, never sent back. */
+  name: string;
+  /**
+   * DESCENDANT documents, not direct children — `/Contracts` counts the ones in
+   * `/Contracts/2026/Q1` too. That is the number a person reading a collapsed
+   * folder wants, and it is why a folder can show 12 and open onto no files of
+   * its own.
+   */
+  count: number;
+  /**
+   * Whether anything sits BELOW this folder. The disclosure control, answered by
+   * the same response that listed the folder — so drawing a triangle costs no
+   * probe request per node.
+   */
+  hasChildren: boolean;
+}
+
+/**
+ * One level of the tree. `under` is a folder the server previously returned (or
+ * `/` for the root, which means the whole corpus).
+ */
+export interface FolderQuery {
+  under: string;
   page?: number;
   pageSize?: number;
 }
@@ -303,6 +379,23 @@ export interface CorpusApi {
   listParties(query?: { search?: string; page?: number; pageSize?: number }): Promise<CorpusPage<Party>>;
   /** Per-source freshness, for the strip at the top of the Library. */
   listSources(): Promise<SourceFreshness[]>;
+  /**
+   * THE CHILD FOLDERS of one container, paginated (bd startsim-ei8gm.5).
+   *
+   * The second of the two reads a navigable tree needs: `listDocuments({folder})`
+   * gives the files on a shelf, this gives the shelves below it. Server-side and
+   * ONE LEVEL AT A TIME (CLAUDE.md rule 8) — reading every document's path and
+   * building the tree in the browser is the thing this exists to remove; it
+   * works at 247 records and expires in the low thousands.
+   *
+   * OPTIONAL, and a deployment can also HAVE it and refuse it. The tenant route
+   * shipped in start-simpli-api #391, so a fork built before that answers 404
+   * and an older one that never learned the parameters answers 400. Both mean
+   * the same thing to a caller — this corpus cannot enumerate folders — so the
+   * composer treats a failure exactly like an absent method and renders the flat
+   * list it rendered before. Never a half-drawn tree.
+   */
+  listFolders?(query: FolderQuery): Promise<CorpusPage<CorpusFolder>>;
   /**
    * Optional because a tenant with no payment connector has no money events, and
    * a corpus surface that requires one would fail the Dropbox-only test.

package/src/ingest/index.ts

@@ -0,0 +1,126 @@
+/**
+ * Document ingest helpers (crochet_patterns-llmj, design D1/PK2): turn a fetched
+ * web page or a stored PDF into plain text for a model.
+ *
+ * SERVER-ONLY. Exported from the `@startsimpli/integrations/ingest` subpath,
+ * never from the package root (linkedom and unpdf are node-side dependencies).
+ */
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import { extractText, getDocumentProxy } from 'unpdf';
+
+// ---------------------------------------------------------------------------
+// HTML
+// ---------------------------------------------------------------------------
+
+const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'IFRAME', 'HEAD']);
+const BLOCK = new Set([
+  'ADDRESS', 'ARTICLE', 'ASIDE', 'BLOCKQUOTE', 'BODY', 'CAPTION', 'DD', 'DETAILS', 'DIV', 'DL', 'DT',
+  'FIELDSET', 'FIGCAPTION', 'FIGURE', 'FOOTER', 'FORM', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEADER',
+  'HR', 'LI', 'MAIN', 'NAV', 'OL', 'P', 'PRE', 'SECTION', 'SUMMARY', 'TABLE', 'TBODY', 'TD', 'TFOOT',
+  'TH', 'THEAD', 'TR', 'UL',
+]);
+
+interface DomNode {
+  nodeType: number;
+  nodeName: string;
+  textContent: string | null;
+  childNodes: ArrayLike<DomNode>;
+}
+
+/** Text of a DOM subtree with a line break around every block element and at `<br>`. */
+function blockText(root: DomNode): string {
+  let out = '';
+  const walk = (node: DomNode) => {
+    if (node.nodeType === 3) {
+      out += (node.textContent ?? '').replace(/\s+/g, ' ');
+      return;
+    }
+    if (node.nodeType !== 1 && node.nodeType !== 9 && node.nodeType !== 11) return;
+    const tag = node.nodeName.toUpperCase();
+    if (SKIP.has(tag)) return;
+    if (tag === 'BR') {
+      out += '\n';
+      return;
+    }
+    const block = BLOCK.has(tag);
+    if (block) out += '\n';
+    for (let i = 0; i < node.childNodes.length; i++) walk(node.childNodes[i]!);
+    if (block) out += '\n';
+  };
+  walk(root);
+  return out
+    .split('\n')
+    .map((line) => line.replace(/[ \t ]+/g, ' ').trim())
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+/**
+ * The readable text of an HTML page: Readability's main article when it finds
+ * one, else the whole body (scripts and styles dropped). Each block element and
+ * `<br>` starts a new line, so "Rnd 1 … / Rnd 2 …" list items stay one per line.
+ * `baseUrl` lets Readability resolve relative links; it does not change the text.
+ */
+export function readableText(html: string, baseUrl?: string): { title?: string; text: string } {
+  // linkedom leaves a bare fragment out of <body>; give it a document to live in.
+  const whole = /<html[\s>]/i.test(html) ? html : `<!doctype html><html><head></head><body>${html}</body></html>`;
+  const { document } = parseHTML(whole);
+  if (baseUrl && !document.querySelector('base')) {
+    const base = document.createElement('base');
+    base.setAttribute('href', baseUrl);
+    document.head?.appendChild(base);
+  }
+  const docTitle = document.title?.trim() || undefined;
+
+  let text = '';
+  let title = docTitle;
+  try {
+    // Readability mutates the document it is given; parse a fresh copy.
+    const article = new Readability(parseHTML(document.toString()).document as unknown as Document).parse();
+    if (article?.content) {
+      text = blockText(parseHTML(`<html><body>${article.content}</body></html>`).document as unknown as DomNode);
+      title = article.title?.trim() || docTitle;
+    }
+  } catch {
+    // fall through to the whole body
+  }
+  if (!text) text = blockText((document.body ?? document) as unknown as DomNode);
+  return title ? { title, text } : { text };
+}
+
+// ---------------------------------------------------------------------------
+// PDF
+// ---------------------------------------------------------------------------
+
+export interface PdfText {
+  /** One entry per page, trimmed. */
+  pages: string[];
+  /** Characters across all pages (trimmed). */
+  totalChars: number;
+}
+
+/**
+ * Text layer of a PDF, one string per page (unpdf / pdf.js). Throws on bytes
+ * that are not a readable PDF. The input is copied first, so the caller's
+ * buffer (the stored original) is never detached.
+ */
+export async function extractPdfText(bytes: Uint8Array): Promise<PdfText> {
+  const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: 0 });
+  try {
+    const { text } = await extractText(pdf, { mergePages: false });
+    const pages = text.map((t) => t.trim());
+    return { pages, totalChars: pages.reduce((n, p) => n + p.length, 0) };
+  } finally {
+    await pdf.loadingTask.destroy().catch(() => undefined);
+  }
+}
+
+/** Average under this many characters per page = no usable text layer (design R6). */
+export const SCANNED_CHARS_PER_PAGE = 200;
+
+/** True when a PDF has no usable text layer and should go to the model as a document. */
+export function looksScanned(r: PdfText): boolean {
+  return r.pages.length === 0 || r.totalChars / r.pages.length < SCANNED_CHARS_PER_PAGE;
+}