@khorsheed/dsh-reader 0.2.0

package/lib/types/service.d.ts

@@ -0,0 +1,633 @@
+/**
+ * The reader service core: owns the fetch, the state document, and the daily
+ * refresh. Everything the browser can ask for is implemented here; the Remote
+ * class next door is a thin adapter with no logic of its own.
+ *
+ * Two seams shape the whole module:
+ *
+ * - **Fetch goes through `ctx.web.fetch`** — the host's sanctioned outbound
+ *   HTTP face, which brings public-address resolution, connection pinning, a
+ *   byte cap and a timeout. Its provider caps the decoded body at 100 000
+ *   characters by default, so a payload that comes back `truncated` is stored
+ *   as-is and FLAGGED; nothing here pretends it is a whole document. A
+ *   truncated feed cannot be parsed and a truncated page yields half an
+ *   article, so the browser half reports "incomplete" rather than guessing.
+ * - **The provider follows same-origin redirects only** — a cross-origin hop is
+ *   refused with `WEB_REDIRECT_BLOCKED` by design (each new origin must be
+ *   validated afresh), and the refusal names the target ORIGIN alone, never the
+ *   path it was headed for. So the service re-enters the seam only for a hop it
+ *   can reconstruct faithfully — the same host under a different scheme or port
+ *   (`http://host/feed` → `https://host/feed`, the common upgrade). A hop that
+ *   changes the HOST is reported as such instead of being followed to the bare
+ *   origin, which would fetch some other page and file it under the reader's
+ *   URL. The final URL becomes the source's canonical URL.
+ * - **A failed fetch is still a saved link.** A URL the reader pasted is one
+ *   they own; the classification below records WHY there is no readable body
+ *   (bot wall, login wall, non-web file, redirect to another site, empty page,
+ *   transport error) instead of discarding the URL, and only the transport
+ *   failure is retried automatically. Losing the link is worse than a card
+ *   that says it can only be opened in a browser.
+ *
+ * With no filesystem mounted the service stays fully functional in memory
+ * (sources still add, refresh and search) and simply does not survive a
+ * restart — a composition must never fail because a deployment has no fs.
+ *
+ * @module @khorsheed/dsh-reader/service
+ */
+import type { Context } from '@deepseek-ai/cordis';
+import { type ReaderAddFailure, type ReaderAddOutcome, type ReaderAddRefusal, type ReaderAnnotationOutcome, type ReaderBackfillCandidate, type ReaderEntryBodyView, type ReaderBody, type ReaderCapabilities, type ReaderMutationOutcome, type ReaderEntryFetchState, type ReaderEntryTranslationView, type ReaderPreviewFailure, type ReaderPreviewFailureCode, type ReaderRecentEntry, type ReaderRefreshResult, type ReaderSentenceLearn, type ReaderSourceSummary, type ReaderStorageStats, type ReaderTag } from './types.ts';
+/**
+ * How many entries one automatic backfill run may fetch.
+ *
+ * A wall of 300 summary-only entries must not turn one refresh into 300
+ * requests to other people's servers; the run takes a slice and the next
+ * refresh takes the next one.
+ */
+export declare const BACKFILL_MAX_PER_RUN = 8;
+/** How long a FAILED fetch is left alone before it may be retried (6 hours). */
+export declare const BACKFILL_RETRY_MS: number;
+/**
+ * Thrown when the FETCH itself failed — a refused connection, a timeout, a
+ * response over the seam's byte cap. Carries the seam's own message: the
+ * difference between "the host has no egress" and "that host timed out" is the
+ * whole diagnosis, and a bare "failed" tells the reader neither.
+ */
+export declare class FetchFailure extends Error {
+    /**
+     * @param message - the seam's message, verbatim.
+     * @param cause - the original error, when there was one.
+     */
+    constructor(message: string, cause?: unknown);
+}
+/**
+ * Classify a fetched payload as a feed or an article page.
+ *
+ * The decision comes from the CONTENT, not from asking the user: nobody pasting
+ * a URL can be expected to know whether it is a feed, and guessing wrong used
+ * to mean silently creating an empty subscription.
+ *
+ * @param raw - the decoded payload.
+ * @returns which kind of source this payload can become.
+ */
+export declare function classifyPayload(raw: string): 'feed' | 'page';
+/** The reader service. */
+export declare class ReaderService {
+    private readonly store;
+    private readonly ctx;
+    private timer;
+    private disposed;
+    /** The document of record when no filesystem is mounted. */
+    private memoryDoc;
+    /**
+     * @param ctx - owning context.
+     * @param config - optional state-root override.
+     */
+    constructor(ctx: Context, config?: {
+        stateRoot?: string;
+    });
+    /** Capabilities the browser may rely on. */
+    capabilities(): Promise<ReaderCapabilities>;
+    /** The source list, without payloads. */
+    listSources(): Promise<{
+        sources: ReaderSourceSummary[];
+    }>;
+    /**
+     * Add a source, deciding what it is from what comes back.
+     *
+     * The one rule this method exists to enforce: **a URL the reader pasted is
+     * never thrown away.** A fetch that fails, a bot challenge, a login wall and
+     * a PDF all end the same way — a `link` source that exists, carries the
+     * reason it has no previewable body, and can be opened, tagged, refreshed
+     * manually or deleted. Only a URL that cannot be a source at all
+     * (`invalid-url`) or one already present (`duplicate`) is refused.
+     *
+     * @param request - the pasted URL and an optional label.
+     * @returns the decision, or a domain refusal.
+     */
+    addSource(request: {
+        url: string;
+        label?: string;
+    }): Promise<ReaderAddOutcome | ReaderAddRefusal | ReaderAddFailure>;
+    /**
+     * Persist a link the reader owns even though its body cannot be previewed.
+     *
+     * Two records, because two consumers ask different questions: the SOURCE
+     * carries the failure so the wall can mark the card and the backfill can
+     * leave a permanent wall alone, while the ENTRY annotation carries it so
+     * `getEntryBody` — the one call the detail view makes — explains itself
+     * instead of answering with a blank article.
+     *
+     * @param request - the final URL, an optional label, and the pasted URL when
+     *   this link was resolved to a canonical version.
+     * @param failure - the classified reason, with its timestamp.
+     * @param now - the commit instant.
+     * @returns the add outcome the browser reports.
+     */
+    private saveLinkOnly;
+    /**
+     * Update a source's mutable fields, or the global refresh time.
+     *
+     * @param request - the id plus the fields to change.
+     * @returns the outcome, or a domain refusal.
+     */
+    updateSource(request: {
+        id: string;
+        enabled?: boolean;
+        label?: string;
+        url?: string;
+        timeOfDay?: string;
+    }): Promise<ReaderMutationOutcome>;
+    /**
+     * Remove a source.
+     *
+     * Its entries' TRANSLATIONS go with it: the entries themselves are unreachable
+     * once the source is gone (the recent page disables them), so their exact-fit
+     * maps are dead weight. The host never parses feeds, so the caller passes the
+     * entry ids it has parsed; a link source's single entry id is derivable here
+     * regardless. The cached BODIES stay — the body budget owns their lifetime,
+     * unchanged.
+     *
+     * @param request - the source id, plus the entries the caller knows belong to it.
+     * @returns the outcome, or a domain refusal.
+     */
+    removeSource(request: {
+        id: string;
+        entryIds?: readonly string[];
+    }): Promise<ReaderMutationOutcome>;
+    /**
+     * Refresh the enabled sources, or the named ones.
+     *
+     * @param request - optional id list.
+     * @returns one result per attempted source.
+     */
+    refresh(request?: {
+        ids?: string[];
+    }): Promise<{
+        results: ReaderRefreshResult[];
+    }>;
+    /**
+     * Hand back raw payloads for the browser to parse.
+     *
+     * @param request - the source ids whose payloads are wanted.
+     * @returns one body per requested id; an error instead of a payload when
+     *   nothing usable is stored.
+     */
+    getBodies(request: {
+        ids: string[];
+    }): Promise<{
+        bodies: ReaderBody[];
+    }>;
+    /**
+     * Pass a ref block to the side-chat service when one is composed.
+     *
+     * @param request - the target context, its label, and the ref text.
+     * @returns `'ok'`, or `'unavailable'` when no side chat is mounted.
+     */
+    quoteToSideChat(request: {
+        contextKey: string;
+        label: string;
+        text: string;
+    }): Promise<'ok' | 'unavailable'>;
+    /**
+     * The body the detail view should render for one entry, if any.
+     *
+     * The order is the whole contract: a fresh cached fetch wins (it is full text
+     * the reader already paid for), then the feed's own payload, and only when
+     * neither exists does the view offer to fetch. A STALE cache entry reports
+     * `fresh: false` and no html — the detail view then offers the fetch again
+     * rather than quietly serving yesterday's copy.
+     *
+     * @param request - the entry and the payload the feed gave for it.
+     * @returns what there is to show, and whether a fetch is worth offering.
+     */
+    getEntryBody(request: {
+        entryId: string;
+        url: string;
+        feedHtml?: string;
+    }): Promise<ReaderEntryBodyView>;
+    /**
+     * Fetch one entry's article and hand the raw HTML to the browser.
+     *
+     * The HOST does not extract: extraction needs a DOM, and this runtime has no
+     * XML/HTML parser at all (the same reason feed parsing lives in the browser
+     * half). So this verb is the network half only — it returns what came back,
+     * and `storeEntryBody` persists what the browser made of it.
+     *
+     * Fetching here is deliberately NOT automatic on refresh: doing it for every
+     * summary-only entry would make this a crawler of other people's sites, which
+     * the design refuses. One entry, one fetch, on the reader's request.
+     *
+     * @param request - the entry and its article URL.
+     * @returns the raw payload, or why there is none.
+     */
+    fetchEntryBody(request: {
+        entryId: string;
+        url: string;
+    }): Promise<{
+        entryId: string;
+        url?: string;
+        raw?: string;
+        truncated?: boolean;
+        error?: string;
+    }>;
+    /**
+     * Which entries have a body, a stored raw payload, or a recorded failure.
+     *
+     * The wall needs this per ENTRY (entries come from the browser's parse of a
+     * feed), so it cannot ride the source summaries. `raw` means "fetched, not yet
+     * extracted" — the browser half turns those into bodies on its next load.
+     *
+     * @param request - the entry ids the caller is showing.
+     * @returns one state per requested id.
+     */
+    entryFetchStates(request: {
+        entryIds: readonly string[];
+    }): Promise<{
+        states: Record<string, ReaderEntryFetchState>;
+    }>;
+    /**
+     * Hand back a stored raw payload so the browser can extract it.
+     *
+     * @param request - the entry whose payload is wanted.
+     * @returns the payload, or why there is none.
+     */
+    getRawBody(request: {
+        entryId: string;
+    }): Promise<{
+        entryId: string;
+        raw?: string;
+        url?: string;
+        truncated?: boolean;
+        error?: string;
+    }>;
+    /** Record that a fetch is in flight, so a wall reopened later can say so. */
+    private markFetching;
+    /**
+     * Store one raw payload for later extraction.
+     *
+     * @param entryId - the entry it belongs to.
+     * @param raw - the payload.
+     * @returns the file name, or `undefined` when the deployment has no fs.
+     */
+    private storeRaw;
+    /**
+     * Cache the markup the browser extracted for one entry.
+     *
+     * The deadline is computed here, not on the client, so the policy has exactly
+     * one owner (this service's document) — a client that guessed the TTL would
+     * be a second implementation of it.
+     *
+     * @param request - the entry, the markup, and the URL it came from.
+     * @returns the stored view.
+     */
+    storeEntryBody(request: {
+        entryId: string;
+        url: string;
+        html: string;
+        truncated?: boolean;
+        scriptFigures?: number;
+        /** True when this body came from the capture rendered fetch — 「重新抓取」 re-renders it. */
+        rendered?: boolean;
+        /** The caller's hash of `html`; the entry's translation map dies with a body it no longer matches. */
+        bodyHash?: string;
+        /** The article's own title, as the extraction read it — a saved link's card upgrade. */
+        title?: string;
+        /** A short excerpt (abstract / first paragraph), with the title. */
+        excerpt?: string;
+    }): Promise<ReaderEntryBodyView>;
+    /**
+     * Which entries still need their full text, and which may be retried.
+     *
+     * The host is the only side that can answer this: it holds the cache, the
+     * recorded failures and the policy. A failure is retried only after
+     * `BACKFILL_RETRY_MS` — an entry whose publisher refuses us must not be
+     * hammered once per refresh, which would make this a crawler with a grudge.
+     *
+     * @param request - the entries the caller is looking at (id, url, label) and
+     *   whether the caller already has full text for them (the feed's own payload).
+     * @returns the candidates, capped so one refresh cannot start a stampede.
+     */
+    listBackfillCandidates(request: {
+        entries: readonly {
+            entryId: string;
+            url: string;
+            label: string;
+            hasBody: boolean;
+        }[];
+        limit?: number;
+    }): Promise<{
+        candidates: ReaderBackfillCandidate[];
+    }>;
+    /**
+     * One entry's exact-fit translation record, with its segment map resolved.
+     *
+     * A record from an older storage schema, or one whose sidecar file is gone,
+     * reads as absent — the caller falls back to the global memory, and the stale
+     * record is carried out by the next write or budget pass (lazy, never wiped).
+     *
+     * @param request - the entry.
+     * @returns the record, or nothing usable.
+     */
+    getEntryTranslation(request: {
+        entryId: string;
+    }): Promise<{
+        translation?: ReaderEntryTranslationView;
+    }>;
+    /**
+     * The translations for exactly the sentences asked about, of one pair.
+     *
+     * The client hashes the body's sentences itself, so the slice is precise and
+     * the whole table never crosses the wire. Reads do not rewrite the file:
+     * the LRU stamp moves when the sentence next serves (the client's batched
+     * report of what it reused), not when a list of hashes arrives.
+     *
+     * @param request - the pair label and the sentence hashes wanted.
+     * @returns hash → translation for the hits.
+     */
+    getSentenceTranslations(request: {
+        pair: string;
+        hashes: readonly string[];
+    }): Promise<{
+        translations: Record<string, string>;
+    }>;
+    /**
+     * Persist what one translation run produced, in one batch.
+     *
+     * `entries` are the sentences the model just translated: they upsert into the
+     * global memory AND the entry's exact-fit map. `recalled` are the sentences
+     * the run was served from memory: their lastUsedAt bumps (that is what makes
+     * the budget's LRU a use-clock rather than a write-clock), and they join the
+     * entry map too, so the map stays the complete record of this body. When
+     * `entryId` + `bodyHash` are present the entry map is REWRITTEN to the run's
+     * full sentence set — the run saw the whole body, so a merge would only
+     * preserve sentences the current body no longer has.
+     *
+     * @param request - the pair, the learned and recalled sentences, and the
+     *   entry identity when this run was an article's.
+     * @returns how many sentences were newly learned.
+     */
+    rememberSentences(request: {
+        pair: string;
+        entries: readonly ReaderSentenceLearn[];
+        recalled?: readonly ReaderSentenceLearn[];
+        entryId?: string;
+        bodyHash?: string;
+    }): Promise<{
+        stored: number;
+    }>;
+    /**
+     * How much the two caches currently hold, aggregated on the host.
+     *
+     * The tables themselves never cross the wire for a settings readout: counts
+     * and characters only. The characters are the store's own accounting unit —
+     * the same numbers the budgets bound.
+     *
+     * @returns per-tier usage.
+     */
+    getStorageStats(): Promise<ReaderStorageStats>;
+    /**
+     * Forget every translation: the global sentence memory AND every entry map.
+     *
+     * The manage page's one gesture for it, after its own confirm. Bodies, tags,
+     * the recent list and the sources are untouched — the reader is clearing a
+     * derived cache, not their library. The commit's own `pruneBodies` sweep
+     * takes the files once the document no longer names them.
+     *
+     * @returns how many entry maps went, and whether the memory table went.
+     */
+    clearTranslations(): Promise<{
+        clearedEntries: number;
+        clearedMemory: boolean;
+    }>;
+    /**
+     * The global sentence memory's table, from its sidecar file.
+     *
+     * Read per call, not cached: the file is the truth, and the service's own
+     * commits are the only writes — a cache keyed on the manifest would save a
+     * few MB of JSON parsing per translation run at the price of a coherence
+     * story. An absent, unreadable, or stale-version file reads as empty (the
+     * lazy half of the schema version).
+     */
+    private translationMemory;
+    /** The tag vocabulary plus which entry ids carry each tag. */
+    listTags(): Promise<{
+        tags: ReaderTag[];
+        counts: Record<string, number>;
+    }>;
+    /**
+     * Create a tag, or return the existing one with the same name.
+     *
+     * Names are matched case-insensitively on purpose: "AI" and "ai" becoming two
+     * tags is what makes a tag system useless for filtering, and the reader
+     * cannot be expected to remember which spelling they used.
+     *
+     * @param request - the name to create or find.
+     * @returns the tag id, or a refusal.
+     */
+    createTag(request: {
+        name: string;
+    }): Promise<ReaderTag | ReaderAnnotationOutcome>;
+    /** Rename a tag (its id, and therefore every entry that carries it, is untouched). */
+    renameTag(request: {
+        id: string;
+        name: string;
+    }): Promise<ReaderAnnotationOutcome>;
+    /** Drop a tag from the vocabulary and from every entry that carried it. */
+    deleteTag(request: {
+        id: string;
+    }): Promise<ReaderAnnotationOutcome>;
+    /** Add or remove one tag on one entry. */
+    tagEntry(request: {
+        entryId: string;
+        tagId: string;
+        on: boolean;
+    }): Promise<ReaderAnnotationOutcome>;
+    /** The tags on one entry. */
+    entryTags(request: {
+        entryId: string;
+    }): Promise<{
+        tags: ReaderTag[];
+    }>;
+    /** How long a fetched body is served, and how large the translation store may grow. */
+    getCachePolicy(): Promise<{
+        ttlHours: number;
+        maxEntries: number;
+        translationBudgetChars: number;
+    }>;
+    /** Change the cache policy (`ttlHours: 0` = keep until the budget evicts). */
+    setCachePolicy(request: {
+        ttlHours: number;
+        maxEntries?: number;
+        translationBudgetChars?: number;
+    }): Promise<ReaderAnnotationOutcome>;
+    /** Drop tags no entry references any more. */
+    pruneTags(): Promise<{
+        removed: number;
+    }>;
+    /**
+     * Record that the reader opened one entry.
+     *
+     * One row per ENTRY (not per source): the question the recent page answers is
+     * about the articles that were read, and an RSS entry has no source-level
+     * page to point at. Reopening an entry moves it to the top rather than
+     * duplicating it, and the list is capped — see `MAX_RECENT_ENTRIES`.
+     *
+     * The write is deliberately not batched: opening is a human-paced gesture,
+     * and a queue that had not flushed yet would lose the item on a restart.
+     *
+     * @param request - the entry, its source, and what to call it in the list.
+     * @returns how many entries the list holds afterwards.
+     */
+    recordRead(request: {
+        entryId: string;
+        sourceId: string;
+        title: string;
+        url?: string;
+    }): Promise<{
+        entries: number;
+    }>;
+    /**
+     * The entries the reader opened, newest first.
+     *
+     * @returns the recent list as the page renders it.
+     */
+    listRecent(): Promise<{
+        entries: ReaderRecentEntry[];
+    }>;
+    /**
+     * Forget every recent entry.
+     *
+     * The reader's own list is theirs to erase: a "recently read" page that can
+     * only grow is one nobody wants to open in company.
+     *
+     * @returns how many entries were dropped.
+     */
+    clearRecent(): Promise<{
+        removed: number;
+    }>;
+    /**
+     * Record why a fetch produced nothing, so the retry is a decision.
+     *
+     * The failure also carries its classified CODE when there is one: the wall's
+     * card button explains a red state with the same sentence the detail view
+     * uses, instead of the seam's raw words.
+     *
+     * @param entryId - the entry.
+     * @param error - the message to keep, verbatim.
+     * @param code - the classified reason, when one was derived.
+     */
+    private recordFetchFailure;
+    /**
+     * Ask the link-resolver table what a pasted URL names.
+     *
+     * Runs on the ADD path only, and never breaks it: the table's own failure
+     * (a resolver bug, an answer shape nobody planned for) reads as "no answer"
+     * and the ordinary fetch path takes over. Resolvers that need the network
+     * ride the same sanctioned seam as every fetch here; with no seam mounted
+     * their fetch throws, they answer `null`, and the ordinary path reports the
+     * missing capability — while a resolver that needs no network (OpenReview's
+     * known-unreadable verdict) still answers.
+     */
+    private resolvePaper;
+    /** The probed web seam (no host import, no boot-time dependency). */
+    private web;
+    /** The probed side-chat service. */
+    private sideChat;
+    /**
+     * The document of record: from disk when one is mounted, else the in-memory
+     * copy the service has been accumulating.
+     */
+    private currentDoc;
+    /**
+     * Apply a mutation to the document of record.
+     *
+     * Without a filesystem the mutation lands in memory so the session still
+     * works; with one it goes through the store's version guard.
+     *
+     * @param mutate - pure function from the current document to the next.
+     */
+    private commit;
+    /**
+     * Fetch a URL, following the cross-origin hops that can be reconstructed.
+     *
+     * The seam refuses a cross-origin hop and names only the target ORIGIN, so
+     * there is exactly one hop this method can follow without inventing a URL:
+     * a same-host scheme/port change, which keeps the path and query
+     * (`http://host/feed` → `https://host/feed`). A hop to a DIFFERENT host is
+     * re-thrown untouched — the caller classifies it, and the reader is told the
+     * address moves to another site rather than being served some other page
+     * under the URL they pasted.
+     *
+     * @param url - the initial URL.
+     * @returns the final URL and its decoded payload.
+     */
+    private fetchFollowing;
+    /** Arm (or re-arm) the daily refresh timer against the current document. */
+    private armSchedule;
+}
+/** Parse and normalize a pasted URL; reject anything that is not http(s). */
+export declare function normalizeUrl(input: string): string | undefined;
+/**
+ * The next URL a cross-origin refusal can be followed to — or `undefined`.
+ *
+ * The host seams throw a `WEB_REDIRECT_BLOCKED` error whose CODE is a property
+ * and whose message names only the target origin. The original implementation
+ * tested the message for the code text, which it never contains, so the whole
+ * follow path was dead code and every cross-origin redirect surfaced as a raw
+ * seam error. This reads the code where it actually lives.
+ *
+ * @param error - the value the seam threw.
+ * @param current - the URL whose request was refused.
+ * @returns the same path and query on the new origin, when the HOST is
+ *   unchanged (a scheme or port upgrade); otherwise `undefined`.
+ */
+export declare function crossOriginRetarget(error: unknown, current: string): string | undefined;
+/**
+ * The smallest amount of visible text that still counts as a readable page.
+ *
+ * Deliberately small: this only runs on a freshly fetched page, and its job is
+ * to catch interstitials — a bot challenge, a cookie notice, an empty shell —
+ * not to judge whether an article is long enough to be worth reading.
+ */
+export declare const MIN_PREVIEW_TEXT_CHARS = 200;
+/**
+ * Whether a fetched page can be previewed at all, and why not when it cannot.
+ *
+ * The host has no DOM, so this is a deliberately crude reading of the payload:
+ * strip script/style/tags, count what is left, and only when that is
+ * vanishingly small look for the words a wall leaves behind. A page with real
+ * text is never second-guessed.
+ *
+ * @param raw - the decoded payload.
+ * @param finalUrl - where the fetch actually landed.
+ * @param requestedUrl - the URL the reader pasted.
+ * @returns the reason when there is nothing to preview, else `undefined`.
+ */
+export declare function inspectPreview(raw: string, finalUrl: string, requestedUrl: string): {
+    code: ReaderPreviewFailureCode;
+    message: string;
+} | undefined;
+/**
+ * Classify a thrown fetch failure into the reason the reader sees.
+ *
+ * Everything here is per-case diagnosable: the seam's own message is kept
+ * verbatim, and only the CODE decides which sentence the browser shows and
+ * whether an automatic retry is allowed.
+ *
+ * @param error - the value the fetch threw.
+ * @param at - the failure instant (ISO-8601).
+ * @returns the recorded failure.
+ */
+export declare function classifyFetchFailure(error: unknown, at: string): ReaderPreviewFailure;
+/**
+ * How much visible text a payload carries.
+ *
+ * Script, style and markup are stripped; entities collapse to a single space.
+ * This is a measurement, not a parser: its only job is to separate "a page"
+ * from "an interstitial that is technically HTML".
+ *
+ * @param raw - the decoded payload.
+ * @returns the visible character count.
+ */
+export declare function visibleTextLength(raw: string): number;
+//# sourceMappingURL=service.d.ts.map