@khorsheed/dsh-reader 0.3.0 → 0.3.2

package/lib/types/document-fetch.d.ts

@@ -0,0 +1,106 @@
+/**
+ * The document fetcher: the reader's own instance of the official HTTP fetch
+ * provider, so feeds and article pages arrive as the origin served them.
+ *
+ * Why not `ctx.web.fetch` alone: the seam runs whichever provider the host
+ * selects, and from 0.2.1-rc.2 the web-search settings switch for the DeepSeek
+ * Search/Fetch API moves `web.fetchProvider` too. That provider answers
+ * Markdown (`body.kind: 'text'`), a constant 200, and the request URL as the
+ * final URL — so `classifyPayload` reads every feed as a page, the extractor
+ * gets no HTML, and the cross-origin redirect handling never fires. The
+ * request carries only a URL; no caller can ask for the HTTP provider (seam
+ * registry S22).
+ *
+ * So when the mounted `web` is the official service (it carries
+ * `registerFetchProvider`), the reader fetches through its own
+ * `HttpFetchProvider` — the same class the default `http` row registers, with
+ * that row's default limits, so the policy (public addresses only, connection
+ * pinned, same-origin redirects, charset decoding) and the errors
+ * (`WEB_REDIRECT_BLOCKED` naming the target origin) are exactly today's.
+ * `ctx.web` stays the permission: without it nothing is fetched. A host
+ * without the provider package (or any other `web` shape, such as a test's
+ * plain `{ fetch }`) is fetched through as before.
+ *
+ * Retire when the seam lets one request name its provider (S22).
+ *
+ * @module @khorsheed/dsh-reader/document-fetch
+ */
+/** The shape of one fetch result, declared structurally (no host import). */
+export interface DocumentFetchResult {
+    readonly url: string;
+    readonly statusCode: number;
+    readonly body: {
+        readonly kind: 'html' | 'text';
+        readonly content: string;
+    };
+    readonly truncated: boolean;
+}
+/** Anything that fetches one URL: the web seam or a provider. */
+export interface DocumentFetcher {
+    fetch(request: {
+        readonly url: string;
+    }, signal?: AbortSignal): Promise<DocumentFetchResult>;
+}
+/** The limits the official provider takes (the `http` row's resolved config). */
+interface HttpFetchLimits {
+    readonly maxResponseBytes: number;
+    readonly maxBodyChars: number;
+    readonly timeoutMs: number;
+    readonly maxRedirects: number;
+    readonly userAgent: string;
+}
+/** The provider package's runtime face, structurally. */
+interface HttpProviderModule {
+    readonly HttpFetchProvider: new (limits: HttpFetchLimits) => DocumentFetcher;
+    readonly DEFAULT_USER_AGENT?: string;
+}
+/**
+ * The reader's own fetch limits: the `http` row's defaults
+ * (`@deepseek-ai/dsh-web-fetch-http` Config, 0.1.5 through 0.2.1-rc.3) except
+ * `maxBodyChars`. The row's 100 000 is sized for a page handed to a model; the
+ * reader extracts the article itself from the raw HTML, and a modern page
+ * carries several hundred KB of inline script before its last paragraph
+ * (measured: claude.com's articles, 620 KB, cut mid-sentence at 100 000). 2 000 000
+ * is what deployments ran the row at while the reader still fetched through
+ * it, and the per-source storage budget (`MAX_BODY_CHARS_PER_SOURCE`) is
+ * sized to match. A deployment changes these with the reader's own
+ * `fetchLimits`: the row's config does not reach this instance.
+ */
+export declare const DOCUMENT_FETCH_DEFAULTS: {
+    readonly maxResponseBytes: 5000000;
+    readonly maxBodyChars: 2000000;
+    readonly timeoutMs: 30000;
+    readonly maxRedirects: 5;
+};
+/** The limits a deployment may override (the user agent stays the provider's). */
+export type DocumentFetchLimits = Partial<Pick<HttpFetchLimits, 'maxResponseBytes' | 'maxBodyChars' | 'timeoutMs' | 'maxRedirects'>>;
+/** Whether a mounted `web` is the official service rather than some other shape. */
+export declare function isOfficialWebService(web: unknown): boolean;
+/** Loads the provider package; injectable so tests never need the real one. */
+export type HttpProviderLoader = () => Promise<HttpProviderModule | undefined>;
+/**
+ * Pick the fetcher for raw documents.
+ *
+ * Built once per service: the first fetch on an official web service loads
+ * the provider package and keeps the instance; a failed load is remembered
+ * too, and every later fetch goes through the seam.
+ */
+export declare class DocumentFetch {
+    private readonly web;
+    private readonly limits;
+    private readonly load;
+    private provider;
+    /**
+     * @param web - the probed web seam, read on every call (it may come and go).
+     * @param limits - the reader's configured fetch limits, over the defaults.
+     * @param load - the provider package loader (tests inject one).
+     */
+    constructor(web: () => unknown, limits?: DocumentFetchLimits, load?: HttpProviderLoader);
+    /**
+     * The fetcher for this call, or undefined when no web capability is mounted.
+     * @returns the reader's own provider on an official web service, else the seam itself.
+     */
+    current(): Promise<DocumentFetcher | undefined>;
+}
+export {};
+//# sourceMappingURL=document-fetch.d.ts.map

package/lib/types/document-fetch.js

@@ -0,0 +1,100 @@
+/**
+ * The document fetcher: the reader's own instance of the official HTTP fetch
+ * provider, so feeds and article pages arrive as the origin served them.
+ *
+ * Why not `ctx.web.fetch` alone: the seam runs whichever provider the host
+ * selects, and from 0.2.1-rc.2 the web-search settings switch for the DeepSeek
+ * Search/Fetch API moves `web.fetchProvider` too. That provider answers
+ * Markdown (`body.kind: 'text'`), a constant 200, and the request URL as the
+ * final URL — so `classifyPayload` reads every feed as a page, the extractor
+ * gets no HTML, and the cross-origin redirect handling never fires. The
+ * request carries only a URL; no caller can ask for the HTTP provider (seam
+ * registry S22).
+ *
+ * So when the mounted `web` is the official service (it carries
+ * `registerFetchProvider`), the reader fetches through its own
+ * `HttpFetchProvider` — the same class the default `http` row registers, with
+ * that row's default limits, so the policy (public addresses only, connection
+ * pinned, same-origin redirects, charset decoding) and the errors
+ * (`WEB_REDIRECT_BLOCKED` naming the target origin) are exactly today's.
+ * `ctx.web` stays the permission: without it nothing is fetched. A host
+ * without the provider package (or any other `web` shape, such as a test's
+ * plain `{ fetch }`) is fetched through as before.
+ *
+ * Retire when the seam lets one request name its provider (S22).
+ *
+ * @module @khorsheed/dsh-reader/document-fetch
+ */
+/**
+ * The reader's own fetch limits: the `http` row's defaults
+ * (`@deepseek-ai/dsh-web-fetch-http` Config, 0.1.5 through 0.2.1-rc.3) except
+ * `maxBodyChars`. The row's 100 000 is sized for a page handed to a model; the
+ * reader extracts the article itself from the raw HTML, and a modern page
+ * carries several hundred KB of inline script before its last paragraph
+ * (measured: claude.com's articles, 620 KB, cut mid-sentence at 100 000). 2 000 000
+ * is what deployments ran the row at while the reader still fetched through
+ * it, and the per-source storage budget (`MAX_BODY_CHARS_PER_SOURCE`) is
+ * sized to match. A deployment changes these with the reader's own
+ * `fetchLimits`: the row's config does not reach this instance.
+ */
+export const DOCUMENT_FETCH_DEFAULTS = {
+    maxResponseBytes: 5_000_000,
+    maxBodyChars: 2_000_000,
+    timeoutMs: 30_000,
+    maxRedirects: 5,
+};
+/** Whether a mounted `web` is the official service rather than some other shape. */
+export function isOfficialWebService(web) {
+    return typeof web?.registerFetchProvider === 'function';
+}
+const loadOfficialProvider = () => import('@deepseek-ai/dsh-web-fetch-http').then(module => module, () => undefined);
+/**
+ * Pick the fetcher for raw documents.
+ *
+ * Built once per service: the first fetch on an official web service loads
+ * the provider package and keeps the instance; a failed load is remembered
+ * too, and every later fetch goes through the seam.
+ */
+export class DocumentFetch {
+    web;
+    limits;
+    load;
+    provider;
+    /**
+     * @param web - the probed web seam, read on every call (it may come and go).
+     * @param limits - the reader's configured fetch limits, over the defaults.
+     * @param load - the provider package loader (tests inject one).
+     */
+    constructor(web, limits = {}, load = loadOfficialProvider) {
+        this.web = web;
+        this.limits = limits;
+        this.load = load;
+    }
+    /**
+     * The fetcher for this call, or undefined when no web capability is mounted.
+     * @returns the reader's own provider on an official web service, else the seam itself.
+     */
+    async current() {
+        const web = this.web();
+        if (web === undefined || web === null)
+            return undefined;
+        const seam = web;
+        if (!isOfficialWebService(web))
+            return seam;
+        this.provider ??= this.load().then(module => {
+            if (module === undefined || typeof module.HttpFetchProvider !== 'function')
+                return undefined;
+            try {
+                return new module.HttpFetchProvider({
+                    ...DOCUMENT_FETCH_DEFAULTS,
+                    ...this.limits,
+                    userAgent: module.DEFAULT_USER_AGENT ?? 'deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)',
+                });
+            }
+            catch {
+                return undefined;
+            }
+        });
+        return (await this.provider) ?? seam;
+    }
+}

package/lib/types/index.d.ts

@@ -12,18 +12,23 @@
  * @module @khorsheed/dsh-reader
  */
 import type { Context } from '@deepseek-ai/cordis';
+import type { DocumentFetchLimits } from './document-fetch.ts';
 import { ReaderService } from './service.ts';
+import { ReaderNotesStore } from './notes.ts';
 export { ReaderService, classifyPayload, normalizeUrl } from './service.ts';
 export { ReaderRemoteService } from './remote.ts';
 export type { ReaderRemoteConfig } from './remote.ts';
 export { MAX_CACHED_BODIES, MAX_TAGS, ReaderStore, ReaderStoreError, boundAnnotations, boundPayloads, emptyStateDoc, normalizeStateDoc, pruneOrphanTags, resolveReaderStateRoot, serializeStateDoc, } from './store.ts';
 export type { ReaderStoreRead } from './store.ts';
+export * from './notes.ts';
 export * from './schedule.ts';
 export * from './types.ts';
 declare module '@deepseek-ai/cordis' {
     interface Context {
         /** The reader plugin's service core. */
         reader: ReaderService;
+        /** The reader's notes, discussions and article snapshots. */
+        readerNotes: ReaderNotesStore;
     }
 }
 /** Cordis plugin name used by loader diagnostics. */
@@ -32,6 +37,16 @@ export declare const name = "reader";
 export interface ReaderConfig {
     /** Directory holding `state.json`; defaults to `$DSH_HOME/state/dsh-reader`. */
     readonly stateRoot?: string;
+    /** A yt-dlp binary to use instead of the reader's managed copy (video metadata and captions). */
+    readonly ytdlpPath?: string;
+    /**
+     * Fetch limits for feeds and pages: `maxResponseBytes` (default 5 000 000),
+     * `maxBodyChars` (2 000 000 — whole pages, not the row's model-sized 100 000),
+     * `timeoutMs` (30 000), `maxRedirects` (5). The reader fetches through its own
+     * instance of the official `http` provider, so limits raised on that row do
+     * not reach it; set them here.
+     */
+    readonly fetchLimits?: DocumentFetchLimits;
 }
 /**
  * Plugin body: provide the service core, then mount the Remote data face.

package/lib/types/index.js

@@ -1,8 +1,10 @@
 import { ReaderService } from "./service.js";
+import { ReaderNotesStore } from "./notes.js";
 import { ReaderRemoteService } from "./remote.js";
 export { ReaderService, classifyPayload, normalizeUrl } from "./service.js";
 export { ReaderRemoteService } from "./remote.js";
 export { MAX_CACHED_BODIES, MAX_TAGS, ReaderStore, ReaderStoreError, boundAnnotations, boundPayloads, emptyStateDoc, normalizeStateDoc, pruneOrphanTags, resolveReaderStateRoot, serializeStateDoc, } from "./store.js";
+export * from "./notes.js";
 export * from "./schedule.js";
 export * from "./types.js";
 /** Cordis plugin name used by loader diagnostics. */
@@ -21,5 +23,6 @@ export const name = 'reader';
 export function apply(ctx, config = {}) {
     const service = new ReaderService(ctx, config);
     ctx.provide('reader', service);
+    ctx.provide('readerNotes', new ReaderNotesStore(config));
     ctx.plugin(ReaderRemoteService, {});
 }

package/lib/types/local-transcribe.d.ts

@@ -0,0 +1,171 @@
+import { type Cue } from './captions.ts';
+import type { ReaderMedia } from './media.ts';
+import type { LocalTranscriptStatus } from './types.ts';
+/** The part of the `transcribe` service the reader calls. */
+export interface TranscribeEngine {
+    modelsFor(request: {
+        language?: 'zh' | 'en' | 'auto';
+        diarize?: false;
+    }): string[];
+    missingBytes(ids: readonly string[]): Promise<number>;
+    run(request: {
+        input: {
+            files: string[];
+        };
+        fromSec?: number;
+        language: 'zh' | 'en' | 'auto';
+        diarize: false;
+        owner: string;
+        label?: string;
+    }, options: {
+        signal?: AbortSignal;
+        onProgress?: (progress: {
+            stage: string;
+            progress: number;
+            ahead?: number;
+        }) => void;
+        onSegments?: (segments: readonly EngineSegment[]) => void;
+    }): Promise<{
+        segments: EngineSegment[];
+        language?: string;
+    }>;
+    /** Who spoke when (an engine older than this has none: transcripts stay one voice). */
+    speakers?(request: {
+        input: {
+            files: string[];
+        };
+        speech?: {
+            start: number;
+            end: number;
+        }[];
+        owner: string;
+        label?: string;
+    }, options: {
+        signal?: AbortSignal;
+        onProgress?: (progress: {
+            stage: string;
+            progress: number;
+            ahead?: number;
+        }) => void;
+    }): Promise<{
+        single: boolean;
+        turns: {
+            cluster: number;
+            start: number;
+            end: number;
+        }[];
+    }>;
+}
+/** A speaker turn as the engine reports it. */
+export interface EngineTurn {
+    readonly cluster: number;
+    readonly start: number;
+    readonly end: number;
+}
+/** One finished stretch as the engine reports it. */
+export interface EngineSegment {
+    readonly start: number;
+    readonly end: number;
+    readonly tokens: readonly {
+        readonly text: string;
+        readonly start: number;
+        readonly end: number;
+    }[];
+}
+/**
+ * Engine stretches as caption-like cues: one per sentence (the engine's
+ * stretches are cut at pauses, not sentence ends), so the transcript merges
+ * into the same paragraphs platform captions do. A sentence that runs on
+ * without a period is closed at a comma or a pause once it is long, or a
+ * single cue would make a minute-and-a-half paragraph (measured 2026-10-20).
+ *
+ * @param segments - the engine's segments, in order.
+ */
+export declare function cuesFromSegments(segments: readonly EngineSegment[], speakers?: readonly (number | undefined)[]): (Cue & {
+    speaker?: number;
+})[];
+/**
+ * Each token's speaker, as an index in order of first appearance (A, B, …):
+ * the turn covering the token's middle, else the nearest one; then runs of a
+ * voice shorter than {@link MIN_TURN_SEC} are folded into the voice before
+ * them — a diarizer's boundary wobble would otherwise split sentences.
+ *
+ * @param segments - the transcript's segments.
+ * @param turns - the engine's turns.
+ */
+export declare function speakerPerToken(segments: readonly EngineSegment[], turns: readonly EngineTurn[]): number[];
+/**
+ * The transcript with its speakers: paragraphs never cross a change of voice,
+ * and each carries `data-spk` (the speaker's index) beside its `data-t`.
+ */
+export declare function renderSpeakerTranscript(segments: readonly EngineSegment[], turns: readonly EngineTurn[]): string;
+/** The marker a locally transcribed paragraph carries (the 来源 line reads it). */
+export declare const LOCAL_MARK = "data-src=\"local\"";
+/** The engine's segments as transcript markup, each paragraph marked local. */
+export declare function renderLocalTranscript(segments: readonly EngineSegment[]): string;
+/**
+ * The language to ask for: always detection (SenseVoice). Measured
+ * 2026-10-20: a platform says nothing reliable about the spoken language — a
+ * Bilibili upload of an English interview came out of the Chinese model as
+ * lower-case English with Chinese punctuation — and on Chinese speech
+ * SenseVoice reads about as well as Paraformer (a 90 s sample: each missed
+ * two or three characters the other got right).
+ */
+export declare function languageFor(_media: ReaderMedia): 'auto';
+/** What the transcriber needs from the reader. */
+export interface LocalTranscriberDeps {
+    /** The engine, probed per start. */
+    readonly engine: () => TranscribeEngine | undefined;
+    /** Where temporary audio goes. */
+    readonly tmpRoot: string;
+    /** Fetch a video's audio track into `dir`. */
+    readonly videoAudio: (url: string, dir: string, options: {
+        signal: AbortSignal;
+        onProgress: (fraction: number) => void;
+    }) => Promise<string>;
+    /** How often a restart checks for the engine before resuming jobs (ms; default 2 s). */
+    readonly resumePollMs?: number;
+    /** Keep the finished transcript as the entry's body (`speakers`: how many voices, once known). */
+    readonly store: (entryId: string, url: string, html: string, language: string | undefined, speakers?: number) => Promise<void>;
+}
+/** The reader's local transcription jobs, by entry. */
+export declare class LocalTranscriber {
+    private readonly deps;
+    private readonly jobs;
+    /** Jobs a previous process left mid-way, waiting for the engine. */
+    private readonly resumable;
+    private resumeTimer;
+    private disposing;
+    constructor(deps: LocalTranscriberDeps);
+    /** An entry's job as the browser sees it. */
+    status(entryId: string): LocalTranscriptStatus;
+    /**
+     * Start (or rejoin) an entry's transcription.
+     *
+     * @param request - the entry, its media, and whether the model download was confirmed.
+     */
+    start(request: {
+        entryId: string;
+        url: string;
+        media: ReaderMedia;
+        title?: string;
+        confirmModels?: boolean;
+    }): Promise<LocalTranscriptStatus>;
+    /**
+     * Leave or stop an entry's job: a queued (or still fetching) job is
+     * withdrawn either way; a running one only with `force`.
+     */
+    cancel(entryId: string, force: boolean): LocalTranscriptStatus;
+    /** Whether the engine is installed right now. */
+    engineAvailable(): boolean;
+    /** Stop everything (plugin disposal) — keeping each job's record, so the next process resumes it. */
+    dispose(): void;
+    /**
+     * Run (or resume) one job: fetch the audio unless the record has it,
+     * transcribe from where the saved segments end, store, then the speakers.
+     *
+     * @param resumeDir - a previous process's directory for this job.
+     */
+    private execute;
+}
+//# sourceMappingURL=local-transcribe.d.ts.map