@sparkelf/dsh-knowledge-base 0.0.0-stage → 0.1.1

package/lib/types/deploy.d.ts

@@ -0,0 +1,209 @@
+/**
+ * Idempotent deployment recipes for the knowledge base.
+ *
+ * Every step here is a *plan*: a named sequence of shell commands plus the
+ * predicate that decides whether the step already ran. The Settings panel shows
+ * each step, runs it over the {@link RemoteRunner} seam, and reports the output.
+ *
+ * Rules these recipes must satisfy:
+ * - Re-runnable. Clicking twice must not install twice, start two servers, or
+ *   fail because something already exists.
+ * - Read-only probes first. A status step never mutates anything.
+ * - The commands are quoted for a POSIX shell and never interpolate untrusted
+ *   input without quoting.
+ *
+ * @module @sparkelf/dsh-knowledge-base/deploy
+ */
+/**
+ * MinerU deployment tiers this plugin can prepare.
+ *
+ * MinerU 4.x recognises exactly these two (`mineru/types.py`: `DEPLOYMENT_TIERS`), and
+ * `models download --tier` validates against that tuple. The `flash` name that also exists in
+ * MinerU is a *quality* tier for the parsers that need no local weights, not a deployment
+ * tier, so it cannot be prepared and is not offered here.
+ */
+export type MineruTier = 'basic' | 'standard';
+/** Small-model backend for MinerU. */
+export type MineruBackend = 'onnx' | 'torch';
+/** One step of a deployment plan. */
+export interface DeployStep {
+    /** Stable id used by the UI. */
+    id: string;
+    /** Human-facing label key. */
+    label: string;
+    /**
+     * Commands to run in order. Each is a complete shell line.
+     * `{{...}}` placeholders are substituted from {@link DeployPlanContext}.
+     */
+    commands: string[];
+    /** How long this step may take; model downloads are slow. */
+    timeoutMs: number;
+}
+/** Values substituted into command templates. */
+export interface DeployPlanContext {
+    /** Absolute path of the authoritative knowledge tree. */
+    actualRoot: string;
+    /** Absolute path of the per-user view tree. */
+    viewRoot: string;
+    /** Tier to deploy. */
+    tier: MineruTier;
+    /** Small-model backend. */
+    backend: MineruBackend;
+    /** TCP port the MinerU API listens on. */
+    port: number;
+    /** Client hosts allowed to mount the NFS export, as an exports spec. */
+    permitted: string;
+    /** Host serving the export, as `address` or `address:/path` (the pseudo root). */
+    server: string;
+    /** Where the client mounts the export, i.e. the browsable knowledge tree. */
+    mountPoint: string;
+    /** Memory ceiling in MiB for the MinerU service; clamped to half the host's RAM. */
+    memoryMaxMiB?: number;
+    /**
+     * How a remote deployment reaches its share.
+     *
+     * Decides which transport steps {@link deploymentPlan} emits. Absent means `nfs`, the
+     * shape this plan had before the protocol existed.
+     */
+    protocol?: 'nfs' | 'sshfs';
+    /** SSHFS: remote host. */
+    sshHost?: string;
+    /** SSHFS: login user on the remote host. */
+    sshUser?: string;
+    /** SSHFS: TCP port the SSH daemon listens on. */
+    sshPort?: number;
+    /**
+     * SSHFS: path of the private key on the HOST, when the credential store resolves to a
+     * file. Optional: an ssh-agent-managed key needs no `-i`.
+     */
+    sshKeyPath?: string;
+    /** SSHFS: absolute path of the shared directory on the remote host. */
+    sshRemotePath?: string;
+    /** SSHFS: extra `-o` options, appended to the ones the step requires. */
+    sshMountOptions?: string;
+}
+/**
+ * Memory ceiling for the MinerU service when the config names none.
+ *
+ * The `basic` tier peaked at 1.86 GB resident while parsing a one-page PDF (measured
+ * 2026-10-15), so a ceiling near the steady-state size leaves `MemoryHigh` no headroom. On a
+ * host without swap that reads as a stalled parse rather than a slow one, because throttling
+ * has nowhere to spill. The service step clamps this to half of RAM, so the number only has to
+ * be generous on a roomy machine.
+ */
+export declare const DEFAULT_MINERU_MEMORY_MAX_MIB = 4096;
+/** Rotate a single-quoted shell string safely. */
+export declare function shquote(value: string): string;
+/** Substitute `{{name}}` placeholders; unknown names are left intact. */
+export declare function renderCommand(template: string, context: DeployPlanContext): string;
+/** Marker the image keeps so a redeploy can prove which tier is present. */
+export declare const KB_STATE_FILE = "/var/lib/dataops-kb/state.json";
+/**
+ * Read-only environment probe.
+ *
+ * Establishes which install path is available so the UI can disable the steps
+ * that cannot work: a host without Python cannot take the pip path.
+ * @returns the probe step.
+ */
+export declare function probeStep(): DeployStep;
+/**
+ * Install the MinerU package and download the selected tier's models.
+ *
+ * The base package (`mineru>=4.0,<5`) is the CPU path: it brings ONNX and
+ * llama.cpp but not vLLM, which needs a GPU this environment does not have.
+ * @param context - deployment values.
+ * @returns the install step.
+ */
+export declare function mineruInstallStep(context: DeployPlanContext): DeployStep;
+/**
+ * Install a systemd unit that serves the MinerU HTTP API.
+ *
+ * A unit rather than a bare background process so the service survives logout
+ * and can be restarted by the same click.
+ * @param context - deployment values.
+ * @returns the service step.
+ */
+export declare function mineruServiceStep(context: DeployPlanContext): DeployStep;
+/**
+ * Verify the MinerU API actually answers.
+ *
+ * This is the check that catches the parameter-compatibility question: the
+ * plugin posts multipart form fields while MinerU 4.x reads a request body, so
+ * a pass here is the evidence that the two agree.
+ * @param context - deployment values.
+ * @returns the verification step.
+ */
+export declare function mineruVerifyStep(context: DeployPlanContext): DeployStep;
+/**
+ * Deploy the NFS export of the knowledge tree.
+ *
+ * Uses the container image rather than host packages: Docker is already present
+ * on the target, the image carries the userspace tools, and its default export
+ * template already includes `no_root_squash`, which hardlinking across NFS
+ * requires.
+ * @param context - deployment values.
+ * @returns the NFS step.
+ */
+export declare function nfsDeployStep(context: DeployPlanContext): DeployStep;
+/**
+ * Prove hardlinks work across the NFS export.
+ *
+ * Creating a hard link between the authoritative tree and a view directory is
+ * the single operation the whole permission model depends on, and it is also
+ * the operation most likely to fail (wrong export scope, `root_squash`, split
+ * filesystems). A pass here is the acceptance test for the storage side.
+ * @param context - deployment values.
+ * @returns the storage verification step.
+ */
+export declare function nfsVerifyStep(context: DeployPlanContext): DeployStep;
+/**
+ * Mount the exported share at the mount point, with the option this deployment needs.
+ *
+ * `noresvport` is not decorative here. This host runs WSL2 with
+ * `networkingMode=mirrored`, where an NFS mount without it hangs until timeout
+ * (`mount.nfs: Connection timed out`) while `rpcinfo`, `showmount -e` and a plain TCP
+ * connect to 2049 all still succeed. Measured 2026-10-16 against a server that mounts
+ * fine from its own kernel, and A/B verified: without the option the mount fails, with
+ * it the mount succeeds, repeatably. Microsoft documents the same failure for mirrored
+ * networking (WSL issue 12508), which is still open.
+ *
+ * The NFS client normally opts into a reserved source port, which requires a reply the
+ * mirrored networking stack does not deliver; `noresvport` drops that requirement.
+ *
+ * @param context - Deployment paths and the export address.
+ * @returns The mount step.
+ */
+export declare function nfsMountStep(context: DeployPlanContext): DeployStep;
+/**
+ * Mount a remote directory over SSHFS on the HOST.
+ *
+ * The same shape as {@link nfsMountStep}, for a host that cannot mount NFS: the container
+ * has no `CAP_SYS_ADMIN` and cannot mount anything itself, so this runs on the Host and the
+ * resulting directory is bound in at `mountPoint` like any other.
+ *
+ * Two details are not incidental:
+ * - `StrictHostKeyChecking=accept-new` is limited to this invocation. Accepting an unknown
+ *   host key globally would silence the one warning that catches a swapped host, so the
+ *   trust-on-first-use decision is scoped to the connection whose key the operator is
+ *   establishing.
+ * - The reconnect options are not optional here. An SSHFS mount whose connection drops
+ *   returns EIO to every reader until someone unmounts it; `reconnect` plus a keepalive is
+ *   what makes a transient network blip a stall instead of a broken knowledge base.
+ * @param context - transport and target paths from the deployment config.
+ * @returns the mount step.
+ */
+export declare function sshfsMountStep(context: DeployPlanContext): DeployStep;
+/**
+ * Verify the SSHFS mount is present, readable, and actually a FUSE mount.
+ *
+ * Separate from the mount step because "the command exited 0" and "the reader will see
+ * files" are different claims: a stale empty directory satisfies the first and not the
+ * second, and the filesystem type is what tells them apart.
+ * @param context - transport and target paths from the deployment config.
+ * @returns the verification step.
+ */
+export declare function sshfsVerifyStep(context: DeployPlanContext): DeployStep;
+/** The full ordered plan the Settings panel presents. */
+export declare function deploymentPlan(context: DeployPlanContext): DeployStep[];
+/** Look up one step by id. */
+export declare function findStep(id: string, context: DeployPlanContext): DeployStep | undefined;

package/lib/types/extract.d.ts

@@ -0,0 +1,144 @@
+/**
+ * Convert an uploaded document into Markdown so its text becomes greppable.
+ *
+ * The knowledge tree doubles as a document store and as what an agent greps. A
+ * `.docx` on disk is neither: its bytes answer no query. Every upload therefore
+ * lands twice — the original, byte for byte, plus a `.md` sibling holding the
+ * extracted text — so a reader can still open the real file while a search sees
+ * the words inside it.
+ *
+ * Extraction is delegated rather than implemented here. Three authorities
+ * already exist in this deployment and each owns one format family:
+ *
+ * - MinerU (`/file_parse`) parses PDF and office documents with layout models.
+ *   It is the best answer for a scanned PDF and the one authority that can
+ *   recognise tables inside one.
+ * - LibreOffice converts the legacy binary formats (`.doc`, `.xls`, `.ppt`) that
+ *   MinerU and the Python readers refuse.
+ * - `python-docx` / `openpyxl` / `python-pptx` / `pypdf` read the modern formats
+ *   directly, without a service round trip.
+ *
+ * None of them is reimplemented in TypeScript: a `.docx` is a ZIP of XML and a
+ * second parser would only disagree with the first. This module decides WHICH
+ * authority to ask and what to do when it cannot answer, which is the part that
+ * is specific to this plugin.
+ *
+ * A failed extraction never fails the upload. The original is already written and
+ * remains the source of truth; the caller learns that no `.md` was produced, and
+ * the file stays readable through the original.
+ *
+ * @module @sparkelf/dsh-knowledge-base/extract
+ */
+/** Formats whose text this module knows how to extract. */
+export declare const EXTRACTABLE_EXTENSIONS: readonly [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".rtf"];
+/** One extractable extension. */
+export type ExtractableExtension = typeof EXTRACTABLE_EXTENSIONS[number];
+/** How one request to the extraction authorities ended. */
+export interface ExtractResult {
+    /** Markdown text, when extraction produced any. */
+    markdown?: string;
+    /**
+     * Which authority answered. Reported so a bad conversion can be traced.
+     *
+     * `mineru-ocr` means the first pass classified the document as text and came back
+     * too thin to search, so the same parser was asked again with OCR forced. It is a
+     * separate value because the two passes can disagree, and a reader debugging a
+     * missing line needs to know which one produced the file.
+     */
+    via?: 'mineru' | 'mineru-ocr' | 'libreoffice' | 'python' | 'native';
+    /** Why no markdown was produced. Present whenever {@link markdown} is absent. */
+    reason?: string;
+}
+/**
+ * Whether this path names a document worth extracting.
+ *
+ * Compared case-insensitively because a browser reports the extension as the
+ * user typed it, and a `.PDF` is the same format as a `.pdf`.
+ *
+ * @param relPath - the uploaded path, relative to the knowledge root.
+ * @returns true when {@link extractDocument} has an authority for it.
+ */
+export declare function isExtractable(relPath: string): boolean;
+/**
+ * The Markdown sibling an extracted document lands beside.
+ *
+ * Appending rather than replacing the extension keeps two same-named uploads
+ * from colliding: `报告.docx` becomes `报告.docx.md`, so it cannot overwrite a
+ * `报告.md` the user wrote by hand.
+ *
+ * @param relPath - the uploaded document's path.
+ * @returns the markdown path to write.
+ */
+export declare function markdownSiblingOf(relPath: string): string;
+/**
+ * Whether MinerU should be asked before the local readers.
+ *
+ * PDF only: a PDF carries no recoverable structure beyond its layout, which is
+ * exactly what MinerU models. An office document still has its XML, and the
+ * direct readers keep more of it (a spreadsheet's cell grid survives
+ * `openpyxl` and is flattened by a layout parser).
+ *
+ * @param ext - lowercased extension including the dot.
+ * @returns true when the remote parser is the better authority.
+ */
+export declare function prefersRemoteParser(ext: string): boolean;
+/**
+ * Ask MinerU to parse one file and return its Markdown.
+ *
+ * The service answers with a task handle for large files and the body directly
+ * for small ones, so both shapes are accepted. A non-200 answer is a miss, not
+ * a failure of the upload.
+ *
+ * @param baseUrl - MinerU service base URL, without a trailing slash.
+ * @param fullPath - absolute path of the document on this host.
+ * @param fileName - name to report to the service.
+ * @returns the Markdown, or undefined when the service could not parse it.
+ */
+/**
+ * Whether an extraction came back too small to be worth searching.
+ *
+ * Not a judgement about document quality: a 200-character memo is a real document,
+ * and treating it as a failure would re-run OCR over something already correct. The
+ * test targets the specific shape of a mis-classified scan -- pages of content that
+ * came back as a title and a page number -- so it counts what is left after markup
+ * rather than trusting a single threshold on the whole answer.
+ *
+ * @param markdown - what the parser returned.
+ * @returns true when the answer is short enough that a classifier mistake is likely.
+ */
+export declare function looksTooThinToSearch(markdown: string): boolean;
+/** Services and commands extraction may use, as configured for this deployment. */
+export interface ExtractOptions {
+    /** MinerU base URL. Empty disables the remote parser. */
+    mineruUrl?: string;
+}
+/**
+ * Extract one document to Markdown.
+ *
+ * Authorities are tried in an order that puts the best answer first for each
+ * format, and every step degrades to the next rather than failing: a PDF whose
+ * MinerU call times out still gets `pypdf`'s text layer.
+ *
+ * @param fullPath - absolute path of the written document.
+ * @param relPath - the same document's knowledge-relative path, for the title.
+ * @param options - services this deployment exposes.
+ * @returns the Markdown and the authority that produced it, or the reason none did.
+ */
+export declare function extractDocument(fullPath: string, relPath: string, options?: ExtractOptions): Promise<ExtractResult>;
+/**
+ * Whether a file exists and is a regular file.
+ *
+ * @param path - absolute path to test.
+ * @returns true when it is there.
+ */
+export declare function fileExists(path: string): Promise<boolean>;
+/**
+ * Write extracted markdown beside its document.
+ *
+ * Kept separate from {@link extractDocument} so the write is testable without an
+ * extraction service.
+ *
+ * @param markdownPath - absolute path of the `.md` sibling.
+ * @param markdown - the text to write.
+ */
+export declare function writeExtractedMarkdown(markdownPath: string, markdown: string): Promise<void>;

package/lib/types/extractionQueue.d.ts

@@ -0,0 +1,154 @@
+/**
+ * Background extraction of a document's Markdown sibling.
+ *
+ * Upload and extraction are two different promises. The bytes are the uploader's
+ * deliverable and must be visible the moment the write lands; the `.md` sibling is
+ * derived text that may take a minute (a scanned PDF goes through MinerU, a legacy
+ * `.doc` through LibreOffice). Doing both inside the upload request made a slow
+ * parse look like a slow upload -- and a failed parse look like a failed upload.
+ *
+ * This queue separates them. `enqueue` never blocks and never throws: a failed
+ * extraction is a status, not an upload error. The queue lives in this process only
+ * (see {@link MAX_FINISHED_TASKS}) because the file itself is already durable; a
+ * restart loses the bookkeeping, and the manual `extract.retry` route is the recovery
+ * path. Persisting a queue would add a second source of truth about files that the
+ * filesystem already answers for.
+ *
+ * @module @sparkelf/dsh-knowledge-base/extractionQueue
+ */
+/** Lifecycle of one extraction task. */
+export type ExtractionState = 'pending' | 'running' | 'succeeded' | 'failed' | 'unsupported';
+/** One extraction task, as a reader observes it. */
+export interface ExtractionTask {
+    /** Knowledge-relative path of the SOURCE document. */
+    relPath: string;
+    /** Where the task is in its lifecycle. */
+    state: ExtractionState;
+    /** When the task was accepted, ISO-8601. */
+    queuedAt: string;
+    /** When the run started, ISO-8601. */
+    startedAt?: string;
+    /** When the run reached a terminal state, ISO-8601. */
+    finishedAt?: string;
+    /** The Markdown sibling that was written, when one was. */
+    markdown?: string;
+    /** Which authority produced the text (`mineru`, `libreoffice`, `python`, `native`). */
+    via?: string;
+    /** Why the extraction produced nothing, verbatim from the extractor. */
+    reason?: string;
+}
+/** What one run reports back to the queue. */
+export interface ExtractionOutcome {
+    /** Terminal state for this run. */
+    state: Exclude<ExtractionState, 'pending' | 'running'>;
+    /** The written sibling, when the run produced one. */
+    markdown?: string;
+    /** The authority that produced the text. */
+    via?: string;
+    /** Why nothing was produced. */
+    reason?: string;
+}
+/**
+ * How many extractions may run at once.
+ *
+ * One, deliberately. Every authority this drives is expensive in a way a second
+ * concurrent run makes worse: LibreOffice spawns a headless process per conversion and
+ * MinerU parses on the same small box the workspace already shares. Serializing also
+ * makes the observable order the submission order, which is what a reader retrying one
+ * file expects.
+ */
+export declare const MAX_CONCURRENT_EXTRACTIONS = 1;
+/**
+ * Most finished tasks remembered for status reads.
+ *
+ * The map is bookkeeping, not storage: an evicted entry loses only its status line,
+ * never a file. Bounded so a long-lived process does not accumulate one record per
+ * upload it ever saw. Running and pending tasks are never evicted.
+ */
+export declare const MAX_FINISHED_TASKS = 200;
+/** What the queue needs from the surrounding plugin, injected so it stays testable. */
+export interface ExtractionQueueOptions {
+    /**
+     * Run one extraction.
+     *
+     * Supplied by the plugin so this module owns scheduling only, not the extraction
+     * itself: the caller keeps the permission-checked `extractBeside` path in one place.
+     * @param relPath - the source document.
+     * @param mineruBaseUrl - the caller's own parser address, when set.
+     * @returns the terminal state of the run.
+     */
+    run: (relPath: string, mineruBaseUrl: string | undefined) => Promise<ExtractionOutcome>;
+    /**
+     * Whether this path can be extracted at all.
+     *
+     * Asked before queueing so an unsupported file answers immediately rather than
+     * occupying the single worker slot to discover it has no reader.
+     */
+    isExtractable: (relPath: string) => boolean;
+}
+/** One extraction queue. One instance per plugin registration. */
+export interface ExtractionQueue {
+    /**
+     * Accept a source document for background extraction.
+     *
+     * Never throws and never waits: the upload that calls this has already written its
+     * bytes, and nothing about a parse may change that answer. A repeat for the same
+     * path while one is still pending or running returns the existing task instead of
+     * queueing a second run.
+     * @param relPath - the source document, relative to the knowledge root.
+     * @param mineruBaseUrl - the caller's own parser address, when set.
+     * @returns the task as it stands right after the call.
+     */
+    enqueue: (relPath: string, mineruBaseUrl?: string) => ExtractionTask;
+    /**
+     * Read one task.
+     * @param relPath - the source document.
+     * @returns the task, or undefined when this process never saw it.
+     */
+    get: (relPath: string) => ExtractionTask | undefined;
+    /**
+     * Snapshot the queue for a status surface.
+     * @returns every remembered task plus what is running.
+     */
+    snapshot: () => {
+        running: string | undefined;
+        size: number;
+        tasks: ExtractionTask[];
+    };
+    /**
+     * Wait for one path's task to reach a terminal state.
+     *
+     * The manual-retry route waits on its own document, and it must wait on the TASK
+     * rather than on `drain()`. Two reasons, both about there being one truth:
+     *
+     * 1. `drain()` also waits for unrelated uploads queued ahead, so a retry would
+     *    report a delay that has nothing to do with this file.
+     * 2. The retry must NOT run its own extraction and then enqueue a second copy to
+     *    keep the status surface current. That is two runs of one document, and an
+     *    answer built from one run while `extract.status` reports the other -- the
+     *    exact split this method exists to remove.
+     *
+     * Resolves with the terminal task, or immediately with `undefined` when this
+     * process never queued that path (a restart, or a path it has not seen). Never
+     * rejects: the caller is a route, and the queue is not allowed to become an error
+     * path for an upload.
+     * @param relPath - the source document.
+     * @returns the terminal task, or undefined when the path was never queued.
+     */
+    settled: (relPath: string) => Promise<ExtractionTask | undefined>;
+    /**
+     * Wait for the queue to go idle.
+     *
+     * For tests and for a caller that asked to wait on one specific run; a reader never
+     * needs it. Resolves when nothing is pending or running.
+     * @returns when the worker has drained.
+     */
+    drain: () => Promise<void>;
+}
+/**
+ * Create an extraction queue.
+ *
+ * @param options - the runner and the predicate that decides whether a path has a reader.
+ * @returns a queue that runs at most {@link MAX_CONCURRENT_EXTRACTIONS} task at a time.
+ */
+export declare function createExtractionQueue(options: ExtractionQueueOptions): ExtractionQueue;