@avantf/dsh-mem 0.6.4 → 0.8.0

package/lib/engine/mem/store/doc_files.d.ts

@@ -1,125 +0,0 @@
-/** The frontmatter a managed file carries; `content_hash` is the body AS INGESTED. */
-interface DocFileMeta {
-    doc_id: number;
-    domain: string;
-    source: string;
-    title: string;
-    source_uri: string | null;
-    ingested_at: string;
-    content_hash: string;
-    /**
-     * The `@avantf/mem-convert` converter that produced the body, when the source was converted
-     * (absent for text and PDF, and for every file written before this field existed — readers must
-     * treat it as optional, which is why it is not `null`).
-     */
-    converter?: string;
-}
-/** One managed file as the file system sees it now. */
-interface DocFileState {
-    path: string;
-    stale: boolean;
-    missing: boolean;
-}
-/** A parsed managed file: frontmatter (possibly partial — a hand-written file may omit it) + body. */
-interface ParsedDocFile {
-    meta: Partial<DocFileMeta>;
-    body: string;
-}
-/** One path segment, made safe and human-readable; `fallback` covers names that sanitize away. */
-export declare function sanitizeSegment(raw: string, fallback: string): string;
-/**
- * A stable title for a body that arrived as TEXT (a paste), where there is no file name to borrow.
- *
- * Pasted text has no `source_uri` to name it after, and `source` defaults to the same value for
- * every paste — so without this, every untitled paste into one domain collapsed onto one identity
- * and the second ingest silently replaced the first. The rule: the first Markdown ATX heading,
- * else the first non-empty line with its block markers stripped.
- *
- * Fenced code is stepped over in BOTH halves: `#include` is not a heading, and a fence opener is
- * not a title. `MAX_SEGMENT` is the same cap the file name uses, so a long line cannot produce a
- * title the managed path has to truncate differently. Empty input falls back to `untitled`, and the
- * result depends only on the text (same body → same title), which is what keeps a re-paste
- * landing on the SAME identity instead of quietly creating a sibling every time.
- */
-export declare function deriveDocTitle(text: string): string;
-/** sha256 of a document body — the staleness key. */
-export declare function bodyHash(body: string): string;
-/**
- * Render one managed file: frontmatter (JSON-encoded values, so any title/URI survives) + body.
- *
- * The body is written VERBATIM — no trailing newline is added. `content_hash` records the ingested
- * text, and `state` compares it against the file's body, so anything this function normalizes
- * would make every document read as edited the moment it was written.
- *
- * An `undefined` value is OMITTED rather than rendered: JSON.stringify(undefined) is the bare token
- * `undefined`, which the tolerant parser would then hand back as the string "undefined". Optional
- * keys (today: `converter`) therefore simply do not appear for the documents that lack them.
- */
-export declare function renderDocFile(meta: DocFileMeta, body: string): string;
-/**
- * Split one managed file into frontmatter and body.
- *
- * Deliberately tolerant: a file without frontmatter (or with a hand-mangled one) is ALL body —
- * the user's content is never thrown away because a header key is missing, and the caller decides
- * what an unknown `doc_id` means.
- */
-export declare function parseDocFile(text: string): ParsedDocFile;
-/**
- * The managed document directory.
- *
- * Paths are `<root>/<domain>/<source>/<title>.md`, sanitized per segment. Collisions resolve by
- * asking the EXISTING file who it belongs to: a file whose frontmatter names another `doc_id` is
- * left alone and this document takes the `<title>~<doc_id>.md` variant. Without that check two
- * documents that sanitize to the same name would take turns clobbering each other.
- */
-export declare class DocFiles {
-    private readonly root;
-    constructor(root: string);
-    /** The directory holding one document's file (created on write, not here). */
-    dirFor(domain: string, source: string): string;
-    /**
-     * The file NAME for a title, minus the directory: a title that is already a Markdown file name
-     * (`kb ingest --uri notes.md` names the document after the file) must not become `notes.md.md`.
-     */
-    fileName(title: string, suffix?: string): string;
-    /** The base path for a title (before collision handling). */
-    basePath(domain: string, source: string, title: string): string;
-    /**
-     * The file this document owns: the base path, or its `~<doc_id>` variant when the base path is
-     * taken by a DIFFERENT document. Purely derived — no state is written to answer this.
-     */
-    pathFor(docId: number, domain: string, source: string, title: string): string;
-    /** Write the managed file for one document (atomic: temp file + rename). Returns its path. */
-    write(meta: DocFileMeta, body: string): string;
-    /** Parse one managed file, or `null` when it does not exist. */
-    read(path: string): ParsedDocFile | null;
-    /**
-     * A cheap per-file stamp — `mtimeMs:size`, no reads — or `null` when the file is absent.
-     *
-     * This is a TRIGGER, never a judgement: same-second edits, coarse filesystem timestamps and
-     * `cp -p` can all leave it unchanged, so a matching stamp means "probably unchanged" and only
-     * `state()` (which hashes the body) decides. The point is that the common case costs one `stat`
-     * per file instead of reading and hashing every file in the corpus.
-     */
-    stamp(path: string): string | null;
-    /** Every `.md` path under the root. A directory walk WITHOUT reads — no frontmatter, no hashing. */
-    listPaths(): string[];
-    /** Is this document's file there, and does it still match what was ingested? */
-    state(docId: number, domain: string, source: string, title: string): DocFileState;
-    /** Delete this document's file. `false` when there was nothing to delete. */
-    remove(docId: number, domain: string, source: string, title: string): boolean;
-    /**
-     * Every `.md` under the root, with the `doc_id` its frontmatter names (when it names one).
-     * Used to report orphans — files left behind by a document that no longer exists.
-     *
-     * ASYNC and batched: the walk itself is an async recursive `readdir` (a `readdirSync` gathers
-     * every entry in one ~41 ms call at 10k files), and the per-file parse is a `read` — at 10k
-     * documents that is a second full read of the corpus on top of `sync`'s staleness pass, and it
-     * used to be one uninterrupted synchronous block (performance review §7.5 / P6).
-     */
-    scan(): Promise<{
-        path: string;
-        doc_id?: number;
-    }[]>;
-}
-export {};

package/lib/engine/mem/store/document_text.d.ts

@@ -1,47 +0,0 @@
-/** What the first bytes say a document is. */
-type DocumentFormat = 'pdf' | 'binary' | 'text';
-/** How text bytes were decoded. Reported so a guess is visible instead of silent. */
-type DocumentEncoding = 'utf-8' | 'utf-8-bom' | 'utf-16le' | 'utf-16be' | 'gb18030';
-/** One document's indexable text, and where it came from. */
-export interface DocumentText {
-    text: string;
-    /**
-     * `pdf` = extracted text layer; `converter` = a document format turned into Markdown (see
-     * {@link DocumentText.converter}); `text` = decoded bytes (see {@link DocumentText.encoding}).
-     */
-    via: 'text' | 'pdf' | 'converter';
-    encoding?: DocumentEncoding;
-    /** Which `@avantf/mem-convert` converter produced the body (only when `via === 'converter'`). */
-    converter?: string;
-    /** What that conversion could not carry over (dropped media, a truncated sheet, …). */
-    warnings?: string[];
-}
-/** Classify one document by its leading bytes. */
-export declare function sniffFormat(bytes: Uint8Array): {
-    format: DocumentFormat;
-    kind?: string;
-};
-/**
- * Restore the ideographs pdfjs sometimes reports as compatibility codepoints.
- *
- * Some CJK PDFs extract `⽹站` (KANGXI RADICAL NET + 站) instead of `网站` — visually identical,
- * different codepoints, so a search for the real word misses. Only those three compatibility ranges
- * are normalized: NFKC over the whole text would also rewrite full-width punctuation (，。！？),
- * which is real content and must survive verbatim.
- */
-export declare function normalizeCjkCompatibility(text: string): string;
-/**
- * The text of one document, whatever its bytes are.
- *
- * CONVERSION RUNS FIRST, before the magic-number sniff and therefore before the binary rejection.
- * docx/xlsx/html/csv are converted to Markdown here, and the order is not cosmetic: docx and xlsx
- * are ZIPs, so a sniff-first pipeline would file them under "known binary" and refuse them. A
- * document no converter claims falls through to the text/PDF/binary paths unchanged.
- *
- * @param bytes - the whole document (already bounded by the byte cap).
- * @param label - the path or URL, used verbatim in error messages and as the sniff's extension hint.
- * @throws when the document is a binary we cannot read, is not decodable, or is a PDF without a
- * text layer.
- */
-export declare function documentText(bytes: Uint8Array, label: string): Promise<DocumentText>;
-export {};

package/lib/engine/mem/store/ingest_guard.d.ts

@@ -1,25 +0,0 @@
-/** The boundary knobs, as resolved from `knowledge.ingest`. */
-export interface IngestLimits {
-    /** Roots a local `source_uri` may be read from; empty ⇒ the process workspace. */
-    local_roots: readonly string[];
-    allow_outside_workspace: boolean;
-    allow_private_network: boolean;
-}
-/**
- * Whether a URL hostname addresses loopback / private / link-local / other
- * non-public space. Names are matched by suffix too (`*.local` mDNS, `*.internal`,
- * `*.home.arpa`), since those resolve inside the LAN by construction.
- */
-export declare function isPrivateHostname(hostname: string): boolean;
-/** Validate one `http(s)` source URL, refusing non-public targets unless opted in. */
-export declare function assertFetchableUrl(uri: string, limits: Pick<IngestLimits, 'allow_private_network'>): URL;
-/** The roots a local `source_uri` may be read from (realpath'd), defaulting to the workspace. */
-export declare function allowedLocalRoots(limits: Pick<IngestLimits, 'local_roots'>): string[];
-/**
- * Resolve a local `source_uri` to the real absolute path it names, refusing one
- * that lands outside the allowed roots.
- *
- * @returns the `realpath` — which is what the caller should BOTH read and store, so
- *   the recorded `source_uri` is exactly the file that was read.
- */
-export declare function resolveLocalSource(raw: string, limits: IngestLimits): string;