@tenphi/akno-core 0.1.0

package/dist/ingest/extract.js

@@ -0,0 +1,255 @@
+import { execFile } from 'node:child_process';
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { sha256 } from "../store/ids.js";
+const run = promisify(execFile);
+const IMAGE_EXTENSIONS = new Set([
+    '.png',
+    '.jpg',
+    '.jpeg',
+    '.heic',
+    '.heif',
+    '.webp',
+    '.tiff',
+    '.tif',
+    '.gif',
+    '.bmp',
+]);
+const PLAIN_EXTENSIONS = new Set([
+    '.txt',
+    '.md',
+    '.markdown',
+    '.csv',
+    '.tsv',
+    '.json',
+    '.yaml',
+    '.yml',
+    '.log',
+]);
+/** Formats `textutil` handles natively — it ships with macOS. */
+const TEXTUTIL_EXTENSIONS = new Set([
+    '.rtf',
+    '.rtfd',
+    '.doc',
+    '.docx',
+    '.odt',
+    '.html',
+    '.htm',
+    '.webarchive',
+]);
+export async function extract(options) {
+    const extension = path.extname(options.absPath).toLowerCase();
+    const stat = await fsp.stat(options.absPath).catch(() => null);
+    if (!stat)
+        return none('the file could not be read');
+    if (stat.size === 0)
+        return none('the file is empty');
+    if (stat.size > options.maxBytes) {
+        return none(`the file is ${Math.round(stat.size / 1_048_576)} MB, over the configured limit`);
+    }
+    if (PLAIN_EXTENSIONS.has(extension)) {
+        const text = await fsp.readFile(options.absPath, 'utf8').catch(() => '');
+        return { text: text.trim(), pageCount: null, ocr: false, confidence: null, via: 'plain', note: null };
+    }
+    if (TEXTUTIL_EXTENSIONS.has(extension))
+        return textutil(options.absPath);
+    if (extension === '.pdf')
+        return pdf(options);
+    if (IMAGE_EXTENSIONS.has(extension))
+        return image(options);
+    return none(`no extractor for ${extension || 'a file with no extension'}`);
+}
+function none(note) {
+    return { text: '', pageCount: null, ocr: false, confidence: null, via: 'none', note };
+}
+async function pdf(options) {
+    const binary = await ensureExtractor();
+    if (!binary) {
+        return none('the extractor could not be built — is the Xcode command line tools package installed?');
+    }
+    const result = await callSwift(binary, [
+        'pdf',
+        options.absPath,
+        '--ocr-pages',
+        String(options.maxOcrPages),
+    ]);
+    if (!result)
+        return none('the extractor did not return a result');
+    if (result.error)
+        return none(result.error);
+    return {
+        text: result.text.trim(),
+        pageCount: result.pages,
+        ocr: result.ocr,
+        confidence: result.confidence ?? null,
+        via: result.ocr ? 'ocr' : 'text-layer',
+        ...(result.sections?.length ? { sections: result.sections } : {}),
+        note: result.ocr && result.pages > options.maxOcrPages
+            ? `scanned: OCR read the first ${options.maxOcrPages} of ${result.pages} pages`
+            : null,
+    };
+}
+async function image(options) {
+    const binary = await ensureExtractor();
+    const result = binary ? await callSwift(binary, ['image', options.absPath]) : null;
+    const text = (result?.text ?? '').trim();
+    if (text.length > 0) {
+        return {
+            text,
+            pageCount: 1,
+            ocr: true,
+            confidence: result?.confidence ?? null,
+            via: 'ocr',
+            note: null,
+            sections: [{ page: 1, text }],
+        };
+    }
+    // **Photos with no text yield no page; OCR still covers scans and
+    // screenshots, which is most of what arrives.** With a vision model configured, a
+    // photo can still be described — but that is a different claim about the file, so
+    // it is labelled `via: 'vision'` and never confused with OCR of real text.
+    if (options.vision?.available) {
+        const described = await describeImage(options.absPath, options.vision);
+        if (described) {
+            return { text: described, pageCount: 1, ocr: false, confidence: null, via: 'vision', note: null };
+        }
+    }
+    return none(binary
+        ? 'no text found in the image, and no vision model is configured to describe it'
+        : 'the extractor could not be built, and no vision model is configured');
+}
+async function describeImage(absPath, vision) {
+    const data = await fsp.readFile(absPath).catch(() => null);
+    if (!data)
+        return null;
+    const mime = mimeFor(path.extname(absPath).toLowerCase());
+    const images = [{ data, mime }];
+    const result = await vision.chat([
+        {
+            role: 'system',
+            content: 'Describe an image for a personal knowledge base index, so it can be found again by search. ' +
+                'Two or three sentences. Name what it is, what is in it, and any text, date, place or amount ' +
+                'you can read. Do not speculate about who took it or why, and do not describe what you cannot ' +
+                'see. If it is a document, transcribe the readable text instead of describing it.',
+        },
+        { role: 'user', content: 'Describe this image.' },
+    ], { images, maxTokens: 600 });
+    return result.ok && result.value ? result.value.trim() : null;
+}
+function mimeFor(extension) {
+    const table = {
+        '.png': 'image/png',
+        '.jpg': 'image/jpeg',
+        '.jpeg': 'image/jpeg',
+        '.heic': 'image/heic',
+        '.heif': 'image/heif',
+        '.webp': 'image/webp',
+        '.tiff': 'image/tiff',
+        '.tif': 'image/tiff',
+        '.gif': 'image/gif',
+        '.bmp': 'image/bmp',
+    };
+    return table[extension] ?? 'application/octet-stream';
+}
+async function callSwift(binary, args) {
+    try {
+        // 10 MB: a fully OCR'd 12-page scan runs to a few hundred KB, so this is slack
+        // rather than a limit anyone will meet.
+        const { stdout } = await run(binary, args, { maxBuffer: 10 * 1_048_576, timeout: 300_000 });
+        return JSON.parse(stdout);
+    }
+    catch {
+        return null;
+    }
+}
+// ─── textutil, which ships with macOS ───────────────────────────────────────
+async function textutil(absPath) {
+    try {
+        const { stdout } = await run('/usr/bin/textutil', ['-convert', 'txt', '-stdout', absPath], {
+            maxBuffer: 10 * 1_048_576,
+            timeout: 60_000,
+        });
+        const text = stdout.trim();
+        return text.length > 0
+            ? { text, pageCount: null, ocr: false, confidence: null, via: 'textutil', note: null }
+            : none('textutil found no text');
+    }
+    catch (err) {
+        return none(`textutil could not read the file: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+// ─── Building and caching the helper ────────────────────────────────────────
+let cachedBinary;
+/**
+ * Compiles `swift/extract.swift` on first use and reuses it after, keyed by a hash of
+ * the source — so upgrading Akno recompiles, and a hand-edited source is picked up.
+ *
+ * Cached in `~/Library/Caches`, **not** in the state directory. The binary is an
+ * artifact of Akno, identical for every knowledge base, and putting it under
+ * `state_dir` meant a second knowledge base compiled its own copy of the same 200
+ * lines. `Caches` is also the directory macOS is allowed to reclaim, which is exactly
+ * right for something rebuildable in six seconds.
+ *
+ * No `-O`: every expensive thing here happens inside Vision and PDFKit, so optimizing
+ * the glue buys nothing and costs 23 of the 29 seconds.
+ */
+async function ensureExtractor() {
+    if (cachedBinary !== undefined)
+        return cachedBinary;
+    const source = findSwiftSource();
+    if (!source) {
+        cachedBinary = null;
+        return null;
+    }
+    const version = sha256(await fsp.readFile(source, 'utf8')).slice(0, 12);
+    const binary = path.join(cacheDir(), `akno-extract-${version}`);
+    if (fs.existsSync(binary)) {
+        cachedBinary = binary;
+        return binary;
+    }
+    try {
+        await fsp.mkdir(path.dirname(binary), { recursive: true });
+        // Compile to a temp name and rename, so two processes racing on first use cannot
+        // leave a half-written binary that the loser then tries to execute.
+        const temp = `${binary}.${process.pid}.tmp`;
+        await run('swiftc', ['-o', temp, source], { timeout: 180_000 });
+        await fsp.rename(temp, binary);
+        await fsp.chmod(binary, 0o755);
+        cachedBinary = binary;
+        return binary;
+    }
+    catch {
+        // Degrade, never fail: plain text and Office formats still extract, and images
+        // fall back to the vision model if one is configured.
+        cachedBinary = null;
+        return null;
+    }
+}
+function cacheDir() {
+    return process.env.AKNO_CACHE_DIR ?? path.join(os.homedir(), 'Library', 'Caches', 'akno', 'bin');
+}
+/** Beside `dist` when installed, at the package root when running from source. */
+function findSwiftSource() {
+    for (const up of ['..', '../..', '../../..', '../../../..']) {
+        const candidate = path.resolve(import.meta.dirname, up, 'swift', 'extract.swift');
+        if (fs.existsSync(candidate))
+            return candidate;
+    }
+    return null;
+}
+/** What `doctor` reports, so a missing capability is visible rather than surprising. */
+export async function extractionCapabilities() {
+    const binary = await ensureExtractor();
+    return {
+        swift: binary !== null,
+        textutil: fs.existsSync('/usr/bin/textutil'),
+        note: binary
+            ? null
+            : 'PDF and image extraction is unavailable: the Swift helper could not be built. ' +
+                'Install the Xcode command line tools (`xcode-select --install`).',
+    };
+}
+//# sourceMappingURL=extract.js.map

package/dist/ingest/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/ingest/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGzC,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AA0ChC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;CACP,CAAC,CAAC;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,MAAM;IACN,KAAK;IACL,WAAW;IACX,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;CACP,CAAC,CAAC;AACH,iEAAiE;AACjE,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,aAAa;CACd,CAAC,CAAC;AAWH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAuB;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IAE9D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACrD,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,gCAAgC,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxG,CAAC;IAED,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACzE,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC;IAE3D,OAAO,IAAI,CAAC,oBAAoB,SAAS,IAAI,0BAA0B,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AACxF,CAAC;AAaD,KAAK,UAAU,GAAG,CAAC,OAAuB;IACxC,MAAM,MAAM,GAAG,MAAM,eAAe,EAAE,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC,uFAAuF,CAAC,CAAC;IACvG,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE;QACrC,KAAK;QACL,OAAO,CAAC,OAAO;QACf,aAAa;QACb,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;KAC5B,CAAC,CAAC;IACH,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE5C,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC,KAAK;QACvB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY;QACtC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,IAAI,EACF,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,WAAW;YAC9C,CAAC,CAAC,+BAA+B,OAAO,CAAC,WAAW,OAAO,MAAM,CAAC,KAAK,QAAQ;YAC/E,CAAC,CAAC,IAAI;KACX,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,OAAuB;IAC1C,MAAM,MAAM,GAAG,MAAM,eAAe,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEnF,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO;YACL,IAAI;YACJ,SAAS,EAAE,CAAC;YACZ,GAAG,EAAE,IAAI;YACT,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,IAAI;YACtC,GAAG,EAAE,KAAK;YACV,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAED,kEAAkE;IAClE,kFAAkF;IAClF,kFAAkF;IAClF,2EAA2E;IAC3E,IAAI,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACvE,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACpG,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CACT,MAAM;QACJ,CAAC,CAAC,8EAA8E;QAChF,CAAC,CAAC,qEAAqE,CAC1E,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,OAAe,EAAE,MAAmB;IAC/D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAC9B;QACE;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EACL,6FAA6F;gBAC7F,8FAA8F;gBAC9F,+FAA+F;gBAC/F,kFAAkF;SACrF;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,sBAAsB,EAAE;KAClD,EACD,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAC3B,CAAC;IACF,OAAO,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAChE,CAAC;AAED,SAAS,OAAO,CAAC,SAAiB;IAChC,MAAM,KAAK,GAA2B;QACpC,MAAM,EAAE,WAAW;QACnB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,YAAY;QACrB,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,WAAW;QACnB,MAAM,EAAE,WAAW;KACpB,CAAC;IACF,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,0BAA0B,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,IAAc;IACrD,IAAI,CAAC;QACH,+EAA+E;QAC/E,wCAAwC;QACxC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAgB,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,+EAA+E;AAE/E,KAAK,UAAU,QAAQ,CAAC,OAAe;IACrC,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,mBAAmB,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE;YACzF,SAAS,EAAE,EAAE,GAAG,SAAS;YACzB,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE;YACtF,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,IAAI,CAAC,qCAAqC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvG,CAAC;AACH,CAAC;AAED,+EAA+E;AAE/E,IAAI,YAAuC,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,eAAe;IAC5B,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,YAAY,CAAC;IAEpD,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,YAAY,GAAG,IAAI,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,gBAAgB,OAAO,EAAE,CAAC,CAAC;IAEhE,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,YAAY,GAAG,MAAM,CAAC;QACtB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,iFAAiF;QACjF,oEAAoE;QACpE,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;QAC5C,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAChE,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/B,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC/B,YAAY,GAAG,MAAM,CAAC;QACtB,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,+EAA+E;QAC/E,sDAAsD;QACtD,YAAY,GAAG,IAAI,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,QAAQ;IACf,OAAO,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;AACnG,CAAC;AAED,kFAAkF;AAClF,SAAS,eAAe;IACtB,KAAK,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,CAAC,EAAE,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QAClF,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAK1C,MAAM,MAAM,GAAG,MAAM,eAAe,EAAE,CAAC;IACvC,OAAO;QACL,KAAK,EAAE,MAAM,KAAK,IAAI;QACtB,QAAQ,EAAE,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAC5C,IAAI,EAAE,MAAM;YACV,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,gFAAgF;gBAChF,kEAAkE;KACvE,CAAC;AACJ,CAAC"}

package/dist/ingest/fetch.d.ts

@@ -0,0 +1,35 @@
+/**
+ * `ingest` pulls documents from a file, a folder, or a URL. This is the URL.
+ *
+ * Downloading is the one thing `ingest` does that reaches outside the machine, so the
+ * limits are deliberate rather than defensive-by-habit:
+ *
+ * - **http and https only.** `file://` would make `ingest({url})` a way to read any
+ *   path on the machine through an interface that looks like it fetches the web, and
+ *   `data:` would smuggle bytes past the size cap.
+ * - **Redirects are followed** by `fetch`, which caps the chain itself; a redirect is
+ *   normal and an infinite one is already handled below us.
+ * - **The size cap applies to what arrives, not to what the server claims.** A
+ *   `Content-Length` is a promise, and the stream is what actually lands on disk.
+ * - **The filename comes from the response, then the URL, then the content type.** A
+ *   URL path is often `/download` or `/`, which says nothing — and a name adding nothing is
+ *   exactly what naming-from-content replaces.
+ */
+export interface Fetched {
+    /** A temp file. The caller ingests it and is responsible for cleaning up. */
+    path: string;
+    /** What the server called it, for the rename guard. */
+    originalName: string;
+    contentType: string | null;
+    bytes: number;
+    finalUrl: string;
+}
+export interface FetchOptions {
+    url: string;
+    maxBytes: number;
+    timeoutMs?: number;
+}
+export declare function fetchDocument(options: FetchOptions): Promise<Fetched>;
+/** The temp directory a fetch created. Removed once the bytes have been stored. */
+export declare function cleanupFetch(fetched: Fetched): Promise<void>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/ingest/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,OAAO;IACtB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CA4D3E;AAED,mFAAmF;AACnF,wBAAsB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE"}

package/dist/ingest/fetch.js

@@ -0,0 +1,142 @@
+import fsp from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { AknoError } from '@tenphi/akno-protocol';
+export async function fetchDocument(options) {
+    const url = parseUrl(options.url);
+    let response;
+    try {
+        response = await fetch(url, {
+            redirect: 'follow',
+            signal: AbortSignal.timeout(options.timeoutMs ?? 60_000),
+            headers: { accept: '*/*' },
+        });
+    }
+    catch (err) {
+        const timedOut = err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError');
+        throw new AknoError('unavailable', timedOut
+            ? `fetching ${url.href} timed out`
+            : `could not fetch ${url.href}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!response.ok) {
+        throw new AknoError('unavailable', `${url.href} returned ${response.status} ${response.statusText}`);
+    }
+    if (!response.body) {
+        throw new AknoError('unavailable', `${url.href} returned no body`);
+    }
+    const contentType = (response.headers.get('content-type') ?? '').split(';')[0].trim() || null;
+    const originalName = filenameFor(response, url, contentType);
+    const directory = await fsp.mkdtemp(path.join(os.tmpdir(), 'akno-fetch-'));
+    const target = path.join(directory, originalName);
+    const handle = await fsp.open(target, 'w');
+    let written = 0;
+    try {
+        for await (const chunk of response.body) {
+            written += chunk.byteLength;
+            // Enforced against the bytes that actually arrive: `Content-Length` is a claim,
+            // and a server willing to lie about the size is exactly the one to guard against.
+            if (written > options.maxBytes) {
+                await handle.close();
+                await fsp.rm(directory, { recursive: true, force: true });
+                throw new AknoError('invalid', `${url.href} is larger than the configured limit of ${Math.round(options.maxBytes / 1_048_576)} MB`);
+            }
+            await handle.write(chunk);
+        }
+    }
+    finally {
+        await handle.close().catch(() => { });
+    }
+    if (written === 0) {
+        await fsp.rm(directory, { recursive: true, force: true });
+        throw new AknoError('unavailable', `${url.href} returned an empty body`);
+    }
+    return { path: target, originalName, contentType, bytes: written, finalUrl: response.url || url.href };
+}
+/** The temp directory a fetch created. Removed once the bytes have been stored. */
+export async function cleanupFetch(fetched) {
+    await fsp.rm(path.dirname(fetched.path), { recursive: true, force: true });
+}
+function parseUrl(raw) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        throw new AknoError('invalid', `not a usable URL: ${raw}`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new AknoError('invalid', `only http and https URLs are fetched, not ${url.protocol} — pass a local path instead`);
+    }
+    return url;
+}
+/**
+ * `Content-Disposition`, then the URL's last path segment, then an extension guessed
+ * from the content type. Every one of these can be useless, which is fine: the page is named
+ * from the content anyway, and a useless name is what triggers the rename.
+ */
+function filenameFor(response, url, contentType) {
+    const disposition = response.headers.get('content-disposition');
+    const candidates = [];
+    if (disposition) {
+        const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(disposition);
+        const plain = /filename="?([^";]+)"?/i.exec(disposition);
+        const raw = star?.[1] ?? plain?.[1];
+        // `path.basename` is what stops `../../../etc/passwd` being a path: a server chooses
+        // this header, so it is untrusted input that becomes a filename.
+        if (raw)
+            candidates.push(sanitize(path.basename(safeDecode(raw.trim()))));
+    }
+    candidates.push(sanitize(path.basename(safeDecode(url.pathname))));
+    const extension = extensionFor(contentType);
+    for (const candidate of candidates) {
+        if (candidate.length === 0 || candidate === '.' || candidate === '..')
+            continue;
+        // The extractor dispatches on the extension, so a name without one is unusable even
+        // though it reads fine. Servers send `filename="report"` often enough that guessing
+        // from the content type is the difference between working and skipping.
+        return path.extname(candidate) ? candidate : `${candidate}${extension}`;
+    }
+    return `download${extension}`;
+}
+/**
+ * A server chooses these bytes, so they are untrusted input on their way to becoming a
+ * filename: separators and control characters are dropped. Spaces survive, because a
+ * filename with spaces is ordinary and the rename handles presentation anyway.
+ */
+function sanitize(name) {
+    return [...name]
+        .filter((char) => char !== '/' && char !== '\\' && char.codePointAt(0) >= 0x20)
+        .join('')
+        .trim();
+}
+function safeDecode(value) {
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return value;
+    }
+}
+function extensionFor(contentType) {
+    const table = {
+        'application/pdf': '.pdf',
+        'text/plain': '.txt',
+        'text/markdown': '.md',
+        'text/html': '.html',
+        'text/csv': '.csv',
+        'application/json': '.json',
+        'image/png': '.png',
+        'image/jpeg': '.jpg',
+        'image/heic': '.heic',
+        'image/webp': '.webp',
+        'image/gif': '.gif',
+        'image/tiff': '.tiff',
+        'application/rtf': '.rtf',
+        'application/msword': '.doc',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+    };
+    // No extension at all is honest: the extractor dispatches on it and will say it has
+    // none, which is better than claiming `.bin` and looking like a decision.
+    return contentType ? (table[contentType] ?? '') : '';
+}
+//# sourceMappingURL=fetch.js.map

package/dist/ingest/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/ingest/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAoClD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAqB;IACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAElC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;YACxD,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;QACpG,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,QAAQ;YACN,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,YAAY;YAClC,CAAC,CAAC,mBAAmB,GAAG,CAAC,IAAI,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,mBAAmB,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IAC/F,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IAE7D,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE3C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAA4C,EAAE,CAAC;YAChF,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC;YAC5B,gFAAgF;YAChF,kFAAkF;YAClF,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,GAAG,GAAG,CAAC,IAAI,2CAA2C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,KAAK,CACpG,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,GAAG,GAAG,CAAC,IAAI,yBAAyB,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;AACzG,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IACjD,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,SAAS,EAAE,qBAAqB,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CACjB,SAAS,EACT,6CAA6C,GAAG,CAAC,QAAQ,8BAA8B,CACxF,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,QAAkB,EAAE,GAAQ,EAAE,WAA0B;IAC3E,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,iCAAiC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,qFAAqF;QACrF,iEAAiE;QACjE,IAAI,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI;YAAE,SAAS;QAChF,oFAAoF;QACpF,oFAAoF;QACpF,wEAAwE;QACxE,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC;IAED,OAAO,WAAW,SAAS,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CAAC,GAAG,IAAI,CAAC;SACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;SAC/E,IAAI,CAAC,EAAE,CAAC;SACR,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,WAA0B;IAC9C,MAAM,KAAK,GAA2B;QACpC,iBAAiB,EAAE,MAAM;QACzB,YAAY,EAAE,MAAM;QACpB,eAAe,EAAE,KAAK;QACtB,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,MAAM;QAClB,kBAAkB,EAAE,OAAO;QAC3B,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO;QACrB,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,OAAO;QACrB,iBAAiB,EAAE,MAAM;QACzB,oBAAoB,EAAE,MAAM;QAC5B,yEAAyE,EAAE,OAAO;KACnF,CAAC;IACF,oFAAoF;IACpF,0EAA0E;IAC1E,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvD,CAAC"}

package/dist/ingest/inbox.d.ts

@@ -0,0 +1,53 @@
+import type { AknoContext } from '../context.ts';
+/**
+ * **The inbox: a folder where you drop anything and it files itself.**
+ *
+ * ```jsonc
+ * "inbox/**": { "ingest": "auto", "route": true }
+ * ```
+ *
+ * `route: true` is what makes it an inbox rather than just a folder. On arrival Akno
+ * extracts, names, summarizes, and then **routes** — the same recall-based routing
+ * `remember` uses, against the same threshold. Above it, the file and its page move
+ * there. Below it, the file **stays in the inbox** with a proposal attached.
+ *
+ * That failure mode is deliberate: an unrouted file sits visibly where you dropped it,
+ * rather than being filed confidently into the wrong place, where you would never look
+ * for it. **An inbox with three things in it is a to-do list. A misfiled document is a
+ * lost one.**
+ *
+ * **The inbox is the only place Akno moves files.** A file dropped straight into
+ * `documents/` was put there on purpose; Akno will name it, page it and index it, but
+ * never relocate it. Automatic filing is opt-in per folder, every move is journalled and
+ * reported in `doctor`, and every one is reversible with `undo`.
+ */
+export interface InboxResult {
+    /** Files that found a home and moved out. */
+    filed: {
+        source: string;
+        slug: string;
+    }[];
+    /** Files still sitting in the inbox, each with the reason. */
+    waiting: {
+        source: string;
+        reason: string;
+        proposalId?: string;
+    }[];
+    /** Files skipped by rule or because they are not documents at all. */
+    skipped: {
+        source: string;
+        reason: string;
+    }[];
+}
+/** True when a relative path sits inside a configured inbox. */
+export declare function isInInbox(ctx: AknoContext, relPath: string): boolean;
+/**
+ * Processes everything currently sitting in the inbox. Safe to call repeatedly: a file
+ * that could not be routed stays put and is simply reconsidered next time, which is what
+ * makes the inbox a to-do list rather than a queue that loses things.
+ */
+export declare function processInbox(ctx: AknoContext, options?: {
+    only?: string[];
+    limit?: number;
+}): Promise<InboxResult>;
+//# sourceMappingURL=inbox.d.ts.map

package/dist/ingest/inbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inbox.d.ts","sourceRoot":"","sources":["../../src/ingest/inbox.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAIjD;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,MAAM,WAAW,WAAW;IAC1B,6CAA6C;IAC7C,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,8DAA8D;IAC9D,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACnE,sEAAsE;IACtE,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/C;AAuBD,gEAAgE;AAChE,wBAAgB,SAAS,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAGpE;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,WAAW,EAChB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAChD,OAAO,CAAC,WAAW,CAAC,CA4EtB"}

package/dist/ingest/inbox.js

@@ -0,0 +1,110 @@
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { effectiveRule } from "../rules/compile.js";
+import { ingestFile } from "../ops/ingest.js";
+/**
+ * Folders that behave as an inbox: the reserved `inbox/`, plus any folder whose rule
+ * carries `route: true`.
+ *
+ * The reserved one is always included, not used as a fallback. Akno *creates* it on
+ * startup with a README saying its contents are extracted, named and routed — so a user
+ * declaring a second inbox somewhere else must not silently turn the first one into a
+ * folder that quietly accumulates files nobody looks at again.
+ */
+function inboxFolders(ctx) {
+    const folders = new Set([ctx.config.paths.inbox]);
+    for (const rule of ctx.config.rules) {
+        if (rule.route !== true)
+            continue;
+        // `inbox/**` → `inbox`. A rule's glob is how the folder is named, so an inbox is
+        // wherever the user said it is rather than a hardcoded path.
+        const folder = rule.glob.replace(/\/\*\*?$/, '').replace(/\/$/, '');
+        if (folder.length > 0 && !folder.includes('*'))
+            folders.add(folder);
+    }
+    return [...folders];
+}
+/** True when a relative path sits inside a configured inbox. */
+export function isInInbox(ctx, relPath) {
+    const normalized = relPath.replace(/\\/g, '/');
+    return inboxFolders(ctx).some((folder) => normalized === folder || normalized.startsWith(`${folder}/`));
+}
+/**
+ * Processes everything currently sitting in the inbox. Safe to call repeatedly: a file
+ * that could not be routed stays put and is simply reconsidered next time, which is what
+ * makes the inbox a to-do list rather than a queue that loses things.
+ */
+export async function processInbox(ctx, options = {}) {
+    const result = { filed: [], waiting: [], skipped: [] };
+    const limit = options.limit ?? 50;
+    let processed = 0;
+    for (const folder of inboxFolders(ctx)) {
+        const absFolder = path.join(ctx.config.aknoPath, folder);
+        let entries;
+        try {
+            entries = (await fsp.readdir(absFolder, { withFileTypes: true }))
+                .filter((entry) => entry.isFile() && !entry.name.startsWith('.'))
+                .map((entry) => entry.name)
+                .sort();
+        }
+        catch {
+            continue; // No inbox on disk yet. Nothing to do, and nothing to report.
+        }
+        for (const name of entries) {
+            if (processed >= limit)
+                break;
+            const relPath = `${folder}/${name}`;
+            if (options.only && !options.only.includes(relPath))
+                continue;
+            // A page written *about* an inbox item, and the README startup puts there, are not
+            // arrivals to be filed.
+            if (ctx.config.pageExtensions.includes(path.extname(name).toLowerCase())) {
+                result.skipped.push({ source: relPath, reason: 'a Markdown page, not a dropped document' });
+                continue;
+            }
+            const rule = effectiveRule(relPath.replace(/\.[^.]+$/, ''), ctx.config.rules);
+            if (rule.ingest === 'ignore') {
+                result.skipped.push({ source: relPath, reason: 'ingest: ignore' });
+                continue;
+            }
+            processed++;
+            try {
+                const ingested = await ingestFile(ctx, 
+                // No `folder`: the inbox exists precisely so routing decides. Passing one would
+                // make it an ordinary folder with extra steps.
+                { route: true }, {
+                    source: path.join(ctx.config.aknoPath, relPath),
+                    originalName: name,
+                    // The one place Akno moves files.
+                    move: true,
+                });
+                if (ingested.outcome === 'ok' && ingested.slug) {
+                    result.filed.push({ source: relPath, slug: ingested.slug });
+                }
+                else if (ingested.outcome === 'duplicate') {
+                    // Already stored under another name. Leaving the copy in the inbox is the
+                    // honest outcome: deleting a file the user put there is not Akno's call.
+                    result.waiting.push({
+                        source: relPath,
+                        reason: `already stored as ${ingested.rel_path} — delete this copy if you meant to`,
+                    });
+                }
+                else {
+                    result.waiting.push({
+                        source: relPath,
+                        reason: ingested.note ?? ingested.outcome,
+                        ...(ingested.approval ? { proposalId: ingested.approval.proposal_id } : {}),
+                    });
+                }
+            }
+            catch (err) {
+                result.waiting.push({
+                    source: relPath,
+                    reason: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=inbox.js.map

package/dist/ingest/inbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inbox.js","sourceRoot":"","sources":["../../src/ingest/inbox.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAkC9C;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,GAAgB;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1D,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,SAAS;QAClC,iFAAiF;QACjF,6DAA6D;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACpE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,SAAS,CAAC,GAAgB,EAAE,OAAe;IACzD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC/C,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,KAAK,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAgB,EAChB,UAA+C,EAAE;IAEjD,MAAM,MAAM,GAAgB,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,MAAM,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzD,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;iBAC9D,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;iBAChE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;iBAC1B,IAAI,EAAE,CAAC;QACZ,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,8DAA8D;QAC1E,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,SAAS,IAAI,KAAK;gBAAE,MAAM;YAC9B,MAAM,OAAO,GAAG,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;YACpC,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAE9D,mFAAmF;YACnF,wBAAwB;YACxB,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBACzE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC,CAAC;gBAC5F,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9E,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBACnE,SAAS;YACX,CAAC;YAED,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,UAAU,CAC/B,GAAG;gBACH,gFAAgF;gBAChF,+CAA+C;gBAC/C,EAAE,KAAK,EAAE,IAAI,EAAE,EACf;oBACE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC;oBAC/C,YAAY,EAAE,IAAI;oBAClB,kCAAkC;oBAClC,IAAI,EAAE,IAAI;iBACX,CACF,CAAC;gBAEF,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC/C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9D,CAAC;qBAAM,IAAI,QAAQ,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;oBAC5C,0EAA0E;oBAC1E,yEAAyE;oBACzE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;wBAClB,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,qBAAqB,QAAQ,CAAC,QAAQ,qCAAqC;qBACpF,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;wBAClB,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,OAAO;wBACzC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC5E,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACzD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}