@tenphi/akno-core 0.1.0

package/dist/index/indexer.d.ts

@@ -0,0 +1,301 @@
+import type { AknoConfig } from '../config/schema.ts';
+import type { Store } from '../store/db.ts';
+import type { ModelClient } from '../models/client.ts';
+import { type IndexRevisionBarrier } from './revision-barrier.ts';
+export interface IndexOptions {
+    /** Hash every file instead of trusting mtime+size. The correctness path. */
+    verify?: boolean;
+    /** Skip the model-backed passes. Structure indexes in milliseconds without them. */
+    structuralOnly?: boolean;
+    /** Re-derive summaries and facts even where the body hash has not moved. */
+    rederive?: boolean;
+    /**
+     * Re-examine the named paths even though nothing about them changed.
+     *
+     * The point is ownership, not content: after a page is written *for* an existing attachment,
+     * the attachment's bytes are the same as ever, so the stat fast path skips it and nothing
+     * re-resolves which page owns it. This forces that one question to be asked again.
+     */
+    reindexUnchanged?: boolean;
+    onProgress?: (progress: IndexProgress) => void;
+    /** Restrict the pass to these relative paths — what the watcher uses. */
+    only?: string[];
+    /**
+     * Restrict the **model-backed** passes to these paths while still walking the
+     * whole tree structurally.
+     *
+     * `undo`, `forget` and `move` all need a full walk, because only a full pass can
+     * conclude a file is gone. But a full walk must not drag the entire embedding and
+     * derivation backlog along with it: reversing a one-line change should not cost
+     * 223 pages of model calls. Structure is ~10ms; model work is seconds per page,
+     * and the two scopes are genuinely different questions.
+     */
+    modelPaths?: string[];
+}
+export interface IndexProgress {
+    phase: 'scan' | 'hash' | 'pages' | 'embed' | 'derive' | 'documents' | 'graph' | 'extract' | 'renditions' | 'summarize' | 'done';
+    done: number;
+    total: number;
+    detail?: string;
+}
+export interface IndexReport {
+    scanned: number;
+    hashed: number;
+    pagesIndexed: number;
+    pagesUnchanged: number;
+    pagesRemoved: number;
+    pagesRenamed: number;
+    chunksWritten: number;
+    chunksEmbedded: number;
+    pagesDerived: number;
+    documentsLinked: number;
+    /** Attachments whose text was read this pass — extraction happens on arrival, always. */
+    documentsExtracted: number;
+    /** Documents given a summary this pass — one per document, not one per file. */
+    documentsSummarized: number;
+    /** `<file>.txt` written beside a document this pass. Zero unless `ingest.text_rendition`. */
+    renditionsWritten: number;
+    eventsIndexed: number;
+    factsDerived: number;
+    graphNodes: number;
+    graphEdges: number;
+    graphEntities: number;
+    graphMentions: number;
+    graphAmbiguousMentions: number;
+    graphUnresolvedMentions: number;
+    graphContextualMentions: number;
+    graphContextualResolved: number;
+    graphContextualAbstained: number;
+    graphContextualCached: number;
+    graphContextualFailed: number;
+    graphFacts: number;
+    graphFactEdges: number;
+    graphNonTraversableFacts: number;
+    ignored: number;
+    /** Non-fatal problems worth reporting rather than throwing. `doctor` prints these. */
+    warnings: string[];
+    durationMs: number;
+}
+/**
+ * Reconciles the index against the knowledge base. Both directions are
+ * first class: the user restructures everything on a Sunday afternoon and this
+ * reconciles, without ever asking them to go through Akno.
+ *
+ * A restart does **not** re-index — it stats. Only files whose mtime or size
+ * moved get hashed, so a full restart with nothing changed is a 1.2ms sweep.
+ */
+export declare class Indexer {
+    #private;
+    constructor(config: AknoConfig, store: Store, models: {
+        embedding: ModelClient;
+        derive: ModelClient;
+    });
+    run(options?: IndexOptions): Promise<IndexReport>;
+    /** Finish a caller-visible memory mutation even when a background dream owns a planner revision. */
+    runForeground(options?: IndexOptions): Promise<IndexReport>;
+    /** Notify a planner barrier about a foreground policy mutation that has no index pass. */
+    invalidateRevisionBarrier(): boolean;
+    /** Hold one indexed revision while a full dream run constructs its complete planner wave. */
+    acquireRevisionBarrier(): Promise<IndexRevisionBarrier>;
+    private runPass;
+    /**
+     * Pages whose policy may have moved because the *rules* moved, not because the file did.
+     *
+     * The resolved rules are fingerprinted in `meta`. While the fingerprint holds this is
+     * one query that finds nothing; when it moves, every page is re-resolved and the ones
+     * are handed back to be re-indexed. Their derivations are dropped in the same breath:
+     * a page that just became `source` was fact-mined under the old
+     * rule, and those facts assert claims the rules now say it never made.
+     *
+     * Re-indexing all pages on the rare rule change is deliberate: rules also supply
+     * management and about metadata, so comparing only role would leave policy stale.
+     */
+    private reclassify;
+    /**
+     * Page files the index has a record of and no page for.
+     *
+     * The stat fast path asks `files` whether anything moved, so a file whose `files` row is
+     * intact is never looked at again — and that is exactly the state a reorganization can
+     * leave behind, with the `pages` row deleted underneath a `files` row that still points at
+     * it. The file is on disk, the index says "unchanged", and recall cannot return it: the
+     * folder and the index disagree, and the index is the one that must yield.
+     *
+     * `reclassify` looks for the same failure one shape over — `page_id IS NULL`, a file the
+     * rules excluded — and misses this one, where the id is set and the page it names is gone.
+     */
+    private pageFilesWithNoPage;
+    /**
+     * What the scan skips: the configured list, plus Akno's own files inside the knowledge
+     * base.
+     *
+     * `akno.jsonc` is not memory — it is the rules that decide what memory *is*. It was
+     * being registered as an attachment of the root, which is how a taxonomy ends up
+     * described in `doctor` as a document whose contents could not be extracted.
+     */
+    private scanIgnore;
+    /** The policy current rules give a slug, ignoring page-local declarations. */
+    private policyFor;
+    private knownFiles;
+    private touchFile;
+    private recordFile;
+    /**
+     * Rename `people/ada.md` to `people/ada-marlow.md` in a file manager
+     * and content hashing sees a delete plus a create. Facts, journal entries and
+     * inbound links all point at a slug that no longer exists.
+     *
+     * In sidecar-identity mode the recovery is the body hash: a vanished file and a
+     * new file with identical content are the same page, so the id — and every fact
+     * and link hanging off it — survives the move. This is what `write_ids: false`
+     * trades away when the database is deleted, and nothing else.
+     */
+    private reconcileDeletions;
+    private indexPage;
+    /**
+     * Identity resolution, in order: a frontmatter `id` the user or a previous run
+     * wrote; the sidecar row for this path; the sidecar row for these exact bytes
+     * (a move the deletion pass did not catch, e.g. a copy); a fresh id.
+     */
+    private resolvePageId;
+    private pageIdForPath;
+    private writeIdIntoPage;
+    private upsertPage;
+    private replaceChunks;
+    /**
+     * FTS5 external-content tables need explicit deletes; there are no triggers. Vectors are
+     * removed per chunk id for the same reason a page's document chunks survive a page edit:
+     * `removeForPage` cannot tell the two kinds apart.
+     */
+    private deleteChunkRows;
+    private replaceEvents;
+    private replaceLinks;
+    /**
+     * A link written before its target existed is not broken. Re-resolving after
+     * every page is present is the difference between a useful broken-link report
+     * and one nobody reads.
+     */
+    private resolveLinks;
+    private removePage;
+    /**
+     * Extraction, OCR and naming land with `ingest`. What this cut does is
+     * *notice* attachments and attach them to their page, so a card can say "this
+     * page has a 9-page PDF" and `doctor` can report how many are un-extracted.
+     * Recording them now also means the write path inherits a populated table
+     * rather than starting from an empty one.
+     */
+    private registerAttachment;
+    private entriesOf;
+    /**
+     * Two ways an attachment belongs to a page. The content-addressed shape
+     * `<page-basename>-<8 hex>.<ext>` is the one Akno creates. A plain
+     * `passport.pdf` beside `passport.md` is the one people already have, and
+     * refusing to recognise it would make the feature useless on an existing
+     * knowledge base.
+     */
+    /**
+     * The page a file belongs to: by Akno's own `<page>-<8 hex>.<ext>` naming, by a matching
+     * stem beside it, or because a page embeds it. Called with the group's part-one path
+     * first, so every part of a multi-part document lands on one page.
+     */
+    private attachmentOwner;
+    /**
+     * **Extraction happens on arrival, always** — and this is what makes that true for
+     * attachments Akno did not place itself: a PDF someone dropped into `documents/` by
+     * hand, or one that predates Akno entirely. Their text is read, chunked, and indexed
+     * against the document, so the file is searchable by its own content.
+     *
+     * The invalidation rule is the *file* hash, which is why `extracted_sha` exists:
+     * re-extract when the bytes change, and never otherwise. Extraction is local — PDFKit,
+     * Vision, `textutil` — so a backlog costs seconds, not model calls.
+     *
+     * A document with no page of its own is chunked with a NULL page id and returned as a
+     * first-class document card. Organization improves the result, but is not a visibility gate.
+     */
+    private extractPending;
+    /**
+     * The extracted text, written beside the file it came from.
+     *
+     * Invalidated by `rendition_sha`, never by whether the file happens to be there: a steady
+     * state costs one indexed comparison per pass rather than a `stat` per document, and a
+     * rendition the user deleted stays deleted, because the folder is theirs and deleting a
+     * file in it is an instruction rather than damage to repair.
+     *
+     * A decline is recorded too — the same reason `extracted_sha` is written when nothing
+     * could be read. Without that, every photograph in the knowledge base is reconsidered on
+     * every pass forever to reach the same answer.
+     */
+    private writeRenditions;
+    /**
+     * Whether Akno may write this path: only if it is free, or holds something Akno
+     * itself put there — which the file says in its own first line.
+     *
+     * Existing and being *recognised* as a rendition is not the same question. The scanner
+     * recognises `contract.pdf.txt` by its name, and a person's own file can have that name.
+     * Authorship has to be read from the contents or the check answers yes to everything.
+     */
+    private renditionIsOurs;
+    /**
+     * Whether each document still is — or has become — a rendition of another.
+     *
+     * `registerAttachment` answers that question for files the pass looked at, and the stat
+     * fast path means an unchanged file is never looked at again. So the answer would be
+     * frozen at whatever was true when the file first arrived: a `contract.txt` indexed before
+     * the rule could recognise it keeps its own chunks and goes on returning the contract's
+     * every phrase a second time, and one whose PDF has since been deleted stays a rendition
+     * of nothing. Neither file changed; what changed is what is around it.
+     *
+     * A query over `documents` rather than over the tree, so it costs one pass over a table
+     * with one row per attachment and nothing per note.
+     */
+    private reconcileRenditionClaims;
+    /**
+     * A rendition policy that moved has to ask again about every document it declined.
+     *
+     * Same problem `reclassify` solves for rules, and for the same reason: declines are
+     * recorded so they are not recomputed, which means lowering the threshold would otherwise
+     * change nothing until the files themselves changed.
+     */
+    private reconsiderRenditions;
+    /**
+     * A stored document has extracted text, a summary and embeddings of its own — one
+     * summary per *document*, so a passport split into two files does not get two
+     * half-summaries describing halves of one thing.
+     *
+     * Kept separate from extraction because the two are invalidated by different things:
+     * extraction by the file's hash, a summary by not having one. A model that was down,
+     * or that failed to answer in JSON, is retried on the next pass instead of waiting for the
+     * bytes on disk to change.
+     */
+    private summarizeDocuments;
+    /**
+     * A document's chunks are `source`: evidence, quoted in a capped
+     * window, never mined for facts. A contract is not the household asserting its terms.
+     */
+    private replaceDocumentChunks;
+    private embedPending;
+    private derivePending;
+    /**
+     * Supersession is structural: a fact whose source line is gone or changed
+     * gets `valid_to` set rather than being deleted, so recall can return it *as
+     * superseded* — "was €28 until June" — instead of as a second competing current
+     * answer.
+     *
+     * The distinction that matters is **why** a fact disappeared:
+     *
+     * - Its source line changed or went away → a real supersession. Retire it.
+     * - Its source line is byte-identical and the deriver merely phrased the claim
+     *   differently → not a supersession at all. Delete it.
+     *
+     * Conflating the two invents history. A fresh derivation
+     * may phrase a claim differently, so retiring on id alone would make every
+     * `--rederive` flood recall with "was X until today" for values that never
+     * changed — and an invented historical claim is worse than none, because a
+     * reader has no way to tell it apart from a real one.
+     *
+     * `presentLines` therefore comes from **the page**, not from the incoming facts. Read
+     * from the facts, an empty derivation looks like every source line vanishing at once, so
+     * a page that merely became `source` — or one where a small model returned no facts
+     * this time — retired its whole history as superseded on lines nobody had touched.
+     */
+    private replaceFacts;
+}
+//# sourceMappingURL=indexer.d.ts.map

package/dist/index/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/index/indexer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA2BtD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,EAA4B,KAAK,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAE5F,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oFAAoF;IACpF,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;IAC/C,yEAAyE;IACzE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EACD,MAAM,GACN,MAAM,GACN,OAAO,GACP,OAAO,GACP,QAAQ,GACR,WAAW,GACX,OAAO,GACP,SAAS,GACT,YAAY,GACZ,WAAW,GACX,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,yFAAyF;IACzF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gFAAgF;IAChF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,6FAA6F;IAC7F,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,uBAAuB,EAAE,MAAM,CAAC;IAChC,uBAAuB,EAAE,MAAM,CAAC;IAChC,uBAAuB,EAAE,MAAM,CAAC;IAChC,wBAAwB,EAAE,MAAM,CAAC;IACjC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,wBAAwB,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,sFAAsF;IACtF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAWD;;;;;;;GAOG;AACH,qBAAa,OAAO;;gBAMN,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;QAAE,SAAS,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,WAAW,CAAA;KAAE;IAM/F,GAAG,CAAC,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;IAI3D,oGAAoG;IAC9F,aAAa,CAAC,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;IAIrE,0FAA0F;IAC1F,yBAAyB,IAAI,OAAO;IAIpC,6FAA6F;IAC7F,sBAAsB,IAAI,OAAO,CAAC,oBAAoB,CAAC;YAIzC,OAAO;IAuOrB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,UAAU;IAuClB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,mBAAmB;IAkB3B;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IAIlB,8EAA8E;IAC9E,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,UAAU;IAclB;;;;;;;;;OASG;IACH,OAAO,CAAC,kBAAkB;YAwEZ,SAAS;IAwDvB;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAgCrB,OAAO,CAAC,aAAa;YAMP,eAAe;IAsB7B,OAAO,CAAC,UAAU;IA8ClB,OAAO,CAAC,aAAa;IA2BrB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,aAAa;IAsBrB,OAAO,CAAC,YAAY;IAiBpB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IASlB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IAyG1B,OAAO,CAAC,SAAS;IAcjB;;;;;;OAMG;IACH;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAiDvB;;;;;;;;;;;;OAYG;YACW,cAAc;IAsG5B;;;;;;;;;;;OAWG;YACW,eAAe;IA4F7B;;;;;;;OAOG;YACW,eAAe;IAO7B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,wBAAwB;IAsDhC;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;IA4B5B;;;;;;;;;OASG;YACW,kBAAkB;IAgDhC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;YAkCf,YAAY;YAkEZ,aAAa;IA2F3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,YAAY;CAwDrB"}