@autorag/librarian 2.13.6 → 2.13.8

package/dist/retrieval/engine.d.ts

@@ -10,6 +10,7 @@
  * after retrieval; local (non-datasource) methods and datasources without
  * source-scope support pass through untouched.
  */
+import type { RecordCut } from "./record-cut.js";
 import { RetrievalMethodRegistry } from "./registry.js";
 import { type Reranker } from "./rerank.js";
 import { type RetrievalSelection } from "./selection.js";
@@ -45,6 +46,11 @@ export interface RetrievalEngineOptions {
      * Only applies when {@link reranker} is set. @default DEFAULT_RERANK_TOP_N
      */
     readonly rerankTopN?: number;
+    /**
+     * Cut the merged pool to `topK` spread across documents instead of by chunk, so one long document cannot
+     * take every slot (see {@link RecordCut}). Unset keeps the plain top-`topK` cut.
+     */
+    readonly recordCut?: RecordCut;
 }
 /**
  * Model-free retrieval pipeline.
@@ -76,6 +82,7 @@ export declare class RetrievalEngine {
     private readonly datasourceIdsProvider;
     private readonly reranker;
     private readonly rerankTopN;
+    private readonly recordCut;
     constructor(options?: RetrievalEngineOptions);
     /** The underlying method registry. Intentionally public for tooling. */
     getMethodRegistry(): RetrievalMethodRegistry;

package/dist/retrieval/merger.d.ts

@@ -1,3 +1,4 @@
+import type { RecordCut } from "./record-cut.js";
 import type { RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalWithDiagnostics } from "./types.js";
 export interface MergeOptions {
     /**
@@ -8,6 +9,11 @@ export interface MergeOptions {
     topK: number;
     /** Retained for API compatibility; pure-duplicate removal is always enabled. */
     dedup: boolean;
+    /**
+     * When set, the ranked pool is cut to `topK` spread across documents (see {@link RecordCut}) instead of
+     * by chunk, so one long document cannot take every slot. Unset keeps the plain top-`topK` cut.
+     */
+    recordCut?: RecordCut;
 }
 /**
  * Cross-method evidence collator.

package/dist/retrieval/record-cut.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Record cut: keep one long document from taking every slot of a ranked list.
+ *
+ * Hybrid retrieval ranks and cuts by passage. A long document that matches the
+ * query in dozens of passages fills the top N, and a document only the keyword
+ * leg found lands just past the cut. With the cut on, a search asks the backend
+ * for `depth` times what the caller wants (never past {@link OVERFETCH_ABSOLUTE_CAP})
+ * and keeps `limit` hits spread across documents: the best half with at most
+ * `perRecord` passages from any one document, then the best passage of each
+ * document not yet shown, then rank order for whatever room is left.
+ *
+ * The same module sizes what the model is *shown*: {@link planReadBudget} splits
+ * a result budget max-min fairly across the results of one tool call, so the
+ * first large result cannot spend all of it and leave the rest empty.
+ *
+ * The technique and its defaults follow PipesHub's `FUSION_RECORD_CUT`
+ * (`backend/python/app/modules/retrieval/record_cut.py`).
+ */
+/** Most hits ever asked of a backend for the deeper fusion. */
+export declare const OVERFETCH_ABSOLUTE_CAP = 300;
+/** Past ten the cap binds for any limit above 30, and a small limit gains nothing from fusing hundreds of hits. */
+export declare const MAX_FUSION_DEPTH = 10;
+/** Bounds the per-record knobs: past the over-fetch cap a larger value changes nothing. */
+export declare const MAX_PER_RECORD = 300;
+/** Characters one search tool result may hold when the cut is on. */
+export declare const DEFAULT_RESULT_CHARS = 60000;
+/** Below this share a result is announced as not read instead of being clipped to a stub. */
+export declare const DEFAULT_MIN_READ_CHARS = 2000;
+export interface CutByRecordOptions<T> {
+    /** Most hits one record may hold in the head of the cut. */
+    readonly perRecord: number;
+    /** Slots at the end of `limit` kept for the best hit of each record not yet shown. */
+    readonly reserve: number;
+    /** The record a hit belongs to; undefined counts as a record of its own. */
+    readonly recordOf: (hit: T) => string | undefined;
+}
+/**
+ * Keep `limit` of `results` (best first), spread across records.
+ *
+ * The best `limit - reserve` results come first, at most `perRecord` from any
+ * one record. The reserve then goes to the best result of each record not yet
+ * represented, and anything left is filled in rank order, so a search confined
+ * to a few records still returns `limit`. Survivors keep their original order.
+ * A `limit` below 1 keeps everything.
+ */
+export declare function cutByRecord<T>(results: readonly T[], limit: number, options: CutByRecordOptions<T>): T[];
+/**
+ * What a caller may tune; every field is optional and clamped by {@link recordCutFromOptions}. The cut is on unless
+ * `enabled` is false.
+ */
+export interface RecordCutOptions {
+    /** Set false to turn the record cut off. Default true. */
+    readonly enabled?: boolean;
+    /** Fusion depth: the backend is asked for `depth` x the limit. 1-10, default 3. */
+    readonly depth?: number;
+    /** Share of the limit kept for documents not yet shown. 0-1, default 0.5. */
+    readonly newRecordShare?: number;
+    /** One record keeps at most `max(perRecordFloor, limit / perRecordDivisor)` of the head. Default 4. */
+    readonly perRecordFloor?: number;
+    /** See `perRecordFloor`. Default 6. */
+    readonly perRecordDivisor?: number;
+    /** Characters one search tool result may hold. Default {@link DEFAULT_RESULT_CHARS}. */
+    readonly resultChars?: number;
+    /** Smallest share worth reading; a result below it is announced as not read. Default {@link DEFAULT_MIN_READ_CHARS}. */
+    readonly minReadChars?: number;
+}
+/** How deep a search fuses, how its hits are cut back by record, and how a result is budgeted. */
+export declare class RecordCut {
+    readonly depth: number;
+    readonly newRecordShare: number;
+    readonly perRecordFloor: number;
+    readonly perRecordDivisor: number;
+    readonly resultChars: number;
+    readonly minReadChars: number;
+    constructor(options?: RecordCutOptions);
+    /** What to ask the backend for: `depth` x `limit`, within `cap`. The cap never lowers the caller's own limit. */
+    fusedLimit(limit: number, cap: number): number;
+    /** Passages one record may hold in the head of a cut to `limit` (8 at a limit of 50). */
+    perRecord(limit: number): number;
+    cut<T>(results: readonly T[], limit: number, recordOf: (hit: T) => string | undefined): T[];
+}
+/** The cut these options describe: on unless `enabled` is false, then undefined. Out-of-range tuning throws. */
+export declare function recordCutFromOptions(options: RecordCutOptions | undefined): RecordCut | undefined;
+/**
+ * Split `total` across results of the given sizes, max-min fair: a result that
+ * needs less than an equal split gets exactly what it needs and what it leaves
+ * is divided among the larger ones. Shares follow input order and never sum
+ * past `total`.
+ */
+export declare function fairShares(sizes: readonly number[], total: number): number[];
+/**
+ * Per-result character limits for one tool call, in input order (best first).
+ * `undefined` means the result is not read: reading it would push a better
+ * result's fair share under `minRead`.
+ *
+ * The largest prefix whose every member can still get `min(minRead, size)`
+ * characters is read, split max-min fairly, so the first large result cannot
+ * spend the budget and leave the rest empty. At least the first result is
+ * always read, with whatever the budget allows.
+ */
+export declare function planReadBudget(sizes: readonly number[], total: number, minRead: number): (number | undefined)[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.13.6",
+  "version": "2.13.8",
   "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {

package/skills/autorag/SKILL.md

@@ -73,6 +73,34 @@ from it instead of probing. `"filee": { "enabled": false }` in `config.json` rem
 tools; `filee.timeoutMs` (default 30 minutes) and `filee.dataDir` (Filee's
 own settings and engines, `FILEE_DATA_DIR`) tune them.
 
+### Calculating numbers and dates
+
+The `calculator` Agent tool does every calculation the model would otherwise
+do in its head; the system prompt and each phase prompt (the fast answer
+included) tell the model never to compute these itself. It is read-only,
+in-process, and needs no configuration. Operations:
+
+- `evaluate`: exact decimal arithmetic to 40 significant digits (`0.1 + 0.2`
+  is `0.3`), `+ - * / % ^ !`, percent and percent change, rounding, sums,
+  means, medians, standard deviations, roots, logs, and trig. `exact: false`
+  in the result means a step rounded; `exp`, `ln`, `log`, trig, and fractional
+  powers are accurate to about 15 digits, not 40. A bare `log(x)` is refused as
+  ambiguous; use `ln`, `log10`, or `log(x, base)`.
+- `date_diff`, `date_add`, `date_info`: day counts, calendar spans
+  (`10 months, 7 days`), month-end clamping (Jan 31 + 1 month is Feb 28),
+  weekday, ISO week, and leap year. Dates read as `YYYY-MM-DD`, `YYYY/MM/DD`,
+  `YYYYMMDD`, `YYYY년 M월 D일`, ISO datetimes with an offset, epoch seconds,
+  `today`, or `now`.
+- `business_days_between` and `add_business_days`: Monday to Friday minus a
+  `holidays` list. The tool knows no public holidays; pass them in.
+- `now` and `convert_timezone`: the current time and DST-aware zone
+  conversion. An omitted `timeZone` means the host zone and the result says so;
+  a datetime with no offset needs its zone. A zone abbreviation other than
+  UTC, KST, and JST is refused, and the error names what to pass instead: PST,
+  PDT, MST, MDT, EST, and EDT are fixed offsets (PST is `-08:00` all year, so
+  it differs from `America/Los_Angeles` in summer), and CST, IST, and BST mean
+  several zones, so the user has to say which.
+
 ### Excluding local files from the index
 
 `"excludePaths"` in `config.json` lists files or folders (absolute, or relative

package/skills/autorag-setup/SKILL.md

@@ -354,6 +354,45 @@ only when `rerank.model` names a model; then the pool is wide (Jikji 30, MinSync
 default) each source contributes its own top 10 to the first answer. An explicit
 value always wins.
 
+### Record cut (`limits.recordCut`)
+
+On by default. A search that keeps only the top N *chunks* lets one long
+document whose dozens of passages all match fill every slot while a document
+only the keyword leg found falls past the cut. The record cut spreads the top N
+across documents instead. Turn it off with `limits.recordCut.enabled: false`.
+When on:
+
+- MinSync and the merged search ask for `depth` times the requested count (3x,
+  never past 300 hits), then keep the requested count spread across documents:
+  the best half with at most `max(perRecordFloor, topK / perRecordDivisor)`
+  chunks per document (8 at `topK` 50), then the best chunk of each document not
+  shown yet (`newRecordShare`, default 0.5 of `topK`), then rank order.
+- A chunk a search tool already showed during the same run is listed by id
+  without its text (`already shown earlier in this run`). A new run resends it.
+- Each search tool result is held to `resultChars` (60000). The budget is split
+  max-min fairly across the chunks, so one huge chunk is cut with a note instead
+  of starving the rest; chunks whose share would fall under `minReadChars`
+  (2000) are listed as `not read` with their ids.
+
+| Field | Default | Controls |
+|---|---|---|
+| `enabled` | true | Set `false` to turn the record cut off |
+| `depth` | 3 | Fusion depth, 1-10 |
+| `newRecordShare` | 0.5 | Share of `topK` reserved for documents not shown yet, 0-1 |
+| `perRecordFloor` | 4 | Fewest chunks one document may keep in the head, 1-300 |
+| `perRecordDivisor` | 6 | One document keeps at most `topK / perRecordDivisor` in the head, 1-300 |
+| `resultChars` | 60000 | Characters one search tool result may hold |
+| `minReadChars` | 2000 | Smallest share worth reading |
+
+```json
+{ "limits": { "recordCut": { "enabled": false } } }
+```
+
+Measured on the maintainer's own corpus (297 sampled passages, a natural question
+and a keyword query each, file-level gold, hybrid mode, top 10): nDCG@10 0.646 →
+0.656, MRR@10 0.588 → 0.600, Recall@10 0.825 → 0.830, and the top 10 holds 8.8
+distinct files instead of 7.5. The Recall difference is within noise.
+
 Ingest caps are trusted connector options under `datasources.<name>.connector`
 and are never settable from model/tool arguments: `maxDocuments`,
 `maxItemsPerFeed`, and `maxContentChars` (RSS). `maxContentChars` defaults to

package/dist/dupey/scan-plan.d.ts

@@ -1,23 +0,0 @@
-import { type CloudFileProbe, type CloudRoot } from "../filesystem/cloud-placeholder.js";
-/** One `dupey scan` to run: the directory and the cloud or network folder names nested in it that must not be walked. */
-export interface DupeyScanTarget {
-    readonly path: string;
-    readonly excludeDirs: readonly string[];
-}
-export interface DupeyScanPlan {
-    readonly targets: readonly DupeyScanTarget[];
-    /** Roots inside a cloud or network folder: never scanned, because `dupey scan` opens every file's contents. */
-    readonly skipped: readonly string[];
-}
-/**
- * The machine's cloud and network roots for the roots about to be scanned. A probe that is missing or fails
- * reads as no roots, never an error: an unknown root is scanned as before.
- */
-export declare function listCloudRoots(probe: CloudFileProbe | undefined, roots: readonly string[]): Promise<readonly CloudRoot[]>;
-/**
- * Which roots `dupey scan` may walk. A root inside a cloud or network folder is skipped whole and a cloud or
- * network folder nested under a root is excluded by name: `dupey scan` reads file contents, which re-downloads
- * a released placeholder (measured: a OneDrive folder of 1.1k placeholders ran past the 120 s scan timeout and
- * every read failed with ETIMEDOUT) and re-reads a network file over the wire.
- */
-export declare function planDupeyScans(roots: readonly string[], cloudRoots: readonly CloudRoot[]): DupeyScanPlan;