@autorag/librarian 2.13.2 → 2.13.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.13.2",
+  "version": "2.13.4",
   "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {
@@ -65,6 +65,7 @@
     "chardet": "^2.2.0",
     "discord.js": "14.27.0",
     "fast-xml-parser": "^5.11.2",
+    "gpt-tokenizer": "^4.1.0",
     "iconv-lite": "^0.7.3",
     "jev-use": "^0.8.0",
     "jszip": "^3.10.2",

package/skills/autorag-doctor/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: autorag-doctor
-description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
+description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl, rclone), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
 license: Apache-2.0
 ---
 
@@ -73,7 +73,6 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
 | Obsidian / notes | `autorag status` (AutoRAG-owned index) | `autorag refresh --method datasources`; if corrupt, delete `.autorag/datasources/obsidian/<instance>` first |
 | Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
 | Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
-| Spotlight (macOS) | `mdutil -s /` | indexed by the OS; no AutoRAG sync |
 | Lark / Feishu | `lark-cli auth status --format json` | no local sync; search is remote |
 
 Rules:

package/skills/autorag-lite-setup/SKILL.md

@@ -137,7 +137,7 @@ Datasource skills belong in trusted config.
 
 Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
 `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
-`mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, `hister`) or connection
+`mailcrawl`, `obsidian`, `rss`, `lark`, `hister`) or connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `scope` narrows a query to a sub-path as ordinary filtering. Tags are

package/skills/autorag-setup/SKILL.md

@@ -330,11 +330,10 @@ size with `prefetch.sectionLimit` / `prefetch.minSyncTopK` /
 
 Ingest caps are trusted connector options under `datasources.<name>.connector`
 and are never settable from model/tool arguments: `maxDocuments`,
-`maxItemsPerFeed`, and `maxContentChars` (RSS); `maxDocuments`,
-`maxContentChars`, `maxResultsPerQuery`, and `maxBytesPerFile` (Spotlight);
+`maxItemsPerFeed`, and `maxContentChars` (RSS);
 `maxDocuments`, `maxContentChars`, `maxBytesPerFile`, `concurrency`,
 `bandwidthLimit`, and `dryRun` (cloud-drive/rclone). `maxContentChars` defaults
-to 20000 for RSS and 100000 for Spotlight and cloud-drive.
+to 20000 for RSS and 100000 for cloud-drive.
 
 ### Refresh safety limits
 
@@ -389,7 +388,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
 Datasource skills belong in trusted config. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
 `clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`,
-`mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, and `hister`. Config keys may be connection
+`mailcrawl`, `obsidian`, `rss`, `lark`, and `hister`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.

package/dist/datasource/skills/spotlight/connector.d.ts

@@ -1,53 +0,0 @@
-import type { ConnectorFetchFail, ConnectorFetchOk } from "../../connector.js";
-export interface SpotlightConnectorFailure extends ConnectorFetchFail {
-    readonly stderr?: string;
-    readonly stdout?: string;
-    readonly code?: number | null;
-}
-export type SpotlightConnectorResult = ConnectorFetchOk | SpotlightConnectorFailure;
-export interface ProcessRunResult {
-    readonly ok: boolean;
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number | null;
-    /** Spawn error code, e.g. `"ENOENT"` when the binary is missing. */
-    readonly error?: string;
-}
-/** Process execution seam; injectable for tests. Never throws. */
-export type ProcessRunner = (command: string, args: readonly string[], timeoutMs: number) => Promise<ProcessRunResult>;
-export type FileReader = (path: string) => Promise<string>;
-export interface SpotlightConnectorOptions {
-    /** Trusted Spotlight filters AND-ed with each user query. Required. */
-    readonly queries?: readonly string[];
-    /** Trusted directory restricting every query (`mdfind -onlyin`). */
-    readonly onlyIn?: string;
-    /** Hits kept per query after dedupe. Default 100. */
-    readonly maxResultsPerQuery?: number;
-    /** Maximum documents hydrated by one live search. Default 300. */
-    readonly maxDocuments?: number;
-    /** Characters kept per document body. Default 100000. */
-    readonly maxContentChars?: number;
-    /** Files larger than this are skipped. Default 256 KiB. */
-    readonly maxBytesPerFile?: number;
-    /** Per-process timeout in ms. Default 15s. */
-    readonly timeoutMs?: number;
-    /** Injectable platform; defaults to `process.platform`. */
-    readonly platform?: NodeJS.Platform;
-    /** Injectable process runner; defaults to a bounded spawn wrapper. */
-    readonly run?: ProcessRunner;
-    /** Injectable file reader; defaults to utf8 `fs.readFile`. */
-    readonly readFile?: FileReader;
-}
-/** Executes live Spotlight queries. macOS owns indexing; no local snapshot is collected. */
-export declare class SpotlightConnector {
-    private readonly options;
-    private readonly run;
-    private readonly readFile;
-    private readonly platform;
-    constructor(options?: SpotlightConnectorOptions);
-    /** Refresh-time health check only: never runs `mdfind` or hydrates files. */
-    probe(): Promise<SpotlightConnectorResult>;
-    /** Search the current macOS Spotlight index and hydrate only the requested hits. */
-    search(query: string, limit?: number, acceptDocument?: (docId: string) => boolean): Promise<SpotlightConnectorResult>;
-    private readinessFailure;
-}

package/dist/datasource/skills/spotlight/index.d.ts

@@ -1,2 +0,0 @@
-export { type FileReader, type ProcessRunner, type ProcessRunResult, SpotlightConnector, type SpotlightConnectorFailure, type SpotlightConnectorOptions, type SpotlightConnectorResult, } from "./connector.js";
-export { SpotlightSkill, type SpotlightSkillOptions } from "./skill.js";

package/dist/datasource/skills/spotlight/skill.d.ts

@@ -1,33 +0,0 @@
-import type { RetrievalMethod, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
-import { SpotlightConnector, type SpotlightConnectorOptions } from "./connector.js";
-export interface SpotlightSkillOptions {
-    /** Config alias used as the independent datasource/skill identity. */
-    readonly skillName?: string;
-    readonly instanceId?: string;
-    readonly instances?: readonly string[];
-    readonly tags?: readonly string[];
-    /** Trusted connector configuration; a pre-built connector wins. */
-    readonly connector?: SpotlightConnector;
-    readonly connectorOptions?: SpotlightConnectorOptions;
-}
-/** Live macOS Spotlight datasource. Refresh probes readiness; retrieval runs `mdfind`. */
-export declare class SpotlightSkill implements DatasourceSkill {
-    private readonly connector;
-    private readonly skillName;
-    private readonly instanceId;
-    private readonly instances;
-    private readonly tags;
-    private lastIndexedAt;
-    private lastError;
-    constructor(options?: SpotlightSkillOptions);
-    describe(): DatasourceSkillDescriptor;
-    polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
-    retrievalMethods(): readonly RetrievalMethod[];
-    describeSources(): readonly SourceDescription[];
-    skillManifest(): DatasourceSkillManifest;
-    search(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
-    private fail;
-    private note;
-}