@autorag/librarian 2.10.0 → 2.10.2

package/dist/datasource/aliased-skill.d.ts

@@ -1,5 +1,5 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, SourceDescription } from "./types.js";
 export interface AliasDatasourceSkillOptions {
     readonly alias: string;
     readonly channelIds?: readonly string[];
@@ -19,7 +19,7 @@ export declare class AliasedDatasourceSkill implements DatasourceSkill {
     constructor(skill: DatasourceSkill, options: AliasDatasourceSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): import("./types.js").PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/connector-skill.d.ts

@@ -22,7 +22,7 @@
 import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
 import { DatasourceChunkStore } from "./chunk-store.js";
 import type { DatasourceConnector } from "./connector.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 /** Static identity/behavior shared by all instances of one skill kind. */
 export interface ConnectorSkillDefinition {
     /** Stable skill name and datasource id, e.g. `"slack"`. */
@@ -86,7 +86,7 @@ export declare class ConnectorDatasourceSkill implements DatasourceSkill {
     constructor(definition: ConnectorSkillDefinition, options: ConnectorSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/crawler-skill.d.ts

@@ -1,6 +1,7 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
 import type { CrawlerSearchOptions, CrawlerSearchResult, CrawlerSyncResult } from "./crawler-types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+import { type CrawlerSidecarOptions } from "./sidecar/index.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 export interface CrawlerSkillClient {
     sync(signal?: AbortSignal): Promise<CrawlerSyncResult>;
     search(query: string, options?: CrawlerSearchOptions): Promise<CrawlerSearchResult>;
@@ -27,6 +28,12 @@ export interface CrawlerDatasourceSkillOptions {
     /** Omitted/empty means search every chat/channel. */
     readonly channelIds?: readonly string[];
     readonly channelNames?: readonly string[];
+    /**
+     * Opt-in read-only sidecar over the crawler's own SQLite archive: adds BM25-ranked,
+     * semantic and hybrid methods next to the CLI-backed FTS method. The crawler database
+     * is never written; sidecar state lives in `sidecarDbPath`.
+     */
+    readonly sidecar?: CrawlerSidecarOptions;
 }
 export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     private readonly definition;
@@ -37,12 +44,22 @@ export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     private readonly tags;
     private readonly channelIds;
     private readonly channelNames;
+    private readonly sidecar;
     private lastIndexedAt;
     constructor(definition: CrawlerDatasourceDefinition, options: CrawlerDatasourceSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
+    /**
+     * Embed new crawler messages into the sidecar. Never fails the index (the CLI sync already succeeded and BM25
+     * stays available): any sidecar problem, including a thrown database error, becomes a warning. A bounded sync
+     * that left a backlog says so, so a partial semantic index is not mistaken for a complete one.
+     */
+    private syncSidecar;
+    /** Release the sidecar's database handles. Safe to call repeatedly; later use reopens them. */
+    close(): void;
     retrievalMethods(): readonly RetrievalMethod[];
+    private cliMethod;
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;
     private fail;

package/dist/datasource/described-skill.d.ts

@@ -1,5 +1,5 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 /**
  * Adds operator-authored context to a configured datasource without changing
  * its backend, access identity, or retrieval methods.
@@ -10,7 +10,7 @@ export declare class DescribedDatasourceSkill implements DatasourceSkill {
     constructor(skill: DatasourceSkill, description: string);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/index.d.ts

@@ -22,4 +22,4 @@ export { SlackSkill, type SlackSkillOptions, SlacrawlClient, type SlacrawlOption
 export { SpotlightConnector, type SpotlightConnectorOptions, SpotlightSkill, type SpotlightSkillOptions, } from "./skills/spotlight/index.js";
 export { TelecrawlClient, type TelecrawlOptions, TelecrawlSkill, type TelecrawlSkillOptions, } from "./skills/telecrawl/index.js";
 export { WacrawlClient, type WacrawlOptions, WacrawlSkill, type WacrawlSkillOptions } from "./skills/wacrawl/index.js";
-export type { DatasourceDiagnostic, DatasourceDiagnosticCode, DatasourceIndexFail, DatasourceIndexOk, DatasourceIndexResult, DatasourceInstance, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription, } from "./types.js";
+export type { DatasourceDiagnostic, DatasourceDiagnosticCode, DatasourceIndexFail, DatasourceIndexOk, DatasourceIndexOptions, DatasourceIndexResult, DatasourceInstance, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription, } from "./types.js";

package/dist/datasource/retry.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Shared retry/backoff policy for datasource `index()` calls.
+ *
+ * Failure causes differ per datasource (HTTP status, CLI exit reason, ...), so
+ * no attempt is made to classify them. {@link indexWithRetry} retries every
+ * failed or thrown `index()` with jittered exponential backoff and, once the
+ * retries are used up, reports that datasource as failed so the refresh moves
+ * on to the next one.
+ *
+ * Process-tree termination of a timed-out CLI is a separate concern (#1587,
+ * #1586); this module never touches the spawn code.
+ */
+import type { DatasourceIndexResult, DatasourceSkill } from "./types.js";
+export interface DatasourceRetryPolicy {
+    /** Re-runs after the first call; `0` disables retrying. */
+    readonly maxRetries: number;
+    /** Backoff before the first retry; doubles for each later retry. */
+    readonly baseDelayMs: number;
+    /** Upper bound on one backoff step (before jitter). */
+    readonly maxDelayMs: number;
+}
+export interface DatasourceRetryOptions extends Partial<DatasourceRetryPolicy> {
+    /** Jitter source in `[0, 1]`; injectable for deterministic tests. */
+    readonly random?: () => number;
+    /** Delay implementation; injectable so tests and hosts need not wait in real time. */
+    readonly sleep?: (ms: number) => Promise<void>;
+}
+/** Two retries (three calls in total) with 1s then 2s of backoff, jittered. */
+export declare const DEFAULT_DATASOURCE_RETRY_POLICY: DatasourceRetryPolicy;
+/**
+ * Backoff before retry number `retry` (1-based): `baseDelayMs * 2^(retry-1)`
+ * capped at `maxDelayMs`, then jittered into the upper half of that step so
+ * simultaneous refreshes do not hammer the same upstream in lockstep.
+ */
+export declare function datasourceRetryDelayMs(retry: number, policy: DatasourceRetryPolicy & {
+    readonly random?: () => number;
+}): number;
+/**
+ * Run `skill.index()`, retrying any failure (a failed result or a throw). A
+ * first-time success is returned untouched. Otherwise the result carries the
+ * total `attempts`; an exhausted failure also carries `attempts` in its message
+ * and last diagnostic, and a late success carries an info diagnostic naming the
+ * last error it recovered from. A thrown `index()` becomes a
+ * `datasource-index-failed` result.
+ *
+ * A cancelled `signal` is not a datasource failure: it is rethrown at once
+ * (also from the backoff wait) instead of being retried or reported.
+ */
+export declare function indexWithRetry(skill: DatasourceSkill, options?: DatasourceRetryOptions, signal?: AbortSignal): Promise<DatasourceIndexResult>;

package/dist/datasource/sidecar/config.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Turns a datasource's `connector.sidecar` config into {@link CrawlerSidecarOptions}.
+ *
+ * On by default for every datasource that has a crawler descriptor; opt out with
+ *
+ *   "connector": { "sidecar": false }
+ *
+ * or tune it:
+ *
+ *   "connector": { "sidecar": { "crawlerDbPath": "~/.slacrawl/slacrawl.db", "semantic": false } }
+ *
+ * The sidecar's own database always lives under the AutoRAG workspace, one file
+ * per datasource instance, never next to (or inside) the crawler's archive.
+ */
+import type { CrawlerDescriptor } from "./descriptors.js";
+import type { CrawlerSidecarOptions } from "./index.js";
+export interface ResolveSidecarInput {
+    readonly descriptor: CrawlerDescriptor;
+    readonly connector: Record<string, unknown> | undefined;
+    readonly workspaceRoot: string | undefined;
+    readonly registrationName: string;
+    readonly instanceId: string;
+    /** Where the crawler keeps its archive when the user did not say. */
+    readonly defaultCrawlerDb: string;
+}
+export declare function resolveSidecarOptions(input: ResolveSidecarInput): CrawlerSidecarOptions | undefined;

package/dist/datasource/sidecar/descriptors.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Declarative description of a crawler's read-only SQLite schema. The sidecar
+ * engine (index.ts) is crawler-agnostic: everything that differs between
+ * slacrawl, telecrawl, wacrawl and notcrawl is expressed here as SQL fragments.
+ *
+ * Every SQL fragment is a trusted constant authored in this file. User input
+ * (the search query, workspace id) is only ever bound as a parameter.
+ *
+ * The crawler database is opened read-only and ATTACHed; the sidecar owns a
+ * separate database file, so a crawler that rebuilds, migrates, purges or
+ * vacuums its own file can never lose or corrupt sidecar state.
+ */
+export interface CrawlerDescriptor {
+    readonly id: string;
+    /**
+     * True when the crawler's own TOML config (`connector.configPath`) can relocate its archive via
+     * `db_path`. The sidecar then follows that setting instead of assuming the default archive.
+     */
+    readonly archivePathInConfig?: boolean;
+    /** `message_key` stored in the sidecar for each row; unique per message within one crawler DB. */
+    readonly keyExpression: string;
+    /**
+     * Public chunk id for a stored key. MUST equal the id the crawler's CLI client derives for the
+     * same message (see each skill's `parseHits`), so the retrieval merger deduplicates a message
+     * found by both the CLI-backed and the sidecar methods.
+     */
+    readonly chunkId: (key: string, metadata: Readonly<Record<string, string>>) => string;
+    /** FROM + JOINs exposing alias `m` (message-like row) and any lookup aliases used below. */
+    readonly fromClause: string;
+    /** Row visibility predicate: excludes soft-deleted rows. May reference a `:scope` parameter when `scopeColumn` is set. */
+    readonly liveWhere: string;
+    /** Column that the optional workspace/account scope filters on; omitted when the crawler has one account per DB. */
+    readonly scopeExpression?: string;
+    /** SQL returning the embeddable/displayed body of the row. */
+    readonly textExpression: string;
+    /** Optional extra text prepended to the embedding input (title, channel). */
+    readonly titleExpression?: string;
+    /** Seconds-since-epoch expression used for `publishedAt`; omitted when unknown. */
+    readonly epochSecondsExpression?: string;
+    /**
+     * FTS5 relation joined to the message row. `joinOn` ties the FTS row to alias `m`;
+     * `table` is the FTS table (also the argument to bm25()).
+     */
+    readonly fts: {
+        readonly table: string;
+        readonly joinOn: string;
+    };
+    /** Extra result columns mapped into hit metadata: a SQL expression and the metadata key it fills. */
+    readonly metadata: readonly {
+        readonly sql: string;
+        readonly key: string;
+    }[];
+    /**
+     * SQL for the chat/channel id and name the allowlist (`channels.ids/names`) matches. Pushed into the
+     * row predicate, so excluded chats are never searched, embedded, or kept in the sidecar.
+     */
+    readonly channelIdExpression: string;
+    readonly channelNameExpression: string;
+    /** Path-opaque label for the hierarchy: ["workspaces", ws, "channels", name] etc. */
+    readonly hierarchy: (metadata: Readonly<Record<string, unknown>>) => readonly string[];
+}
+export declare const CRAWLER_DESCRIPTORS: {
+    readonly slack: CrawlerDescriptor;
+    readonly telegram: CrawlerDescriptor;
+    readonly whatsapp: CrawlerDescriptor;
+};
+export type SidecarCrawlerId = keyof typeof CRAWLER_DESCRIPTORS;

package/dist/datasource/sidecar/fusion.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Pure ranking primitives shared by every crawler sidecar: float32 vector
+ * (de)serialization, cosine top-k, and reciprocal rank fusion. No I/O.
+ */
+export declare function encodeVector(vector: readonly number[]): Uint8Array;
+/** Inverse of {@link encodeVector}; undefined when the blob is not a whole number of float32s. */
+export declare function decodeVector(blob: Uint8Array): Float32Array | undefined;
+/** Cosine similarity; 0 (never NaN) for zero-length, zero-norm, or mismatched-dimension inputs. */
+export declare function cosine(a: ArrayLike<number>, b: ArrayLike<number>): number;
+export interface KeyedVector {
+    readonly key: string;
+    readonly vector: ArrayLike<number>;
+}
+export interface ScoredKey {
+    readonly key: string;
+    readonly score: number;
+}
+/** Best `k` positive-similarity keys, descending; ties broken by key for determinism. */
+export declare function topKByCosine(query: ArrayLike<number>, rows: Iterable<KeyedVector>, k: number): ScoredKey[];
+/**
+ * Reciprocal rank fusion over ranked key lists (best first). A key repeated
+ * inside one list counts once, at its best rank. Ties keep first-seen order,
+ * so earlier lists win.
+ */
+export declare function reciprocalRankFusion(lists: readonly (readonly string[])[]): ScoredKey[];

package/dist/datasource/sidecar/index.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Crawler sidecar: BM25 + semantic + hybrid retrieval over a crawler's own
+ * SQLite archive without touching it.
+ *
+ * The sidecar owns a separate database file (MAIN). The crawler's database is
+ * ATTACHed read-only, so a crawler that migrates, purges, vacuums or rebuilds
+ * its file cannot lose sidecar state, and the sidecar cannot corrupt the
+ * crawler. Only vectors keyed by the crawler's message key are stored; message
+ * bodies are always read from the crawler database at query time.
+ *
+ * Everything crawler-specific lives in a {@link CrawlerDescriptor}; this file
+ * is shared by every crawler.
+ */
+import type { Embedder } from "../../embedding-runtime/gateway-embedder.js";
+import type { CrawlerDescriptor } from "./descriptors.js";
+export interface SidecarHit {
+    readonly key: string;
+    readonly content: string;
+    readonly score: number;
+    readonly title?: string;
+    /** Milliseconds since epoch. */
+    readonly publishedAt?: number;
+    readonly metadata: Readonly<Record<string, string>>;
+}
+export type SidecarSyncResult = {
+    readonly ok: true;
+    readonly embedded: number;
+    /** Messages the embedder rejected on their own; retried only after their text changes. */
+    readonly skipped: number;
+    readonly pruned: number;
+    /** Messages still waiting for a vector after this sync (0 when fully caught up). */
+    readonly remaining: number;
+} | {
+    readonly ok: false;
+    readonly message: string;
+};
+export interface CrawlerSidecarOptions {
+    readonly descriptor: CrawlerDescriptor;
+    readonly crawlerDbPath: string;
+    readonly sidecarDbPath: string;
+    /** Restricts every query to one workspace/account when the descriptor supports scoping. */
+    readonly workspaceId?: string;
+    /**
+     * Chat/channel allowlist (`channels.ids/names`). Empty means every chat. Applied in SQL, so excluded chats are
+     * never searched, embedded, or kept as vectors.
+     */
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
+    /** Required for semantic and hybrid search; lexical search works without it. */
+    readonly embedder?: Embedder;
+    /** Messages per embedding request and per write transaction. Default {@link DEFAULT_BATCH_SIZE}. */
+    readonly batchSize?: number;
+    /**
+     * Most messages one `sync()` will embed. A first sync over a large archive otherwise holds the whole
+     * refresh for minutes; the remainder is picked up by the next sync, and BM25 never waits for vectors.
+     * Default {@link DEFAULT_MAX_EMBED_PER_SYNC}.
+     */
+    readonly maxEmbedPerSync?: number;
+}
+export declare class CrawlerSidecar {
+    readonly descriptor: CrawlerDescriptor;
+    private readonly options;
+    private readonly select;
+    private readonly where;
+    /** Opened on first use: building a skill must not create files, directories, or file descriptors. */
+    private opened;
+    /** Identity of the attached crawler file; a crawler that replaces or deletes its archive forces a re-attach. */
+    private attached;
+    /**
+     * The last query's embedding and ranking. The semantic and hybrid methods of one search ask for the same
+     * query, so they share one embedding call and one vector scan. Short-lived, and dropped on any write.
+     */
+    private lastSemantic;
+    constructor(options: CrawlerSidecarOptions);
+    /** The sidecar database, created on first call. Only reached once the crawler archive exists. */
+    private get db();
+    /**
+     * Attach the crawler archive read-only, once it exists and has the expected
+     * shape. The crawler may not have synced yet, or may have replaced or deleted
+     * its file, so this is re-checked on every call: a changed file is re-attached,
+     * a missing one detached. Nothing is created on disk until the archive exists.
+     */
+    private attachCrawler;
+    private detachCrawler;
+    /** True when a semantic index can be built: an embedder is configured. */
+    get semanticEnabled(): boolean;
+    /** Release the database handle. Safe to call repeatedly; a later search reopens and re-attaches. */
+    close(): void;
+    /** Bound parameters for {@link where}, in the order its predicates appear. */
+    private whereParams;
+    /** BM25-ranked lexical search. Natural-language queries are OR-ed term by term. */
+    searchLexical(query: string, topK: number): SidecarHit[];
+    /** Cosine search over embedded messages. Empty when there is no embedder or the query embedding fails. */
+    searchSemantic(query: string, topK: number): Promise<SidecarHit[]>;
+    /** Best `pool` stored vectors for `query`, reusing the previous ranking when it already covers `pool`. */
+    private rankSemantic;
+    /** Reciprocal-rank fusion of lexical and semantic rankings; lexical-only when semantic is unavailable. */
+    searchHybrid(query: string, topK: number): Promise<SidecarHit[]>;
+    /**
+     * Bring the vector table in line with the crawler archive: drop vectors of
+     * purged/deleted messages, embed new or edited ones, re-embed all on an
+     * embedding identity change. An embedder failure keeps what was already
+     * stored. A message the embedder keeps rejecting while others succeed is
+     * skipped until its text changes, so it cannot stall the backlog behind it.
+     * Database errors (for example a lock held past the busy timeout) throw.
+     */
+    sync(): Promise<SidecarSyncResult>;
+    private adoptIdentity;
+    /** Delete vectors whose message no longer exists, is deleted, or fell out of scope. */
+    private prune;
+    /** Keys with no vector yet, or whose embedded text changed since. */
+    private pendingKeys;
+    /** Live rows for the given keys, in the order given. */
+    private hydrate;
+    /**
+     * Upsert vectors keyed by message. `undefined` stores an empty vector: the message is recorded as tried at
+     * this text hash (so it is not retried every sync) but never ranked.
+     */
+    private store;
+    private storedVectors;
+    private transaction;
+    private toRow;
+    private toHit;
+}
+/**
+ * FTS5 MATCH expression for free text: every alphanumeric term quoted and
+ * OR-ed. Quoting neutralizes FTS operators (AND, NOT, NEAR, *, parentheses),
+ * so user text can never change the query's structure. Undefined when the
+ * text contains no searchable term.
+ */
+export declare function toMatchQuery(text: string): string | undefined;

package/dist/datasource/sidecar/methods.d.ts

@@ -0,0 +1,27 @@
+/**
+ * RetrievalMethod adapters over a {@link CrawlerSidecar}. One class serves every
+ * crawler and every mode; the only per-instance input is the datasource identity
+ * (the channel allowlist is applied by the sidecar's SQL).
+ */
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../retrieval/types.js";
+import type { CrawlerDescriptor } from "./descriptors.js";
+import type { CrawlerSidecar } from "./index.js";
+export type SidecarMode = "bm25" | "semantic" | "hybrid";
+export interface SidecarMethodOptions {
+    readonly sidecar: CrawlerSidecar;
+    readonly descriptor: CrawlerDescriptor;
+    readonly mode: SidecarMode;
+    readonly datasourceId: string;
+    readonly skillType: string;
+    readonly instanceId: string;
+    readonly tags: readonly string[];
+}
+/** Result method name; `bm25` keeps the `-sidecar-` infix so it never collides with the CLI-backed `-fts` method. */
+export declare function sidecarMethodName(datasourceId: string, mode: SidecarMode): string;
+export declare class SidecarRetrievalMethod implements RetrievalMethod {
+    private readonly options;
+    constructor(options: SidecarMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+    private toResult;
+}

package/dist/datasource/skills/clawgallery/skill.d.ts

@@ -1,10 +1,10 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type ClawGallerySearchClient } from "./methods.js";
 import type { ClawGalleryIndexResult, ClawGalleryOptions, ClawGallerySearchMode, ClawGalleryVdrResult } from "./types.js";
 export interface ClawGallerySkillClient extends ClawGallerySearchClient {
-    bootstrap(): Promise<ClawGalleryIndexResult>;
-    syncVisual(): Promise<ClawGalleryVdrResult>;
+    bootstrap(signal?: AbortSignal): Promise<ClawGalleryIndexResult>;
+    syncVisual(signal?: AbortSignal): Promise<ClawGalleryVdrResult>;
 }
 export interface ClawGallerySkillOptions {
     readonly client: ClawGallerySkillClient;
@@ -28,7 +28,7 @@ export declare class ClawGallerySkill implements DatasourceSkill {
     constructor(options: ClawGallerySkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/discrawl/skill.d.ts

@@ -1,11 +1,11 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type DiscrawlSearchClient } from "./methods.js";
 import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlSearchMode, DiscrawlSyncResult } from "./types.js";
 export interface DiscrawlSkillClient extends DiscrawlSearchClient {
-    doctor(): Promise<DiscrawlDoctorResult>;
-    sync(): Promise<DiscrawlSyncResult>;
-    embed(limit?: number, rebuild?: boolean): Promise<DiscrawlEmbedResult>;
+    doctor(signal?: AbortSignal): Promise<DiscrawlDoctorResult>;
+    sync(signal?: AbortSignal): Promise<DiscrawlSyncResult>;
+    embed(limit?: number, rebuild?: boolean, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
     prepareEmbeddings?(): Promise<boolean>;
 }
 export interface DiscrawlSkillOptions {
@@ -47,7 +47,7 @@ export declare class DiscrawlSkill implements DatasourceSkill {
     constructor(options: DiscrawlSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/github-gist/skill.d.ts

@@ -10,7 +10,7 @@
 import { type Embedder, type GatewayEmbedderOptions } from "../../../embedding-runtime/gateway-embedder.js";
 import type { RetrievalMethod } from "../../../retrieval/types.js";
 import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import type { DatasourceIndexResult } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult } from "../../types.js";
 import { GitHubGistConnector, type GitHubGistConnectorOptions } from "./connector.js";
 export declare const GITHUB_GIST_SKILL_DEFINITION: ConnectorSkillDefinition;
 export declare function gistDatasourceDir(workspaceRoot: string, skillName: string, instanceId: string): string;
@@ -34,6 +34,6 @@ export declare class GitHubGistSkill extends ConnectorDatasourceSkill {
     private readonly semanticEmbedder;
     private readonly semanticMethod;
     constructor(options?: GitHubGistSkillOptions);
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
 }

package/dist/datasource/skills/lazykatok/skill.d.ts

@@ -1,11 +1,11 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type LazykatokSearchClient } from "./methods.js";
 import type { LazykatokDoctorResult, LazykatokIndexResult, LazykatokSyncResult } from "./types.js";
 export interface LazykatokSkillClient extends LazykatokSearchClient {
-    doctor(): Promise<LazykatokDoctorResult>;
-    sync(): Promise<LazykatokSyncResult>;
-    index(): Promise<LazykatokIndexResult>;
+    doctor(signal?: AbortSignal): Promise<LazykatokDoctorResult>;
+    sync(signal?: AbortSignal): Promise<LazykatokSyncResult>;
+    index(signal?: AbortSignal): Promise<LazykatokIndexResult>;
 }
 export interface LazykatokSkillOptions {
     readonly client: LazykatokSkillClient;
@@ -25,7 +25,7 @@ export declare class LazykatokSkill implements DatasourceSkill {
     constructor(options: LazykatokSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/mailcrawl/skill.d.ts

@@ -1,5 +1,5 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type MailcrawlIndexClient } from "./client.js";
 import type { MailcrawlOptions, MailcrawlSyncResult } from "./types.js";
 export interface MailcrawlSkillClient extends MailcrawlIndexClient {
@@ -26,7 +26,7 @@ export declare class MailcrawlSkill implements DatasourceSkill {
     constructor(options?: MailcrawlSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/obsidian/skill.d.ts

@@ -1,11 +1,11 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type ObsidianSearchClient } from "./methods.js";
 import type { QmdEmbedResult, QmdEnsureResult, QmdUpdateResult } from "./types.js";
 export interface ObsidianSkillClient extends ObsidianSearchClient {
-    ensureCollection(): Promise<QmdEnsureResult>;
-    update(): Promise<QmdUpdateResult>;
-    embed(): Promise<QmdEmbedResult>;
+    ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
+    update(signal?: AbortSignal): Promise<QmdUpdateResult>;
+    embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
 }
 export interface ObsidianSkillOptions {
     readonly datasourceId?: string;
@@ -37,7 +37,7 @@ export declare class ObsidianSkill implements DatasourceSkill {
     constructor(options?: ObsidianSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/types.d.ts

@@ -52,6 +52,11 @@ export interface DatasourceSkillDescriptor {
     /** Optional list of instance ids this skill reports. */
     readonly instances?: readonly string[];
 }
+/** Per-call options for {@link DatasourceSkill.index}. */
+export interface DatasourceIndexOptions {
+    /** Aborting stops the index run and kills any CLI it spawned. An aborted run may reject instead of returning a fail variant. */
+    readonly signal?: AbortSignal;
+}
 /**
  * A pluggable datasource skill. Skills are server-bound: they are constructed
  * with trusted configuration and never derive authority from model/tool args.
@@ -67,7 +72,7 @@ export interface DatasourceSkill {
      * never throws. Remote embedding egress MUST be rejected here and reported
      * via a `datasource-embedding-egress-rejected` diagnostic.
      */
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     /** Retrieval methods exposed by this skill, fed into the shared pipeline. */
     retrievalMethods(): readonly RetrievalMethod[];
     /** Opaque source descriptions for the skill's current evidence set. */
@@ -128,6 +133,11 @@ export interface DatasourceIndexOk {
     readonly chunkCount: number;
     readonly indexedAt: number;
     readonly diagnostics: readonly DatasourceDiagnostic[];
+    /**
+     * Total `index()` calls it took (set only when the refresh retried; a
+     * first-time success omits it).
+     */
+    readonly attempts?: number;
 }
 /** Failed indexing outcome. Never thrown; returned with diagnostics. */
 export interface DatasourceIndexFail {
@@ -139,13 +149,17 @@ export interface DatasourceIndexFail {
     readonly error?: string;
     readonly code?: DatasourceDiagnosticCode;
     readonly message?: string;
+    /** Total `index()` calls made before this result was reported (set by the refresh retry policy). */
+    readonly attempts?: number;
 }
 /**
  * Index result — an ok/fail discriminated union. Skills MUST NOT throw from
  * `index()`; they return a fail variant instead.
  */
 export type DatasourceIndexResult = DatasourceIndexOk | DatasourceIndexFail;
-export type DatasourceDiagnosticCode = "datasource-unavailable" | "datasource-cli-error" | "datasource-empty" | "datasource-rate-limited" | "datasource-auth-error" | "datasource-embedding-egress-rejected" | "datasource-index-failed" | "datasource-permission-denied" | "datasource-remote-embedding-rejected";
+export type DatasourceDiagnosticCode = "datasource-unavailable" | "datasource-cli-error" | "datasource-empty" | "datasource-rate-limited" | "datasource-auth-error" | "datasource-embedding-egress-rejected" | "datasource-index-failed"
+/** Index succeeded but is incomplete (e.g. a semantic backlog still draining); not a failure. */
+ | "datasource-index-partial" | "datasource-permission-denied" | "datasource-remote-embedding-rejected";
 /** Path-opaque diagnostic emitted by the datasource layer. */
 export interface DatasourceDiagnostic {
     readonly code: DatasourceDiagnosticCode;

package/dist/embedding-runtime/cache.d.ts

@@ -16,6 +16,15 @@ export interface CacheOptions {
     readonly downloadTimeoutMs?: number;
 }
 export declare function verifyCacheEntry(path: string, expectedSha256: string): Promise<string>;
+/**
+ * Marker a distributor writes inside `<archive>.extracted/` when it ships the
+ * runtime already extracted instead of as the pinned archive. Its content is the
+ * archive's SHA-256, so a tree provisioned for another runtime build is never
+ * taken for this one. A signed and notarized macOS app needs this: every Mach-O
+ * it carries must be re-signed with the app's Developer ID, so the binaries can
+ * no longer be packed back into an archive that matches the pinned digest.
+ */
+export declare const PROVISIONED_RUNTIME_MARKER = ".autorag-archive-sha256";
 export declare function hasVerifiedAsset(asset: CacheAsset, cacheRoot?: string): Promise<boolean>;
 export declare function downloadAsset(asset: CacheAsset, options?: CacheOptions): Promise<string>;
 export declare function importAsset(sourcePath: string, asset: CacheAsset, options?: CacheOptions): Promise<string>;