@autorag/librarian 2.10.1 → 2.10.3

package/dist/datasource/aliased-skill.d.ts

@@ -1,5 +1,5 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, SourceDescription } from "./types.js";
 export interface AliasDatasourceSkillOptions {
     readonly alias: string;
     readonly channelIds?: readonly string[];
@@ -19,7 +19,7 @@ export declare class AliasedDatasourceSkill implements DatasourceSkill {
     constructor(skill: DatasourceSkill, options: AliasDatasourceSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): import("./types.js").PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/connector-skill.d.ts

@@ -22,7 +22,7 @@
 import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
 import { DatasourceChunkStore } from "./chunk-store.js";
 import type { DatasourceConnector } from "./connector.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 /** Static identity/behavior shared by all instances of one skill kind. */
 export interface ConnectorSkillDefinition {
     /** Stable skill name and datasource id, e.g. `"slack"`. */
@@ -86,7 +86,7 @@ export declare class ConnectorDatasourceSkill implements DatasourceSkill {
     constructor(definition: ConnectorSkillDefinition, options: ConnectorSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/crawler-skill.d.ts

@@ -1,7 +1,7 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
 import type { CrawlerSearchOptions, CrawlerSearchResult, CrawlerSyncResult } from "./crawler-types.js";
 import { type CrawlerSidecarOptions } from "./sidecar/index.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 export interface CrawlerSkillClient {
     sync(signal?: AbortSignal): Promise<CrawlerSyncResult>;
     search(query: string, options?: CrawlerSearchOptions): Promise<CrawlerSearchResult>;
@@ -49,7 +49,7 @@ export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     constructor(definition: CrawlerDatasourceDefinition, options: CrawlerDatasourceSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     /**
      * Embed new crawler messages into the sidecar. Never fails the index (the CLI sync already succeeded and BM25
      * stays available): any sidecar problem, including a thrown database error, becomes a warning. A bounded sync

package/dist/datasource/described-skill.d.ts

@@ -1,5 +1,5 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 /**
  * Adds operator-authored context to a configured datasource without changing
  * its backend, access identity, or retrieval methods.
@@ -10,7 +10,7 @@ export declare class DescribedDatasourceSkill implements DatasourceSkill {
     constructor(skill: DatasourceSkill, description: string);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/index.d.ts

@@ -22,4 +22,4 @@ export { SlackSkill, type SlackSkillOptions, SlacrawlClient, type SlacrawlOption
 export { SpotlightConnector, type SpotlightConnectorOptions, SpotlightSkill, type SpotlightSkillOptions, } from "./skills/spotlight/index.js";
 export { TelecrawlClient, type TelecrawlOptions, TelecrawlSkill, type TelecrawlSkillOptions, } from "./skills/telecrawl/index.js";
 export { WacrawlClient, type WacrawlOptions, WacrawlSkill, type WacrawlSkillOptions } from "./skills/wacrawl/index.js";
-export type { DatasourceDiagnostic, DatasourceDiagnosticCode, DatasourceIndexFail, DatasourceIndexOk, DatasourceIndexResult, DatasourceInstance, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription, } from "./types.js";
+export type { DatasourceDiagnostic, DatasourceDiagnosticCode, DatasourceIndexFail, DatasourceIndexOk, DatasourceIndexOptions, DatasourceIndexResult, DatasourceInstance, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription, } from "./types.js";

package/dist/datasource/retry.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Shared retry/backoff policy for datasource `index()` calls.
+ *
+ * Failure causes differ per datasource (HTTP status, CLI exit reason, ...), so
+ * no attempt is made to classify them. {@link indexWithRetry} retries every
+ * failed or thrown `index()` with jittered exponential backoff and, once the
+ * retries are used up, reports that datasource as failed so the refresh moves
+ * on to the next one.
+ *
+ * Process-tree termination of a timed-out CLI is a separate concern (#1587,
+ * #1586); this module never touches the spawn code.
+ */
+import type { DatasourceIndexResult, DatasourceSkill } from "./types.js";
+export interface DatasourceRetryPolicy {
+    /** Re-runs after the first call; `0` disables retrying. */
+    readonly maxRetries: number;
+    /** Backoff before the first retry; doubles for each later retry. */
+    readonly baseDelayMs: number;
+    /** Upper bound on one backoff step (before jitter). */
+    readonly maxDelayMs: number;
+}
+export interface DatasourceRetryOptions extends Partial<DatasourceRetryPolicy> {
+    /** Jitter source in `[0, 1]`; injectable for deterministic tests. */
+    readonly random?: () => number;
+    /** Delay implementation; injectable so tests and hosts need not wait in real time. */
+    readonly sleep?: (ms: number) => Promise<void>;
+}
+/** Two retries (three calls in total) with 1s then 2s of backoff, jittered. */
+export declare const DEFAULT_DATASOURCE_RETRY_POLICY: DatasourceRetryPolicy;
+/**
+ * Backoff before retry number `retry` (1-based): `baseDelayMs * 2^(retry-1)`
+ * capped at `maxDelayMs`, then jittered into the upper half of that step so
+ * simultaneous refreshes do not hammer the same upstream in lockstep.
+ */
+export declare function datasourceRetryDelayMs(retry: number, policy: DatasourceRetryPolicy & {
+    readonly random?: () => number;
+}): number;
+/**
+ * Run `skill.index()`, retrying any failure (a failed result or a throw). A
+ * first-time success is returned untouched. Otherwise the result carries the
+ * total `attempts`; an exhausted failure also carries `attempts` in its message
+ * and last diagnostic, and a late success carries an info diagnostic naming the
+ * last error it recovered from. A thrown `index()` becomes a
+ * `datasource-index-failed` result.
+ *
+ * A cancelled `signal` is not a datasource failure: it is rethrown at once
+ * (also from the backoff wait) instead of being retried or reported.
+ */
+export declare function indexWithRetry(skill: DatasourceSkill, options?: DatasourceRetryOptions, signal?: AbortSignal): Promise<DatasourceIndexResult>;

package/dist/datasource/skills/clawgallery/skill.d.ts

@@ -1,10 +1,10 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type ClawGallerySearchClient } from "./methods.js";
 import type { ClawGalleryIndexResult, ClawGalleryOptions, ClawGallerySearchMode, ClawGalleryVdrResult } from "./types.js";
 export interface ClawGallerySkillClient extends ClawGallerySearchClient {
-    bootstrap(): Promise<ClawGalleryIndexResult>;
-    syncVisual(): Promise<ClawGalleryVdrResult>;
+    bootstrap(signal?: AbortSignal): Promise<ClawGalleryIndexResult>;
+    syncVisual(signal?: AbortSignal): Promise<ClawGalleryVdrResult>;
 }
 export interface ClawGallerySkillOptions {
     readonly client: ClawGallerySkillClient;
@@ -28,7 +28,7 @@ export declare class ClawGallerySkill implements DatasourceSkill {
     constructor(options: ClawGallerySkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/discrawl/skill.d.ts

@@ -1,11 +1,11 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type DiscrawlSearchClient } from "./methods.js";
 import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlSearchMode, DiscrawlSyncResult } from "./types.js";
 export interface DiscrawlSkillClient extends DiscrawlSearchClient {
-    doctor(): Promise<DiscrawlDoctorResult>;
-    sync(): Promise<DiscrawlSyncResult>;
-    embed(limit?: number, rebuild?: boolean): Promise<DiscrawlEmbedResult>;
+    doctor(signal?: AbortSignal): Promise<DiscrawlDoctorResult>;
+    sync(signal?: AbortSignal): Promise<DiscrawlSyncResult>;
+    embed(limit?: number, rebuild?: boolean, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
     prepareEmbeddings?(): Promise<boolean>;
 }
 export interface DiscrawlSkillOptions {
@@ -47,7 +47,7 @@ export declare class DiscrawlSkill implements DatasourceSkill {
     constructor(options: DiscrawlSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/github-gist/skill.d.ts

@@ -10,7 +10,7 @@
 import { type Embedder, type GatewayEmbedderOptions } from "../../../embedding-runtime/gateway-embedder.js";
 import type { RetrievalMethod } from "../../../retrieval/types.js";
 import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import type { DatasourceIndexResult } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult } from "../../types.js";
 import { GitHubGistConnector, type GitHubGistConnectorOptions } from "./connector.js";
 export declare const GITHUB_GIST_SKILL_DEFINITION: ConnectorSkillDefinition;
 export declare function gistDatasourceDir(workspaceRoot: string, skillName: string, instanceId: string): string;
@@ -34,6 +34,6 @@ export declare class GitHubGistSkill extends ConnectorDatasourceSkill {
     private readonly semanticEmbedder;
     private readonly semanticMethod;
     constructor(options?: GitHubGistSkillOptions);
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
 }

package/dist/datasource/skills/lazykatok/skill.d.ts

@@ -1,11 +1,11 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type LazykatokSearchClient } from "./methods.js";
 import type { LazykatokDoctorResult, LazykatokIndexResult, LazykatokSyncResult } from "./types.js";
 export interface LazykatokSkillClient extends LazykatokSearchClient {
-    doctor(): Promise<LazykatokDoctorResult>;
-    sync(): Promise<LazykatokSyncResult>;
-    index(): Promise<LazykatokIndexResult>;
+    doctor(signal?: AbortSignal): Promise<LazykatokDoctorResult>;
+    sync(signal?: AbortSignal): Promise<LazykatokSyncResult>;
+    index(signal?: AbortSignal): Promise<LazykatokIndexResult>;
 }
 export interface LazykatokSkillOptions {
     readonly client: LazykatokSkillClient;
@@ -25,7 +25,7 @@ export declare class LazykatokSkill implements DatasourceSkill {
     constructor(options: LazykatokSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/mailcrawl/skill.d.ts

@@ -1,5 +1,5 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type MailcrawlIndexClient } from "./client.js";
 import type { MailcrawlOptions, MailcrawlSyncResult } from "./types.js";
 export interface MailcrawlSkillClient extends MailcrawlIndexClient {
@@ -26,7 +26,7 @@ export declare class MailcrawlSkill implements DatasourceSkill {
     constructor(options?: MailcrawlSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/skills/obsidian/skill.d.ts

@@ -1,11 +1,11 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
-import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
 import { type ObsidianSearchClient } from "./methods.js";
 import type { QmdEmbedResult, QmdEnsureResult, QmdUpdateResult } from "./types.js";
 export interface ObsidianSkillClient extends ObsidianSearchClient {
-    ensureCollection(): Promise<QmdEnsureResult>;
-    update(): Promise<QmdUpdateResult>;
-    embed(): Promise<QmdEmbedResult>;
+    ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
+    update(signal?: AbortSignal): Promise<QmdUpdateResult>;
+    embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
 }
 export interface ObsidianSkillOptions {
     readonly datasourceId?: string;
@@ -37,7 +37,7 @@ export declare class ObsidianSkill implements DatasourceSkill {
     constructor(options?: ObsidianSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     retrievalMethods(): readonly RetrievalMethod[];
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;

package/dist/datasource/types.d.ts

@@ -52,6 +52,11 @@ export interface DatasourceSkillDescriptor {
     /** Optional list of instance ids this skill reports. */
     readonly instances?: readonly string[];
 }
+/** Per-call options for {@link DatasourceSkill.index}. */
+export interface DatasourceIndexOptions {
+    /** Aborting stops the index run and kills any CLI it spawned. An aborted run may reject instead of returning a fail variant. */
+    readonly signal?: AbortSignal;
+}
 /**
  * A pluggable datasource skill. Skills are server-bound: they are constructed
  * with trusted configuration and never derive authority from model/tool args.
@@ -67,7 +72,7 @@ export interface DatasourceSkill {
      * never throws. Remote embedding egress MUST be rejected here and reported
      * via a `datasource-embedding-egress-rejected` diagnostic.
      */
-    index(): Promise<DatasourceIndexResult>;
+    index(options?: DatasourceIndexOptions): Promise<DatasourceIndexResult>;
     /** Retrieval methods exposed by this skill, fed into the shared pipeline. */
     retrievalMethods(): readonly RetrievalMethod[];
     /** Opaque source descriptions for the skill's current evidence set. */
@@ -128,6 +133,11 @@ export interface DatasourceIndexOk {
     readonly chunkCount: number;
     readonly indexedAt: number;
     readonly diagnostics: readonly DatasourceDiagnostic[];
+    /**
+     * Total `index()` calls it took (set only when the refresh retried; a
+     * first-time success omits it).
+     */
+    readonly attempts?: number;
 }
 /** Failed indexing outcome. Never thrown; returned with diagnostics. */
 export interface DatasourceIndexFail {
@@ -139,6 +149,8 @@ export interface DatasourceIndexFail {
     readonly error?: string;
     readonly code?: DatasourceDiagnosticCode;
     readonly message?: string;
+    /** Total `index()` calls made before this result was reported (set by the refresh retry policy). */
+    readonly attempts?: number;
 }
 /**
  * Index result — an ok/fail discriminated union. Skills MUST NOT throw from

package/dist/embedding-runtime/cache.d.ts

@@ -16,6 +16,15 @@ export interface CacheOptions {
     readonly downloadTimeoutMs?: number;
 }
 export declare function verifyCacheEntry(path: string, expectedSha256: string): Promise<string>;
+/**
+ * Marker a distributor writes inside `<archive>.extracted/` when it ships the
+ * runtime already extracted instead of as the pinned archive. Its content is the
+ * archive's SHA-256, so a tree provisioned for another runtime build is never
+ * taken for this one. A signed and notarized macOS app needs this: every Mach-O
+ * it carries must be re-signed with the app's Developer ID, so the binaries can
+ * no longer be packed back into an archive that matches the pinned digest.
+ */
+export declare const PROVISIONED_RUNTIME_MARKER = ".autorag-archive-sha256";
 export declare function hasVerifiedAsset(asset: CacheAsset, cacheRoot?: string): Promise<boolean>;
 export declare function downloadAsset(asset: CacheAsset, options?: CacheOptions): Promise<string>;
 export declare function importAsset(sourcePath: string, asset: CacheAsset, options?: CacheOptions): Promise<string>;

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export { AutoRAGAgent, type AutoRAGAgentOptions, type AutoRAGRefreshComponentStatus, type AutoRAGRefreshOptions, type AutoRAGRefreshResult, type AutoRAGRefreshStatus, type AutoRAGWatchRefreshHandle, type AutoRAGWatchRefreshOptions, type AutoRefreshOptions, type RefreshMethod, } from "./agent/agent.js";
+export { AutoRAGAgent, type AutoRAGAgentOptions, type AutoRAGRefreshComponentStatus, type AutoRAGRefreshOptions, type AutoRAGRefreshProgress, type AutoRAGRefreshResult, type AutoRAGRefreshStatus, type AutoRAGWatchRefreshHandle, type AutoRAGWatchRefreshOptions, type AutoRefreshOptions, type RefreshMethod, } from "./agent/agent.js";
 export { type AutoRAGResultsDetails, reportSchema, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentsStreamEvent, type SearchDocumentWarning, } from "./agent/index.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./agent/system-prompt.js";
 export * from "./cloud/index.js";
@@ -8,7 +8,6 @@ export * from "./dupey/index.js";
 export * from "./everything/index.js";
 export * from "./fsearch/index.js";
 export * from "./jikji/index.js";
-export * from "./manifest/index.js";
 export { type AutoRAGMcpServerOptions, createAutoRAGMcpServer } from "./mcp/server.js";
 export * from "./memory/index.js";
 export * from "./minsync/index.js";