@autorag/librarian 2.9.0 → 2.10.1

package/dist/datasource/crawler-skill.d.ts

@@ -1,5 +1,6 @@
 import type { RetrievalMethod } from "../retrieval/types.js";
 import type { CrawlerSearchOptions, CrawlerSearchResult, CrawlerSyncResult } from "./crawler-types.js";
+import { type CrawlerSidecarOptions } from "./sidecar/index.js";
 import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
 export interface CrawlerSkillClient {
     sync(signal?: AbortSignal): Promise<CrawlerSyncResult>;
@@ -27,6 +28,12 @@ export interface CrawlerDatasourceSkillOptions {
     /** Omitted/empty means search every chat/channel. */
     readonly channelIds?: readonly string[];
     readonly channelNames?: readonly string[];
+    /**
+     * Opt-in read-only sidecar over the crawler's own SQLite archive: adds BM25-ranked,
+     * semantic and hybrid methods next to the CLI-backed FTS method. The crawler database
+     * is never written; sidecar state lives in `sidecarDbPath`.
+     */
+    readonly sidecar?: CrawlerSidecarOptions;
 }
 export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     private readonly definition;
@@ -37,12 +44,22 @@ export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     private readonly tags;
     private readonly channelIds;
     private readonly channelNames;
+    private readonly sidecar;
     private lastIndexedAt;
     constructor(definition: CrawlerDatasourceDefinition, options: CrawlerDatasourceSkillOptions);
     describe(): DatasourceSkillDescriptor;
     polling(): PollingMetadata;
     index(): Promise<DatasourceIndexResult>;
+    /**
+     * Embed new crawler messages into the sidecar. Never fails the index (the CLI sync already succeeded and BM25
+     * stays available): any sidecar problem, including a thrown database error, becomes a warning. A bounded sync
+     * that left a backlog says so, so a partial semantic index is not mistaken for a complete one.
+     */
+    private syncSidecar;
+    /** Release the sidecar's database handles. Safe to call repeatedly; later use reopens them. */
+    close(): void;
     retrievalMethods(): readonly RetrievalMethod[];
+    private cliMethod;
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;
     private fail;

package/dist/datasource/sidecar/config.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Turns a datasource's `connector.sidecar` config into {@link CrawlerSidecarOptions}.
+ *
+ * On by default for every datasource that has a crawler descriptor; opt out with
+ *
+ *   "connector": { "sidecar": false }
+ *
+ * or tune it:
+ *
+ *   "connector": { "sidecar": { "crawlerDbPath": "~/.slacrawl/slacrawl.db", "semantic": false } }
+ *
+ * The sidecar's own database always lives under the AutoRAG workspace, one file
+ * per datasource instance, never next to (or inside) the crawler's archive.
+ */
+import type { CrawlerDescriptor } from "./descriptors.js";
+import type { CrawlerSidecarOptions } from "./index.js";
+export interface ResolveSidecarInput {
+    readonly descriptor: CrawlerDescriptor;
+    readonly connector: Record<string, unknown> | undefined;
+    readonly workspaceRoot: string | undefined;
+    readonly registrationName: string;
+    readonly instanceId: string;
+    /** Where the crawler keeps its archive when the user did not say. */
+    readonly defaultCrawlerDb: string;
+}
+export declare function resolveSidecarOptions(input: ResolveSidecarInput): CrawlerSidecarOptions | undefined;

package/dist/datasource/sidecar/descriptors.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Declarative description of a crawler's read-only SQLite schema. The sidecar
+ * engine (index.ts) is crawler-agnostic: everything that differs between
+ * slacrawl, telecrawl, wacrawl and notcrawl is expressed here as SQL fragments.
+ *
+ * Every SQL fragment is a trusted constant authored in this file. User input
+ * (the search query, workspace id) is only ever bound as a parameter.
+ *
+ * The crawler database is opened read-only and ATTACHed; the sidecar owns a
+ * separate database file, so a crawler that rebuilds, migrates, purges or
+ * vacuums its own file can never lose or corrupt sidecar state.
+ */
+export interface CrawlerDescriptor {
+    readonly id: string;
+    /**
+     * True when the crawler's own TOML config (`connector.configPath`) can relocate its archive via
+     * `db_path`. The sidecar then follows that setting instead of assuming the default archive.
+     */
+    readonly archivePathInConfig?: boolean;
+    /** `message_key` stored in the sidecar for each row; unique per message within one crawler DB. */
+    readonly keyExpression: string;
+    /**
+     * Public chunk id for a stored key. MUST equal the id the crawler's CLI client derives for the
+     * same message (see each skill's `parseHits`), so the retrieval merger deduplicates a message
+     * found by both the CLI-backed and the sidecar methods.
+     */
+    readonly chunkId: (key: string, metadata: Readonly<Record<string, string>>) => string;
+    /** FROM + JOINs exposing alias `m` (message-like row) and any lookup aliases used below. */
+    readonly fromClause: string;
+    /** Row visibility predicate: excludes soft-deleted rows. May reference a `:scope` parameter when `scopeColumn` is set. */
+    readonly liveWhere: string;
+    /** Column that the optional workspace/account scope filters on; omitted when the crawler has one account per DB. */
+    readonly scopeExpression?: string;
+    /** SQL returning the embeddable/displayed body of the row. */
+    readonly textExpression: string;
+    /** Optional extra text prepended to the embedding input (title, channel). */
+    readonly titleExpression?: string;
+    /** Seconds-since-epoch expression used for `publishedAt`; omitted when unknown. */
+    readonly epochSecondsExpression?: string;
+    /**
+     * FTS5 relation joined to the message row. `joinOn` ties the FTS row to alias `m`;
+     * `table` is the FTS table (also the argument to bm25()).
+     */
+    readonly fts: {
+        readonly table: string;
+        readonly joinOn: string;
+    };
+    /** Extra result columns mapped into hit metadata: a SQL expression and the metadata key it fills. */
+    readonly metadata: readonly {
+        readonly sql: string;
+        readonly key: string;
+    }[];
+    /**
+     * SQL for the chat/channel id and name the allowlist (`channels.ids/names`) matches. Pushed into the
+     * row predicate, so excluded chats are never searched, embedded, or kept in the sidecar.
+     */
+    readonly channelIdExpression: string;
+    readonly channelNameExpression: string;
+    /** Path-opaque label for the hierarchy: ["workspaces", ws, "channels", name] etc. */
+    readonly hierarchy: (metadata: Readonly<Record<string, unknown>>) => readonly string[];
+}
+export declare const CRAWLER_DESCRIPTORS: {
+    readonly slack: CrawlerDescriptor;
+    readonly telegram: CrawlerDescriptor;
+    readonly whatsapp: CrawlerDescriptor;
+};
+export type SidecarCrawlerId = keyof typeof CRAWLER_DESCRIPTORS;

package/dist/datasource/sidecar/fusion.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Pure ranking primitives shared by every crawler sidecar: float32 vector
+ * (de)serialization, cosine top-k, and reciprocal rank fusion. No I/O.
+ */
+export declare function encodeVector(vector: readonly number[]): Uint8Array;
+/** Inverse of {@link encodeVector}; undefined when the blob is not a whole number of float32s. */
+export declare function decodeVector(blob: Uint8Array): Float32Array | undefined;
+/** Cosine similarity; 0 (never NaN) for zero-length, zero-norm, or mismatched-dimension inputs. */
+export declare function cosine(a: ArrayLike<number>, b: ArrayLike<number>): number;
+export interface KeyedVector {
+    readonly key: string;
+    readonly vector: ArrayLike<number>;
+}
+export interface ScoredKey {
+    readonly key: string;
+    readonly score: number;
+}
+/** Best `k` positive-similarity keys, descending; ties broken by key for determinism. */
+export declare function topKByCosine(query: ArrayLike<number>, rows: Iterable<KeyedVector>, k: number): ScoredKey[];
+/**
+ * Reciprocal rank fusion over ranked key lists (best first). A key repeated
+ * inside one list counts once, at its best rank. Ties keep first-seen order,
+ * so earlier lists win.
+ */
+export declare function reciprocalRankFusion(lists: readonly (readonly string[])[]): ScoredKey[];

package/dist/datasource/sidecar/index.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Crawler sidecar: BM25 + semantic + hybrid retrieval over a crawler's own
+ * SQLite archive without touching it.
+ *
+ * The sidecar owns a separate database file (MAIN). The crawler's database is
+ * ATTACHed read-only, so a crawler that migrates, purges, vacuums or rebuilds
+ * its file cannot lose sidecar state, and the sidecar cannot corrupt the
+ * crawler. Only vectors keyed by the crawler's message key are stored; message
+ * bodies are always read from the crawler database at query time.
+ *
+ * Everything crawler-specific lives in a {@link CrawlerDescriptor}; this file
+ * is shared by every crawler.
+ */
+import type { Embedder } from "../../embedding-runtime/gateway-embedder.js";
+import type { CrawlerDescriptor } from "./descriptors.js";
+export interface SidecarHit {
+    readonly key: string;
+    readonly content: string;
+    readonly score: number;
+    readonly title?: string;
+    /** Milliseconds since epoch. */
+    readonly publishedAt?: number;
+    readonly metadata: Readonly<Record<string, string>>;
+}
+export type SidecarSyncResult = {
+    readonly ok: true;
+    readonly embedded: number;
+    /** Messages the embedder rejected on their own; retried only after their text changes. */
+    readonly skipped: number;
+    readonly pruned: number;
+    /** Messages still waiting for a vector after this sync (0 when fully caught up). */
+    readonly remaining: number;
+} | {
+    readonly ok: false;
+    readonly message: string;
+};
+export interface CrawlerSidecarOptions {
+    readonly descriptor: CrawlerDescriptor;
+    readonly crawlerDbPath: string;
+    readonly sidecarDbPath: string;
+    /** Restricts every query to one workspace/account when the descriptor supports scoping. */
+    readonly workspaceId?: string;
+    /**
+     * Chat/channel allowlist (`channels.ids/names`). Empty means every chat. Applied in SQL, so excluded chats are
+     * never searched, embedded, or kept as vectors.
+     */
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
+    /** Required for semantic and hybrid search; lexical search works without it. */
+    readonly embedder?: Embedder;
+    /** Messages per embedding request and per write transaction. Default {@link DEFAULT_BATCH_SIZE}. */
+    readonly batchSize?: number;
+    /**
+     * Most messages one `sync()` will embed. A first sync over a large archive otherwise holds the whole
+     * refresh for minutes; the remainder is picked up by the next sync, and BM25 never waits for vectors.
+     * Default {@link DEFAULT_MAX_EMBED_PER_SYNC}.
+     */
+    readonly maxEmbedPerSync?: number;
+}
+export declare class CrawlerSidecar {
+    readonly descriptor: CrawlerDescriptor;
+    private readonly options;
+    private readonly select;
+    private readonly where;
+    /** Opened on first use: building a skill must not create files, directories, or file descriptors. */
+    private opened;
+    /** Identity of the attached crawler file; a crawler that replaces or deletes its archive forces a re-attach. */
+    private attached;
+    /**
+     * The last query's embedding and ranking. The semantic and hybrid methods of one search ask for the same
+     * query, so they share one embedding call and one vector scan. Short-lived, and dropped on any write.
+     */
+    private lastSemantic;
+    constructor(options: CrawlerSidecarOptions);
+    /** The sidecar database, created on first call. Only reached once the crawler archive exists. */
+    private get db();
+    /**
+     * Attach the crawler archive read-only, once it exists and has the expected
+     * shape. The crawler may not have synced yet, or may have replaced or deleted
+     * its file, so this is re-checked on every call: a changed file is re-attached,
+     * a missing one detached. Nothing is created on disk until the archive exists.
+     */
+    private attachCrawler;
+    private detachCrawler;
+    /** True when a semantic index can be built: an embedder is configured. */
+    get semanticEnabled(): boolean;
+    /** Release the database handle. Safe to call repeatedly; a later search reopens and re-attaches. */
+    close(): void;
+    /** Bound parameters for {@link where}, in the order its predicates appear. */
+    private whereParams;
+    /** BM25-ranked lexical search. Natural-language queries are OR-ed term by term. */
+    searchLexical(query: string, topK: number): SidecarHit[];
+    /** Cosine search over embedded messages. Empty when there is no embedder or the query embedding fails. */
+    searchSemantic(query: string, topK: number): Promise<SidecarHit[]>;
+    /** Best `pool` stored vectors for `query`, reusing the previous ranking when it already covers `pool`. */
+    private rankSemantic;
+    /** Reciprocal-rank fusion of lexical and semantic rankings; lexical-only when semantic is unavailable. */
+    searchHybrid(query: string, topK: number): Promise<SidecarHit[]>;
+    /**
+     * Bring the vector table in line with the crawler archive: drop vectors of
+     * purged/deleted messages, embed new or edited ones, re-embed all on an
+     * embedding identity change. An embedder failure keeps what was already
+     * stored. A message the embedder keeps rejecting while others succeed is
+     * skipped until its text changes, so it cannot stall the backlog behind it.
+     * Database errors (for example a lock held past the busy timeout) throw.
+     */
+    sync(): Promise<SidecarSyncResult>;
+    private adoptIdentity;
+    /** Delete vectors whose message no longer exists, is deleted, or fell out of scope. */
+    private prune;
+    /** Keys with no vector yet, or whose embedded text changed since. */
+    private pendingKeys;
+    /** Live rows for the given keys, in the order given. */
+    private hydrate;
+    /**
+     * Upsert vectors keyed by message. `undefined` stores an empty vector: the message is recorded as tried at
+     * this text hash (so it is not retried every sync) but never ranked.
+     */
+    private store;
+    private storedVectors;
+    private transaction;
+    private toRow;
+    private toHit;
+}
+/**
+ * FTS5 MATCH expression for free text: every alphanumeric term quoted and
+ * OR-ed. Quoting neutralizes FTS operators (AND, NOT, NEAR, *, parentheses),
+ * so user text can never change the query's structure. Undefined when the
+ * text contains no searchable term.
+ */
+export declare function toMatchQuery(text: string): string | undefined;

package/dist/datasource/sidecar/methods.d.ts

@@ -0,0 +1,27 @@
+/**
+ * RetrievalMethod adapters over a {@link CrawlerSidecar}. One class serves every
+ * crawler and every mode; the only per-instance input is the datasource identity
+ * (the channel allowlist is applied by the sidecar's SQL).
+ */
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../retrieval/types.js";
+import type { CrawlerDescriptor } from "./descriptors.js";
+import type { CrawlerSidecar } from "./index.js";
+export type SidecarMode = "bm25" | "semantic" | "hybrid";
+export interface SidecarMethodOptions {
+    readonly sidecar: CrawlerSidecar;
+    readonly descriptor: CrawlerDescriptor;
+    readonly mode: SidecarMode;
+    readonly datasourceId: string;
+    readonly skillType: string;
+    readonly instanceId: string;
+    readonly tags: readonly string[];
+}
+/** Result method name; `bm25` keeps the `-sidecar-` infix so it never collides with the CLI-backed `-fts` method. */
+export declare function sidecarMethodName(datasourceId: string, mode: SidecarMode): string;
+export declare class SidecarRetrievalMethod implements RetrievalMethod {
+    private readonly options;
+    constructor(options: SidecarMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+    private toResult;
+}

package/dist/datasource/types.d.ts

@@ -145,7 +145,9 @@ export interface DatasourceIndexFail {
  * `index()`; they return a fail variant instead.
  */
 export type DatasourceIndexResult = DatasourceIndexOk | DatasourceIndexFail;
-export type DatasourceDiagnosticCode = "datasource-unavailable" | "datasource-cli-error" | "datasource-empty" | "datasource-rate-limited" | "datasource-auth-error" | "datasource-embedding-egress-rejected" | "datasource-index-failed" | "datasource-permission-denied" | "datasource-remote-embedding-rejected";
+export type DatasourceDiagnosticCode = "datasource-unavailable" | "datasource-cli-error" | "datasource-empty" | "datasource-rate-limited" | "datasource-auth-error" | "datasource-embedding-egress-rejected" | "datasource-index-failed"
+/** Index succeeded but is incomplete (e.g. a semantic backlog still draining); not a failure. */
+ | "datasource-index-partial" | "datasource-permission-denied" | "datasource-remote-embedding-rejected";
 /** Path-opaque diagnostic emitted by the datasource layer. */
 export interface DatasourceDiagnostic {
     readonly code: DatasourceDiagnosticCode;

package/dist/embedding-runtime/index.d.ts

@@ -87,15 +87,15 @@ export declare function createEmbeddingRuntime(options?: EmbeddingRuntimeOptions
     stopRuntime: () => Promise<void>;
     runtimeStatus: () => Promise<RuntimeStatus>;
     prefetchModel: (profileId?: ProfileId) => Promise<{
-        profileId: ProfileId;
+        profileId: "qwen3-embedding-0.6b";
         path: string;
     }>;
     importModel: (profileId: ProfileId | undefined, sourcePath: string) => Promise<{
-        profileId: ProfileId;
+        profileId: "qwen3-embedding-0.6b";
         path: string;
     }>;
     verifyModel: (profileId?: ProfileId) => Promise<{
-        profileId: ProfileId;
+        profileId: "qwen3-embedding-0.6b";
         path: string;
         hash: string;
     }>;
@@ -106,15 +106,15 @@ export declare const releaseRuntimeHandles: () => Promise<void>;
 export declare const stopRuntime: () => Promise<void>;
 export declare const runtimeStatus: () => Promise<RuntimeStatus>;
 export declare const prefetchModel: (profileId?: ProfileId) => Promise<{
-    profileId: ProfileId;
+    profileId: "qwen3-embedding-0.6b";
     path: string;
 }>;
 export declare const importModel: (profileId: ProfileId | undefined, sourcePath: string) => Promise<{
-    profileId: ProfileId;
+    profileId: "qwen3-embedding-0.6b";
     path: string;
 }>;
 export declare const verifyModel: (profileId?: ProfileId) => Promise<{
-    profileId: ProfileId;
+    profileId: "qwen3-embedding-0.6b";
     path: string;
     hash: string;
 }>;

package/dist/embedding-runtime/manifest.d.ts

@@ -14,19 +14,7 @@ export declare const MODEL_ASSETS: {
         readonly upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B";
         readonly modelCard: "docs/model-cards/qwen3-embedding-0.6b.md";
     };
-    readonly embeddinggemma: {
-        readonly url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf";
-        readonly revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12";
-        readonly sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63";
-        readonly license: "Gemma Terms of Use";
-        readonly noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use";
-        readonly intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.";
-        readonly unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.";
-        readonly upstreamCardUrl: "https://ai.google.dev/gemma/docs/embeddinggemma/model_card";
-        readonly modelCard: "docs/model-cards/embeddinggemma-300m.md";
-    };
 };
 export declare const MODEL_ASSET_URLS: {
     readonly qwen3: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf";
-    readonly embeddinggemma: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf";
 };

package/dist/embedding-runtime/types.d.ts

@@ -1,6 +1,6 @@
 export type BackendKind = "auto" | "cpu" | "vulkan";
 export type PlatformId = "darwin-arm64-metal" | "win-x64-cpu" | "win-x64-vulkan";
-export type ProfileId = "qwen3-embedding-0.6b" | "embeddinggemma-300m";
+export type ProfileId = "qwen3-embedding-0.6b";
 export interface ProtocolCapabilities {
     readonly healthz: boolean;
     readonly embed: boolean;

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 export { AutoRAGAgent, type AutoRAGAgentOptions, type AutoRAGRefreshComponentStatus, type AutoRAGRefreshOptions, type AutoRAGRefreshResult, type AutoRAGRefreshStatus, type AutoRAGWatchRefreshHandle, type AutoRAGWatchRefreshOptions, type AutoRefreshOptions, type RefreshMethod, } from "./agent/agent.js";
-export { type AutoRAGResultsDetails, createEmitResultsTool, EMIT_AUTORAG_RESULTS_TOOL_NAME, emitResultsSchema, reportSchema, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentsStreamEvent, type SearchDocumentWarning, } from "./agent/index.js";
+export { type AutoRAGResultsDetails, reportSchema, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentsStreamEvent, type SearchDocumentWarning, } from "./agent/index.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./agent/system-prompt.js";
+export * from "./cloud/index.js";
 export { AutoRAGLite, type AutoRAGLiteOptions, buildAgentOptions, type CliConfig, createAutoRAGLite, type ResolveConfigInput, resolveConfig, resolveConfigReadOnly, writeConfigObject, writeDefaultConfig, } from "./core.js";
 export * from "./datasource/index.js";
 export * from "./dupey/index.js";