@karmaniverous/jeeves-watcher 0.19.1 → 0.20.0

package/dist/index.d.ts

@@ -1,10 +1,11 @@
-import { JeevesComponentDescriptor } from '@karmaniverous/jeeves';
 import { FastifyInstance } from 'fastify';
+import { JeevesComponentDescriptor } from '@karmaniverous/jeeves';
 import pino from 'pino';
-import { EmbeddingConfig, InferenceRule, SchemaEntry, VectorStoreConfig, VcsConfig, JeevesWatcherConfig, WatchConfig, LoggingConfig, VcsRetentionConfig } from '@karmaniverous/jeeves-watcher-core';
-export { ApiConfig, ConfigWatchConfig, EmbeddingConfig, InferenceRule, JeevesWatcherConfig, JeevesWatcherConfigInput, LoggingConfig, VectorStoreConfig, WatchConfig, apiConfigSchema, configWatchConfigSchema, embeddingConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema, loggingConfigSchema, vectorStoreConfigSchema, watchConfigSchema } from '@karmaniverous/jeeves-watcher-core';
-import { z } from 'zod';
+import { EmbeddingConfig, IssueRecord, InferenceRule, SchemaEntry, SourcesConfig, VectorStoreConfig, DocumentResult, DocumentRejectionError, DocumentFailure, VcsConfig, JeevesWatcherConfig, WatchConfig, LoggingConfig, VcsRetentionConfig } from '@karmaniverous/jeeves-watcher-core';
+export { ApiConfig, ConfigWatchConfig, EmbeddingConfig, InferenceRule, IssueRecord, JeevesWatcherConfig, JeevesWatcherConfigInput, LoggingConfig, VectorStoreConfig, WatchConfig, apiConfigSchema, configWatchConfigSchema, embeddingConfigSchema, inferenceRuleSchema, issueRecordSchema, jeevesWatcherConfigSchema, loggingConfigSchema, vectorStoreConfigSchema, watchConfigSchema } from '@karmaniverous/jeeves-watcher-core';
+import { ErrorObject } from 'ajv';
 import Handlebars from 'handlebars';
+import { z } from 'zod';
 import { JsonMapMap } from '@karmaniverous/jsonmap';
 import { Stats } from 'node:fs';
 
@@ -13,12 +14,45 @@ import { Stats } from 'node:fs';
  * Embedding provider type definitions.
  */
 
+/**
+ * Per-call budget for the shared rate limiter. Providers that do not
+ * rate-limit ignore the budget; providers that retry internally await
+ * `beforeRetry` before each retry.
+ *
+ * The budget bounds the limiter, not the provider's own network retries:
+ * each of the `attempts` provider calls may itself retry a transient
+ * (non-429) failure up to `NETWORK_RETRY_POLICY.attempts` times in all, with
+ * backoff sleeps, so one `embed()` can issue up to `attempts` x 5 HTTP
+ * requests.
+ */
+interface EmbedOptions {
+    /**
+     * Maximum provider calls for this call: a 429 ends a provider call and
+     * costs one attempt. Transient network retries inside a provider call do
+     * not count against it (see the interface note).
+     */
+    attempts?: number;
+    /**
+     * Maximum total time to wait for the limiter (token or `Retry-After`
+     * pause), in ms. When a wait would exceed it, the call fails fast. The
+     * provider's own network-retry backoff sleeps are not counted.
+     */
+    maxWaitMs?: number;
+    /**
+     * Awaited by a provider before each retry it makes internally (any HTTP
+     * request after the first of an `embed()` call), so every request draws
+     * from the shared limiter. Supplied by the rate-limited wrapper; it may
+     * reject (e.g. `EmbeddingUnavailableError`), which the provider must
+     * propagate without retrying.
+     */
+    beforeRetry?: () => Promise<void>;
+}
 /**
  * An embedding provider that converts text into vector representations.
  */
 interface EmbeddingProvider {
-    /** Generate embedding vectors for the given texts. */
-    embed(texts: string[]): Promise<number[][]>;
+    /** Generate embedding vectors for the given texts (one provider request). */
+    embed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
     /** The dimensionality of the embedding vectors. */
     dimensions: number;
 }
@@ -48,7 +82,7 @@ declare function createEmbeddingProvider(config: EmbeddingConfig, logger?: pino.
 
 /**
  * @module enrichment/EnrichmentStore
- * SQLite-backed enrichment metadata store. Persists path-keyed metadata at stateDir/enrichments.sqlite. Atomic writes, supports move.
+ * SQLite-backed enrichment metadata store. Persists path- or source-reference-keyed metadata (references verbatim) at stateDir/enrichments.sqlite. Atomic writes, supports move.
  */
 
 /**
@@ -188,95 +222,44 @@ declare abstract class JsonFileStore<T> {
 
 /**
  * @module issues/types
- * Zod schemas and TypeScript types for the issues tracking system. Defines IssueRecord and IssuesFile structures.
+ * Issue record schema and types for the issues tracking system. IssueRecord is defined in core (GET /issues response); IssuesFile is the on-disk shape.
  */
 
-/** Schema for a single issue record tracking a processing failure. */
-declare const issueRecordSchema: z.ZodObject<{
-    type: z.ZodUnion<readonly [z.ZodLiteral<"type_collision">, z.ZodLiteral<"interpolation_error">]>;
-    property: z.ZodOptional<z.ZodString>;
-    rules: z.ZodOptional<z.ZodArray<z.ZodString>>;
-    rule: z.ZodOptional<z.ZodString>;
-    types: z.ZodOptional<z.ZodArray<z.ZodString>>;
-    message: z.ZodString;
-    timestamp: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
-}, z.core.$strip>;
-/** A single issue record tracking a processing failure for a file. */
-type IssueRecord = z.infer<typeof issueRecordSchema>;
 /** Issues file: array of issue records per file path. */
 type IssuesFile = Partial<Record<string, IssueRecord[]>>;
 
 /**
  * @module issues/IssuesManager
- * Manages persistent issue tracking for file processing failures. Read-modify-write with in-memory cache.
+ * Manages persistent issue tracking for processing failures, keyed by document key (file path or source reference). Read-modify-write with in-memory cache.
  */
 
 /**
- * Manages a persistent issues.json file tracking processing failures per file.
+ * Manages a persistent issues.json file tracking processing failures per
+ * document key (file path or source reference).
  */
 declare class IssuesManager extends JsonFileStore<IssuesFile> {
     constructor(stateDir: string, logger: pino.Logger);
     protected createEmpty(): IssuesFile;
-    /** Record or update an issue for a file path. */
-    record(filePath: string, type: IssueRecord['type'], message: string, options?: {
-        property?: string;
-        rules?: string[];
-        types?: string[];
-    }): void;
-    /** Clear an issue for a file path (called on successful processing). */
-    clear(filePath: string): void;
+    /**
+     * Record an issue for a document key (file path or source reference).
+     *
+     * @param key - Document key (file path or source reference).
+     * @param type - Canonical issue code.
+     * @param message - Error message.
+     * @param options - Optional operation, property, rule(s), types and
+     *   structured `details` (stored as JSON, e.g. AJV errors).
+     */
+    record(key: string, type: IssueRecord['type'], message: string, options?: Pick<IssueRecord, 'operation' | 'property' | 'rules' | 'rule' | 'types' | 'details'>): void;
+    /** Clear the issues of a document key (called on successful processing). */
+    clear(key: string): void;
     /** Wipe all issues (called on full reindex start). */
     clearAll(): void;
+    /** Get the current issues of one document key, if any. */
+    get(key: string): IssueRecord[] | undefined;
     /** Get all current issues. */
     getAll(): IssuesFile;
 }
 
-/**
- * @module cache/ContentHashCache
- * In-memory cache mapping normalized file paths to content hashes.
- * Supports reverse lookup by hash for move correlation.
- */
-/**
- * In-memory content hash cache for move detection.
- *
- * Maps normalized file paths to SHA-256 content hashes.
- * Supports reverse lookup (hash → paths) for correlating
- * unlink+add events as file moves.
- */
-declare class ContentHashCache {
-    private readonly pathToHash;
-    private readonly hashToPaths;
-    /**
-     * Store or update the content hash for a file path.
-     *
-     * @param filePath - The file path (will be normalized).
-     * @param hash - The SHA-256 content hash.
-     */
-    set(filePath: string, hash: string): void;
-    /**
-     * Get the content hash for a file path.
-     *
-     * @param filePath - The file path (will be normalized).
-     * @returns The content hash, or undefined if not cached.
-     */
-    get(filePath: string): string | undefined;
-    /**
-     * Remove a file path from the cache.
-     *
-     * @param filePath - The file path (will be normalized).
-     */
-    delete(filePath: string): void;
-    /**
-     * Reverse lookup: get all file paths with a given content hash.
-     *
-     * @param hash - The content hash to look up.
-     * @returns Array of normalized file paths with that hash.
-     */
-    getByHash(hash: string): string[];
-    /** Number of cached entries. */
-    get size(): number;
-}
-
 /**
  * @module templates/engine
  * Handlebars template compilation, caching, and resolution (file path vs named ref vs inline).
@@ -391,6 +374,29 @@ declare function buildTemplateEngine(rules: InferenceRule[], namedTemplates?: Re
  */
 declare function registerBuiltinHelpers(hbs: typeof Handlebars): void;
 
+/**
+ * @module rules/sourceAttributes
+ * Zod schema and builder for source-document rule attributes (`source` identity block + `json` record; no `file` key). Pure, no I/O.
+ */
+
+/**
+ * Attributes a source document is matched and resolved against. There is no
+ * `file` key, so file rules (`file.path` globs) never match a source
+ * document, and a source rule (`required: ["source"]`) never matches a file.
+ */
+declare const sourceAttributesSchema: z.ZodObject<{
+    source: z.ZodObject<{
+        ref: z.ZodString;
+        name: z.ZodString;
+        id: z.ZodString;
+        rule: z.ZodString;
+        retrievedAt: z.ZodString;
+    }, z.core.$strip>;
+    json: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+/** Attributes of a source document. */
+type SourceAttributes = z.infer<typeof sourceAttributesSchema>;
+
 /**
  * @module rules/attributes
  * Builds file attribute objects for rule matching. Pure function: derives attributes from path, stats, and extracted data.
@@ -420,6 +426,8 @@ interface FileAttributes {
     /** Parsed JSON content, if any. */
     json?: Record<string, unknown>;
 }
+/** Attributes rules are matched and resolved against: a file or a source document. */
+type RuleAttributes = FileAttributes | SourceAttributes;
 /**
  * Build {@link FileAttributes} from a file path and stat info.
  *
@@ -442,8 +450,13 @@ declare function buildAttributes(filePath: string, stats: Stats, extractedFrontm
 interface CompiledRule {
     /** The original rule definition. */
     rule: InferenceRule;
-    /** The compiled AJV validate function. */
-    validate: (data: unknown) => boolean;
+    /**
+     * The compiled AJV validate function. After a failed call, `errors` holds
+     * the AJV errors (used for `match_failed` issues).
+     */
+    validate: ((data: unknown) => boolean) & {
+        errors?: ErrorObject[] | null;
+    };
 }
 /**
  * Compile an array of inference rules into executable validators.
@@ -454,9 +467,31 @@ interface CompiledRule {
  */
 declare function compileRules(rules: InferenceRule[]): CompiledRule[];
 
+/**
+ * @module rules/ruleErrors
+ * Interpolation errors collected while applying rules (schema `set` resolution, `render`, `template`): Zod schema, type and one-line formatting. Pure.
+ */
+
+/**
+ * One interpolation failure of a matched rule. The rule's output for that
+ * stage is missing (schema failure: the rule's metadata, map and render are
+ * skipped; render/template failure: the raw content is used instead).
+ */
+declare const ruleApplicationErrorSchema: z.ZodObject<{
+    rule: z.ZodString;
+    stage: z.ZodEnum<{
+        schema: "schema";
+        render: "render";
+        template: "template";
+    }>;
+    message: z.ZodString;
+}, z.core.$strip>;
+/** One interpolation failure of a matched rule. */
+type RuleApplicationError = z.infer<typeof ruleApplicationErrorSchema>;
+
 /**
  * @module rules/apply
- * Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
+ * Applies compiled inference rules to file or source attributes, producing merged metadata via schema resolution, JsonMap transforms and render/template output; collects interpolation errors.
  */
 
 /**
@@ -478,6 +513,11 @@ interface ApplyRulesResult {
     matchedRules: string[];
     /** The renderAs value from the last matching rule that declares it, or null. */
     renderAs: string | null;
+    /**
+     * Interpolation failures (schema `set` resolution, `render`, `template`)
+     * of matched rules, in order. Each was also logged as a warning.
+     */
+    errors: RuleApplicationError[];
 }
 /**
  * Optional parameters for applyRules beyond the required compiledRules and attributes.
@@ -503,14 +543,16 @@ interface ApplyRulesOptions {
  *
  * Rules are evaluated in order; later rules override earlier ones.
  * If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
- * and map output overrides set output on conflict.
+ * and map output overrides set output on conflict. A failing schema, render
+ * or template is logged and reported in `errors`; the rule's output for that
+ * stage is skipped (a schema failure skips the rest of the rule).
  *
  * @param compiledRules - The compiled rules to evaluate.
- * @param attributes - The file attributes to match against.
+ * @param attributes - The file or source attributes to match against.
  * @param options - Optional configuration for rule application.
- * @returns The merged metadata and optional rendered content.
+ * @returns The merged metadata, optional rendered content and errors.
  */
-declare function applyRules(compiledRules: CompiledRule[], attributes: FileAttributes, options?: ApplyRulesOptions): Promise<ApplyRulesResult>;
+declare function applyRules(compiledRules: CompiledRule[], attributes: RuleAttributes, options?: ApplyRulesOptions): Promise<ApplyRulesResult>;
 
 /**
  * @module rules/virtualRules
@@ -560,6 +602,188 @@ declare class VirtualRuleStore {
     get size(): number;
 }
 
+/**
+ * @module cache/ContentHashCache
+ * In-memory cache mapping normalized file paths to content hashes.
+ * Supports reverse lookup by hash for move correlation.
+ */
+/**
+ * In-memory content hash cache for move detection.
+ *
+ * Maps normalized file paths to SHA-256 content hashes.
+ * Supports reverse lookup (hash → paths) for correlating
+ * unlink+add events as file moves.
+ */
+declare class ContentHashCache {
+    private readonly pathToHash;
+    private readonly hashToPaths;
+    /**
+     * Store or update the content hash for a file path.
+     *
+     * @param filePath - The file path (will be normalized).
+     * @param hash - The SHA-256 content hash.
+     */
+    set(filePath: string, hash: string): void;
+    /**
+     * Get the content hash for a file path.
+     *
+     * @param filePath - The file path (will be normalized).
+     * @returns The content hash, or undefined if not cached.
+     */
+    get(filePath: string): string | undefined;
+    /**
+     * Remove a file path from the cache.
+     *
+     * @param filePath - The file path (will be normalized).
+     */
+    delete(filePath: string): void;
+    /**
+     * Reverse lookup: get all file paths with a given content hash.
+     *
+     * @param hash - The content hash to look up.
+     * @returns Array of normalized file paths with that hash.
+     */
+    getByHash(hash: string): string[];
+    /** Number of cached entries. */
+    get size(): number;
+}
+
+/**
+ * @module sources/types
+ * Zod schemas and derived types for source callback outcomes (document / gone / error) and reference rejections. Pure declarations; no I/O.
+ */
+
+/**
+ * Error codes a source callback can produce: the subset of the canonical
+ * issue codes that belong to retrieval.
+ */
+declare const sourceErrorCodeSchema: z.ZodEnum<{
+    source_unavailable: "source_unavailable";
+    unsupported_content_type: "unsupported_content_type";
+}>;
+/** A source callback error code. */
+type SourceErrorCode = z.infer<typeof sourceErrorCodeSchema>;
+/** Outcome of one source callback (discriminated on `kind`). */
+declare const sourceRetrievalResultSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"document">;
+    body: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"gone">;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"error">;
+    code: z.ZodEnum<{
+        source_unavailable: "source_unavailable";
+        unsupported_content_type: "unsupported_content_type";
+    }>;
+    message: z.ZodString;
+}, z.core.$strip>], "kind">;
+/** Outcome of one source callback. */
+type SourceRetrievalResult = z.infer<typeof sourceRetrievalResultSchema>;
+/**
+ * Why a reference was rejected before any request was made:
+ * `invalid_reference` (not `<name>://<id>`), `unknown_source` (name not in
+ * config `sources`), `invalid_id` (empty, `.` or `..` segment, or the built
+ * URL would leave the configured origin). The reference-level subset of
+ * core `documentRejectionCodeSchema`.
+ */
+declare const sourceRejectionCodeSchema: z.ZodEnum<{
+    invalid_reference: "invalid_reference";
+    unknown_source: "unknown_source";
+    invalid_id: "invalid_id";
+}>;
+/** A reference rejection code. */
+type SourceRejectionCode = z.infer<typeof sourceRejectionCodeSchema>;
+/** A parsed `<name>://<id>` reference. */
+declare const parsedSourceReferenceSchema: z.ZodObject<{
+    ref: z.ZodString;
+    name: z.ZodString;
+    id: z.ZodString;
+}, z.core.$strip>;
+/** A parsed `<name>://<id>` reference. */
+type ParsedSourceReference = z.infer<typeof parsedSourceReferenceSchema>;
+/** A reference resolved against config: parsed parts plus the URL to request. */
+declare const resolvedSourceReferenceSchema: z.ZodObject<{
+    ref: z.ZodString;
+    name: z.ZodString;
+    id: z.ZodString;
+    url: z.ZodString;
+}, z.core.$strip>;
+/** A reference resolved against config. */
+type ResolvedSourceReference = z.infer<typeof resolvedSourceReferenceSchema>;
+/**
+ * Thrown (or rejected) when a reference cannot be retrieved for a reason
+ * known before any request is made. No request is ever sent for a rejected
+ * reference.
+ */
+declare class SourceReferenceError extends Error {
+    /** Rejection code. */
+    readonly code: SourceRejectionCode;
+    /**
+     * @param code - Rejection code.
+     * @param message - Human-readable reason (never contains header values).
+     */
+    constructor(code: SourceRejectionCode, message: string);
+}
+
+/**
+ * @module sources/SourceClient
+ * Registry of configured sources: resolves `<name>://<id>` to a config-built URL and retrieves it with per-source concurrency caps. Network I/O via requestSource.
+ */
+
+/**
+ * Retrieves source documents by callback. Injected into later pipeline
+ * stages (resolver, notify queue, reindex) so tests can substitute it.
+ */
+interface SourceRetriever {
+    /**
+     * Validate a reference and build its URL without making a request.
+     *
+     * @throws {@link SourceReferenceError}
+     */
+    resolve(reference: string): ResolvedSourceReference;
+    /**
+     * Retrieve a document. Rejects with {@link SourceReferenceError} (and
+     * sends nothing) for an invalid reference; otherwise never rejects.
+     */
+    retrieve(reference: string): Promise<SourceRetrievalResult>;
+}
+/**
+ * The source callback client. Constructed once from the parsed `sources`
+ * config (defaults applied, `${ENV}` already substituted at load). Each
+ * source gets its own concurrency semaphore; the `timeoutMs` clock starts
+ * when a request leaves the queue, not while it waits for a slot. Callbacks
+ * do not draw from the embedding rate limiter.
+ */
+declare class SourceClient implements SourceRetriever {
+    private readonly entries;
+    /**
+     * @param sources - The parsed `sources` config section.
+     * @throws Error when a `url` template is not an absolute http(s) URL or
+     *   places `{id}` in its origin (scheme, host or port).
+     */
+    constructor(sources?: SourcesConfig);
+    /** {@inheritDoc SourceRetriever.resolve} */
+    resolve(reference: string): ResolvedSourceReference;
+    /** {@inheritDoc SourceRetriever.retrieve} */
+    retrieve(reference: string): Promise<SourceRetrievalResult>;
+    private resolveEntry;
+}
+
+/**
+ * @module sources/sourceUrl
+ * Pure helpers: parse `<name>://<id>` references, encode ids segment-by-segment, and build callback URLs from a configured template. No I/O.
+ */
+
+/**
+ * Parse a source reference.
+ *
+ * @param reference - A `<name>://<id>` string.
+ * @returns The parsed reference.
+ * @throws {@link SourceReferenceError} `invalid_reference` when the string
+ *   does not match the core reference schema.
+ */
+declare function parseSourceReference(reference: string): ParsedSourceReference;
+
 /**
  * @module util/BinaryFileStore
  * Binary-backed read/modify/write store with in-memory caching and debounced flush.
@@ -619,7 +843,7 @@ declare abstract class BinaryFileStore<T> {
 /** Per-rule distinct values: rule name → field name → sorted unique values. */
 type ValuesIndex = Record<string, Record<string, unknown[]>>;
 /**
- * Manages a persistent values.json file tracking distinct metadata values per rule.
+ * Manages a persistent values.v8 file tracking distinct metadata values per rule.
  */
 declare class ValuesManager extends BinaryFileStore<ValuesIndex> {
     constructor(stateDir: string, logger: pino.Logger);
@@ -730,11 +954,19 @@ interface VectorStore {
      * @param payload - Payload fields to set.
      */
     setPayload(ids: string[], payload: Record<string, unknown>): Promise<void>;
+    /**
+     * Remove payload keys from the specified point IDs (other keys kept).
+     *
+     * @param ids - Point IDs to update.
+     * @param keys - Payload keys to remove.
+     */
+    deletePayload(ids: string[], keys: string[]): Promise<void>;
     /**
      * Get the payload of a point by ID.
      *
      * @param id - The point ID.
      * @returns The payload, or `null` if the point doesn't exist.
+     * @throws When the read fails (never reported as a missing point).
      */
     getPayload(id: string): Promise<Record<string, unknown> | null>;
     /**
@@ -796,19 +1028,26 @@ interface VectorStore {
     getPointsWithVectors(ids: string[]): Promise<VectorPoint[]>;
 }
 
+/**
+ * @module vectorStore
+ * Qdrant-backed {@link VectorStore}: collection setup, retried writes (see `writes`), payload reads, search, hybrid search and scrolling. Qdrant I/O.
+ */
+
 /**
  * Client wrapper for Qdrant vector store operations.
  *
  * Implements the {@link VectorStore} interface for dependency inversion.
+ * Writes (upsert, delete, set/delete payload) are retried, each retry on a
+ * fresh client (see QdrantWriter).
  */
 declare class VectorStoreClient implements VectorStore {
     private readonly client;
-    private writeClient;
+    private readonly writer;
     private readonly clientConfig;
     private readonly collectionName;
     private readonly dims;
-    private readonly log;
     private readonly pinoLogger?;
+    private readonly payloadIndexes;
     /**
      * Create a new VectorStoreClient.
      *
@@ -823,7 +1062,8 @@ declare class VectorStoreClient implements VectorStore {
      * Used to avoid stale HTTP keep-alive connections. The Qdrant JS client's
      * internal undici Agent uses keepAliveTimeout: 10s, which causes ECONNRESET
      * when connections sit idle during slow embedding calls (Gemini p99 ~8s).
-     * Creating a fresh client for write operations ensures clean TCP connections.
+     * Writes therefore run on their own client, replaced on every retry
+     * (see QdrantWriter).
      */
     private createClient;
     /**
@@ -834,48 +1074,24 @@ declare class VectorStoreClient implements VectorStore {
      */
     count(filter?: Record<string, unknown>): Promise<number>;
     /**
-     * Ensure the collection exists with correct dimensions and Cosine distance.
+     * Ensure the collection exists (Cosine, configured dimensions) and create
+     * any missing payload indexes (configured plus `source`/`source_rule`).
      */
     ensureCollection(): Promise<void>;
-    /**
-     * Retry a Qdrant operation with standardized config and logging.
-     *
-     * @param operation - Operation name for logging (e.g., 'upsert', 'delete').
-     * @param fn - Async function to retry.
-     */
-    private retryOperation;
-    private getWriteClient;
-    /**
-     * Upsert points into the collection.
-     *
-     * Uses the shared client. On retry (after ECONNRESET from stale connections),
-     * creates a fresh client to recover.
-     *
-     * @param points - The points to upsert.
-     */
+    /** {@inheritDoc VectorStore.upsert} */
     upsert(points: VectorPoint[]): Promise<void>;
-    /**
-     * Delete points by their IDs.
-     *
-     * Uses the shared client. On retry, creates a fresh client to recover.
-     *
-     * @param ids - The point IDs to delete.
-     */
+    /** {@inheritDoc VectorStore.delete} */
     delete(ids: string[]): Promise<void>;
-    /**
-     * Set payload fields for the specified point IDs.
-     *
-     * This merges the given payload object into each point's existing payload.
-     *
-     * @param ids - Point IDs to update.
-     * @param payload - Payload fields to set.
-     */
+    /** {@inheritDoc VectorStore.setPayload} */
     setPayload(ids: string[], payload: Record<string, unknown>): Promise<void>;
+    /** {@inheritDoc VectorStore.deletePayload} */
+    deletePayload(ids: string[], keys: string[]): Promise<void>;
     /**
      * Get the payload of a point by ID.
      *
      * @param id - The point ID.
      * @returns The payload, or `null` if the point doesn't exist.
+     * @throws When the read fails (never reported as a missing point).
      */
     getPayload(id: string): Promise<Record<string, unknown> | null>;
     /**
@@ -918,6 +1134,7 @@ declare class VectorStoreClient implements VectorStore {
      *
      * @param ids - The point IDs to retrieve.
      * @returns Points with vectors and payloads; missing IDs are omitted.
+     * @throws When the read fails (never reported as missing points).
      */
     getPointsWithVectors(ids: string[]): Promise<VectorPoint[]>;
     /**
@@ -963,10 +1180,44 @@ interface ProcessorConfig {
     globalSchemas?: Record<string, SchemaEntry>;
 }
 
+/**
+ * @module processor/DocumentProcessorDeps
+ * Constructor dependencies for DocumentProcessor (config, embedding provider, vector store, rules, optional stores), extracted for single-responsibility.
+ */
+
+/**
+ * Dependencies injected into DocumentProcessor.
+ */
+interface DocumentProcessorDeps {
+    /** Processor configuration (chunk sizes, directories, maps). */
+    config: ProcessorConfig;
+    /** Provider for generating text embeddings. */
+    embeddingProvider: EmbeddingProvider;
+    /** Vector store for persistence. */
+    vectorStore: VectorStore;
+    /** Pre-compiled inference rules for metadata extraction. */
+    compiledRules: CompiledRule[];
+    /** Pino logger instance. */
+    logger: pino.Logger;
+    /** Optional Handlebars template engine for content templates. */
+    templateEngine?: TemplateEngine;
+    /** Optional enrichment store for persisted enrichment metadata. */
+    enrichmentStore?: EnrichmentStoreInterface;
+    /** Optional issues manager; processing failures are recorded as typed issues (#255). */
+    issuesManager?: IssuesManager;
+    /** Optional values manager for tracking rule-extracted values. */
+    valuesManager?: ValuesManager;
+    /** Optional content hash cache for move detection. */
+    contentHashCache?: ContentHashCache;
+    /** Optional source callback client; without it, source documents fail `source_unavailable`. */
+    sourceRetriever?: SourceRetriever;
+}
+
 /**
  * @module processor/renderResult
- * Type definition for the render endpoint result.
+ * The render endpoint result and its one constructor from a resolved document (file or source). Pure, no I/O.
  */
+
 /**
  * Result returned by {@link DocumentProcessorInterface.renderFile}.
  */
@@ -983,6 +1234,63 @@ interface RenderResult {
     transformed: boolean;
 }
 
+/**
+ * @module processor/documentPipeline
+ * The shared indexing pipeline for resolved documents (file or source): onResolved hook, empty check, hash compare, chunk, embed, upsert, orphan cleanup; plus point deletion.
+ */
+
+/**
+ * Options for indexing one document (see
+ * {@link SourceOperationsInterface.process}).
+ */
+interface IndexOptions {
+    /**
+     * Re-embed and rewrite even when the content hash is unchanged (source
+     * documents on a `full` reindex).
+     */
+    force?: boolean;
+}
+
+/**
+ * @module processor/SourceOperations
+ * Source-document operations for the API: notify validation (reference, configured source, rule), stored designated rule lookup, uncached render, metadata enrichment and processing. Qdrant and callback I/O.
+ */
+
+/** Outcome of rendering a source document (`POST /render { source }`). */
+type SourceRenderOutcome = {
+    kind: 'rendered';
+    result: RenderResult;
+} | {
+    kind: 'gone';
+} | {
+    kind: 'failed';
+    error: DocumentFailure;
+};
+/** Source-document operations used by the API. */
+interface SourceOperationsInterface {
+    /** Process a reference with its designated rule (see `processSourceDocument`); `force` re-embeds unchanged content. */
+    process(ref: string, rule: string, options?: IndexOptions): Promise<DocumentResult>;
+    /** Synchronously validate a notification; `undefined` when acceptable. */
+    validate(ref: string, rule: string): DocumentRejectionError | undefined;
+    /** Designated rule stored on the document's chunk-0 point, or `null` if not indexed. */
+    storedRule(ref: string): Promise<string | null>;
+    /**
+     * The compiled rule a designated rule name resolves to in the merged rule
+     * set (config rules, then virtual rules: first of that name), or
+     * `undefined` when no such rule exists.
+     */
+    compiledRule(name: string): CompiledRule | undefined;
+    /** Retrieve and render a document without embedding or writing anything. */
+    render(ref: string, rule: string): Promise<SourceRenderOutcome>;
+    /** Merge enrichment metadata into an indexed document's points; `null` if not indexed. */
+    updateMetadata(ref: string, metadata: Record<string, unknown>): Promise<Record<string, unknown> | null>;
+}
+/** A processor exposing source operations (`DocumentProcessor` implements it). */
+interface SourceOperationsProvider {
+    /** Source operations bound to the processor's current rules and stores. */
+    sourceOperations(): SourceOperationsInterface;
+}
+
 /**
  * @module processor/types
  * Document processor interface definitions.
@@ -1009,46 +1317,26 @@ interface DocumentProcessorInterface {
     /** Update compiled inference rules and associated engines. */
     updateRules(compiledRules: CompiledRule[], templateEngine?: TemplateEngine, customMapLib?: Record<string, (...args: unknown[]) => unknown>): void;
 }
-
 /**
- * @module processor
- *
- * Core document processing pipeline. Handles extracting text, computing embeddings, syncing with vector store.
+ * The processor the service runs on: file operations plus source operations.
+ * `createDocumentProcessor` returns it; the API server and the notify queue
+ * depend on it (`DocumentProcessor` implements it). Adapt a custom file-only
+ * processor with `withoutSourceSupport`, whose source operations fail.
  */
+type WatcherProcessor = DocumentProcessorInterface & SourceOperationsProvider;
 
 /**
- * Core document processing pipeline.
+ * @module processor
  *
- * Handles extracting text, computing embeddings, and syncing with the vector store.
+ * Core document processing pipeline. Resolves files and source documents, computes embeddings, syncs with vector store.
  */
-interface DocumentProcessorDeps {
-    /** Processor configuration (chunk sizes, directories, maps). */
-    config: ProcessorConfig;
-    /** Provider for generating text embeddings. */
-    embeddingProvider: EmbeddingProvider;
-    /** Vector store for persistence. */
-    vectorStore: VectorStore;
-    /** Pre-compiled inference rules for metadata extraction. */
-    compiledRules: CompiledRule[];
-    /** Pino logger instance. */
-    logger: pino.Logger;
-    /** Optional Handlebars template engine for content templates. */
-    templateEngine?: TemplateEngine;
-    /** Optional enrichment store for persisted enrichment metadata. */
-    enrichmentStore?: EnrichmentStoreInterface;
-    /** Optional issues manager for tracking processing errors. */
-    issuesManager?: IssuesManager;
-    /** Optional values manager for tracking rule-extracted values. */
-    valuesManager?: ValuesManager;
-    /** Optional content hash cache for move detection. */
-    contentHashCache?: ContentHashCache;
-}
+
 /**
  * Core document processing pipeline.
  *
  * Handles extracting text, computing embeddings, and syncing with the vector store.
  */
-declare class DocumentProcessor implements DocumentProcessorInterface {
+declare class DocumentProcessor implements WatcherProcessor {
     private config;
     private readonly embeddingProvider;
     private readonly vectorStore;
@@ -1059,26 +1347,35 @@ declare class DocumentProcessor implements DocumentProcessorInterface {
     private readonly issuesManager?;
     private readonly valuesManager?;
     private readonly contentHashCache?;
+    private readonly sourceRetriever?;
     /**
      * Create a new DocumentProcessor.
      *
      * @param deps - The processor dependencies.
      */
-    constructor({ config, embeddingProvider, vectorStore, compiledRules, logger, templateEngine, enrichmentStore, issuesManager, valuesManager, contentHashCache, }: DocumentProcessorDeps);
-    /**
-     * Build merged metadata for a file and add matched_rules.
-     */
-    private buildMetadataWithRules;
+    constructor(deps: DocumentProcessorDeps);
+    /** Rule context reflecting the current rules, templates and config. */
+    private ruleContext;
+    /** Shared pipeline dependencies. */
+    private pipelineContext;
+    /** File resolver bound to the current rule context. */
+    private fileResolver;
+    /** Dependencies of the payload-only file operations. */
+    private fileOperations;
     /**
-     * Execute an async operation with standardized file error handling.
+     * Run a file operation with standard failure handling: log, and record a
+     * typed issue when `context.issueType` (the code for untagged failures)
+     * is given; return `fallback` on failure.
      */
-    private withFileErrorHandling;
+    private withFailureHandling;
     /**
      * Process a file through the full pipeline: extract, hash, chunk, embed, upsert.
      *
      * @param filePath - The file to process.
      */
     processFile(filePath: string): Promise<void>;
+    /** {@inheritDoc SourceOperationsProvider.sourceOperations} */
+    sourceOperations(): SourceOperationsInterface;
     /**
      * Delete all chunks for a file from the vector store and remove metadata.
      *
@@ -1086,7 +1383,9 @@ declare class DocumentProcessor implements DocumentProcessorInterface {
      */
     deleteFile(filePath: string): Promise<void>;
     /**
-     * Process a metadata update: merge into enrichment store, update Qdrant payloads (no re-embed).
+     * Process a metadata update: merge into enrichment store, update Qdrant
+     * payloads (no re-embed). The stored `file_path` (casing included) and
+     * `matched_rules` are kept.
      *
      * @param filePath - The file whose metadata to update.
      * @param metadata - The new metadata to merge.
@@ -1094,9 +1393,9 @@ declare class DocumentProcessor implements DocumentProcessorInterface {
      */
     processMetadataUpdate(filePath: string, metadata: Record<string, unknown>): Promise<Record<string, unknown> | null>;
     /**
-     * Re-apply inference rules to a file without re-embedding.
-     * Reads file attributes, applies current rules, merges with enrichment metadata,
-     * and updates Qdrant payloads.
+     * Re-apply inference rules to a file without re-embedding (see
+     * `reapplyFileRules`: system fields and the stored `file_path` are
+     * re-asserted; interpolation failures are recorded as an issue).
      *
      * @param filePath - The file to update.
      * @returns The merged metadata, or `null` if the file is not indexed.
@@ -1128,10 +1427,29 @@ declare class DocumentProcessor implements DocumentProcessorInterface {
     updateRules(compiledRules: CompiledRule[], templateEngine?: TemplateEngine, customMapLib?: Record<string, (...args: unknown[]) => unknown>): void;
 }
 
+/**
+ * @module queue/pausable
+ * The pause/resume/quiesce surface shared by the file event queue and the source notify queue, and a combinator that drives several queues as one (used by prune). No I/O.
+ */
+/**
+ * A queue that can stop starting new work, resume, and report when its
+ * in-flight work has finished. Prune uses this to quiesce every writer
+ * before it computes and executes its plan.
+ */
+interface PausableQueue {
+    /** Stop starting new work; enqueues are still accepted. */
+    pause(): void;
+    /** Resume starting work. */
+    resume(): void;
+    /** Resolve once no work is in flight (queued work may remain). */
+    waitForIdleWorkers(): Promise<void>;
+}
+
 /**
  * @module queue
- * Debounced, rate-limited, concurrent event queue for file watchers. Manages priority queuing and async callbacks. No direct I/O; orchestrates processing.
+ * Debounced, concurrent event queue for file watchers. Manages priority queuing and async callbacks. Embedding request rate is limited by the shared embedding provider wrapper, not here. No direct I/O.
  */
+
 /**
  * A watch event.
  */
@@ -1157,27 +1475,22 @@ interface EventQueueOptions {
     debounceMs: number;
     /** Maximum number of concurrent processors. */
     concurrency: number;
-    /** Optional max processed events per minute (token bucket). */
-    rateLimitPerMinute?: number;
 }
 /**
- * A debounced, rate-limited, concurrent event queue.
+ * A debounced, concurrent event queue.
  */
 declare class EventQueue {
     private readonly debounceMs;
     private readonly concurrency;
-    private readonly rateLimitPerMinute?;
     private started;
     private paused;
+    private stopped;
     private active;
     private readonly debounceTimers;
     private readonly latestByKey;
     private readonly normalQueue;
     private readonly lowQueue;
-    private tokens;
-    private lastRefillMs;
-    private drainWaiters;
-    private idleWorkerWaiters;
+    private readonly signals;
     /**
      * Create an event queue.
      *
@@ -1185,16 +1498,25 @@ declare class EventQueue {
      */
     constructor(options: EventQueueOptions);
     /**
-     * Enqueue an event, debounced per path+priority.
+     * Enqueue an event, debounced per path+priority. Ignored once stopped.
      *
      * @param event - The watch event.
      * @param fn - The processing function to invoke when dequeued.
      */
     enqueue(event: WatchEvent, fn: ProcessFn): void;
     /**
-     * Start processing events.
+     * Start processing events (events enqueued earlier are held until now).
+     */
+    start(): void;
+    /**
+     * Alias of {@link EventQueue.start}, kept for existing callers.
      */
     process(): void;
+    /**
+     * Stop: cancel pending debounces, drop queued events, ignore new ones, and
+     * resolve once in-flight work finishes. Drain waiters resolve then too.
+     */
+    stop(): Promise<void>;
     /**
      * Pause processing events. Events can still be enqueued but will not be processed.
      */
@@ -1218,13 +1540,136 @@ declare class EventQueue {
      */
     waitForIdleWorkers(): Promise<void>;
     private push;
-    private refillTokens;
-    private takeToken;
     private nextItem;
     private pump;
     private isIdle;
-    private maybeResolveDrain;
-    private maybeResolveIdleWorkers;
+    private settle;
+}
+
+/**
+ * @module queue/sourceNotifyTypes
+ * Options, errors and job bookkeeping types of the source notify queue (validated options schema, stopped error, coalesced jobs and their waiters). No I/O.
+ */
+
+/**
+ * Default number of references processed at once: the core
+ * `embedding.concurrency` default, which the service passes in.
+ */
+declare const DEFAULT_SOURCE_NOTIFY_CONCURRENCY: number;
+/** Options for the source notify queue. */
+declare const sourceNotifyQueueOptionsSchema: z.ZodObject<{
+    concurrency: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+/** Options for the source notify queue (input form; defaults applied). */
+type SourceNotifyQueueOptions = z.input<typeof sourceNotifyQueueOptionsSchema>;
+/** Issues-store operations the queue needs. */
+type SourceQueueIssues = Pick<IssuesManager, 'record' | 'clear'>;
+/** Per-run options of `SourceNotifyQueue.processNow`. */
+interface ProcessNowOptions {
+    /**
+     * Re-embed even when content is unchanged (reindex scope `full`). Merged
+     * as OR into a run it joins.
+     */
+    force?: boolean;
+}
+/** Rejection for `SourceNotifyQueue.processNow` after `stop()`. */
+declare class SourceNotifyQueueStoppedError extends Error {
+    /** Create the error. */
+    constructor();
+}
+
+/**
+ * @module queue/SourceNotifyQueue
+ * In-memory notify queue for source documents, keyed by reference: pending notifications merge, in-flight ones mark the reference dirty for exactly one re-run. Runs each reference through the source operations; records an issue when a run rejects unexpectedly.
+ */
+
+/** What the queue runs: one reference with its designated rule. */
+type SourceProcess = Pick<SourceOperationsInterface, 'process'>;
+/**
+ * Coalescing queue for source-document notifications (Decision 12).
+ *
+ * - A notification for a **pending** reference is merged into it (the latest
+ *   rule wins; the reference keeps its place in line).
+ * - One for an **in-flight** reference marks it dirty: when the current run
+ *   ends the reference is queued once more (at the back), however many
+ *   notifications arrived meanwhile.
+ * - Every run calls `process`, which retrieves current state, so a
+ *   burst on one reference costs at most two retrievals and the final state
+ *   wins. Runs never overlap for the same reference.
+ *
+ * References are compared exactly (case-sensitive), like source point IDs.
+ * The queue holds work until {@link start}; {@link pause} (prune) stops new
+ * runs without rejecting enqueues. Failures never escape: `process`
+ * records issues itself, and an unexpected rejection becomes a `failed`
+ * result and an `extraction_error` issue (with the designated rule).
+ *
+ * Reindex runs go through {@link processNow} as well, so a notification and
+ * a reindex of one reference never run in parallel and the latest state wins.
+ */
+declare class SourceNotifyQueue implements PausableQueue {
+    private readonly processor;
+    private readonly logger?;
+    private readonly issues?;
+    private readonly concurrency;
+    private started;
+    private paused;
+    private stopped;
+    private readonly pending;
+    private readonly inFlight;
+    private readonly signals;
+    /**
+     * @param processor - Processes one reference with its designated rule
+     *   (source operations).
+     * @param options - Queue options (validated).
+     * @param logger - Logger for unexpected processing rejections.
+     * @param issues - Issues store for unexpected processing rejections.
+     */
+    constructor(processor: SourceProcess, options?: SourceNotifyQueueOptions, logger?: pino.Logger | undefined, issues?: SourceQueueIssues | undefined);
+    /**
+     * Notify that a source document changed (fire and forget).
+     *
+     * @param ref - The exact `<name>://<id>` reference.
+     * @param rule - The designated rule name.
+     * @returns `false` when the queue is stopped and the notification was dropped.
+     */
+    enqueue(ref: string, rule: string): boolean;
+    /**
+     * Process a reference and resolve with the result of a run that started
+     * after this call, so the result reflects the source's state at or after
+     * the call (the `?wait=true` path). Coalesces with other notifications:
+     * if the reference is pending, this joins that run; if it is in flight,
+     * this waits for the current run and then the one dirty re-run.
+     *
+     * @param ref - The exact `<name>://<id>` reference.
+     * @param rule - The designated rule name.
+     * @param options - `force` re-embeds unchanged content.
+     * @returns The per-document result.
+     * @throws {@link SourceNotifyQueueStoppedError} (as a rejection) when the
+     *   queue is stopped before the run starts.
+     */
+    processNow(ref: string, rule: string, options?: ProcessNowOptions): Promise<DocumentResult>;
+    /** Start processing (work enqueued earlier is held until now). */
+    start(): void;
+    /** {@inheritDoc PausableQueue.pause} */
+    pause(): void;
+    /** {@inheritDoc PausableQueue.resume} */
+    resume(): void;
+    /** {@inheritDoc PausableQueue.waitForIdleWorkers} */
+    waitForIdleWorkers(): Promise<void>;
+    /** Resolve once nothing is pending, owed (dirty) or in flight. */
+    drain(): Promise<void>;
+    /**
+     * Stop: drop pending and owed runs (their `processNow` callers reject with
+     * {@link SourceNotifyQueueStoppedError}), refuse new work, and resolve once
+     * in-flight runs finish.
+     */
+    stop(): Promise<void>;
+    private schedule;
+    private pump;
+    private run;
+    private execute;
+    private isIdle;
+    private settle;
 }
 
 /**
@@ -1724,24 +2169,30 @@ declare class ReindexTracker {
 }
 
 /**
- * @module api
- * Fastify API server factory. Registers all route handlers and returns an unstarted server instance.
+ * @module api/apiServerOptions
+ * Options accepted by createApiServer: processor, stores, queues, config access, trackers and optional collaborators (virtual rules, VCS, source queue). Types only.
  */
 
 /**
- * Options for {@link createApiServer}.
+ * Options for `createApiServer`.
  */
 interface ApiServerOptions {
+    /**
+     * Queues paused while a prune deletes points. The service passes
+     * `combinePausableQueues(fileQueue, sourceQueue)` (`app/index.ts`), so a
+     * prune quiesces both the file event queue and the source notify queue.
+     */
+    queue: PausableQueue;
+    /** Source notify queue (`POST /documents`); absent in file-only setups. */
+    sourceQueue?: SourceNotifyQueue;
     /** The component descriptor (used for config-apply handler). */
     descriptor: JeevesComponentDescriptor;
-    /** The document processor. */
-    processor: DocumentProcessorInterface;
+    /** The document processor: file and source operations. */
+    processor: WatcherProcessor;
     /** The vector store client. */
     vectorStore: VectorStoreClient;
     /** The embedding provider. */
     embeddingProvider: EmbeddingProvider;
-    /** The event queue. */
-    queue: EventQueue;
     /** The application configuration (used as initial/fallback value). */
     config: JeevesWatcherConfig;
     /** Config getter for live config access after hot-reload. */
@@ -1775,16 +2226,58 @@ interface ApiServerOptions {
     /** VCS coordinator for git-backed versioning API routes. */
     vcsCoordinator?: VcsCoordinator;
 }
+
+/**
+ * @module api
+ * Fastify API server factory. Registers all route handlers (status, query, documents, config, rules, VCS) and returns an unstarted server instance.
+ */
+
 /**
  * Create the Fastify API server with all routes registered.
  *
- * The returned instance is not yet listening — call `server.listen()` to start.
+ * The returned instance is not yet listening; call `server.listen()` to start.
  *
  * @param options - The server options.
  * @returns A configured Fastify instance.
  */
 declare function createApiServer(options: ApiServerOptions): FastifyInstance;
 
+/**
+ * @module config/warnings
+ * The non-fatal config warning shape shared by config loading (unknown source keys, reserved rule field names) and `POST /config/validate`. Types only.
+ */
+/** A non-fatal config validation warning. */
+interface ConfigWarning {
+    /** Config location the warning refers to. */
+    path: string;
+    /** Human-readable warning message. */
+    message: string;
+}
+
+/**
+ * @module config/loadConfig
+ * Loads the watcher config via cosmiconfig: schema validation, rule/map file resolution, defaults, env substitution and secret checks; reports non-fatal warnings (unknown source keys, reserved rule field names) through onWarning. Reads files.
+ */
+
+/** Options of {@link loadConfig}. */
+interface LoadConfigOptions {
+    /**
+     * Receives non-fatal warnings once the config has loaded successfully:
+     * unknown keys under `sources.<name>` (which validation drops), then rule
+     * schema properties named like reserved system fields.
+     */
+    onWarning?: (warning: ConfigWarning) => void;
+}
+/**
+ * Load the jeeves-watcher configuration.
+ *
+ * @param configPath - Optional explicit path to a config file.
+ * @param options - Optional warning sink.
+ * @returns The loaded configuration.
+ * @throws If no configuration is found or validation fails.
+ */
+declare function loadConfig(configPath?: string, options?: LoadConfigOptions): Promise<JeevesWatcherConfig>;
+
 /**
  * @module logger
  * Creates pino logger instances. I/O: optionally writes logs to file via pino/file transport. Defaults to stdout at info level.
@@ -1803,12 +2296,17 @@ declare function createLogger(config?: LoggingConfig): pino.Logger;
  * Component factory interfaces and defaults for {@link JeevesWatcher}. Override in tests to inject mocks.
  */
 
+/** Runtime options for {@link JeevesWatcher} that aren't serializable in config. */
+interface JeevesWatcherRuntimeOptions {
+    /** Callback invoked on unrecoverable system error. If not set, throws. */
+    onFatalError?: (error: unknown) => void;
+}
 /**
  * Component factories for {@link JeevesWatcher}. Override in tests to inject mocks.
  */
 interface JeevesWatcherFactories {
     /** Load and validate a {@link JeevesWatcherConfig} from disk. */
-    loadConfig: (configPath?: string) => Promise<JeevesWatcherConfig>;
+    loadConfig: (configPath?: string, options?: LoadConfigOptions) => Promise<JeevesWatcherConfig>;
     /** Create a pino logger instance. */
     createLogger: typeof createLogger;
     /** Create an embedding provider from config. */
@@ -1818,7 +2316,7 @@ interface JeevesWatcherFactories {
     /** Compile inference rules from config. */
     compileRules: typeof compileRules;
     /** Create a document processor for file ingestion. */
-    createDocumentProcessor: (deps: ConstructorParameters<typeof DocumentProcessor>[0]) => DocumentProcessorInterface;
+    createDocumentProcessor: (deps: ConstructorParameters<typeof DocumentProcessor>[0]) => WatcherProcessor;
     /** Create an event queue for batching file-system events. */
     createEventQueue: (options: ConstructorParameters<typeof EventQueue>[0]) => EventQueue;
     /** Create a file-system watcher for the configured watch paths. */
@@ -1827,6 +2325,27 @@ interface JeevesWatcherFactories {
     createApiServer: typeof createApiServer;
 }
 
+/**
+ * @module app/sources
+ * Startup wiring for source documents: builds the source callback client from `sources` config (read at startup only), the notify queue over the processor's source operations, and the adapter giving a file-only processor failing source operations. No I/O.
+ */
+
+/**
+ * Adapt a custom file-only processor to the {@link WatcherProcessor} the
+ * service runs on. File operations are forwarded unchanged. Its source
+ * operations fail: every notify-queue run returns a `failed` result with
+ * `source_unavailable` (the run's issue, as for a failing callback), and
+ * every other source operation throws the API's `503 not_available`, so
+ * `POST /documents`, `POST /metadata { source }`, `POST /render { source }`
+ * and `POST /reindex { source }` answer as before source support existed.
+ * Use it in a custom `createDocumentProcessor` factory; never wrap a
+ * processor that has source operations of its own (they would be hidden).
+ *
+ * @param processor - The file-only processor.
+ * @returns The processor with failing source operations.
+ */
+declare function withoutSourceSupport(processor: DocumentProcessorInterface): WatcherProcessor;
+
 /**
  * @module app/startFromConfig
  * Convenience entry point: loads config from disk and starts a {@link JeevesWatcher}.
@@ -1850,11 +2369,6 @@ declare function startFromConfig(configPath?: string, descriptor?: JeevesCompone
  * Main application orchestrator. Wires components, manages lifecycle (start/stop/reload).
  */
 
-/** Runtime options for {@link JeevesWatcher} that aren't serializable in config. */
-interface JeevesWatcherRuntimeOptions {
-    /** Callback invoked on unrecoverable system error. If not set, throws. */
-    onFatalError?: (error: unknown) => void;
-}
 /**
  * Main application class that wires together all components.
  */
@@ -1867,13 +2381,14 @@ declare class JeevesWatcher {
     private logger;
     private watcher;
     private queue;
+    private sourceQueue;
     private server;
     private processor;
     private configWatcher;
     private issuesManager;
     private valuesManager;
     private helperIntrospection;
-    private virtualRuleStore;
+    private readonly virtualRuleStore;
     private vectorStore;
     private embeddingProvider;
     private gitignoreFilter;
@@ -1884,9 +2399,7 @@ declare class JeevesWatcher {
     private readonly version;
     /** Create a new JeevesWatcher instance. */
     constructor(config: JeevesWatcherConfig, configPath?: string, descriptor?: JeevesComponentDescriptor, factories?: Partial<JeevesWatcherFactories>, runtimeOptions?: JeevesWatcherRuntimeOptions);
-    /**
-     * Start the watcher, API server, and all components.
-     */
+    /** Start the watcher, API server, and all components. */
     start(): Promise<void>;
     /** Set up logger, VCS, embedding provider, and vector store. */
     private initInfrastructure;
@@ -1895,24 +2408,15 @@ declare class JeevesWatcher {
     /** Wire up watcher and VCS coordinator, then start all services. */
     private startServicesAndWatcher;
     /**
-     * Gracefully stop all components.
+     * Gracefully stop all components: the API server closes first (no new
+     * notifications), then producers stop and queues drain (see
+     * shutdownComponents).
      */
     stop(): Promise<void>;
-    private startApiServer;
-    private startConfigWatch;
-    private stopConfigWatch;
+    private apiServerOptions;
     private reloadConfig;
 }
 
-/**
- * Load the jeeves-watcher configuration.
- *
- * @param configPath - Optional explicit path to a config file.
- * @returns The loaded configuration.
- * @throws If no configuration is found or validation fails.
- */
-declare function loadConfig(configPath?: string): Promise<JeevesWatcherConfig>;
-
 /**
  * @module descriptor
  * Jeeves Component Descriptor for the watcher service. Single source of truth
@@ -2090,5 +2594,5 @@ declare class SquashManager {
     private forcePushIfConfigured;
 }
 
-export { CommitMessageGenerator, ContentHashCache, DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SquashManager, SystemHealth, TemplateEngine, ValuesManager, VcsCoordinator, VcsManager, VectorStoreClient, VirtualRuleStore, applyRules, buildAttributes, buildTemplateEngine, compileRules, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, extractText, issueRecordSchema, loadConfig, loadCustomHelpers, mergeEnrichment, pointId, registerBuiltinHelpers, resolveTemplateSource, startFromConfig, watcherDescriptor };
-export type { AllHelpersIntrospection, ApiServerOptions, ApplyRulesOptions, ApplyRulesResult, CollectionInfo, CommitInfo, CompiledRule, CompiledTemplate, DocumentProcessorDeps, DocumentProcessorInterface, EmbeddingProvider, EnrichmentStoreInterface, EventQueueOptions, ExtractedText, Extractor, FileAttributes, FileSystemWatcherOptions, HelperModuleIntrospection, InitialScanStatus, IssueRecord, IssuesFile, JeevesWatcherFactories, JeevesWatcherRuntimeOptions, PayloadFieldSchema, PendingReversion, ProcessFn, ProcessorConfig, ProviderFactory, PushError, ReindexStatus, RenderResult, ResolvedWatchRoot, RuleLogger, ScrollPageResult, ScrolledPoint, SearchResult, SquashManagerOptions, SquashResult, SystemHealthOptions, ValuesIndex, VectorPoint, VectorStore, WatchEvent };
+export { CommitMessageGenerator, ContentHashCache, DEFAULT_SOURCE_NOTIFY_CONCURRENCY, DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SourceClient, SourceNotifyQueue, SourceNotifyQueueStoppedError, SourceReferenceError, SquashManager, SystemHealth, TemplateEngine, ValuesManager, VcsCoordinator, VcsManager, VectorStoreClient, VirtualRuleStore, applyRules, buildAttributes, buildTemplateEngine, compileRules, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, extractText, loadConfig, loadCustomHelpers, mergeEnrichment, parseSourceReference, parsedSourceReferenceSchema, pointId, registerBuiltinHelpers, resolveTemplateSource, resolvedSourceReferenceSchema, sourceAttributesSchema, sourceErrorCodeSchema, sourceNotifyQueueOptionsSchema, sourceRejectionCodeSchema, sourceRetrievalResultSchema, startFromConfig, watcherDescriptor, withoutSourceSupport };
+export type { AllHelpersIntrospection, ApiServerOptions, ApplyRulesOptions, ApplyRulesResult, CircuitBreakerSnapshot, CollectionInfo, CommitInfo, CompiledRule, CompiledTemplate, ConfigWarning, DocumentProcessorDeps, DocumentProcessorInterface, EmbedOptions, EmbeddingProvider, EnrichmentStoreInterface, EventQueueOptions, ExtractedText, Extractor, FileAttributes, FileSystemWatcherOptions, HelperModuleIntrospection, IndexOptions, InitialScanStatus, IssuesFile, JeevesWatcherFactories, JeevesWatcherRuntimeOptions, LoadConfigOptions, ParsedSourceReference, PausableQueue, PayloadFieldSchema, PendingReversion, ProcessFn, ProcessNowOptions, ProcessorConfig, ProviderFactory, PushError, ReindexStatus, RenderResult, ResolvedSourceReference, ResolvedWatchRoot, RuleAttributes, RuleLogger, ScrollPageResult, ScrolledPoint, SearchResult, SourceAttributes, SourceErrorCode, SourceNotifyQueueOptions, SourceOperationsInterface, SourceOperationsProvider, SourceProcess, SourceQueueIssues, SourceRejectionCode, SourceRenderOutcome, SourceRetrievalResult, SourceRetriever, SquashManagerOptions, SquashResult, SystemHealthOptions, ValuesIndex, VcsBreakerState, VectorPoint, VectorStore, WatchEvent, WatcherProcessor };