@karmaniverous/jeeves-watcher 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,615 @@
+import { FastifyInstance } from 'fastify';
+import pino from 'pino';
+import { Stats } from 'node:fs';
+
+/**
+ * Watch configuration for file system monitoring.
+ */
+interface WatchConfig {
+    /** Glob patterns to watch. */
+    paths: string[];
+    /** Glob patterns to ignore. */
+    ignored?: string[];
+    /** Polling interval in milliseconds. */
+    pollIntervalMs?: number;
+    /** Whether to use polling instead of native watchers. */
+    usePolling?: boolean;
+    /** Debounce delay in milliseconds for file change events. */
+    debounceMs?: number;
+    /** Time in milliseconds a file must be stable before processing. */
+    stabilityThresholdMs?: number;
+}
+/**
+ * Configuration watch settings.
+ */
+interface ConfigWatchConfig {
+    /** Whether config file watching is enabled. */
+    enabled?: boolean;
+    /** Debounce delay in milliseconds for config change events. */
+    debounceMs?: number;
+}
+/**
+ * Embedding model configuration.
+ */
+interface EmbeddingConfig {
+    /** The embedding model provider. */
+    provider: string;
+    /** The embedding model name. */
+    model: string;
+    /** Maximum tokens per chunk for splitting. */
+    chunkSize?: number;
+    /** Overlap between chunks in tokens. */
+    chunkOverlap?: number;
+    /** Embedding vector dimensions. */
+    dimensions?: number;
+    /** API key for the embedding provider. */
+    apiKey?: string;
+    /** Maximum embedding requests per minute. */
+    rateLimitPerMinute?: number;
+    /** Maximum concurrent embedding requests. */
+    concurrency?: number;
+}
+/**
+ * Vector store configuration for Qdrant.
+ */
+interface VectorStoreConfig {
+    /** Qdrant server URL. */
+    url: string;
+    /** Qdrant collection name. */
+    collectionName: string;
+    /** Qdrant API key. */
+    apiKey?: string;
+}
+/**
+ * API server configuration.
+ */
+interface ApiConfig {
+    /** Host to bind to. */
+    host?: string;
+    /** Port to listen on. */
+    port?: number;
+}
+/**
+ * Logging configuration.
+ */
+interface LoggingConfig {
+    /** Log level. */
+    level?: string;
+    /** Log file path. */
+    file?: string;
+}
+/**
+ * An inference rule that enriches document metadata.
+ */
+interface InferenceRule {
+    /** JSON Schema object to match against document metadata. */
+    match: Record<string, unknown>;
+    /** Metadata fields to set when the rule matches. */
+    set: Record<string, unknown>;
+}
+/**
+ * Top-level configuration for jeeves-watcher.
+ */
+interface JeevesWatcherConfig {
+    /** File system watch configuration. */
+    watch: WatchConfig;
+    /** Configuration file watch settings. */
+    configWatch?: ConfigWatchConfig;
+    /** Embedding model configuration. */
+    embedding: EmbeddingConfig;
+    /** Vector store configuration. */
+    vectorStore: VectorStoreConfig;
+    /** Directory for persisted metadata. */
+    metadataDir?: string;
+    /** API server configuration. */
+    api?: ApiConfig;
+    /** Extractor configurations keyed by name. */
+    extractors?: Record<string, unknown>;
+    /** Rules for inferring metadata from document properties. */
+    inferenceRules?: InferenceRule[];
+    /** Logging configuration. */
+    logging?: LoggingConfig;
+    /** Timeout in milliseconds for graceful shutdown. */
+    shutdownTimeoutMs?: number;
+}
+
+/**
+ * An embedding provider that converts text into vector representations.
+ */
+interface EmbeddingProvider {
+    /** Generate embedding vectors for the given texts. */
+    embed(texts: string[]): Promise<number[][]>;
+    /** The dimensionality of the embedding vectors. */
+    dimensions: number;
+}
+/**
+ * Create an embedding provider based on the given configuration.
+ *
+ * @param config - The embedding configuration.
+ * @returns An {@link EmbeddingProvider} instance.
+ * @throws If the configured provider is not supported.
+ */
+declare function createEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvider;
+
+/**
+ * Attributes derived from a watched file for rule matching.
+ */
+interface FileAttributes {
+    /** File-level properties. */
+    file: {
+        /** Full file path (forward slashes). */
+        path: string;
+        /** Directory containing the file. */
+        directory: string;
+        /** File name with extension. */
+        filename: string;
+        /** File extension including the leading dot. */
+        extension: string;
+        /** File size in bytes. */
+        sizeBytes: number;
+        /** ISO-8601 last-modified timestamp. */
+        modified: string;
+    };
+    /** Extracted YAML frontmatter, if any. */
+    frontmatter?: Record<string, unknown>;
+    /** Parsed JSON content, if any. */
+    json?: Record<string, unknown>;
+}
+/**
+ * A compiled inference rule ready for evaluation.
+ */
+interface CompiledRule {
+    /** The original rule definition. */
+    rule: InferenceRule;
+    /** The compiled ajv validate function. */
+    validate: (data: unknown) => boolean;
+}
+/**
+ * Build {@link FileAttributes} from a file path and stat info.
+ *
+ * @param filePath - The file path.
+ * @param stats - The file stats.
+ * @param extractedFrontmatter - Optional extracted frontmatter.
+ * @param extractedJson - Optional parsed JSON content.
+ * @returns The constructed file attributes.
+ */
+declare function buildAttributes(filePath: string, stats: Stats, extractedFrontmatter?: Record<string, unknown>, extractedJson?: Record<string, unknown>): FileAttributes;
+/**
+ * Compile an array of inference rules into executable validators.
+ *
+ * @param rules - The inference rule definitions.
+ * @returns An array of compiled rules.
+ */
+declare function compileRules(rules: InferenceRule[]): CompiledRule[];
+/**
+ * Apply compiled inference rules to file attributes, returning merged metadata.
+ *
+ * Rules are evaluated in order; later rules override earlier ones.
+ *
+ * @param compiledRules - The compiled rules to evaluate.
+ * @param attributes - The file attributes to match against.
+ * @returns The merged metadata from all matching rules.
+ */
+declare function applyRules(compiledRules: CompiledRule[], attributes: FileAttributes): Record<string, unknown>;
+
+/**
+ * A point to upsert into the vector store.
+ */
+interface VectorPoint {
+    /** The point ID. */
+    id: string;
+    /** The embedding vector. */
+    vector: number[];
+    /** The payload metadata. */
+    payload: Record<string, unknown>;
+}
+/**
+ * A search result from the vector store.
+ */
+interface SearchResult {
+    /** The point ID. */
+    id: string;
+    /** The similarity score. */
+    score: number;
+    /** The payload metadata. */
+    payload: Record<string, unknown>;
+}
+/**
+ * A scrolled point from the vector store.
+ */
+interface ScrolledPoint {
+    /** The point ID. */
+    id: string;
+    /** The payload metadata. */
+    payload: Record<string, unknown>;
+}
+/**
+ * Client wrapper for Qdrant vector store operations.
+ */
+declare class VectorStoreClient {
+    private readonly client;
+    private readonly collectionName;
+    private readonly dims;
+    /**
+     * Create a new VectorStoreClient.
+     *
+     * @param config - Vector store configuration.
+     * @param dimensions - The embedding vector dimensions.
+     */
+    constructor(config: VectorStoreConfig, dimensions: number);
+    /**
+     * Ensure the collection exists with correct dimensions and Cosine distance.
+     */
+    ensureCollection(): Promise<void>;
+    /**
+     * Upsert points into the collection.
+     *
+     * @param points - The points to upsert.
+     */
+    upsert(points: VectorPoint[]): Promise<void>;
+    /**
+     * Delete points by their IDs.
+     *
+     * @param ids - The point IDs to delete.
+     */
+    delete(ids: string[]): Promise<void>;
+    /**
+     * Set payload fields for the specified point IDs.
+     *
+     * This merges the given payload object into each point's existing payload.
+     *
+     * @param ids - Point IDs to update.
+     * @param payload - Payload fields to set.
+     */
+    setPayload(ids: string[], payload: Record<string, unknown>): Promise<void>;
+    /**
+     * Get the payload of a point by ID.
+     *
+     * @param id - The point ID.
+     * @returns The payload, or `null` if the point doesn't exist.
+     */
+    getPayload(id: string): Promise<Record<string, unknown> | null>;
+    /**
+     * Search for similar vectors.
+     *
+     * @param vector - The query vector.
+     * @param limit - Maximum results to return.
+     * @param filter - Optional Qdrant filter.
+     * @returns An array of search results.
+     */
+    search(vector: number[], limit: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    /**
+     * Scroll through all points matching a filter.
+     *
+     * @param filter - Optional Qdrant filter.
+     * @param limit - Page size for scrolling.
+     * @yields Scrolled points.
+     */
+    scroll(filter?: Record<string, unknown>, limit?: number): AsyncGenerator<ScrolledPoint>;
+}
+
+/**
+ * Core document processing pipeline.
+ *
+ * Handles extracting text, computing embeddings, and syncing with the vector store.
+ */
+declare class DocumentProcessor {
+    private readonly config;
+    private readonly embeddingProvider;
+    private readonly vectorStore;
+    private compiledRules;
+    private readonly logger;
+    private readonly metadataDir;
+    /**
+     * Create a new DocumentProcessor.
+     *
+     * @param config - The application configuration.
+     * @param embeddingProvider - The embedding provider.
+     * @param vectorStore - The vector store client.
+     * @param compiledRules - The compiled inference rules.
+     * @param logger - The logger instance.
+     */
+    constructor(config: JeevesWatcherConfig, embeddingProvider: EmbeddingProvider, vectorStore: VectorStoreClient, compiledRules: CompiledRule[], logger: pino.Logger);
+    /**
+     * Process a file through the full pipeline: extract, hash, chunk, embed, upsert.
+     *
+     * @param filePath - The file to process.
+     */
+    processFile(filePath: string): Promise<void>;
+    /**
+     * Delete all chunks for a file from the vector store and remove metadata.
+     *
+     * @param filePath - The file to delete.
+     */
+    deleteFile(filePath: string): Promise<void>;
+    /**
+     * Process a metadata update: merge metadata, write to disk, update Qdrant payloads (no re-embed).
+     *
+     * @param filePath - The file whose metadata to update.
+     * @param metadata - The new metadata to merge.
+     * @returns The merged payload, or `null` if the file is not indexed.
+     */
+    processMetadataUpdate(filePath: string, metadata: Record<string, unknown>): Promise<Record<string, unknown> | null>;
+    /**
+     * Re-apply inference rules to a file without re-embedding.
+     * Reads file attributes, applies current rules, merges with enrichment metadata,
+     * and updates Qdrant payloads.
+     *
+     * @param filePath - The file to update.
+     * @returns The merged metadata, or `null` if the file is not indexed.
+     */
+    processRulesUpdate(filePath: string): Promise<Record<string, unknown> | null>;
+    /**
+     * Update compiled inference rules for subsequent file processing.
+     *
+     * @param compiledRules - The newly compiled rules.
+     */
+    updateRules(compiledRules: CompiledRule[]): void;
+    /**
+     * Create the appropriate text splitter for the given file extension.
+     *
+     * @param ext - File extension.
+     * @param chunkSize - Maximum chunk size in characters.
+     * @param chunkOverlap - Overlap between chunks in characters.
+     * @returns A text splitter instance.
+     */
+    private createSplitter;
+}
+
+/**
+ * A watch event.
+ */
+type WatchEvent = {
+    /** Event type. */
+    type: 'create' | 'modify' | 'delete';
+    /** File path associated with the event. */
+    path: string;
+    /** Event priority. */
+    priority: 'normal' | 'low';
+};
+/**
+ * Function invoked when a queued event is processed.
+ */
+type ProcessFn = (event: WatchEvent) => Promise<void> | void;
+/**
+ * Options for {@link EventQueue}.
+ */
+interface EventQueueOptions {
+    /** Debounce delay in milliseconds (applied per path+priority key). */
+    debounceMs: number;
+    /** Maximum number of concurrent processors. */
+    concurrency: number;
+    /** Optional max processed events per minute (token bucket). */
+    rateLimitPerMinute?: number;
+}
+/**
+ * A debounced, rate-limited, concurrent event queue.
+ */
+declare class EventQueue {
+    private readonly debounceMs;
+    private readonly concurrency;
+    private readonly rateLimitPerMinute?;
+    private started;
+    private active;
+    private readonly debounceTimers;
+    private readonly latestByKey;
+    private readonly normalQueue;
+    private readonly lowQueue;
+    private tokens;
+    private lastRefillMs;
+    private drainWaiters;
+    /**
+     * Create an event queue.
+     *
+     * @param options - Queue options.
+     */
+    constructor(options: EventQueueOptions);
+    /**
+     * Enqueue an event, debounced per path+priority.
+     *
+     * @param event - The watch event.
+     * @param fn - The processing function to invoke when dequeued.
+     */
+    enqueue(event: WatchEvent, fn: ProcessFn): void;
+    /**
+     * Start processing events.
+     */
+    process(): void;
+    /**
+     * Wait for the queue to become idle (no pending debounces, no queued items, no active work).
+     *
+     * @returns A promise that resolves when the queue is drained.
+     */
+    drain(): Promise<void>;
+    private push;
+    private refillTokens;
+    private takeToken;
+    private nextItem;
+    private pump;
+    private isIdle;
+    private maybeResolveDrain;
+}
+
+/**
+ * Options for {@link createApiServer}.
+ */
+interface ApiServerOptions {
+    /** The document processor. */
+    processor: DocumentProcessor;
+    /** The vector store client. */
+    vectorStore: VectorStoreClient;
+    /** The embedding provider. */
+    embeddingProvider: EmbeddingProvider;
+    /** The event queue. */
+    queue: EventQueue;
+    /** The application configuration. */
+    config: JeevesWatcherConfig;
+    /** The logger instance. */
+    logger: pino.Logger;
+}
+/**
+ * Create the Fastify API server with all routes registered.
+ *
+ * The returned instance is not yet listening — call `server.listen()` to start.
+ *
+ * @param options - The server options.
+ * @returns A configured Fastify instance.
+ */
+declare function createApiServer(options: ApiServerOptions): FastifyInstance;
+
+/**
+ * Main application class that wires together all components.
+ */
+declare class JeevesWatcher {
+    private config;
+    private readonly configPath?;
+    private logger;
+    private watcher;
+    private queue;
+    private server;
+    private processor;
+    private configWatcher;
+    private configDebounce;
+    /**
+     * Create a new JeevesWatcher instance.
+     *
+     * @param config - The application configuration.
+     * @param configPath - Optional config file path to watch for changes.
+     */
+    constructor(config: JeevesWatcherConfig, configPath?: string);
+    /**
+     * Start the watcher, API server, and all components.
+     */
+    start(): Promise<void>;
+    /**
+     * Gracefully stop all components.
+     */
+    stop(): Promise<void>;
+    private startConfigWatch;
+    private stopConfigWatch;
+    private reloadConfig;
+}
+/**
+ * Create and start a JeevesWatcher from a config file path.
+ *
+ * @param configPath - Optional path to the configuration file.
+ * @returns The running JeevesWatcher instance.
+ */
+declare function startFromConfig(configPath?: string): Promise<JeevesWatcher>;
+
+/**
+ * Load the jeeves-watcher configuration.
+ *
+ * @param configPath - Optional explicit path to a config file.
+ * @returns The loaded configuration.
+ * @throws If no configuration is found or validation fails.
+ */
+declare function loadConfig(configPath?: string): Promise<JeevesWatcherConfig>;
+
+/**
+ * Result of extracting text and structured data from a file.
+ */
+interface ExtractedText {
+    /** Extracted plain text content. */
+    text: string;
+    /** Extracted YAML frontmatter (Markdown). */
+    frontmatter?: Record<string, unknown>;
+    /** Parsed JSON object (JSON files). */
+    json?: Record<string, unknown>;
+}
+/**
+ * Extract text from a file based on extension.
+ *
+ * @param filePath - Path to the file.
+ * @param extension - File extension (including leading dot).
+ * @returns Extracted text and optional structured data.
+ */
+declare function extractText(filePath: string, extension: string): Promise<ExtractedText>;
+
+/**
+ * Compute a SHA-256 hex digest of the given text.
+ *
+ * @param text - The input text to hash.
+ * @returns The hex-encoded SHA-256 hash.
+ */
+declare function contentHash(text: string): string;
+
+/**
+ * Create a pino logger instance.
+ *
+ * @param config - Optional logging configuration.
+ * @returns A configured pino logger.
+ */
+declare function createLogger(config?: LoggingConfig): pino.Logger;
+
+/**
+ * Derive a deterministic `.meta.json` path for a given file.
+ *
+ * @param filePath - The watched file path.
+ * @param metadataDir - The root metadata directory.
+ * @returns The full path to the metadata file.
+ */
+declare function metadataPath(filePath: string, metadataDir: string): string;
+/**
+ * Read persisted metadata for a file.
+ *
+ * @param filePath - The watched file path.
+ * @param metadataDir - The root metadata directory.
+ * @returns The parsed metadata object, or `null` if not found.
+ */
+declare function readMetadata(filePath: string, metadataDir: string): Promise<Record<string, unknown> | null>;
+/**
+ * Write metadata for a file.
+ *
+ * @param filePath - The watched file path.
+ * @param metadataDir - The root metadata directory.
+ * @param metadata - The metadata to persist.
+ */
+declare function writeMetadata(filePath: string, metadataDir: string, metadata: Record<string, unknown>): Promise<void>;
+/**
+ * Delete metadata for a file.
+ *
+ * @param filePath - The watched file path.
+ * @param metadataDir - The root metadata directory.
+ */
+declare function deleteMetadata(filePath: string, metadataDir: string): Promise<void>;
+
+/**
+ * Generate a deterministic UUID v5 point ID for a file (and optional chunk index).
+ *
+ * @param filePath - The file path.
+ * @param chunkIndex - Optional chunk index within the file.
+ * @returns A deterministic UUID v5 string.
+ */
+declare function pointId(filePath: string, chunkIndex?: number): string;
+
+/**
+ * Filesystem watcher that maps chokidar events to the processing queue.
+ */
+declare class FileSystemWatcher {
+    private readonly config;
+    private readonly queue;
+    private readonly processor;
+    private readonly logger;
+    private watcher;
+    /**
+     * Create a new FileSystemWatcher.
+     *
+     * @param config - Watch configuration.
+     * @param queue - The event queue.
+     * @param processor - The document processor.
+     * @param logger - The logger instance.
+     */
+    constructor(config: WatchConfig, queue: EventQueue, processor: DocumentProcessor, logger: pino.Logger);
+    /**
+     * Start watching the filesystem and processing events.
+     */
+    start(): void;
+    /**
+     * Stop the filesystem watcher.
+     */
+    stop(): Promise<void>;
+}
+
+export { DocumentProcessor, EventQueue, FileSystemWatcher, JeevesWatcher, VectorStoreClient, applyRules, buildAttributes, compileRules, contentHash, createApiServer, createEmbeddingProvider, createLogger, deleteMetadata, extractText, loadConfig, metadataPath, pointId, readMetadata, startFromConfig, writeMetadata };
+export type { ApiConfig, ApiServerOptions, CompiledRule, ConfigWatchConfig, EmbeddingConfig, EmbeddingProvider, EventQueueOptions, ExtractedText, FileAttributes, InferenceRule, JeevesWatcherConfig, LoggingConfig, ProcessFn, ScrolledPoint, SearchResult, VectorPoint, VectorStoreConfig, WatchConfig, WatchEvent };